mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-10-12 02:24:05 +00:00
feat(xo-server-openmetrics): estimate per-VM power consumption from CPU (#10031)
* feat(xo-server-openmetrics): estimate per-VM power consumption from CPU usage * refactor(xo-server-openmetrics): move hostPowerWatts undefined guard to caller * fix(xo-server-openmetrics): log IPMI power read failure as info
This commit is contained in:
@@ -24,6 +24,7 @@
|
||||
- [VM]: Add possibility to duplicate a VM (PR [#9580](https://github.com/vatesfr/xen-orchestra/pull/9580))
|
||||
- [RPU] Trace rolling pool updates/reboots to disk to allow diagnosis even after xo-server restarts (PR [#10078](https://github.com/vatesfr/xen-orchestra/pull/10078))
|
||||
- [XO6/VM] Group VM power actions in a new "Change state" submenu and isolate the Delete action (PR [#10036](https://github.com/vatesfr/xen-orchestra/pull/10036))
|
||||
- [OpenMetrics] Add an estimated per-VM power consumption metric (`xcp_vm_power_consumption_watts`), splitting each host's IPMI power across its running VMs proportionally to CPU load (PR [#10031](https://github.com/vatesfr/xen-orchestra/pull/10031))
|
||||
|
||||
### Bug fixes
|
||||
|
||||
@@ -60,5 +61,6 @@
|
||||
- xapi-explore-sr patch
|
||||
- xen-api minor
|
||||
- xo-server minor
|
||||
- xo-server-openmetrics minor
|
||||
|
||||
<!--packages-end-->
|
||||
|
||||
@@ -14,6 +14,8 @@ This plugin creates an HTTP server that exposes a `/metrics` endpoint compatible
|
||||
- VM status metric (`xcp_vm_status`) with `power_state` label, including non-running VMs
|
||||
- VM uptime metric (`xcp_vm_uptime_seconds`) for running VMs
|
||||
- VM metrics: CPU, memory, network, disk, runstate
|
||||
- Host power metric (`xcp_host_power_consumption_watts`) read from IPMI, carrying a `source="ipmi"` label
|
||||
- Estimated per-VM power metric (`xcp_vm_power_consumption_watts`, label `estimate="true"`): each running guest's share of its host's power, split proportionally to CPU load (`cpu_usage × vCPUs`); dom0 is excluded and the per-host total is preserved; flagged `data_complete="false"` when a VM's `cpu_usage` is missing
|
||||
- `is_control_domain` label on all VM metrics to distinguish dom0 from user VMs
|
||||
- `tags` label on host, VM and SR metrics carrying the XO tags of the object (comma-separated, sorted), enabling selective monitoring with regex matchers such as `{tags=~"(^|.*,)production(,.*|$)"}`
|
||||
- VDI disk size metrics: virtual size and physical usage per VDI (`xcp_vdi_virtual_size_bytes`, `xcp_vdi_physical_usage_bytes`)
|
||||
|
||||
@@ -22,6 +22,8 @@ This plugin creates an HTTP server that exposes a `/metrics` endpoint compatible
|
||||
- VM status metric (`xcp_vm_status`) with `power_state` label, including non-running VMs
|
||||
- VM uptime metric (`xcp_vm_uptime_seconds`) for running VMs
|
||||
- VM metrics: CPU, memory, network, disk, runstate
|
||||
- Host power metric (`xcp_host_power_consumption_watts`) read from IPMI, carrying a `source="ipmi"` label
|
||||
- Estimated per-VM power metric (`xcp_vm_power_consumption_watts`, label `estimate="true"`): each running guest's share of its host's power, split proportionally to CPU load (`cpu_usage × vCPUs`); dom0 is excluded and the per-host total is preserved; flagged `data_complete="false"` when a VM's `cpu_usage` is missing
|
||||
- `is_control_domain` label on all VM metrics to distinguish dom0 from user VMs
|
||||
- `tags` label on host, VM and SR metrics carrying the XO tags of the object (comma-separated, sorted), enabling selective monitoring with regex matchers such as `{tags=~"(^|.*,)production(,.*|$)"}`
|
||||
- VDI disk size metrics: virtual size and physical usage per VDI (`xcp_vdi_virtual_size_bytes`, `xcp_vdi_physical_usage_bytes`)
|
||||
|
||||
@@ -74,6 +74,8 @@ export interface VmLabelInfo {
|
||||
vifIndexToNetworkName: Record<string, string> // { "0": "Pool-wide network" }
|
||||
startTime: number | null // Unix timestamp of VM boot (from vm.startTime)
|
||||
power_state: string // VM power state (Running, Paused, Halted, Suspended)
|
||||
vcpus?: number // vCPU count (vm.CPUs.max || number || 1), used to weight power share
|
||||
host_id?: string // residence host/pool id (vm.$container); informative label only
|
||||
pool_id: string
|
||||
pool_name: string
|
||||
tags?: string[]
|
||||
@@ -197,6 +199,8 @@ interface VmStatusPayload {
|
||||
vms: VmStatusItem[]
|
||||
}
|
||||
|
||||
type HostPowerPayload = Record<string, number>
|
||||
|
||||
/**
|
||||
* XOSTOR cluster node entry.
|
||||
*
|
||||
@@ -458,6 +462,12 @@ const XOSTOR_UPDATES_CACHE_TTL_MS = 3_600_000
|
||||
/** Timeout for a single `updater.py check_update` plugin call (ms). */
|
||||
const XOSTOR_UPDATES_TIMEOUT_MS = 30_000
|
||||
|
||||
const IPMI_POWER_CACHE_TTL_MS = 60_000
|
||||
|
||||
const IPMI_POWER_TIMEOUT_MS = 10_000
|
||||
|
||||
export const POWER_SENSOR_REGEX = /^(pwr consumption|sys power|system power|power consumption|total_power)$/i
|
||||
|
||||
// ============================================================================
|
||||
// Configuration Schema (exported for xo-server)
|
||||
// ============================================================================
|
||||
@@ -712,6 +722,7 @@ class OpenMetricsPlugin {
|
||||
#xostorHealthCheckCache = new TtlCache<XostorPayload>(XOSTOR_CACHE_TTL_MS)
|
||||
#xostorSmartCache = new TtlCache<XostorSmartPayload>(XOSTOR_SMART_CACHE_TTL_MS)
|
||||
#xostorUpdatesCache = new TtlCache<XostorUpdatesPayload>(XOSTOR_UPDATES_CACHE_TTL_MS)
|
||||
#hostPowerCache = new TtlCache<HostPowerPayload>(IPMI_POWER_CACHE_TTL_MS)
|
||||
|
||||
constructor(xo: XoApp) {
|
||||
this.#xo = xo
|
||||
@@ -939,6 +950,26 @@ class OpenMetricsPlugin {
|
||||
break
|
||||
}
|
||||
|
||||
case 'GET_HOST_POWER': {
|
||||
this.#getHostPowerData()
|
||||
.then((hostPower: HostPowerPayload) => {
|
||||
this.#sendToChildNoWait({
|
||||
type: 'HOST_POWER',
|
||||
requestId: message.requestId,
|
||||
payload: hostPower,
|
||||
})
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
logger.error('Failed to collect host power data', { error: err })
|
||||
this.#sendToChildNoWait({
|
||||
type: 'HOST_POWER',
|
||||
requestId: message.requestId,
|
||||
payload: {},
|
||||
})
|
||||
})
|
||||
break
|
||||
}
|
||||
|
||||
case 'GET_XOSTOR_DATA': {
|
||||
this.#getXostorData()
|
||||
.then((xostorPayload: XostorPayload) => {
|
||||
@@ -1549,6 +1580,58 @@ class OpenMetricsPlugin {
|
||||
}
|
||||
}
|
||||
|
||||
#getHostPowerData(): Promise<HostPowerPayload> {
|
||||
return this.#hostPowerCache.get(() => this.#collectHostPowerData())
|
||||
}
|
||||
|
||||
async #collectHostPowerData(): Promise<HostPowerPayload> {
|
||||
const allHosts = this.#xo.getObjects({ filter: { type: 'host' } }) as Record<string, XoHost>
|
||||
const readings = await Promise.all(Object.values(allHosts).map(host => this.#fetchHostPower(host)))
|
||||
|
||||
const watts: HostPowerPayload = {}
|
||||
for (const reading of readings) {
|
||||
if (reading !== undefined) {
|
||||
watts[reading.hostId] = reading.watts
|
||||
}
|
||||
}
|
||||
logger.debug('Returning host power data', { hostCount: Object.keys(watts).length })
|
||||
return watts
|
||||
}
|
||||
|
||||
async #fetchHostPower(host: XoHost): Promise<{ hostId: string; watts: number } | undefined> {
|
||||
try {
|
||||
const xapi = this.#xo.getXapi(host)
|
||||
const available = await withTimeout(
|
||||
xapi.callAsync<string>('host.call_plugin', host._xapiRef, 'ipmitool.py', 'is_ipmi_device_available', {}),
|
||||
IPMI_POWER_TIMEOUT_MS,
|
||||
'ipmitool.py is_ipmi_device_available timed out'
|
||||
)
|
||||
if (available === 'false') {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const raw = await withTimeout(
|
||||
xapi.callAsync<string>('host.call_plugin', host._xapiRef, 'ipmitool.py', 'get_all_sensors', {}),
|
||||
IPMI_POWER_TIMEOUT_MS,
|
||||
'ipmitool.py get_all_sensors timed out'
|
||||
)
|
||||
const sensors = JSON.parse(raw) as Array<{ name?: string; value?: string }>
|
||||
const sensor = Array.isArray(sensors)
|
||||
? sensors.find(s => typeof s?.name === 'string' && POWER_SENSOR_REGEX.test(s.name))
|
||||
: undefined
|
||||
// ipmitool reports power as a numeric string, sometimes unit-suffixed
|
||||
// (e.g. "210 Watts"); parseFloat strips the suffix, Number() would not.
|
||||
const watts = sensor !== undefined ? Number.parseFloat(String(sensor.value)) : NaN
|
||||
if (!Number.isFinite(watts)) {
|
||||
return undefined
|
||||
}
|
||||
return { hostId: host.uuid, watts }
|
||||
} catch (error) {
|
||||
logger.info('ipmitool.py power read failed', { hostUuid: host.uuid, error })
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect pending XOSTOR-related package updates for every host backing a
|
||||
* XOSTOR PBD.
|
||||
@@ -1912,6 +1995,8 @@ class OpenMetricsPlugin {
|
||||
vifIndexToNetworkName,
|
||||
startTime: vm.startTime ?? null,
|
||||
power_state: vm.power_state,
|
||||
vcpus: vm.CPUs.max || vm.CPUs.number || 1,
|
||||
host_id: vm.$container,
|
||||
pool_id: vm.$poolId,
|
||||
pool_name: poolLabelMap.get(vm.$poolId) ?? '',
|
||||
tags: vm.tags,
|
||||
|
||||
@@ -188,6 +188,7 @@ function handleParentMessage(rawMessage: unknown): void {
|
||||
case 'VDI_DATA':
|
||||
case 'HOST_STATUS':
|
||||
case 'VM_STATUS':
|
||||
case 'HOST_POWER':
|
||||
case 'XO_METRICS':
|
||||
case 'XOSTOR_DATA':
|
||||
case 'XOSTOR_ALARMS':
|
||||
@@ -362,6 +363,25 @@ async function requestVmStatusData(): Promise<VmStatusPayload> {
|
||||
})
|
||||
}
|
||||
|
||||
async function requestHostPowerData(): Promise<Record<string, number>> {
|
||||
const requestId = `host-power-${++requestIdCounter}`
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
pendingRequests.delete(requestId)
|
||||
reject(new Error('Timeout waiting for host power data from parent'))
|
||||
}, IPC_REQUEST_TIMEOUT_MS)
|
||||
|
||||
pendingRequests.set(requestId, {
|
||||
resolve: value => resolve(value as Record<string, number>),
|
||||
reject,
|
||||
timer,
|
||||
})
|
||||
|
||||
sendToParent({ type: 'GET_HOST_POWER', requestId })
|
||||
})
|
||||
}
|
||||
|
||||
async function requestXoMetrics(): Promise<XoMetricsData> {
|
||||
const requestId = `xo-metrics-${++requestIdCounter}`
|
||||
|
||||
@@ -472,7 +492,7 @@ async function requestXostorUpdates(): Promise<XostorUpdatesPayload> {
|
||||
* @returns ParsedRrdData or null on error
|
||||
*/
|
||||
async function fetchRrdFromHost(host: HostCredentials): Promise<ParsedRrdData | null> {
|
||||
const { hostAddress, hostLabel, sessionId, poolId, poolLabel, protocol } = host
|
||||
const { hostId, hostAddress, hostLabel, sessionId, poolId, poolLabel, protocol } = host
|
||||
|
||||
// Calculate start time: current time minus 2 intervals (to ensure we get recent data)
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
@@ -510,7 +530,7 @@ async function fetchRrdFromHost(host: HostCredentials): Promise<ParsedRrdData |
|
||||
|
||||
try {
|
||||
// Parse RRD response using the dedicated parser (handles JSON5 fallback)
|
||||
return parseRrdResponse(text, poolId)
|
||||
return parseRrdResponse(text, poolId, hostId)
|
||||
} catch (parseError) {
|
||||
logger.warn('RRD parse failed', { hostLabel, poolLabel, error: parseError })
|
||||
return null
|
||||
@@ -558,6 +578,7 @@ async function collectMetrics(): Promise<string> {
|
||||
xostorAlarms,
|
||||
xostorSmart,
|
||||
xostorUpdates,
|
||||
hostPowerData,
|
||||
] = await Promise.all([
|
||||
requestXapiCredentials(),
|
||||
requestSrData(),
|
||||
@@ -569,6 +590,10 @@ async function collectMetrics(): Promise<string> {
|
||||
safeXostorRequest(requestXostorAlarms(), 'alarms', { clusters: [] } as XostorAlarmsPayload),
|
||||
safeXostorRequest(requestXostorSmart(), 'SMART', { hosts: [] } as XostorSmartPayload),
|
||||
safeXostorRequest(requestXostorUpdates(), 'updates', { hosts: [] } as XostorUpdatesPayload),
|
||||
requestHostPowerData().catch((err: unknown) => {
|
||||
logger.warn('Host power request failed; emitting no host power', { error: err })
|
||||
return {} as Record<string, number>
|
||||
}),
|
||||
])
|
||||
|
||||
logger.debug('Collecting metrics', {
|
||||
@@ -627,7 +652,7 @@ async function collectMetrics(): Promise<string> {
|
||||
hostCount: rrdDataList.length,
|
||||
totalMetrics: rrdDataList.reduce((sum, d) => sum + d.metrics.length, 0),
|
||||
})
|
||||
const rrdMetrics = formatAllPoolsToOpenMetrics(rrdDataList, credentials)
|
||||
const rrdMetrics = formatAllPoolsToOpenMetrics(rrdDataList, credentials, hostPowerData)
|
||||
logger.debug('Formatted metrics', { outputLength: rrdMetrics.length })
|
||||
|
||||
// Format SR capacity metrics
|
||||
|
||||
@@ -735,6 +735,139 @@ function computeVmCpuUsageFallback(metrics: FormattedMetric[]): FormattedMetric[
|
||||
return syntheticMetrics
|
||||
}
|
||||
|
||||
function emitHostPowerMetrics(
|
||||
metrics: FormattedMetric[],
|
||||
hostPowerWatts: Record<string, number>,
|
||||
labelContext?: LabelContext
|
||||
): void {
|
||||
for (const [hostId, watts] of Object.entries(hostPowerWatts)) {
|
||||
// Drop any existing host power series for this host (dedup, IPMI preferred).
|
||||
for (let i = metrics.length - 1; i >= 0; i--) {
|
||||
const m = metrics[i]
|
||||
if (m !== undefined && m.name === 'xcp_host_power_consumption_watts' && m.labels.uuid === hostId) {
|
||||
metrics.splice(i, 1)
|
||||
}
|
||||
}
|
||||
|
||||
const labels: Record<string, string> = { uuid: hostId, type: 'host', source: 'ipmi' }
|
||||
const hostName = labelContext?.labels.hosts[hostId]?.name_label
|
||||
if (hostName !== undefined && hostName !== '') {
|
||||
labels.host_name = hostName
|
||||
}
|
||||
|
||||
metrics.push({
|
||||
name: 'xcp_host_power_consumption_watts',
|
||||
help: 'Host power consumption in watts (IPMI)',
|
||||
type: 'gauge',
|
||||
labels,
|
||||
value: watts,
|
||||
timestamp: Math.floor(Date.now() / 1000),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function computeVmPowerConsumption(metrics: FormattedMetric[], labelContext?: LabelContext): FormattedMetric[] {
|
||||
// Labels carried by per-resource VM metrics that must not leak onto the
|
||||
// aggregate per-VM power series.
|
||||
const PER_RESOURCE_LABELS = new Set([
|
||||
'core',
|
||||
'device',
|
||||
'vdi_name',
|
||||
'sr_name',
|
||||
'sr_type',
|
||||
'vif',
|
||||
'network_name',
|
||||
'interface',
|
||||
])
|
||||
|
||||
const hostPower = new Map<string, { watts: number; timestamp: number }>()
|
||||
const guestsByHost = new Map<string, Map<string, FormattedMetric>>() // hostId -> vmUuid -> representative metric
|
||||
const cpuByVm = new Map<string, { value: number; timestamp: number }>()
|
||||
|
||||
for (const metric of metrics) {
|
||||
const { labels } = metric
|
||||
if (labels.uuid === undefined) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (labels.type === 'host' && metric.name === 'xcp_host_power_consumption_watts') {
|
||||
hostPower.set(labels.uuid, { watts: metric.value, timestamp: metric.timestamp })
|
||||
continue
|
||||
}
|
||||
|
||||
if (labels.type !== 'vm') {
|
||||
continue
|
||||
}
|
||||
const hostId = labels.host_id
|
||||
// No residence host or dom0 → takes no share.
|
||||
if (hostId === undefined || hostId === '' || labels.is_control_domain === 'true') {
|
||||
continue
|
||||
}
|
||||
|
||||
let guests = guestsByHost.get(hostId)
|
||||
if (guests === undefined) {
|
||||
guests = new Map()
|
||||
guestsByHost.set(hostId, guests)
|
||||
}
|
||||
if (!guests.has(labels.uuid)) {
|
||||
guests.set(labels.uuid, metric)
|
||||
}
|
||||
|
||||
if (metric.name === 'xcp_vm_cpu_usage') {
|
||||
cpuByVm.set(labels.uuid, { value: metric.value, timestamp: metric.timestamp })
|
||||
}
|
||||
}
|
||||
|
||||
const out: FormattedMetric[] = []
|
||||
|
||||
for (const [hostId, power] of hostPower) {
|
||||
const guests = guestsByHost.get(hostId)
|
||||
if (guests === undefined || guests.size === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
const loadByVm = new Map<string, { load: number; complete: boolean }>()
|
||||
let totalLoad = 0
|
||||
for (const vmUuid of guests.keys()) {
|
||||
const cpu = cpuByVm.get(vmUuid)?.value
|
||||
const vcpus = labelContext?.labels.vms[vmUuid]?.vcpus ?? 1
|
||||
const load = (cpu ?? 0) * vcpus
|
||||
loadByVm.set(vmUuid, { load, complete: cpu !== undefined })
|
||||
totalLoad += load
|
||||
}
|
||||
|
||||
const equalSplit = totalLoad < 0.01
|
||||
|
||||
for (const [vmUuid, representative] of guests) {
|
||||
const entry = loadByVm.get(vmUuid)
|
||||
const load = entry?.load ?? 0
|
||||
const watts = equalSplit ? power.watts / guests.size : (power.watts * load) / totalLoad
|
||||
|
||||
const labels: Record<string, string> = {}
|
||||
for (const [key, value] of Object.entries(representative.labels)) {
|
||||
if (!PER_RESOURCE_LABELS.has(key)) {
|
||||
labels[key] = value
|
||||
}
|
||||
}
|
||||
labels.estimate = 'true'
|
||||
if (entry?.complete === false) {
|
||||
labels.data_complete = 'false'
|
||||
}
|
||||
|
||||
out.push({
|
||||
name: 'xcp_vm_power_consumption_watts',
|
||||
help: 'Estimated VM power consumption in watts (estimate, host power split by CPU load)',
|
||||
type: 'gauge',
|
||||
labels,
|
||||
value: watts,
|
||||
timestamp: cpuByVm.get(vmUuid)?.timestamp ?? power.timestamp,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Formatting Functions
|
||||
// ============================================================================
|
||||
@@ -805,7 +938,8 @@ function formatLabels(labels: Record<string, string>): string {
|
||||
export function transformMetric(
|
||||
metric: ParsedMetric,
|
||||
poolId: string,
|
||||
labelContext?: LabelContext
|
||||
labelContext?: LabelContext,
|
||||
hostId?: string
|
||||
): FormattedMetric | null {
|
||||
const { legend, value, timestamp } = metric
|
||||
|
||||
@@ -839,6 +973,10 @@ export function transformMetric(
|
||||
type: legend.objectType,
|
||||
}
|
||||
|
||||
if (legend.objectType === 'vm' && hostId !== undefined && hostId !== '') {
|
||||
labels.host_id = hostId
|
||||
}
|
||||
|
||||
let objectTags: readonly string[] | undefined
|
||||
|
||||
// Add pool_name from host credentials
|
||||
@@ -886,6 +1024,12 @@ export function transformMetric(
|
||||
}
|
||||
|
||||
if (legend.objectType === 'vm') {
|
||||
if (hostId !== undefined && hostId !== '') {
|
||||
const residentHost = labelContext.labels.hosts[hostId]
|
||||
if (residentHost !== undefined && residentHost.name_label !== '') {
|
||||
labels.host_name = residentHost.name_label
|
||||
}
|
||||
}
|
||||
const vmInfo = labelContext.labels.vms[legend.uuid]
|
||||
if (vmInfo !== undefined) {
|
||||
if (vmInfo.name_label !== '') {
|
||||
@@ -1003,15 +1147,20 @@ export function formatToOpenMetrics(metrics: FormattedMetric[]): string {
|
||||
*
|
||||
* @param rrdDataList - Array of ParsedRrdData from all pools
|
||||
* @param labelContext - Optional label context for enriching metrics with human-readable names
|
||||
* @param hostPowerWatts - Optional map of host UUID -> power draw in watts (IPMI)
|
||||
* @returns Complete OpenMetrics output string with EOF marker
|
||||
*/
|
||||
export function formatAllPoolsToOpenMetrics(rrdDataList: ParsedRrdData[], labelContext?: LabelContext): string {
|
||||
export function formatAllPoolsToOpenMetrics(
|
||||
rrdDataList: ParsedRrdData[],
|
||||
labelContext?: LabelContext,
|
||||
hostPowerWatts?: Record<string, number>
|
||||
): string {
|
||||
const allMetrics: FormattedMetric[] = []
|
||||
const unmatchedMetrics: Set<string> = new Set()
|
||||
|
||||
for (const rrdData of rrdDataList) {
|
||||
for (const metric of rrdData.metrics) {
|
||||
const formatted = transformMetric(metric, rrdData.poolId, labelContext)
|
||||
const formatted = transformMetric(metric, rrdData.poolId, labelContext, rrdData.hostId)
|
||||
if (formatted !== null) {
|
||||
allMetrics.push(formatted)
|
||||
} else if (metric.value !== null) {
|
||||
@@ -1026,10 +1175,15 @@ export function formatAllPoolsToOpenMetrics(rrdDataList: ParsedRrdData[], labelC
|
||||
logger.debug('Unmatched RRD metrics', { metrics: Array.from(unmatchedMetrics).sort() })
|
||||
}
|
||||
|
||||
if (hostPowerWatts !== undefined) {
|
||||
emitHostPowerMetrics(allMetrics, hostPowerWatts, labelContext)
|
||||
}
|
||||
|
||||
// Compute fallback vm_cpu_usage for VMs that don't have the native metric
|
||||
// This handles XCP-ng 8.2 and other versions that only report per-core metrics
|
||||
const syntheticCpuMetrics = computeVmCpuUsageFallback(allMetrics)
|
||||
allMetrics.push(...syntheticCpuMetrics)
|
||||
allMetrics.push(...computeVmPowerConsumption(allMetrics, labelContext))
|
||||
|
||||
// Sort metrics by name for consistent output
|
||||
allMetrics.sort((a, b) => {
|
||||
|
||||
@@ -27,7 +27,7 @@ import {
|
||||
type LabelContext,
|
||||
} from './openmetric-formatter.mjs'
|
||||
|
||||
import { indexSrUuidTruncations } from './index.mjs'
|
||||
import { indexSrUuidTruncations, POWER_SENSOR_REGEX } from './index.mjs'
|
||||
|
||||
import type {
|
||||
HostStatusItem,
|
||||
@@ -1240,6 +1240,225 @@ describe('formatAllPoolsToOpenMetrics with labelContext', () => {
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// VM power consumption (estimate) tests
|
||||
// ============================================================================
|
||||
|
||||
describe('xcp_vm_power_consumption_watts', () => {
|
||||
// Label context whose VMs carry vcpus / host_id, keyed by uuid.
|
||||
const labelContext = (
|
||||
vms: Record<string, { vcpus: number; is_control_domain?: boolean; host_id?: string }>
|
||||
): LabelContext => ({
|
||||
hosts: [
|
||||
{
|
||||
hostId: 'host-1',
|
||||
hostAddress: '10.0.0.1',
|
||||
hostLabel: 'Host 1',
|
||||
poolId: 'pool-1',
|
||||
poolLabel: 'Prod',
|
||||
sessionId: 's',
|
||||
protocol: 'https:',
|
||||
},
|
||||
{
|
||||
hostId: 'host-2',
|
||||
hostAddress: '10.0.0.2',
|
||||
hostLabel: 'Host 2',
|
||||
poolId: 'pool-1',
|
||||
poolLabel: 'Prod',
|
||||
sessionId: 's',
|
||||
protocol: 'https:',
|
||||
},
|
||||
],
|
||||
labels: {
|
||||
vms: Object.fromEntries(
|
||||
Object.entries(vms).map(([uuid, v]) => [
|
||||
uuid,
|
||||
{
|
||||
name_label: uuid,
|
||||
is_control_domain: v.is_control_domain ?? false,
|
||||
vbdDeviceToVdiName: {},
|
||||
vbdDeviceToVdiUuid: {},
|
||||
vifIndexToNetworkName: {},
|
||||
startTime: null,
|
||||
power_state: 'Running',
|
||||
vcpus: v.vcpus,
|
||||
host_id: v.host_id ?? 'host-1',
|
||||
pool_id: 'pool-1',
|
||||
pool_name: 'Prod',
|
||||
},
|
||||
])
|
||||
),
|
||||
hosts: {
|
||||
'host-1': { name_label: 'Host 1', pifDeviceToNetworkName: {}, startTime: null },
|
||||
'host-2': { name_label: 'Host 2', pifDeviceToNetworkName: {}, startTime: null },
|
||||
},
|
||||
srs: {},
|
||||
srTruncatedToUuid: {},
|
||||
vdiUuidToSrUuid: {},
|
||||
},
|
||||
})
|
||||
|
||||
const cpuUsage = (uuid: string, value: number): ParsedMetric => ({
|
||||
legend: {
|
||||
cf: 'AVERAGE',
|
||||
objectType: 'vm',
|
||||
uuid,
|
||||
metricName: 'cpu_usage',
|
||||
rawLegend: `AVERAGE:vm:${uuid}:cpu_usage`,
|
||||
},
|
||||
value,
|
||||
timestamp: 1700000000,
|
||||
})
|
||||
const memory = (uuid: string, value: number): ParsedMetric => ({
|
||||
legend: {
|
||||
cf: 'AVERAGE',
|
||||
objectType: 'vm',
|
||||
uuid,
|
||||
metricName: 'memory_internal_free',
|
||||
rawLegend: `AVERAGE:vm:${uuid}:memory_internal_free`,
|
||||
},
|
||||
value,
|
||||
timestamp: 1700000000,
|
||||
})
|
||||
const hostFetch = (hostId: string, metrics: ParsedMetric[]): ParsedRrdData => ({
|
||||
poolId: 'pool-1',
|
||||
hostId,
|
||||
timestamp: 1700000000,
|
||||
metrics,
|
||||
})
|
||||
|
||||
// Parse `name{labels} value timestamp` lines for a metric name.
|
||||
const series = (output: string, name: string): Array<{ line: string; value: number }> =>
|
||||
output
|
||||
.split('\n')
|
||||
.filter(l => l.startsWith(`${name}{`))
|
||||
.map(l => ({
|
||||
line: l,
|
||||
value: Number(
|
||||
l
|
||||
.slice(l.lastIndexOf('}') + 1)
|
||||
.trim()
|
||||
.split(/\s+/)[0]
|
||||
),
|
||||
}))
|
||||
const uuidOf = (line: string): string => line.match(/uuid="([^"]+)"/)![1]!
|
||||
|
||||
it('splits host power across running guests proportional to cpu_usage × vCPUs (conservation)', () => {
|
||||
// loads: vm-a = 0.25×4 = 1, vm-b = 0.75×4 = 3, total 4 → 50 / 150 W of 200 W
|
||||
const out = formatAllPoolsToOpenMetrics(
|
||||
[hostFetch('host-1', [cpuUsage('vm-a', 0.25), cpuUsage('vm-b', 0.75)])],
|
||||
labelContext({ 'vm-a': { vcpus: 4 }, 'vm-b': { vcpus: 4 } }),
|
||||
{ 'host-1': 200 }
|
||||
)
|
||||
const lines = series(out, 'xcp_vm_power_consumption_watts')
|
||||
assert.equal(lines.length, 2)
|
||||
const byVm = Object.fromEntries(lines.map(l => [uuidOf(l.line), l.value]))
|
||||
assert.equal(byVm['vm-a'], 50)
|
||||
assert.equal(byVm['vm-b'], 150)
|
||||
assert.equal(
|
||||
lines.reduce((s, l) => s + l.value, 0),
|
||||
200
|
||||
) // Σ == host power
|
||||
assert.ok(lines.every(l => l.line.includes('estimate="true"')))
|
||||
assert.ok(lines.every(l => l.line.includes('host_id="host-1"')))
|
||||
})
|
||||
|
||||
it('emits no VM series for a host without a power reading', () => {
|
||||
const out = formatAllPoolsToOpenMetrics(
|
||||
[hostFetch('host-1', [cpuUsage('vm-a', 0.5)])],
|
||||
labelContext({ 'vm-a': { vcpus: 2 } }),
|
||||
{} // no power for host-1
|
||||
)
|
||||
assert.equal(series(out, 'xcp_vm_power_consumption_watts').length, 0)
|
||||
})
|
||||
|
||||
it('excludes dom0 from numerator and denominator', () => {
|
||||
const out = formatAllPoolsToOpenMetrics(
|
||||
[hostFetch('host-1', [cpuUsage('vm-a', 0.5), cpuUsage('dom0', 5)])],
|
||||
labelContext({ 'vm-a': { vcpus: 1 }, dom0: { vcpus: 1, is_control_domain: true } }),
|
||||
{ 'host-1': 100 }
|
||||
)
|
||||
const lines = series(out, 'xcp_vm_power_consumption_watts')
|
||||
assert.equal(lines.length, 1)
|
||||
assert.ok(lines[0]!.line.includes('uuid="vm-a"'))
|
||||
assert.equal(lines[0]!.value, 100) // dom0 took no share
|
||||
})
|
||||
|
||||
it('falls back to equal split when the host is quasi-idle (Σ load < 0.01)', () => {
|
||||
const out = formatAllPoolsToOpenMetrics(
|
||||
[hostFetch('host-1', [cpuUsage('vm-a', 0), cpuUsage('vm-b', 0)])],
|
||||
labelContext({ 'vm-a': { vcpus: 2 }, 'vm-b': { vcpus: 2 } }),
|
||||
{ 'host-1': 100 }
|
||||
)
|
||||
const lines = series(out, 'xcp_vm_power_consumption_watts')
|
||||
assert.equal(lines.length, 2)
|
||||
assert.ok(lines.every(l => l.value === 50))
|
||||
assert.equal(
|
||||
lines.reduce((s, l) => s + l.value, 0),
|
||||
100
|
||||
)
|
||||
})
|
||||
|
||||
it('flags a running VM with an RRD gap (no cpu_usage) as data_complete="false" with 0 W', () => {
|
||||
// vm-a load 1 (takes all power); vm-b present only via a memory metric → load 0, incomplete
|
||||
const out = formatAllPoolsToOpenMetrics(
|
||||
[hostFetch('host-1', [cpuUsage('vm-a', 1), memory('vm-b', 1024)])],
|
||||
labelContext({ 'vm-a': { vcpus: 1 }, 'vm-b': { vcpus: 1 } }),
|
||||
{ 'host-1': 100 }
|
||||
)
|
||||
const lines = series(out, 'xcp_vm_power_consumption_watts')
|
||||
const byVm = Object.fromEntries(lines.map(l => [uuidOf(l.line), l]))
|
||||
assert.equal(byVm['vm-a']!.value, 100)
|
||||
assert.equal(byVm['vm-b']!.value, 0)
|
||||
assert.ok(byVm['vm-b']!.line.includes('data_complete="false"'))
|
||||
assert.ok(!byVm['vm-a']!.line.includes('data_complete'))
|
||||
assert.equal(
|
||||
lines.reduce((s, l) => s + l.value, 0),
|
||||
100
|
||||
) // conservation preserved
|
||||
})
|
||||
|
||||
it('does not mix VMs across hosts', () => {
|
||||
const out = formatAllPoolsToOpenMetrics(
|
||||
[hostFetch('host-1', [cpuUsage('vm-a', 0.5)]), hostFetch('host-2', [cpuUsage('vm-b', 0.5)])],
|
||||
labelContext({ 'vm-a': { vcpus: 1, host_id: 'host-1' }, 'vm-b': { vcpus: 1, host_id: 'host-2' } }),
|
||||
{ 'host-1': 200, 'host-2': 100 }
|
||||
)
|
||||
const byVm = Object.fromEntries(series(out, 'xcp_vm_power_consumption_watts').map(l => [uuidOf(l.line), l.value]))
|
||||
assert.equal(byVm['vm-a'], 200)
|
||||
assert.equal(byVm['vm-b'], 100)
|
||||
})
|
||||
|
||||
it('emits nothing for powered-off VMs and labels IPMI host power source="ipmi"', () => {
|
||||
const out = formatAllPoolsToOpenMetrics(
|
||||
[hostFetch('host-1', [cpuUsage('vm-a', 0.5)])],
|
||||
labelContext({ 'vm-a': { vcpus: 1 }, 'vm-off': { vcpus: 1 } }), // vm-off absent from RRD
|
||||
{ 'host-1': 100 }
|
||||
)
|
||||
const lines = series(out, 'xcp_vm_power_consumption_watts')
|
||||
assert.equal(lines.length, 1)
|
||||
assert.ok(!lines.some(l => l.line.includes('vm-off')))
|
||||
assert.ok(/xcp_host_power_consumption_watts\{[^}]*source="ipmi"/.test(out))
|
||||
})
|
||||
})
|
||||
|
||||
describe('POWER_SENSOR_REGEX', () => {
|
||||
it('matches the supported vendor power-sensor names and nothing else', () => {
|
||||
for (const name of ['pwr consumption', 'Sys Power', 'system power', 'power consumption', 'total_power']) {
|
||||
assert.ok(POWER_SENSOR_REGEX.test(name), name)
|
||||
}
|
||||
for (const name of ['cpu temp', 'fan1', 'psu1 status', 'voltage 1']) {
|
||||
assert.ok(!POWER_SENSOR_REGEX.test(name), name)
|
||||
}
|
||||
})
|
||||
|
||||
it('parseFloat strips a unit suffix from the reading', () => {
|
||||
assert.equal(Number.parseFloat('210 Watts'), 210)
|
||||
assert.equal(Number.parseFloat('195'), 195)
|
||||
assert.ok(Number.isNaN(Number.parseFloat('N/A')))
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// CPU Usage Fallback Tests
|
||||
// ============================================================================
|
||||
|
||||
@@ -62,6 +62,10 @@ export interface ParsedMetric {
|
||||
export interface ParsedRrdData {
|
||||
/** Pool UUID */
|
||||
poolId: string
|
||||
/**
|
||||
* UUID of the host this data was fetched from, when known.
|
||||
*/
|
||||
hostId?: string
|
||||
/** Timestamp of the data in seconds */
|
||||
timestamp: number
|
||||
/** All parsed metrics */
|
||||
@@ -177,7 +181,7 @@ export function parseRrdText(text: string): RrdResponse {
|
||||
* @param poolId - Pool UUID for labeling
|
||||
* @returns ParsedRrdData with all valid metrics
|
||||
*/
|
||||
export function parseRrdData(rrd: RrdResponse, poolId: string): ParsedRrdData {
|
||||
export function parseRrdData(rrd: RrdResponse, poolId: string, hostId?: string): ParsedRrdData {
|
||||
const { meta, data } = rrd
|
||||
|
||||
// Use the most recent data point (last in array after reversal)
|
||||
@@ -185,6 +189,7 @@ export function parseRrdData(rrd: RrdResponse, poolId: string): ParsedRrdData {
|
||||
if (data.length === 0) {
|
||||
return {
|
||||
poolId,
|
||||
hostId,
|
||||
timestamp: meta.end,
|
||||
metrics: [],
|
||||
}
|
||||
@@ -195,6 +200,7 @@ export function parseRrdData(rrd: RrdResponse, poolId: string): ParsedRrdData {
|
||||
if (latestData === undefined) {
|
||||
return {
|
||||
poolId,
|
||||
hostId,
|
||||
timestamp: meta.end,
|
||||
metrics: [],
|
||||
}
|
||||
@@ -225,6 +231,7 @@ export function parseRrdData(rrd: RrdResponse, poolId: string): ParsedRrdData {
|
||||
|
||||
return {
|
||||
poolId,
|
||||
hostId,
|
||||
timestamp,
|
||||
metrics,
|
||||
}
|
||||
@@ -239,7 +246,7 @@ export function parseRrdData(rrd: RrdResponse, poolId: string): ParsedRrdData {
|
||||
* @param poolId - Pool UUID for labeling
|
||||
* @returns ParsedRrdData with all valid metrics
|
||||
*/
|
||||
export function parseRrdResponse(text: string, poolId: string): ParsedRrdData {
|
||||
export function parseRrdResponse(text: string, poolId: string, hostId?: string): ParsedRrdData {
|
||||
const rrd = parseRrdText(text)
|
||||
return parseRrdData(rrd, poolId)
|
||||
return parseRrdData(rrd, poolId, hostId)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user