Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
428 lines
15 KiB
Rust
428 lines
15 KiB
Rust
//! Metrics poll loop and observable gauges.
|
|
//!
|
|
//! A background task polls the sources every `metrics_interval_secs` and
|
|
//! replaces a shared snapshot. Observable gauge callbacks read the latest
|
|
//! snapshot, so units or containers that vanish stop being reported.
|
|
|
|
use std::collections::HashMap;
|
|
use std::sync::{Arc, RwLock};
|
|
use std::time::Duration;
|
|
|
|
use futures::StreamExt;
|
|
use opentelemetry::metrics::MeterProvider as _;
|
|
use opentelemetry::KeyValue;
|
|
use opentelemetry_sdk::metrics::SdkMeterProvider;
|
|
use tokio::task::JoinHandle;
|
|
|
|
use super::any_glob;
|
|
use crate::config::OtelMetricsConfig;
|
|
use crate::sources::{ContainerSummary, TimerInfo, UnitSummary};
|
|
use crate::state::AppState;
|
|
|
|
/// Per-call bound so a hung backend cannot stall the poll loop.
|
|
const CALL_TIMEOUT: Duration = Duration::from_secs(10);
|
|
|
|
/// Metric name and UCUM unit.
|
|
pub const METRICS: [(&str, &str); 8] = [
|
|
("systemd.unit.active", "1"),
|
|
("systemd.unit.failed", "1"),
|
|
("systemd.unit.restarts", "{restart}"),
|
|
// Absolute unix timestamp of the next elapse, not a duration.
|
|
("systemd.timer.next_elapse_timestamp_seconds", "s"),
|
|
("docker.container.running", "1"),
|
|
("docker.container.healthy", "1"),
|
|
("docker.container.uptime_seconds", "s"),
|
|
("docker.container.restarts", "{restart}"),
|
|
];
|
|
|
|
/// Overall budget for the per-unit restart-count polling.
|
|
const RESTARTS_BUDGET: Duration = Duration::from_secs(15);
|
|
const RESTARTS_CONCURRENCY: usize = 8;
|
|
|
|
/// Latest poll result, already filtered by the allowlists.
|
|
#[derive(Debug, Default, Clone)]
|
|
pub struct Snapshot {
|
|
pub units: Vec<UnitSummary>,
|
|
/// Automatic restart count per unit name (services only).
|
|
pub restarts: HashMap<String, u32>,
|
|
pub timers: Vec<TimerInfo>,
|
|
pub containers: Vec<ContainerSummary>,
|
|
}
|
|
|
|
/// One gauge data point.
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub struct Obs {
|
|
pub metric: &'static str,
|
|
pub value: f64,
|
|
pub attrs: Vec<(&'static str, String)>,
|
|
}
|
|
|
|
/// empty list means `*.service` and `*.timer`; otherwise a glob allowlist.
|
|
pub fn unit_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool {
|
|
if cfg.units.is_empty() {
|
|
name.ends_with(".service") || name.ends_with(".timer")
|
|
} else {
|
|
any_glob(&cfg.units, name)
|
|
}
|
|
}
|
|
|
|
/// Empty list means all containers.
|
|
pub fn container_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool {
|
|
cfg.containers.is_empty() || any_glob(&cfg.containers, name)
|
|
}
|
|
|
|
fn unit_id(name: &str) -> Vec<(&'static str, String)> {
|
|
vec![("systemd.unit", name.to_owned())]
|
|
}
|
|
|
|
fn container_id(c: &ContainerSummary) -> Vec<(&'static str, String)> {
|
|
vec![("container.name", c.name.clone())]
|
|
}
|
|
|
|
/// Map a snapshot to gauge observations. Series carry identity attributes
|
|
/// only (`systemd.unit` / `container.name`) so state changes do not churn
|
|
/// series. The `active` gauge also carries sub/load state, and `running`
|
|
/// carries the image name.
|
|
pub fn observations(s: &Snapshot) -> Vec<Obs> {
|
|
let b = |v: bool| if v { 1.0 } else { 0.0 };
|
|
let mut out = Vec::new();
|
|
for u in &s.units {
|
|
let mut active_attrs = unit_id(&u.name);
|
|
active_attrs.push(("systemd.sub_state", u.sub_state.clone()));
|
|
active_attrs.push(("systemd.load_state", u.load_state.clone()));
|
|
out.push(Obs {
|
|
metric: "systemd.unit.active",
|
|
value: b(u.active_state == "active"),
|
|
attrs: active_attrs,
|
|
});
|
|
out.push(Obs {
|
|
metric: "systemd.unit.failed",
|
|
value: b(u.active_state == "failed"),
|
|
attrs: unit_id(&u.name),
|
|
});
|
|
if let Some(n) = s.restarts.get(&u.name) {
|
|
out.push(Obs {
|
|
metric: "systemd.unit.restarts",
|
|
value: f64::from(*n),
|
|
attrs: unit_id(&u.name),
|
|
});
|
|
}
|
|
}
|
|
for t in &s.timers {
|
|
// Unscheduled timers are omitted.
|
|
if let Some(next) = t.next_elapse {
|
|
out.push(Obs {
|
|
metric: "systemd.timer.next_elapse_timestamp_seconds",
|
|
value: next.as_second() as f64,
|
|
attrs: unit_id(&t.name),
|
|
});
|
|
}
|
|
}
|
|
for c in &s.containers {
|
|
let mut running_attrs = container_id(c);
|
|
running_attrs.push(("container.image.name", c.image.clone()));
|
|
out.push(Obs {
|
|
metric: "docker.container.running",
|
|
value: b(c.state == "running"),
|
|
attrs: running_attrs,
|
|
});
|
|
if let Some(h) = &c.health {
|
|
out.push(Obs {
|
|
metric: "docker.container.healthy",
|
|
value: b(h == "healthy"),
|
|
attrs: container_id(c),
|
|
});
|
|
}
|
|
if let Some(up) = c.uptime_seconds {
|
|
out.push(Obs {
|
|
metric: "docker.container.uptime_seconds",
|
|
value: up as f64,
|
|
attrs: container_id(c),
|
|
});
|
|
}
|
|
if let Some(r) = c.restart_count {
|
|
out.push(Obs {
|
|
metric: "docker.container.restarts",
|
|
value: r as f64,
|
|
attrs: container_id(c),
|
|
});
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Poll the sources. A section whose call fails or times out keeps its value
|
|
/// from `prev` instead of being emptied.
|
|
async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot {
|
|
let cfg = &state.config.otel.metrics;
|
|
let mut snap = prev.clone();
|
|
|
|
match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_units(None)).await {
|
|
Ok(Ok(units)) => {
|
|
snap.units = units
|
|
.into_iter()
|
|
.filter(|u| unit_in_scope(cfg, &u.name))
|
|
.collect();
|
|
}
|
|
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_units failed"),
|
|
Err(_) => tracing::debug!("otel metrics: list_units timed out"),
|
|
}
|
|
match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_timers()).await {
|
|
Ok(Ok(timers)) => {
|
|
snap.timers = timers
|
|
.into_iter()
|
|
.filter(|t| unit_in_scope(cfg, &t.name))
|
|
.collect();
|
|
}
|
|
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_timers failed"),
|
|
Err(_) => tracing::debug!("otel metrics: list_timers timed out"),
|
|
}
|
|
|
|
// Restart counts: bounded concurrency and an overall budget. Entries not
|
|
// refreshed in time keep their previous value; vanished units are dropped.
|
|
let services: Vec<String> = snap
|
|
.units
|
|
.iter()
|
|
.filter(|u| u.name.ends_with(".service") && u.load_state == "loaded")
|
|
.map(|u| u.name.clone())
|
|
.collect();
|
|
let mut restarts: HashMap<String, u32> = services
|
|
.iter()
|
|
.filter_map(|n| prev.restarts.get(n).map(|v| (n.clone(), *v)))
|
|
.collect();
|
|
let fetch = futures::stream::iter(services)
|
|
.map(|name| async move {
|
|
let r = tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await;
|
|
(name, r)
|
|
})
|
|
.buffer_unordered(RESTARTS_CONCURRENCY)
|
|
.for_each(|(name, r)| {
|
|
if let Ok(Ok(st)) = r {
|
|
if let Some(n) = st.n_restarts {
|
|
restarts.insert(name, n);
|
|
}
|
|
}
|
|
std::future::ready(())
|
|
});
|
|
if tokio::time::timeout(RESTARTS_BUDGET, fetch).await.is_err() {
|
|
tracing::debug!("otel metrics: restart polling hit its time budget");
|
|
}
|
|
snap.restarts = restarts;
|
|
|
|
match tokio::time::timeout(CALL_TIMEOUT, state.docker.ps(true)).await {
|
|
Ok(Ok(cs)) => {
|
|
snap.containers = cs
|
|
.into_iter()
|
|
.filter(|c| container_in_scope(cfg, &c.name))
|
|
.collect();
|
|
}
|
|
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: docker ps failed"),
|
|
Err(_) => tracing::debug!("otel metrics: docker ps timed out"),
|
|
}
|
|
snap
|
|
}
|
|
|
|
/// Register the gauges and start the poll loop.
|
|
pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>> {
|
|
// Observations are computed once per poll; callbacks only filter.
|
|
let current: Arc<RwLock<Arc<Vec<Obs>>>> = Arc::default();
|
|
let meter = provider.meter("skyg-linux-tracker");
|
|
for (name, unit) in METRICS {
|
|
let cur = Arc::clone(¤t);
|
|
// Gauges live as long as the provider; the instrument handle is not needed.
|
|
let _ = meter
|
|
.f64_observable_gauge(name)
|
|
.with_unit(unit)
|
|
.with_callback(move |o| {
|
|
let obs = cur.read().map(|g| Arc::clone(&g)).unwrap_or_default();
|
|
for ob in obs.iter().filter(|x| x.metric == name) {
|
|
let attrs: Vec<KeyValue> = ob
|
|
.attrs
|
|
.iter()
|
|
.map(|(k, v)| KeyValue::new(*k, v.clone()))
|
|
.collect();
|
|
o.observe(ob.value, &attrs);
|
|
}
|
|
})
|
|
.build();
|
|
}
|
|
|
|
let interval = Duration::from_secs(state.config.otel.metrics_interval_secs.max(1));
|
|
let h = tokio::spawn(async move {
|
|
let mut tick = tokio::time::interval(interval);
|
|
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
|
let mut snap = Snapshot::default();
|
|
loop {
|
|
tokio::select! {
|
|
_ = state.shutdown.cancelled() => break,
|
|
_ = tick.tick() => {}
|
|
}
|
|
snap = tokio::select! {
|
|
_ = state.shutdown.cancelled() => break,
|
|
s = collect(&state, &snap) => s,
|
|
};
|
|
if let Ok(mut g) = current.write() {
|
|
*g = Arc::new(observations(&snap));
|
|
}
|
|
}
|
|
});
|
|
vec![h]
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::config::Config;
|
|
use crate::sources::fake::fake_state;
|
|
|
|
fn unit(name: &str, active: &str) -> UnitSummary {
|
|
UnitSummary {
|
|
name: name.into(),
|
|
description: String::new(),
|
|
load_state: "loaded".into(),
|
|
active_state: active.into(),
|
|
sub_state: "running".into(),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn default_scope_is_service_and_timer() {
|
|
let c = OtelMetricsConfig::default();
|
|
assert!(unit_in_scope(&c, "a.service"));
|
|
assert!(unit_in_scope(&c, "a.timer"));
|
|
assert!(!unit_in_scope(&c, "a.socket"));
|
|
assert!(!unit_in_scope(&c, "a.mount"));
|
|
assert!(container_in_scope(&c, "anything"));
|
|
}
|
|
|
|
#[test]
|
|
fn allowlist_replaces_default() {
|
|
let c = OtelMetricsConfig {
|
|
units: vec!["*.socket".into(), "ssh*".into()],
|
|
containers: vec!["web-*".into()],
|
|
};
|
|
assert!(unit_in_scope(&c, "dbus.socket"));
|
|
assert!(unit_in_scope(&c, "sshd.service"));
|
|
assert!(!unit_in_scope(&c, "cron.service"));
|
|
assert!(container_in_scope(&c, "web-1"));
|
|
assert!(!container_in_scope(&c, "db"));
|
|
}
|
|
|
|
#[test]
|
|
fn maps_units_and_restarts() {
|
|
let mut s = Snapshot {
|
|
units: vec![unit("a.service", "active"), unit("b.service", "failed")],
|
|
..Default::default()
|
|
};
|
|
s.restarts.insert("b.service".into(), 3);
|
|
let o = observations(&s);
|
|
let get = |m: &str, unit: &str| {
|
|
o.iter()
|
|
.find(|x| x.metric == m && x.attrs[0].1 == unit)
|
|
.map(|x| x.value)
|
|
};
|
|
assert_eq!(get("systemd.unit.active", "a.service"), Some(1.0));
|
|
assert_eq!(get("systemd.unit.failed", "a.service"), Some(0.0));
|
|
assert_eq!(get("systemd.unit.active", "b.service"), Some(0.0));
|
|
assert_eq!(get("systemd.unit.failed", "b.service"), Some(1.0));
|
|
assert_eq!(get("systemd.unit.restarts", "b.service"), Some(3.0));
|
|
assert_eq!(get("systemd.unit.restarts", "a.service"), None);
|
|
let a = o
|
|
.iter()
|
|
.find(|x| x.metric == "systemd.unit.active")
|
|
.unwrap();
|
|
let keys: Vec<_> = a.attrs.iter().map(|(k, _)| *k).collect();
|
|
assert_eq!(
|
|
keys,
|
|
["systemd.unit", "systemd.sub_state", "systemd.load_state"]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn maps_timers_and_containers() {
|
|
let c = ContainerSummary {
|
|
id: "abc".into(),
|
|
name: "web".into(),
|
|
image: "nginx:1".into(),
|
|
state: "running".into(),
|
|
health: Some("unhealthy".into()),
|
|
created: None,
|
|
started_at: None,
|
|
uptime_seconds: Some(42),
|
|
restart_count: Some(2),
|
|
};
|
|
let nohealth = ContainerSummary {
|
|
health: None,
|
|
uptime_seconds: None,
|
|
restart_count: None,
|
|
state: "exited".into(),
|
|
name: "job".into(),
|
|
..c.clone()
|
|
};
|
|
let s = Snapshot {
|
|
timers: vec![
|
|
TimerInfo {
|
|
name: "t.timer".into(),
|
|
unit: "t.service".into(),
|
|
active_state: "active".into(),
|
|
next_elapse: Some("2026-10-15T12:00:00Z".parse().unwrap()),
|
|
last_trigger: None,
|
|
},
|
|
TimerInfo {
|
|
name: "u.timer".into(),
|
|
unit: "u.service".into(),
|
|
active_state: "inactive".into(),
|
|
next_elapse: None,
|
|
last_trigger: None,
|
|
},
|
|
],
|
|
containers: vec![c, nohealth],
|
|
..Default::default()
|
|
};
|
|
let o = observations(&s);
|
|
let timers: Vec<_> = o
|
|
.iter()
|
|
.filter(|x| x.metric == "systemd.timer.next_elapse_timestamp_seconds")
|
|
.collect();
|
|
assert_eq!(timers.len(), 1);
|
|
assert_eq!(timers[0].value, 1_792_065_600.0);
|
|
let of = |m: &str| o.iter().filter(|x| x.metric == m).count();
|
|
assert_eq!(of("docker.container.running"), 2);
|
|
assert_eq!(of("docker.container.healthy"), 1);
|
|
assert_eq!(of("docker.container.uptime_seconds"), 1);
|
|
assert_eq!(of("docker.container.restarts"), 1);
|
|
let h = o
|
|
.iter()
|
|
.find(|x| x.metric == "docker.container.healthy")
|
|
.unwrap();
|
|
assert_eq!(h.value, 0.0);
|
|
assert_eq!(h.attrs[0], ("container.name", "web".to_string()));
|
|
assert_eq!(h.attrs.len(), 1);
|
|
let r = o
|
|
.iter()
|
|
.find(|x| x.metric == "docker.container.running")
|
|
.unwrap();
|
|
assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string()));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn collect_applies_scope_with_fakes() {
|
|
let mut cfg = Config::default();
|
|
cfg.otel.metrics.units = vec!["__nomatch__*".into()];
|
|
cfg.otel.metrics.containers = vec!["__nomatch__*".into()];
|
|
let st = fake_state(cfg);
|
|
let snap = collect(&st, &Snapshot::default()).await;
|
|
assert!(snap.units.is_empty());
|
|
assert!(snap.timers.is_empty());
|
|
assert!(snap.containers.is_empty());
|
|
|
|
let st = fake_state(Config::default());
|
|
let snap = collect(&st, &Snapshot::default()).await;
|
|
assert!(snap
|
|
.units
|
|
.iter()
|
|
.all(|u| u.name.ends_with(".service") || u.name.ends_with(".timer")));
|
|
assert!(!snap.units.is_empty());
|
|
}
|
|
}
|