WS-G: OpenTelemetry export (traces, metrics, logs)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
ariandClaude Sonnet 5.5 committed 2026-10-15 18:37:40 -04:00
1 parent 2d57fb1a78
commit 2aeb9ae154
6 files changed
+1175 -25

No files matched your search

+398 -2
View File
@@ -1,3 +1,399 @@
//! Metrics poll loop and observable gauges (WS-G)
//! Metrics poll loop and observable gauges (WS-G).
//!
//! Placeholder created in P0 so the owning workstream only edits its own files.
//! A background task polls the sources every `metrics_interval_secs` and
//! replaces a shared snapshot. Observable gauge callbacks read the latest
//! snapshot, so units or containers that vanish stop being reported.
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::time::Duration;
use opentelemetry::metrics::MeterProvider as _;
use opentelemetry::KeyValue;
use opentelemetry_sdk::metrics::SdkMeterProvider;
use tokio::task::JoinHandle;
use super::any_glob;
use crate::config::OtelMetricsConfig;
use crate::sources::{ContainerSummary, TimerInfo, UnitSummary};
use crate::state::AppState;
/// Per-call bound so a hung backend cannot stall the poll loop.
const CALL_TIMEOUT: Duration = Duration::from_secs(10);
pub const METRIC_NAMES: [&str; 8] = [
"systemd.unit.active",
"systemd.unit.failed",
"systemd.unit.restarts",
"systemd.timer.next_elapse_seconds",
"docker.container.running",
"docker.container.healthy",
"docker.container.uptime_seconds",
"docker.container.restarts",
];
/// Latest poll result, already filtered by the allowlists.
#[derive(Debug, Default, Clone)]
pub struct Snapshot {
pub units: Vec<UnitSummary>,
/// Automatic restart count per unit name (services only).
pub restarts: HashMap<String, u32>,
pub timers: Vec<TimerInfo>,
pub containers: Vec<ContainerSummary>,
}
/// One gauge data point.
#[derive(Debug, Clone, PartialEq)]
pub struct Obs {
pub metric: &'static str,
pub value: f64,
pub attrs: Vec<(&'static str, String)>,
}
/// DEC-7: empty list means `*.service` and `*.timer`; otherwise a glob allowlist.
pub fn unit_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool {
if cfg.units.is_empty() {
name.ends_with(".service") || name.ends_with(".timer")
} else {
any_glob(&cfg.units, name)
}
}
/// Empty list means all containers.
pub fn container_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool {
cfg.containers.is_empty() || any_glob(&cfg.containers, name)
}
fn unit_attrs(u: &UnitSummary) -> Vec<(&'static str, String)> {
vec![
("systemd.unit", u.name.clone()),
("systemd.sub_state", u.sub_state.clone()),
("systemd.load_state", u.load_state.clone()),
]
}
fn container_attrs(c: &ContainerSummary) -> Vec<(&'static str, String)> {
vec![
("container.name", c.name.clone()),
("container.id", c.id.clone()),
("container.image.name", c.image.clone()),
]
}
/// Map a snapshot to gauge observations.
pub fn observations(s: &Snapshot) -> Vec<Obs> {
let b = |v: bool| if v { 1.0 } else { 0.0 };
let mut out = Vec::new();
for u in &s.units {
out.push(Obs {
metric: "systemd.unit.active",
value: b(u.active_state == "active"),
attrs: unit_attrs(u),
});
out.push(Obs {
metric: "systemd.unit.failed",
value: b(u.active_state == "failed"),
attrs: unit_attrs(u),
});
if let Some(n) = s.restarts.get(&u.name) {
out.push(Obs {
metric: "systemd.unit.restarts",
value: f64::from(*n),
attrs: unit_attrs(u),
});
}
}
for t in &s.timers {
// Absolute unix time of the next elapse; unscheduled timers are omitted.
if let Some(next) = t.next_elapse {
out.push(Obs {
metric: "systemd.timer.next_elapse_seconds",
value: next.as_second() as f64,
attrs: vec![
("systemd.unit", t.name.clone()),
("systemd.sub_state", t.active_state.clone()),
],
});
}
}
for c in &s.containers {
out.push(Obs {
metric: "docker.container.running",
value: b(c.state == "running"),
attrs: container_attrs(c),
});
if let Some(h) = &c.health {
out.push(Obs {
metric: "docker.container.healthy",
value: b(h == "healthy"),
attrs: container_attrs(c),
});
}
if let Some(up) = c.uptime_seconds {
out.push(Obs {
metric: "docker.container.uptime_seconds",
value: up as f64,
attrs: container_attrs(c),
});
}
if let Some(r) = c.restart_count {
out.push(Obs {
metric: "docker.container.restarts",
value: r as f64,
attrs: container_attrs(c),
});
}
}
out
}
async fn collect(state: &AppState) -> Snapshot {
let cfg = &state.config.otel.metrics;
let mut snap = Snapshot::default();
match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_units(None)).await {
Ok(Ok(units)) => {
snap.units = units
.into_iter()
.filter(|u| unit_in_scope(cfg, &u.name))
.collect();
}
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_units failed"),
Err(_) => tracing::debug!("otel metrics: list_units timed out"),
}
match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_timers()).await {
Ok(Ok(timers)) => {
snap.timers = timers
.into_iter()
.filter(|t| unit_in_scope(cfg, &t.name))
.collect();
}
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_timers failed"),
Err(_) => tracing::debug!("otel metrics: list_timers timed out"),
}
let services: Vec<String> = snap
.units
.iter()
.filter(|u| u.name.ends_with(".service") && u.load_state == "loaded")
.map(|u| u.name.clone())
.collect();
for name in services {
if state.shutdown.is_cancelled() {
break;
}
if let Ok(Ok(st)) =
tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await
{
if let Some(n) = st.n_restarts {
snap.restarts.insert(name, n);
}
}
}
match tokio::time::timeout(CALL_TIMEOUT, state.docker.ps(true)).await {
Ok(Ok(cs)) => {
snap.containers = cs
.into_iter()
.filter(|c| container_in_scope(cfg, &c.name))
.collect();
}
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: docker ps failed"),
Err(_) => tracing::debug!("otel metrics: docker ps timed out"),
}
snap
}
/// Register the gauges and start the poll loop.
pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>> {
let snapshot: Arc<RwLock<Arc<Snapshot>>> = Arc::default();
let meter = provider.meter("skyg-linux-tracker");
for name in METRIC_NAMES {
let snap = Arc::clone(&snapshot);
// Gauges live as long as the provider; the instrument handle is not needed.
let _ = meter
.f64_observable_gauge(name)
.with_callback(move |o| {
let cur = snap.read().map(|g| Arc::clone(&g)).unwrap_or_default();
for ob in observations(&cur).into_iter().filter(|x| x.metric == name) {
let attrs: Vec<KeyValue> = ob
.attrs
.into_iter()
.map(|(k, v)| KeyValue::new(k, v))
.collect();
o.observe(ob.value, &attrs);
}
})
.build();
}
let interval = Duration::from_secs(state.config.otel.metrics_interval_secs.max(1));
let h = tokio::spawn(async move {
let mut tick = tokio::time::interval(interval);
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
_ = state.shutdown.cancelled() => break,
_ = tick.tick() => {}
}
let snap = tokio::select! {
_ = state.shutdown.cancelled() => break,
s = collect(&state) => s,
};
if let Ok(mut g) = snapshot.write() {
*g = Arc::new(snap);
}
}
});
vec![h]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::Config;
use crate::sources::fake::fake_state;
fn unit(name: &str, active: &str) -> UnitSummary {
UnitSummary {
name: name.into(),
description: String::new(),
load_state: "loaded".into(),
active_state: active.into(),
sub_state: "running".into(),
}
}
#[test]
fn default_scope_is_service_and_timer() {
let c = OtelMetricsConfig::default();
assert!(unit_in_scope(&c, "a.service"));
assert!(unit_in_scope(&c, "a.timer"));
assert!(!unit_in_scope(&c, "a.socket"));
assert!(!unit_in_scope(&c, "a.mount"));
assert!(container_in_scope(&c, "anything"));
}
#[test]
fn allowlist_replaces_default() {
let c = OtelMetricsConfig {
units: vec!["*.socket".into(), "ssh*".into()],
containers: vec!["web-*".into()],
};
assert!(unit_in_scope(&c, "dbus.socket"));
assert!(unit_in_scope(&c, "sshd.service"));
assert!(!unit_in_scope(&c, "cron.service"));
assert!(container_in_scope(&c, "web-1"));
assert!(!container_in_scope(&c, "db"));
}
#[test]
fn maps_units_and_restarts() {
let mut s = Snapshot {
units: vec![unit("a.service", "active"), unit("b.service", "failed")],
..Default::default()
};
s.restarts.insert("b.service".into(), 3);
let o = observations(&s);
let get = |m: &str, unit: &str| {
o.iter()
.find(|x| x.metric == m && x.attrs[0].1 == unit)
.map(|x| x.value)
};
assert_eq!(get("systemd.unit.active", "a.service"), Some(1.0));
assert_eq!(get("systemd.unit.failed", "a.service"), Some(0.0));
assert_eq!(get("systemd.unit.active", "b.service"), Some(0.0));
assert_eq!(get("systemd.unit.failed", "b.service"), Some(1.0));
assert_eq!(get("systemd.unit.restarts", "b.service"), Some(3.0));
assert_eq!(get("systemd.unit.restarts", "a.service"), None);
let a = o
.iter()
.find(|x| x.metric == "systemd.unit.active")
.unwrap();
let keys: Vec<_> = a.attrs.iter().map(|(k, _)| *k).collect();
assert_eq!(
keys,
["systemd.unit", "systemd.sub_state", "systemd.load_state"]
);
}
#[test]
fn maps_timers_and_containers() {
let c = ContainerSummary {
id: "abc".into(),
name: "web".into(),
image: "nginx:1".into(),
state: "running".into(),
health: Some("unhealthy".into()),
created: None,
started_at: None,
uptime_seconds: Some(42),
restart_count: Some(2),
};
let nohealth = ContainerSummary {
health: None,
uptime_seconds: None,
restart_count: None,
state: "exited".into(),
name: "job".into(),
..c.clone()
};
let s = Snapshot {
timers: vec![
TimerInfo {
name: "t.timer".into(),
unit: "t.service".into(),
active_state: "active".into(),
next_elapse: Some("2026-10-15T12:00:00Z".parse().unwrap()),
last_trigger: None,
},
TimerInfo {
name: "u.timer".into(),
unit: "u.service".into(),
active_state: "inactive".into(),
next_elapse: None,
last_trigger: None,
},
],
containers: vec![c, nohealth],
..Default::default()
};
let o = observations(&s);
let timers: Vec<_> = o
.iter()
.filter(|x| x.metric == "systemd.timer.next_elapse_seconds")
.collect();
assert_eq!(timers.len(), 1);
assert_eq!(timers[0].value, 1_792_065_600.0);
let of = |m: &str| o.iter().filter(|x| x.metric == m).count();
assert_eq!(of("docker.container.running"), 2);
assert_eq!(of("docker.container.healthy"), 1);
assert_eq!(of("docker.container.uptime_seconds"), 1);
assert_eq!(of("docker.container.restarts"), 1);
let h = o
.iter()
.find(|x| x.metric == "docker.container.healthy")
.unwrap();
assert_eq!(h.value, 0.0);
assert_eq!(h.attrs[0], ("container.name", "web".to_string()));
assert_eq!(h.attrs[2], ("container.image.name", "nginx:1".to_string()));
}
#[tokio::test]
async fn collect_applies_scope_with_fakes() {
let mut cfg = Config::default();
cfg.otel.metrics.units = vec!["__nomatch__*".into()];
cfg.otel.metrics.containers = vec!["__nomatch__*".into()];
let st = fake_state(cfg);
let snap = collect(&st).await;
assert!(snap.units.is_empty());
assert!(snap.timers.is_empty());
assert!(snap.containers.is_empty());
let st = fake_state(Config::default());
let snap = collect(&st).await;
assert!(snap
.units
.iter()
.all(|u| u.name.ends_with(".service") || u.name.ends_with(".timer")));
assert!(!snap.units.is_empty());
}
}