//! Metrics poll loop and observable gauges. //! //! A background task polls the sources every `metrics_interval_secs` and //! replaces a shared snapshot. Observable gauge callbacks read the latest //! snapshot, so units or containers that vanish stop being reported. use std::collections::HashMap; use std::sync::{Arc, RwLock}; use std::time::Duration; use futures::StreamExt; use opentelemetry::metrics::MeterProvider as _; use opentelemetry::KeyValue; use opentelemetry_sdk::metrics::SdkMeterProvider; use tokio::task::JoinHandle; use super::any_glob; use crate::config::OtelMetricsConfig; use crate::sources::{ContainerSummary, TimerInfo, UnitSummary}; use crate::state::AppState; /// Per-call bound so a hung backend cannot stall the poll loop. const CALL_TIMEOUT: Duration = Duration::from_secs(10); /// Metric name and UCUM unit. pub const METRICS: [(&str, &str); 8] = [ ("systemd.unit.active", "1"), ("systemd.unit.failed", "1"), ("systemd.unit.restarts", "{restart}"), // Absolute unix timestamp of the next elapse, not a duration. ("systemd.timer.next_elapse_timestamp_seconds", "s"), ("docker.container.running", "1"), ("docker.container.healthy", "1"), ("docker.container.uptime_seconds", "s"), ("docker.container.restarts", "{restart}"), ]; /// Overall budget for the per-unit restart-count polling. const RESTARTS_BUDGET: Duration = Duration::from_secs(15); const RESTARTS_CONCURRENCY: usize = 8; /// Latest poll result, already filtered by the allowlists. #[derive(Debug, Default, Clone)] pub struct Snapshot { pub units: Vec, /// Automatic restart count per unit name (services only). pub restarts: HashMap, pub timers: Vec, pub containers: Vec, } /// One gauge data point. #[derive(Debug, Clone, PartialEq)] pub struct Obs { pub metric: &'static str, pub value: f64, pub attrs: Vec<(&'static str, String)>, } /// empty list means `*.service` and `*.timer`; otherwise a glob allowlist. pub fn unit_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool { if cfg.units.is_empty() { name.ends_with(".service") || name.ends_with(".timer") } else { any_glob(&cfg.units, name) } } /// Empty list means all containers. pub fn container_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool { cfg.containers.is_empty() || any_glob(&cfg.containers, name) } fn unit_id(name: &str) -> Vec<(&'static str, String)> { vec![("systemd.unit", name.to_owned())] } fn container_id(c: &ContainerSummary) -> Vec<(&'static str, String)> { vec![("container.name", c.name.clone())] } /// Map a snapshot to gauge observations. Series carry identity attributes /// only (`systemd.unit` / `container.name`) so state changes do not churn /// series. The `active` gauge also carries sub/load state, and `running` /// carries the image name. pub fn observations(s: &Snapshot) -> Vec { let b = |v: bool| if v { 1.0 } else { 0.0 }; let mut out = Vec::new(); for u in &s.units { let mut active_attrs = unit_id(&u.name); active_attrs.push(("systemd.sub_state", u.sub_state.clone())); active_attrs.push(("systemd.load_state", u.load_state.clone())); out.push(Obs { metric: "systemd.unit.active", value: b(u.active_state == "active"), attrs: active_attrs, }); out.push(Obs { metric: "systemd.unit.failed", value: b(u.active_state == "failed"), attrs: unit_id(&u.name), }); if let Some(n) = s.restarts.get(&u.name) { out.push(Obs { metric: "systemd.unit.restarts", value: f64::from(*n), attrs: unit_id(&u.name), }); } } for t in &s.timers { // Unscheduled timers are omitted. if let Some(next) = t.next_elapse { out.push(Obs { metric: "systemd.timer.next_elapse_timestamp_seconds", value: next.as_second() as f64, attrs: unit_id(&t.name), }); } } for c in &s.containers { let mut running_attrs = container_id(c); running_attrs.push(("container.image.name", c.image.clone())); out.push(Obs { metric: "docker.container.running", value: b(c.state == "running"), attrs: running_attrs, }); if let Some(h) = &c.health { out.push(Obs { metric: "docker.container.healthy", value: b(h == "healthy"), attrs: container_id(c), }); } if let Some(up) = c.uptime_seconds { out.push(Obs { metric: "docker.container.uptime_seconds", value: up as f64, attrs: container_id(c), }); } if let Some(r) = c.restart_count { out.push(Obs { metric: "docker.container.restarts", value: r as f64, attrs: container_id(c), }); } } out } /// Poll the sources. A section whose call fails or times out keeps its value /// from `prev` instead of being emptied. async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot { let cfg = &state.config.otel.metrics; let mut snap = prev.clone(); match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_units(None)).await { Ok(Ok(units)) => { snap.units = units .into_iter() .filter(|u| unit_in_scope(cfg, &u.name)) .collect(); } Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_units failed"), Err(_) => tracing::debug!("otel metrics: list_units timed out"), } match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_timers()).await { Ok(Ok(timers)) => { snap.timers = timers .into_iter() .filter(|t| unit_in_scope(cfg, &t.name)) .collect(); } Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_timers failed"), Err(_) => tracing::debug!("otel metrics: list_timers timed out"), } // Restart counts: bounded concurrency and an overall budget. Entries not // refreshed in time keep their previous value; vanished units are dropped. let services: Vec = snap .units .iter() .filter(|u| u.name.ends_with(".service") && u.load_state == "loaded") .map(|u| u.name.clone()) .collect(); let mut restarts: HashMap = services .iter() .filter_map(|n| prev.restarts.get(n).map(|v| (n.clone(), *v))) .collect(); let fetch = futures::stream::iter(services) .map(|name| async move { let r = tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await; (name, r) }) .buffer_unordered(RESTARTS_CONCURRENCY) .for_each(|(name, r)| { if let Ok(Ok(st)) = r { if let Some(n) = st.n_restarts { restarts.insert(name, n); } } std::future::ready(()) }); if tokio::time::timeout(RESTARTS_BUDGET, fetch).await.is_err() { tracing::debug!("otel metrics: restart polling hit its time budget"); } snap.restarts = restarts; match tokio::time::timeout(CALL_TIMEOUT, state.docker.ps(true)).await { Ok(Ok(cs)) => { snap.containers = cs .into_iter() .filter(|c| container_in_scope(cfg, &c.name)) .collect(); } Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: docker ps failed"), Err(_) => tracing::debug!("otel metrics: docker ps timed out"), } snap } /// Register the gauges and start the poll loop. pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec> { // Observations are computed once per poll; callbacks only filter. let current: Arc>>> = Arc::default(); let meter = provider.meter("skyg-linux-tracker"); for (name, unit) in METRICS { let cur = Arc::clone(¤t); // Gauges live as long as the provider; the instrument handle is not needed. let _ = meter .f64_observable_gauge(name) .with_unit(unit) .with_callback(move |o| { let obs = cur.read().map(|g| Arc::clone(&g)).unwrap_or_default(); for ob in obs.iter().filter(|x| x.metric == name) { let attrs: Vec = ob .attrs .iter() .map(|(k, v)| KeyValue::new(*k, v.clone())) .collect(); o.observe(ob.value, &attrs); } }) .build(); } let interval = Duration::from_secs(state.config.otel.metrics_interval_secs.max(1)); let h = tokio::spawn(async move { let mut tick = tokio::time::interval(interval); tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); let mut snap = Snapshot::default(); loop { tokio::select! { _ = state.shutdown.cancelled() => break, _ = tick.tick() => {} } snap = tokio::select! { _ = state.shutdown.cancelled() => break, s = collect(&state, &snap) => s, }; if let Ok(mut g) = current.write() { *g = Arc::new(observations(&snap)); } } }); vec![h] } #[cfg(test)] mod tests { use super::*; use crate::config::Config; use crate::sources::fake::fake_state; fn unit(name: &str, active: &str) -> UnitSummary { UnitSummary { name: name.into(), description: String::new(), load_state: "loaded".into(), active_state: active.into(), sub_state: "running".into(), } } #[test] fn default_scope_is_service_and_timer() { let c = OtelMetricsConfig::default(); assert!(unit_in_scope(&c, "a.service")); assert!(unit_in_scope(&c, "a.timer")); assert!(!unit_in_scope(&c, "a.socket")); assert!(!unit_in_scope(&c, "a.mount")); assert!(container_in_scope(&c, "anything")); } #[test] fn allowlist_replaces_default() { let c = OtelMetricsConfig { units: vec!["*.socket".into(), "ssh*".into()], containers: vec!["web-*".into()], }; assert!(unit_in_scope(&c, "dbus.socket")); assert!(unit_in_scope(&c, "sshd.service")); assert!(!unit_in_scope(&c, "cron.service")); assert!(container_in_scope(&c, "web-1")); assert!(!container_in_scope(&c, "db")); } #[test] fn maps_units_and_restarts() { let mut s = Snapshot { units: vec![unit("a.service", "active"), unit("b.service", "failed")], ..Default::default() }; s.restarts.insert("b.service".into(), 3); let o = observations(&s); let get = |m: &str, unit: &str| { o.iter() .find(|x| x.metric == m && x.attrs[0].1 == unit) .map(|x| x.value) }; assert_eq!(get("systemd.unit.active", "a.service"), Some(1.0)); assert_eq!(get("systemd.unit.failed", "a.service"), Some(0.0)); assert_eq!(get("systemd.unit.active", "b.service"), Some(0.0)); assert_eq!(get("systemd.unit.failed", "b.service"), Some(1.0)); assert_eq!(get("systemd.unit.restarts", "b.service"), Some(3.0)); assert_eq!(get("systemd.unit.restarts", "a.service"), None); let a = o .iter() .find(|x| x.metric == "systemd.unit.active") .unwrap(); let keys: Vec<_> = a.attrs.iter().map(|(k, _)| *k).collect(); assert_eq!( keys, ["systemd.unit", "systemd.sub_state", "systemd.load_state"] ); } #[test] fn maps_timers_and_containers() { let c = ContainerSummary { id: "abc".into(), name: "web".into(), image: "nginx:1".into(), state: "running".into(), health: Some("unhealthy".into()), created: None, started_at: None, uptime_seconds: Some(42), restart_count: Some(2), }; let nohealth = ContainerSummary { health: None, uptime_seconds: None, restart_count: None, state: "exited".into(), name: "job".into(), ..c.clone() }; let s = Snapshot { timers: vec![ TimerInfo { name: "t.timer".into(), unit: "t.service".into(), active_state: "active".into(), next_elapse: Some("2026-10-15T12:00:00Z".parse().unwrap()), last_trigger: None, }, TimerInfo { name: "u.timer".into(), unit: "u.service".into(), active_state: "inactive".into(), next_elapse: None, last_trigger: None, }, ], containers: vec![c, nohealth], ..Default::default() }; let o = observations(&s); let timers: Vec<_> = o .iter() .filter(|x| x.metric == "systemd.timer.next_elapse_timestamp_seconds") .collect(); assert_eq!(timers.len(), 1); assert_eq!(timers[0].value, 1_792_065_600.0); let of = |m: &str| o.iter().filter(|x| x.metric == m).count(); assert_eq!(of("docker.container.running"), 2); assert_eq!(of("docker.container.healthy"), 1); assert_eq!(of("docker.container.uptime_seconds"), 1); assert_eq!(of("docker.container.restarts"), 1); let h = o .iter() .find(|x| x.metric == "docker.container.healthy") .unwrap(); assert_eq!(h.value, 0.0); assert_eq!(h.attrs[0], ("container.name", "web".to_string())); assert_eq!(h.attrs.len(), 1); let r = o .iter() .find(|x| x.metric == "docker.container.running") .unwrap(); assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string())); } #[tokio::test] async fn collect_applies_scope_with_fakes() { let mut cfg = Config::default(); cfg.otel.metrics.units = vec!["__nomatch__*".into()]; cfg.otel.metrics.containers = vec!["__nomatch__*".into()]; let st = fake_state(cfg); let snap = collect(&st, &Snapshot::default()).await; assert!(snap.units.is_empty()); assert!(snap.timers.is_empty()); assert!(snap.containers.is_empty()); let st = fake_state(Config::default()); let snap = collect(&st, &Snapshot::default()).await; assert!(snap .units .iter() .all(|u| u.name.ends_with(".service") || u.name.ends_with(".timer"))); assert!(!snap.units.is_empty()); } }