From eee253b162d584b28ace74d49a8ac36774270a2d Mon Sep 17 00:00:00 2001 From: ari Date: Mon, 19 Oct 2026 20:48:15 -0400 Subject: [PATCH] E: E1 peer health gauges from the registry snapshot --- src/telemetry/metrics_export.rs | 442 ++++++++++++++++++++++++++++++-- 1 file changed, 425 insertions(+), 17 deletions(-) diff --git a/src/telemetry/metrics_export.rs b/src/telemetry/metrics_export.rs index 8b7f31e..a6fd225 100644 --- a/src/telemetry/metrics_export.rs +++ b/src/telemetry/metrics_export.rs @@ -9,13 +9,14 @@ use std::sync::{Arc, RwLock}; use std::time::Duration; use futures::StreamExt; -use opentelemetry::metrics::MeterProvider as _; +use opentelemetry::metrics::{Meter, MeterProvider as _}; use opentelemetry::KeyValue; use opentelemetry_sdk::metrics::SdkMeterProvider; use tokio::task::JoinHandle; use super::any_glob; use crate::config::OtelMetricsConfig; +use crate::peers::types::{AuthState, PeerStatus}; use crate::sources::{ContainerSummary, TimerInfo, UnitSummary}; use crate::state::AppState; @@ -35,6 +36,16 @@ pub const METRICS: [(&str, &str); 8] = [ ("docker.container.restarts", "{restart}"), ]; +/// Peer health gauges. Registered only when the peers feature is on; names, +/// units and attributes are a contract with dashboards and alerts. +pub const PEER_METRICS: [(&str, &str); 4] = [ + ("peer.up", "1"), + ("peer.api.auth_ok", "1"), + ("peer.check.duration_seconds", "s"), + // Absolute unix timestamp of the last state change, not a duration. + ("peer.last_change_timestamp_seconds", "s"), +]; + /// Overall budget for the per-unit restart-count polling. const RESTARTS_BUDGET: Duration = Duration::from_secs(15); const RESTARTS_CONCURRENCY: usize = 8; @@ -151,6 +162,61 @@ pub fn observations(s: &Snapshot) -> Vec { out } +/// Map a registry snapshot to peer gauge observations. +/// +/// Series carry `peer.name` and, per check, `peer.check` only: URLs can embed +/// hostnames an operator treats as sensitive, and a version attribute would +/// churn series on upgrade. A missing latency or `since` (no check yet, or a +/// failed one) yields no point rather than a stale or invented value. +/// +/// Disabled peers are skipped: their checks never run, so a `0` would read as +/// "down" and page someone for a peer that was switched off deliberately. +pub fn peer_observations(peers: &[PeerStatus]) -> Vec { + let b = |v: bool| if v { 1.0 } else { 0.0 }; + let mut out = Vec::new(); + for p in peers.iter().filter(|p| p.enabled) { + let name = p.name.as_str(); + let check_attrs = |check: &str| { + vec![ + ("peer.name", name.to_owned()), + ("peer.check", check.to_owned()), + ] + }; + for (check, up, latency, since) in [ + ("server", p.server.up, p.server.latency_ms, p.server.since), + ("api", p.api.up, p.api.latency_ms, p.api.since), + ] { + out.push(Obs { + metric: "peer.up", + value: b(up), + attrs: check_attrs(check), + }); + if let Some(ms) = latency { + out.push(Obs { + metric: "peer.check.duration_seconds", + value: ms as f64 / 1000.0, + attrs: check_attrs(check), + }); + } + if let Some(ts) = since { + out.push(Obs { + metric: "peer.last_change_timestamp_seconds", + value: ts.as_second() as f64, + attrs: check_attrs(check), + }); + } + } + // "server up + api up + auth_ok = 0" is the revoked-credential state, + // kept separate from an API outage so alerts can tell them apart. + out.push(Obs { + metric: "peer.api.auth_ok", + value: b(p.api.auth == AuthState::Ok), + attrs: vec![("peer.name", name.to_owned())], + }); + } + out +} + /// Poll the sources. A section whose call fails or times out keeps its value /// from `prev` instead of being emptied. async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot { @@ -222,28 +288,52 @@ async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot { snap } +/// Register one observable f64 gauge fed by `obs`. The callback filters by +/// metric name, so a single observation list can feed several gauges. +fn register_gauge( + meter: &Meter, + name: &'static str, + unit: &'static str, + obs: impl Fn() -> Arc> + Send + Sync + 'static, +) { + // Gauges live as long as the provider; the instrument handle is not needed. + let _ = meter + .f64_observable_gauge(name) + .with_unit(unit) + .with_callback(move |o| { + let obs = obs(); + for ob in obs.iter().filter(|x| x.metric == name) { + let attrs: Vec = ob + .attrs + .iter() + .map(|(k, v)| KeyValue::new(*k, v.clone())) + .collect(); + o.observe(ob.value, &attrs); + } + }) + .build(); +} + pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec> { // Observations are computed once per poll; callbacks only filter. let current: Arc>>> = Arc::default(); let meter = provider.meter("skyg-linux-tracker"); for (name, unit) in METRICS { let cur = Arc::clone(¤t); - // Gauges live as long as the provider; the instrument handle is not needed. - let _ = meter - .f64_observable_gauge(name) - .with_unit(unit) - .with_callback(move |o| { - let obs = cur.read().map(|g| Arc::clone(&g)).unwrap_or_default(); - for ob in obs.iter().filter(|x| x.metric == name) { - let attrs: Vec = ob - .attrs - .iter() - .map(|(k, v)| KeyValue::new(*k, v.clone())) - .collect(); - o.observe(ob.value, &attrs); - } - }) - .build(); + register_gauge(&meter, name, unit, move || { + cur.read().map(|g| Arc::clone(&g)).unwrap_or_default() + }); + } + // Peer state already lives in memory, so peer gauges read the registry on + // each collection instead of polling anything; a peer removed from config + // therefore disappears from the next export. + if let Some(peers) = state.peers.clone() { + for (name, unit) in PEER_METRICS { + let peers = Arc::clone(&peers); + register_gauge(&meter, name, unit, move || { + Arc::new(peer_observations(&peers.snapshot())) + }); + } } let interval = Duration::from_secs(state.config.otel.metrics_interval_secs.max(1)); @@ -404,6 +494,324 @@ mod tests { assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string())); } + fn ts(s: &str) -> jiff::Timestamp { + s.parse().expect("valid timestamp") + } + + fn peer_status(name: &str, server_up: bool, api_up: bool, auth: AuthState) -> PeerStatus { + PeerStatus { + name: crate::peers::types::PeerName::parse(name).unwrap(), + enabled: true, + url: format!("http://{name}:1984"), + server: crate::peers::types::ServerStatus { + up: server_up, + since: Some(ts("2026-10-19T10:00:00Z")), + last_checked: Some(ts("2026-10-19T10:00:09Z")), + last_error: None, + latency_ms: Some(1500), + }, + api: crate::peers::types::ApiStatus { + up: api_up, + since: Some(ts("2026-10-19T10:00:00Z")), + last_checked: Some(ts("2026-10-19T10:00:05Z")), + last_error: None, + latency_ms: Some(7), + auth, + peer_version: Some("0.1.0".into()), + }, + } + } + + fn attrs(ob: &Obs) -> Vec<(&str, &str)> { + ob.attrs.iter().map(|(k, v)| (*k, v.as_str())).collect() + } + + fn find<'a>(o: &'a [Obs], metric: &str, peer: &str, check: &str) -> Option<&'a Obs> { + o.iter().find(|x| { + x.metric == metric + && x.attrs.iter().any(|(k, v)| *k == "peer.name" && v == peer) + && x.attrs + .iter() + .any(|(k, v)| *k == "peer.check" && v == check) + }) + } + + #[test] + fn peer_metric_names_and_units_are_the_documented_contract() { + assert_eq!( + PEER_METRICS.map(|(name, _)| name), + [ + "peer.up", + "peer.api.auth_ok", + "peer.check.duration_seconds", + "peer.last_change_timestamp_seconds", + ] + ); + assert_eq!(PEER_METRICS.map(|(_, unit)| unit), ["1", "1", "s", "s"]); + } + + #[test] + fn maps_peer_status_to_gauges() { + let o = peer_observations(&[peer_status("beta", true, true, AuthState::Ok)]); + + let up = find(&o, "peer.up", "beta", "server").unwrap(); + assert_eq!(up.value, 1.0); + assert_eq!(attrs(up), [("peer.name", "beta"), ("peer.check", "server")]); + assert_eq!(find(&o, "peer.up", "beta", "api").unwrap().value, 1.0); + + let auth = o.iter().find(|x| x.metric == "peer.api.auth_ok").unwrap(); + assert_eq!(auth.value, 1.0); + assert_eq!(attrs(auth), [("peer.name", "beta")]); + + assert_eq!( + find(&o, "peer.check.duration_seconds", "beta", "server") + .unwrap() + .value, + 1.5 + ); + assert_eq!( + find(&o, "peer.check.duration_seconds", "beta", "api") + .unwrap() + .value, + 0.007 + ); + assert_eq!( + find(&o, "peer.last_change_timestamp_seconds", "beta", "api") + .unwrap() + .value, + ts("2026-10-19T10:00:00Z").as_second() as f64 + ); + + // URLs and versions stay out of attributes: cardinality and hostnames + // the operator may treat as sensitive. + for ob in &o { + assert!(!ob.attrs.iter().any(|(_, v)| v.contains("http"))); + assert!(!ob.attrs.iter().any(|(_, v)| v.contains("0.1.0"))); + } + } + + #[test] + fn rejected_credentials_are_visible_in_auth_ok() { + let o = peer_observations(&[peer_status("beta", true, true, AuthState::Rejected)]); + assert_eq!(find(&o, "peer.up", "beta", "api").unwrap().value, 1.0); + let auth = o.iter().find(|x| x.metric == "peer.api.auth_ok").unwrap(); + assert_eq!(auth.value, 0.0); + + let unknown = peer_observations(&[peer_status("beta", true, true, AuthState::Unknown)]); + let auth = unknown + .iter() + .find(|x| x.metric == "peer.api.auth_ok") + .unwrap(); + assert_eq!(auth.value, 0.0); + } + + #[test] + fn unchecked_peer_reports_up_only() { + let mut p = peer_status("beta", false, false, AuthState::Unknown); + p.server.latency_ms = None; + p.server.since = None; + p.api.latency_ms = None; + p.api.since = None; + let o = peer_observations(&[p]); + let metrics: Vec<&str> = o.iter().map(|x| x.metric).collect(); + assert_eq!(metrics, ["peer.up", "peer.up", "peer.api.auth_ok"]); + assert!(o.iter().all(|x| x.value == 0.0)); + } + + #[test] + fn disabled_peer_is_not_reported() { + let mut p = peer_status("beta", true, true, AuthState::Ok); + p.enabled = false; + assert!(peer_observations(&[p]).is_empty()); + } + + /// Collects exports and keeps the samples of the last one. + #[derive(Default, Clone)] + struct Capture(std::sync::Arc>>>); + + #[derive(Debug, Clone, PartialEq)] + struct Sample { + metric: String, + unit: String, + value: f64, + attrs: Vec<(String, String)>, + } + + impl Capture { + fn last(&self) -> Vec { + self.0.lock().unwrap().last().cloned().unwrap_or_default() + } + } + + impl opentelemetry_sdk::metrics::exporter::PushMetricExporter for Capture { + fn export( + &self, + metrics: &opentelemetry_sdk::metrics::data::ResourceMetrics, + ) -> impl std::future::Future + Send + { + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; + let mut samples = Vec::new(); + for scope in metrics.scope_metrics() { + for m in scope.metrics() { + let AggregatedMetrics::F64(MetricData::Gauge(g)) = m.data() else { + continue; + }; + for p in g.data_points() { + samples.push(Sample { + metric: m.name().to_owned(), + unit: m.unit().to_owned(), + value: p.value(), + attrs: p + .attributes() + .map(|kv| { + (kv.key.as_str().to_owned(), kv.value.as_str().into_owned()) + }) + .collect(), + }); + } + } + } + self.0.lock().unwrap().push(samples); + std::future::ready(Ok(())) + } + + fn force_flush(&self) -> opentelemetry_sdk::error::OTelSdkResult { + Ok(()) + } + + fn shutdown_with_timeout( + &self, + _timeout: Duration, + ) -> opentelemetry_sdk::error::OTelSdkResult { + Ok(()) + } + + fn temporality(&self) -> opentelemetry_sdk::metrics::Temporality { + opentelemetry_sdk::metrics::Temporality::Cumulative + } + } + + /// Provider with a capture exporter; export happens on `force_flush`. + fn capture_provider() -> (SdkMeterProvider, Capture) { + let capture = Capture::default(); + let reader = opentelemetry_sdk::metrics::PeriodicReader::builder(capture.clone()) + .with_interval(Duration::from_secs(3600)) + .build(); + let provider = SdkMeterProvider::builder().with_reader(reader).build(); + (provider, capture) + } + + fn sample<'a>( + s: &'a [Sample], + metric: &str, + peer: &str, + check: Option<&str>, + ) -> Option<&'a Sample> { + s.iter().find(|x| { + x.metric == metric + && x.attrs.iter().any(|(k, v)| k == "peer.name" && v == peer) + && check.is_none_or(|c| x.attrs.iter().any(|(k, v)| k == "peer.check" && v == c)) + }) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn exported_peer_gauges_read_the_registry_snapshot() { + let reg = crate::peers::testing::static_registry(vec![ + peer_status("beta", true, true, AuthState::Ok), + peer_status("gamma", true, false, AuthState::Unknown), + ]); + let state = crate::peers::testing::fake_state_with_peers(Config::default(), reg.clone()); + let (provider, capture) = capture_provider(); + let handles = spawn(state.clone(), &provider); + provider.force_flush().unwrap(); + + let samples = capture.last(); + let beta_up = sample(&samples, "peer.up", "beta", Some("server")).unwrap(); + assert_eq!(beta_up.value, 1.0); + assert_eq!(beta_up.unit, "1"); + assert_eq!( + sample(&samples, "peer.up", "gamma", Some("api")) + .unwrap() + .value, + 0.0 + ); + assert_eq!( + sample(&samples, "peer.api.auth_ok", "gamma", None) + .unwrap() + .value, + 0.0 + ); + assert_eq!( + sample( + &samples, + "peer.check.duration_seconds", + "beta", + Some("server") + ) + .unwrap() + .value, + 1.5 + ); + + // Removing a peer from the snapshot drops its series; it is not cached. + reg.reload(crate::peers::testing::fake_peers_config(&["beta"])) + .await + .unwrap(); + provider.force_flush().unwrap(); + let after = capture.last(); + assert!(sample(&after, "peer.up", "beta", Some("server")).is_some()); + assert!(sample(&after, "peer.up", "gamma", Some("server")).is_none()); + + state.shutdown.cancel(); + for h in handles { + let _ = h.await; + } + provider.shutdown().unwrap(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn collecting_peer_gauges_does_not_poll_the_transport() { + let fake = crate::peers::testing::FakeTransport::new(); + let reg = crate::peers::testing::registry_with( + fake.clone(), + crate::peers::testing::fake_peers_config(&["beta"]), + ); + let state = crate::peers::testing::fake_state_with_peers(Config::default(), reg); + let (provider, capture) = capture_provider(); + let handles = spawn(state.clone(), &provider); + provider.force_flush().unwrap(); + + let samples = capture.last(); + assert!(sample(&samples, "peer.up", "beta", Some("server")).is_some()); + assert!(fake.calls().is_empty(), "gauges must not poll peers"); + + state.shutdown.cancel(); + for h in handles { + let _ = h.await; + } + provider.shutdown().unwrap(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn peerless_state_exports_no_peer_series() { + let state = fake_state(Config::default()); + let (provider, capture) = capture_provider(); + let handles = spawn(state.clone(), &provider); + provider.force_flush().unwrap(); + + let samples = capture.last(); + assert!( + samples.iter().all(|s| !s.metric.starts_with("peer.")), + "no peer series without the feature: {samples:?}" + ); + + state.shutdown.cancel(); + for h in handles { + let _ = h.await; + } + provider.shutdown().unwrap(); + } + #[tokio::test] async fn collect_applies_scope_with_fakes() { let mut cfg = Config::default();