E: E1 peer health gauges from the registry snapshot

This commit is contained in:
ari committed 2026-10-19 20:57:52 -04:00
1 parent f0b6c29eac
commit eee253b162
1 file changed
+425 -17
+425 -17
View File
@@ -9,13 +9,14 @@ use std::sync::{Arc, RwLock};
use std::time::Duration;
use futures::StreamExt;
use opentelemetry::metrics::MeterProvider as _;
use opentelemetry::metrics::{Meter, MeterProvider as _};
use opentelemetry::KeyValue;
use opentelemetry_sdk::metrics::SdkMeterProvider;
use tokio::task::JoinHandle;
use super::any_glob;
use crate::config::OtelMetricsConfig;
use crate::peers::types::{AuthState, PeerStatus};
use crate::sources::{ContainerSummary, TimerInfo, UnitSummary};
use crate::state::AppState;
@@ -35,6 +36,16 @@ pub const METRICS: [(&str, &str); 8] = [
("docker.container.restarts", "{restart}"),
];
/// Peer health gauges. Registered only when the peers feature is on; names,
/// units and attributes are a contract with dashboards and alerts.
pub const PEER_METRICS: [(&str, &str); 4] = [
("peer.up", "1"),
("peer.api.auth_ok", "1"),
("peer.check.duration_seconds", "s"),
// Absolute unix timestamp of the last state change, not a duration.
("peer.last_change_timestamp_seconds", "s"),
];
/// Overall budget for the per-unit restart-count polling.
const RESTARTS_BUDGET: Duration = Duration::from_secs(15);
const RESTARTS_CONCURRENCY: usize = 8;
@@ -151,6 +162,61 @@ pub fn observations(s: &Snapshot) -> Vec<Obs> {
out
}
/// Map a registry snapshot to peer gauge observations.
///
/// Series carry `peer.name` and, per check, `peer.check` only: URLs can embed
/// hostnames an operator treats as sensitive, and a version attribute would
/// churn series on upgrade. A missing latency or `since` (no check yet, or a
/// failed one) yields no point rather than a stale or invented value.
///
/// Disabled peers are skipped: their checks never run, so a `0` would read as
/// "down" and page someone for a peer that was switched off deliberately.
pub fn peer_observations(peers: &[PeerStatus]) -> Vec<Obs> {
let b = |v: bool| if v { 1.0 } else { 0.0 };
let mut out = Vec::new();
for p in peers.iter().filter(|p| p.enabled) {
let name = p.name.as_str();
let check_attrs = |check: &str| {
vec![
("peer.name", name.to_owned()),
("peer.check", check.to_owned()),
]
};
for (check, up, latency, since) in [
("server", p.server.up, p.server.latency_ms, p.server.since),
("api", p.api.up, p.api.latency_ms, p.api.since),
] {
out.push(Obs {
metric: "peer.up",
value: b(up),
attrs: check_attrs(check),
});
if let Some(ms) = latency {
out.push(Obs {
metric: "peer.check.duration_seconds",
value: ms as f64 / 1000.0,
attrs: check_attrs(check),
});
}
if let Some(ts) = since {
out.push(Obs {
metric: "peer.last_change_timestamp_seconds",
value: ts.as_second() as f64,
attrs: check_attrs(check),
});
}
}
// "server up + api up + auth_ok = 0" is the revoked-credential state,
// kept separate from an API outage so alerts can tell them apart.
out.push(Obs {
metric: "peer.api.auth_ok",
value: b(p.api.auth == AuthState::Ok),
attrs: vec![("peer.name", name.to_owned())],
});
}
out
}
/// Poll the sources. A section whose call fails or times out keeps its value
/// from `prev` instead of being emptied.
async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot {
@@ -222,28 +288,52 @@ async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot {
snap
}
/// Register one observable f64 gauge fed by `obs`. The callback filters by
/// metric name, so a single observation list can feed several gauges.
fn register_gauge(
meter: &Meter,
name: &'static str,
unit: &'static str,
obs: impl Fn() -> Arc<Vec<Obs>> + Send + Sync + 'static,
) {
// Gauges live as long as the provider; the instrument handle is not needed.
let _ = meter
.f64_observable_gauge(name)
.with_unit(unit)
.with_callback(move |o| {
let obs = obs();
for ob in obs.iter().filter(|x| x.metric == name) {
let attrs: Vec<KeyValue> = ob
.attrs
.iter()
.map(|(k, v)| KeyValue::new(*k, v.clone()))
.collect();
o.observe(ob.value, &attrs);
}
})
.build();
}
pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>> {
// Observations are computed once per poll; callbacks only filter.
let current: Arc<RwLock<Arc<Vec<Obs>>>> = Arc::default();
let meter = provider.meter("skyg-linux-tracker");
for (name, unit) in METRICS {
let cur = Arc::clone(&current);
// Gauges live as long as the provider; the instrument handle is not needed.
let _ = meter
.f64_observable_gauge(name)
.with_unit(unit)
.with_callback(move |o| {
let obs = cur.read().map(|g| Arc::clone(&g)).unwrap_or_default();
for ob in obs.iter().filter(|x| x.metric == name) {
let attrs: Vec<KeyValue> = ob
.attrs
.iter()
.map(|(k, v)| KeyValue::new(*k, v.clone()))
.collect();
o.observe(ob.value, &attrs);
}
})
.build();
register_gauge(&meter, name, unit, move || {
cur.read().map(|g| Arc::clone(&g)).unwrap_or_default()
});
}
// Peer state already lives in memory, so peer gauges read the registry on
// each collection instead of polling anything; a peer removed from config
// therefore disappears from the next export.
if let Some(peers) = state.peers.clone() {
for (name, unit) in PEER_METRICS {
let peers = Arc::clone(&peers);
register_gauge(&meter, name, unit, move || {
Arc::new(peer_observations(&peers.snapshot()))
});
}
}
let interval = Duration::from_secs(state.config.otel.metrics_interval_secs.max(1));
@@ -404,6 +494,324 @@ mod tests {
assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string()));
}
fn ts(s: &str) -> jiff::Timestamp {
s.parse().expect("valid timestamp")
}
fn peer_status(name: &str, server_up: bool, api_up: bool, auth: AuthState) -> PeerStatus {
PeerStatus {
name: crate::peers::types::PeerName::parse(name).unwrap(),
enabled: true,
url: format!("http://{name}:1984"),
server: crate::peers::types::ServerStatus {
up: server_up,
since: Some(ts("2026-10-19T10:00:00Z")),
last_checked: Some(ts("2026-10-19T10:00:09Z")),
last_error: None,
latency_ms: Some(1500),
},
api: crate::peers::types::ApiStatus {
up: api_up,
since: Some(ts("2026-10-19T10:00:00Z")),
last_checked: Some(ts("2026-10-19T10:00:05Z")),
last_error: None,
latency_ms: Some(7),
auth,
peer_version: Some("0.1.0".into()),
},
}
}
fn attrs(ob: &Obs) -> Vec<(&str, &str)> {
ob.attrs.iter().map(|(k, v)| (*k, v.as_str())).collect()
}
fn find<'a>(o: &'a [Obs], metric: &str, peer: &str, check: &str) -> Option<&'a Obs> {
o.iter().find(|x| {
x.metric == metric
&& x.attrs.iter().any(|(k, v)| *k == "peer.name" && v == peer)
&& x.attrs
.iter()
.any(|(k, v)| *k == "peer.check" && v == check)
})
}
#[test]
fn peer_metric_names_and_units_are_the_documented_contract() {
assert_eq!(
PEER_METRICS.map(|(name, _)| name),
[
"peer.up",
"peer.api.auth_ok",
"peer.check.duration_seconds",
"peer.last_change_timestamp_seconds",
]
);
assert_eq!(PEER_METRICS.map(|(_, unit)| unit), ["1", "1", "s", "s"]);
}
#[test]
fn maps_peer_status_to_gauges() {
let o = peer_observations(&[peer_status("beta", true, true, AuthState::Ok)]);
let up = find(&o, "peer.up", "beta", "server").unwrap();
assert_eq!(up.value, 1.0);
assert_eq!(attrs(up), [("peer.name", "beta"), ("peer.check", "server")]);
assert_eq!(find(&o, "peer.up", "beta", "api").unwrap().value, 1.0);
let auth = o.iter().find(|x| x.metric == "peer.api.auth_ok").unwrap();
assert_eq!(auth.value, 1.0);
assert_eq!(attrs(auth), [("peer.name", "beta")]);
assert_eq!(
find(&o, "peer.check.duration_seconds", "beta", "server")
.unwrap()
.value,
1.5
);
assert_eq!(
find(&o, "peer.check.duration_seconds", "beta", "api")
.unwrap()
.value,
0.007
);
assert_eq!(
find(&o, "peer.last_change_timestamp_seconds", "beta", "api")
.unwrap()
.value,
ts("2026-10-19T10:00:00Z").as_second() as f64
);
// URLs and versions stay out of attributes: cardinality and hostnames
// the operator may treat as sensitive.
for ob in &o {
assert!(!ob.attrs.iter().any(|(_, v)| v.contains("http")));
assert!(!ob.attrs.iter().any(|(_, v)| v.contains("0.1.0")));
}
}
#[test]
fn rejected_credentials_are_visible_in_auth_ok() {
let o = peer_observations(&[peer_status("beta", true, true, AuthState::Rejected)]);
assert_eq!(find(&o, "peer.up", "beta", "api").unwrap().value, 1.0);
let auth = o.iter().find(|x| x.metric == "peer.api.auth_ok").unwrap();
assert_eq!(auth.value, 0.0);
let unknown = peer_observations(&[peer_status("beta", true, true, AuthState::Unknown)]);
let auth = unknown
.iter()
.find(|x| x.metric == "peer.api.auth_ok")
.unwrap();
assert_eq!(auth.value, 0.0);
}
#[test]
fn unchecked_peer_reports_up_only() {
let mut p = peer_status("beta", false, false, AuthState::Unknown);
p.server.latency_ms = None;
p.server.since = None;
p.api.latency_ms = None;
p.api.since = None;
let o = peer_observations(&[p]);
let metrics: Vec<&str> = o.iter().map(|x| x.metric).collect();
assert_eq!(metrics, ["peer.up", "peer.up", "peer.api.auth_ok"]);
assert!(o.iter().all(|x| x.value == 0.0));
}
#[test]
fn disabled_peer_is_not_reported() {
let mut p = peer_status("beta", true, true, AuthState::Ok);
p.enabled = false;
assert!(peer_observations(&[p]).is_empty());
}
/// Collects exports and keeps the samples of the last one.
#[derive(Default, Clone)]
struct Capture(std::sync::Arc<std::sync::Mutex<Vec<Vec<Sample>>>>);
#[derive(Debug, Clone, PartialEq)]
struct Sample {
metric: String,
unit: String,
value: f64,
attrs: Vec<(String, String)>,
}
impl Capture {
fn last(&self) -> Vec<Sample> {
self.0.lock().unwrap().last().cloned().unwrap_or_default()
}
}
impl opentelemetry_sdk::metrics::exporter::PushMetricExporter for Capture {
fn export(
&self,
metrics: &opentelemetry_sdk::metrics::data::ResourceMetrics,
) -> impl std::future::Future<Output = opentelemetry_sdk::error::OTelSdkResult> + Send
{
use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
let mut samples = Vec::new();
for scope in metrics.scope_metrics() {
for m in scope.metrics() {
let AggregatedMetrics::F64(MetricData::Gauge(g)) = m.data() else {
continue;
};
for p in g.data_points() {
samples.push(Sample {
metric: m.name().to_owned(),
unit: m.unit().to_owned(),
value: p.value(),
attrs: p
.attributes()
.map(|kv| {
(kv.key.as_str().to_owned(), kv.value.as_str().into_owned())
})
.collect(),
});
}
}
}
self.0.lock().unwrap().push(samples);
std::future::ready(Ok(()))
}
fn force_flush(&self) -> opentelemetry_sdk::error::OTelSdkResult {
Ok(())
}
fn shutdown_with_timeout(
&self,
_timeout: Duration,
) -> opentelemetry_sdk::error::OTelSdkResult {
Ok(())
}
fn temporality(&self) -> opentelemetry_sdk::metrics::Temporality {
opentelemetry_sdk::metrics::Temporality::Cumulative
}
}
/// Provider with a capture exporter; export happens on `force_flush`.
fn capture_provider() -> (SdkMeterProvider, Capture) {
let capture = Capture::default();
let reader = opentelemetry_sdk::metrics::PeriodicReader::builder(capture.clone())
.with_interval(Duration::from_secs(3600))
.build();
let provider = SdkMeterProvider::builder().with_reader(reader).build();
(provider, capture)
}
fn sample<'a>(
s: &'a [Sample],
metric: &str,
peer: &str,
check: Option<&str>,
) -> Option<&'a Sample> {
s.iter().find(|x| {
x.metric == metric
&& x.attrs.iter().any(|(k, v)| k == "peer.name" && v == peer)
&& check.is_none_or(|c| x.attrs.iter().any(|(k, v)| k == "peer.check" && v == c))
})
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn exported_peer_gauges_read_the_registry_snapshot() {
let reg = crate::peers::testing::static_registry(vec![
peer_status("beta", true, true, AuthState::Ok),
peer_status("gamma", true, false, AuthState::Unknown),
]);
let state = crate::peers::testing::fake_state_with_peers(Config::default(), reg.clone());
let (provider, capture) = capture_provider();
let handles = spawn(state.clone(), &provider);
provider.force_flush().unwrap();
let samples = capture.last();
let beta_up = sample(&samples, "peer.up", "beta", Some("server")).unwrap();
assert_eq!(beta_up.value, 1.0);
assert_eq!(beta_up.unit, "1");
assert_eq!(
sample(&samples, "peer.up", "gamma", Some("api"))
.unwrap()
.value,
0.0
);
assert_eq!(
sample(&samples, "peer.api.auth_ok", "gamma", None)
.unwrap()
.value,
0.0
);
assert_eq!(
sample(
&samples,
"peer.check.duration_seconds",
"beta",
Some("server")
)
.unwrap()
.value,
1.5
);
// Removing a peer from the snapshot drops its series; it is not cached.
reg.reload(crate::peers::testing::fake_peers_config(&["beta"]))
.await
.unwrap();
provider.force_flush().unwrap();
let after = capture.last();
assert!(sample(&after, "peer.up", "beta", Some("server")).is_some());
assert!(sample(&after, "peer.up", "gamma", Some("server")).is_none());
state.shutdown.cancel();
for h in handles {
let _ = h.await;
}
provider.shutdown().unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn collecting_peer_gauges_does_not_poll_the_transport() {
let fake = crate::peers::testing::FakeTransport::new();
let reg = crate::peers::testing::registry_with(
fake.clone(),
crate::peers::testing::fake_peers_config(&["beta"]),
);
let state = crate::peers::testing::fake_state_with_peers(Config::default(), reg);
let (provider, capture) = capture_provider();
let handles = spawn(state.clone(), &provider);
provider.force_flush().unwrap();
let samples = capture.last();
assert!(sample(&samples, "peer.up", "beta", Some("server")).is_some());
assert!(fake.calls().is_empty(), "gauges must not poll peers");
state.shutdown.cancel();
for h in handles {
let _ = h.await;
}
provider.shutdown().unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn peerless_state_exports_no_peer_series() {
let state = fake_state(Config::default());
let (provider, capture) = capture_provider();
let handles = spawn(state.clone(), &provider);
provider.force_flush().unwrap();
let samples = capture.last();
assert!(
samples.iter().all(|s| !s.metric.starts_with("peer.")),
"no peer series without the feature: {samples:?}"
);
state.shutdown.cancel();
for h in handles {
let _ = h.await;
}
provider.shutdown().unwrap();
}
#[tokio::test]
async fn collect_applies_scope_with_fakes() {
let mut cfg = Config::default();