E: E1 peer health gauges from the registry snapshot
This commit is contained in:
1 parent
f0b6c29eac
commit
eee253b162
1 file changed
+425
-17
+425
-17
@@ -9,13 +9,14 @@ use std::sync::{Arc, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt;
|
||||
use opentelemetry::metrics::MeterProvider as _;
|
||||
use opentelemetry::metrics::{Meter, MeterProvider as _};
|
||||
use opentelemetry::KeyValue;
|
||||
use opentelemetry_sdk::metrics::SdkMeterProvider;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use super::any_glob;
|
||||
use crate::config::OtelMetricsConfig;
|
||||
use crate::peers::types::{AuthState, PeerStatus};
|
||||
use crate::sources::{ContainerSummary, TimerInfo, UnitSummary};
|
||||
use crate::state::AppState;
|
||||
|
||||
@@ -35,6 +36,16 @@ pub const METRICS: [(&str, &str); 8] = [
|
||||
("docker.container.restarts", "{restart}"),
|
||||
];
|
||||
|
||||
/// Peer health gauges. Registered only when the peers feature is on; names,
|
||||
/// units and attributes are a contract with dashboards and alerts.
|
||||
pub const PEER_METRICS: [(&str, &str); 4] = [
|
||||
("peer.up", "1"),
|
||||
("peer.api.auth_ok", "1"),
|
||||
("peer.check.duration_seconds", "s"),
|
||||
// Absolute unix timestamp of the last state change, not a duration.
|
||||
("peer.last_change_timestamp_seconds", "s"),
|
||||
];
|
||||
|
||||
/// Overall budget for the per-unit restart-count polling.
|
||||
const RESTARTS_BUDGET: Duration = Duration::from_secs(15);
|
||||
const RESTARTS_CONCURRENCY: usize = 8;
|
||||
@@ -151,6 +162,61 @@ pub fn observations(s: &Snapshot) -> Vec<Obs> {
|
||||
out
|
||||
}
|
||||
|
||||
/// Map a registry snapshot to peer gauge observations.
|
||||
///
|
||||
/// Series carry `peer.name` and, per check, `peer.check` only: URLs can embed
|
||||
/// hostnames an operator treats as sensitive, and a version attribute would
|
||||
/// churn series on upgrade. A missing latency or `since` (no check yet, or a
|
||||
/// failed one) yields no point rather than a stale or invented value.
|
||||
///
|
||||
/// Disabled peers are skipped: their checks never run, so a `0` would read as
|
||||
/// "down" and page someone for a peer that was switched off deliberately.
|
||||
pub fn peer_observations(peers: &[PeerStatus]) -> Vec<Obs> {
|
||||
let b = |v: bool| if v { 1.0 } else { 0.0 };
|
||||
let mut out = Vec::new();
|
||||
for p in peers.iter().filter(|p| p.enabled) {
|
||||
let name = p.name.as_str();
|
||||
let check_attrs = |check: &str| {
|
||||
vec![
|
||||
("peer.name", name.to_owned()),
|
||||
("peer.check", check.to_owned()),
|
||||
]
|
||||
};
|
||||
for (check, up, latency, since) in [
|
||||
("server", p.server.up, p.server.latency_ms, p.server.since),
|
||||
("api", p.api.up, p.api.latency_ms, p.api.since),
|
||||
] {
|
||||
out.push(Obs {
|
||||
metric: "peer.up",
|
||||
value: b(up),
|
||||
attrs: check_attrs(check),
|
||||
});
|
||||
if let Some(ms) = latency {
|
||||
out.push(Obs {
|
||||
metric: "peer.check.duration_seconds",
|
||||
value: ms as f64 / 1000.0,
|
||||
attrs: check_attrs(check),
|
||||
});
|
||||
}
|
||||
if let Some(ts) = since {
|
||||
out.push(Obs {
|
||||
metric: "peer.last_change_timestamp_seconds",
|
||||
value: ts.as_second() as f64,
|
||||
attrs: check_attrs(check),
|
||||
});
|
||||
}
|
||||
}
|
||||
// "server up + api up + auth_ok = 0" is the revoked-credential state,
|
||||
// kept separate from an API outage so alerts can tell them apart.
|
||||
out.push(Obs {
|
||||
metric: "peer.api.auth_ok",
|
||||
value: b(p.api.auth == AuthState::Ok),
|
||||
attrs: vec![("peer.name", name.to_owned())],
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Poll the sources. A section whose call fails or times out keeps its value
|
||||
/// from `prev` instead of being emptied.
|
||||
async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot {
|
||||
@@ -222,28 +288,52 @@ async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot {
|
||||
snap
|
||||
}
|
||||
|
||||
/// Register one observable f64 gauge fed by `obs`. The callback filters by
|
||||
/// metric name, so a single observation list can feed several gauges.
|
||||
fn register_gauge(
|
||||
meter: &Meter,
|
||||
name: &'static str,
|
||||
unit: &'static str,
|
||||
obs: impl Fn() -> Arc<Vec<Obs>> + Send + Sync + 'static,
|
||||
) {
|
||||
// Gauges live as long as the provider; the instrument handle is not needed.
|
||||
let _ = meter
|
||||
.f64_observable_gauge(name)
|
||||
.with_unit(unit)
|
||||
.with_callback(move |o| {
|
||||
let obs = obs();
|
||||
for ob in obs.iter().filter(|x| x.metric == name) {
|
||||
let attrs: Vec<KeyValue> = ob
|
||||
.attrs
|
||||
.iter()
|
||||
.map(|(k, v)| KeyValue::new(*k, v.clone()))
|
||||
.collect();
|
||||
o.observe(ob.value, &attrs);
|
||||
}
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>> {
|
||||
// Observations are computed once per poll; callbacks only filter.
|
||||
let current: Arc<RwLock<Arc<Vec<Obs>>>> = Arc::default();
|
||||
let meter = provider.meter("skyg-linux-tracker");
|
||||
for (name, unit) in METRICS {
|
||||
let cur = Arc::clone(¤t);
|
||||
// Gauges live as long as the provider; the instrument handle is not needed.
|
||||
let _ = meter
|
||||
.f64_observable_gauge(name)
|
||||
.with_unit(unit)
|
||||
.with_callback(move |o| {
|
||||
let obs = cur.read().map(|g| Arc::clone(&g)).unwrap_or_default();
|
||||
for ob in obs.iter().filter(|x| x.metric == name) {
|
||||
let attrs: Vec<KeyValue> = ob
|
||||
.attrs
|
||||
.iter()
|
||||
.map(|(k, v)| KeyValue::new(*k, v.clone()))
|
||||
.collect();
|
||||
o.observe(ob.value, &attrs);
|
||||
}
|
||||
})
|
||||
.build();
|
||||
register_gauge(&meter, name, unit, move || {
|
||||
cur.read().map(|g| Arc::clone(&g)).unwrap_or_default()
|
||||
});
|
||||
}
|
||||
// Peer state already lives in memory, so peer gauges read the registry on
|
||||
// each collection instead of polling anything; a peer removed from config
|
||||
// therefore disappears from the next export.
|
||||
if let Some(peers) = state.peers.clone() {
|
||||
for (name, unit) in PEER_METRICS {
|
||||
let peers = Arc::clone(&peers);
|
||||
register_gauge(&meter, name, unit, move || {
|
||||
Arc::new(peer_observations(&peers.snapshot()))
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let interval = Duration::from_secs(state.config.otel.metrics_interval_secs.max(1));
|
||||
@@ -404,6 +494,324 @@ mod tests {
|
||||
assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string()));
|
||||
}
|
||||
|
||||
fn ts(s: &str) -> jiff::Timestamp {
|
||||
s.parse().expect("valid timestamp")
|
||||
}
|
||||
|
||||
fn peer_status(name: &str, server_up: bool, api_up: bool, auth: AuthState) -> PeerStatus {
|
||||
PeerStatus {
|
||||
name: crate::peers::types::PeerName::parse(name).unwrap(),
|
||||
enabled: true,
|
||||
url: format!("http://{name}:1984"),
|
||||
server: crate::peers::types::ServerStatus {
|
||||
up: server_up,
|
||||
since: Some(ts("2026-10-19T10:00:00Z")),
|
||||
last_checked: Some(ts("2026-10-19T10:00:09Z")),
|
||||
last_error: None,
|
||||
latency_ms: Some(1500),
|
||||
},
|
||||
api: crate::peers::types::ApiStatus {
|
||||
up: api_up,
|
||||
since: Some(ts("2026-10-19T10:00:00Z")),
|
||||
last_checked: Some(ts("2026-10-19T10:00:05Z")),
|
||||
last_error: None,
|
||||
latency_ms: Some(7),
|
||||
auth,
|
||||
peer_version: Some("0.1.0".into()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn attrs(ob: &Obs) -> Vec<(&str, &str)> {
|
||||
ob.attrs.iter().map(|(k, v)| (*k, v.as_str())).collect()
|
||||
}
|
||||
|
||||
fn find<'a>(o: &'a [Obs], metric: &str, peer: &str, check: &str) -> Option<&'a Obs> {
|
||||
o.iter().find(|x| {
|
||||
x.metric == metric
|
||||
&& x.attrs.iter().any(|(k, v)| *k == "peer.name" && v == peer)
|
||||
&& x.attrs
|
||||
.iter()
|
||||
.any(|(k, v)| *k == "peer.check" && v == check)
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_metric_names_and_units_are_the_documented_contract() {
|
||||
assert_eq!(
|
||||
PEER_METRICS.map(|(name, _)| name),
|
||||
[
|
||||
"peer.up",
|
||||
"peer.api.auth_ok",
|
||||
"peer.check.duration_seconds",
|
||||
"peer.last_change_timestamp_seconds",
|
||||
]
|
||||
);
|
||||
assert_eq!(PEER_METRICS.map(|(_, unit)| unit), ["1", "1", "s", "s"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_peer_status_to_gauges() {
|
||||
let o = peer_observations(&[peer_status("beta", true, true, AuthState::Ok)]);
|
||||
|
||||
let up = find(&o, "peer.up", "beta", "server").unwrap();
|
||||
assert_eq!(up.value, 1.0);
|
||||
assert_eq!(attrs(up), [("peer.name", "beta"), ("peer.check", "server")]);
|
||||
assert_eq!(find(&o, "peer.up", "beta", "api").unwrap().value, 1.0);
|
||||
|
||||
let auth = o.iter().find(|x| x.metric == "peer.api.auth_ok").unwrap();
|
||||
assert_eq!(auth.value, 1.0);
|
||||
assert_eq!(attrs(auth), [("peer.name", "beta")]);
|
||||
|
||||
assert_eq!(
|
||||
find(&o, "peer.check.duration_seconds", "beta", "server")
|
||||
.unwrap()
|
||||
.value,
|
||||
1.5
|
||||
);
|
||||
assert_eq!(
|
||||
find(&o, "peer.check.duration_seconds", "beta", "api")
|
||||
.unwrap()
|
||||
.value,
|
||||
0.007
|
||||
);
|
||||
assert_eq!(
|
||||
find(&o, "peer.last_change_timestamp_seconds", "beta", "api")
|
||||
.unwrap()
|
||||
.value,
|
||||
ts("2026-10-19T10:00:00Z").as_second() as f64
|
||||
);
|
||||
|
||||
// URLs and versions stay out of attributes: cardinality and hostnames
|
||||
// the operator may treat as sensitive.
|
||||
for ob in &o {
|
||||
assert!(!ob.attrs.iter().any(|(_, v)| v.contains("http")));
|
||||
assert!(!ob.attrs.iter().any(|(_, v)| v.contains("0.1.0")));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejected_credentials_are_visible_in_auth_ok() {
|
||||
let o = peer_observations(&[peer_status("beta", true, true, AuthState::Rejected)]);
|
||||
assert_eq!(find(&o, "peer.up", "beta", "api").unwrap().value, 1.0);
|
||||
let auth = o.iter().find(|x| x.metric == "peer.api.auth_ok").unwrap();
|
||||
assert_eq!(auth.value, 0.0);
|
||||
|
||||
let unknown = peer_observations(&[peer_status("beta", true, true, AuthState::Unknown)]);
|
||||
let auth = unknown
|
||||
.iter()
|
||||
.find(|x| x.metric == "peer.api.auth_ok")
|
||||
.unwrap();
|
||||
assert_eq!(auth.value, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unchecked_peer_reports_up_only() {
|
||||
let mut p = peer_status("beta", false, false, AuthState::Unknown);
|
||||
p.server.latency_ms = None;
|
||||
p.server.since = None;
|
||||
p.api.latency_ms = None;
|
||||
p.api.since = None;
|
||||
let o = peer_observations(&[p]);
|
||||
let metrics: Vec<&str> = o.iter().map(|x| x.metric).collect();
|
||||
assert_eq!(metrics, ["peer.up", "peer.up", "peer.api.auth_ok"]);
|
||||
assert!(o.iter().all(|x| x.value == 0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_peer_is_not_reported() {
|
||||
let mut p = peer_status("beta", true, true, AuthState::Ok);
|
||||
p.enabled = false;
|
||||
assert!(peer_observations(&[p]).is_empty());
|
||||
}
|
||||
|
||||
/// Collects exports and keeps the samples of the last one.
|
||||
#[derive(Default, Clone)]
|
||||
struct Capture(std::sync::Arc<std::sync::Mutex<Vec<Vec<Sample>>>>);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct Sample {
|
||||
metric: String,
|
||||
unit: String,
|
||||
value: f64,
|
||||
attrs: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl Capture {
|
||||
fn last(&self) -> Vec<Sample> {
|
||||
self.0.lock().unwrap().last().cloned().unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
impl opentelemetry_sdk::metrics::exporter::PushMetricExporter for Capture {
|
||||
fn export(
|
||||
&self,
|
||||
metrics: &opentelemetry_sdk::metrics::data::ResourceMetrics,
|
||||
) -> impl std::future::Future<Output = opentelemetry_sdk::error::OTelSdkResult> + Send
|
||||
{
|
||||
use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
|
||||
let mut samples = Vec::new();
|
||||
for scope in metrics.scope_metrics() {
|
||||
for m in scope.metrics() {
|
||||
let AggregatedMetrics::F64(MetricData::Gauge(g)) = m.data() else {
|
||||
continue;
|
||||
};
|
||||
for p in g.data_points() {
|
||||
samples.push(Sample {
|
||||
metric: m.name().to_owned(),
|
||||
unit: m.unit().to_owned(),
|
||||
value: p.value(),
|
||||
attrs: p
|
||||
.attributes()
|
||||
.map(|kv| {
|
||||
(kv.key.as_str().to_owned(), kv.value.as_str().into_owned())
|
||||
})
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
self.0.lock().unwrap().push(samples);
|
||||
std::future::ready(Ok(()))
|
||||
}
|
||||
|
||||
fn force_flush(&self) -> opentelemetry_sdk::error::OTelSdkResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn shutdown_with_timeout(
|
||||
&self,
|
||||
_timeout: Duration,
|
||||
) -> opentelemetry_sdk::error::OTelSdkResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn temporality(&self) -> opentelemetry_sdk::metrics::Temporality {
|
||||
opentelemetry_sdk::metrics::Temporality::Cumulative
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider with a capture exporter; export happens on `force_flush`.
|
||||
fn capture_provider() -> (SdkMeterProvider, Capture) {
|
||||
let capture = Capture::default();
|
||||
let reader = opentelemetry_sdk::metrics::PeriodicReader::builder(capture.clone())
|
||||
.with_interval(Duration::from_secs(3600))
|
||||
.build();
|
||||
let provider = SdkMeterProvider::builder().with_reader(reader).build();
|
||||
(provider, capture)
|
||||
}
|
||||
|
||||
fn sample<'a>(
|
||||
s: &'a [Sample],
|
||||
metric: &str,
|
||||
peer: &str,
|
||||
check: Option<&str>,
|
||||
) -> Option<&'a Sample> {
|
||||
s.iter().find(|x| {
|
||||
x.metric == metric
|
||||
&& x.attrs.iter().any(|(k, v)| k == "peer.name" && v == peer)
|
||||
&& check.is_none_or(|c| x.attrs.iter().any(|(k, v)| k == "peer.check" && v == c))
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn exported_peer_gauges_read_the_registry_snapshot() {
|
||||
let reg = crate::peers::testing::static_registry(vec![
|
||||
peer_status("beta", true, true, AuthState::Ok),
|
||||
peer_status("gamma", true, false, AuthState::Unknown),
|
||||
]);
|
||||
let state = crate::peers::testing::fake_state_with_peers(Config::default(), reg.clone());
|
||||
let (provider, capture) = capture_provider();
|
||||
let handles = spawn(state.clone(), &provider);
|
||||
provider.force_flush().unwrap();
|
||||
|
||||
let samples = capture.last();
|
||||
let beta_up = sample(&samples, "peer.up", "beta", Some("server")).unwrap();
|
||||
assert_eq!(beta_up.value, 1.0);
|
||||
assert_eq!(beta_up.unit, "1");
|
||||
assert_eq!(
|
||||
sample(&samples, "peer.up", "gamma", Some("api"))
|
||||
.unwrap()
|
||||
.value,
|
||||
0.0
|
||||
);
|
||||
assert_eq!(
|
||||
sample(&samples, "peer.api.auth_ok", "gamma", None)
|
||||
.unwrap()
|
||||
.value,
|
||||
0.0
|
||||
);
|
||||
assert_eq!(
|
||||
sample(
|
||||
&samples,
|
||||
"peer.check.duration_seconds",
|
||||
"beta",
|
||||
Some("server")
|
||||
)
|
||||
.unwrap()
|
||||
.value,
|
||||
1.5
|
||||
);
|
||||
|
||||
// Removing a peer from the snapshot drops its series; it is not cached.
|
||||
reg.reload(crate::peers::testing::fake_peers_config(&["beta"]))
|
||||
.await
|
||||
.unwrap();
|
||||
provider.force_flush().unwrap();
|
||||
let after = capture.last();
|
||||
assert!(sample(&after, "peer.up", "beta", Some("server")).is_some());
|
||||
assert!(sample(&after, "peer.up", "gamma", Some("server")).is_none());
|
||||
|
||||
state.shutdown.cancel();
|
||||
for h in handles {
|
||||
let _ = h.await;
|
||||
}
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn collecting_peer_gauges_does_not_poll_the_transport() {
|
||||
let fake = crate::peers::testing::FakeTransport::new();
|
||||
let reg = crate::peers::testing::registry_with(
|
||||
fake.clone(),
|
||||
crate::peers::testing::fake_peers_config(&["beta"]),
|
||||
);
|
||||
let state = crate::peers::testing::fake_state_with_peers(Config::default(), reg);
|
||||
let (provider, capture) = capture_provider();
|
||||
let handles = spawn(state.clone(), &provider);
|
||||
provider.force_flush().unwrap();
|
||||
|
||||
let samples = capture.last();
|
||||
assert!(sample(&samples, "peer.up", "beta", Some("server")).is_some());
|
||||
assert!(fake.calls().is_empty(), "gauges must not poll peers");
|
||||
|
||||
state.shutdown.cancel();
|
||||
for h in handles {
|
||||
let _ = h.await;
|
||||
}
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn peerless_state_exports_no_peer_series() {
|
||||
let state = fake_state(Config::default());
|
||||
let (provider, capture) = capture_provider();
|
||||
let handles = spawn(state.clone(), &provider);
|
||||
provider.force_flush().unwrap();
|
||||
|
||||
let samples = capture.last();
|
||||
assert!(
|
||||
samples.iter().all(|s| !s.metric.starts_with("peer.")),
|
||||
"no peer series without the feature: {samples:?}"
|
||||
);
|
||||
|
||||
state.shutdown.cancel();
|
||||
for h in handles {
|
||||
let _ = h.await;
|
||||
}
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_applies_scope_with_fakes() {
|
||||
let mut cfg = Config::default();
|
||||
|
||||
Reference in new issue
Block a user