WS-G: apply review fixes F1-F8

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
ariandClaude Sonnet 5.5 committed 2026-10-15 18:43:21 -04:00
1 parent 2aeb9ae154
commit 1eb7ffce19
4 files changed
+219 -99

No files matched your search

+90 -62
View File
@@ -8,6 +8,7 @@ use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::time::Duration;
use futures::StreamExt;
use opentelemetry::metrics::MeterProvider as _;
use opentelemetry::KeyValue;
use opentelemetry_sdk::metrics::SdkMeterProvider;
@@ -21,17 +22,23 @@ use crate::state::AppState;
/// Per-call bound so a hung backend cannot stall the poll loop.
const CALL_TIMEOUT: Duration = Duration::from_secs(10);
pub const METRIC_NAMES: [&str; 8] = [
"systemd.unit.active",
"systemd.unit.failed",
"systemd.unit.restarts",
"systemd.timer.next_elapse_seconds",
"docker.container.running",
"docker.container.healthy",
"docker.container.uptime_seconds",
"docker.container.restarts",
/// Metric name and UCUM unit.
pub const METRICS: [(&str, &str); 8] = [
("systemd.unit.active", "1"),
("systemd.unit.failed", "1"),
("systemd.unit.restarts", "{restart}"),
// Absolute unix timestamp of the next elapse, not a duration.
("systemd.timer.next_elapse_timestamp_seconds", "s"),
("docker.container.running", "1"),
("docker.container.healthy", "1"),
("docker.container.uptime_seconds", "s"),
("docker.container.restarts", "{restart}"),
];
/// Overall budget for the per-unit restart-count polling.
const RESTARTS_BUDGET: Duration = Duration::from_secs(15);
const RESTARTS_CONCURRENCY: usize = 8;
/// Latest poll result, already filtered by the allowlists.
#[derive(Debug, Default, Clone)]
pub struct Snapshot {
@@ -64,92 +71,91 @@ pub fn container_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool {
cfg.containers.is_empty() || any_glob(&cfg.containers, name)
}
fn unit_attrs(u: &UnitSummary) -> Vec<(&'static str, String)> {
vec![
("systemd.unit", u.name.clone()),
("systemd.sub_state", u.sub_state.clone()),
("systemd.load_state", u.load_state.clone()),
]
fn unit_id(name: &str) -> Vec<(&'static str, String)> {
vec![("systemd.unit", name.to_owned())]
}
fn container_attrs(c: &ContainerSummary) -> Vec<(&'static str, String)> {
vec![
("container.name", c.name.clone()),
("container.id", c.id.clone()),
("container.image.name", c.image.clone()),
]
fn container_id(c: &ContainerSummary) -> Vec<(&'static str, String)> {
vec![("container.name", c.name.clone())]
}
/// Map a snapshot to gauge observations.
/// Map a snapshot to gauge observations. Series carry identity attributes
/// only (`systemd.unit` / `container.name`) so state changes do not churn
/// series. The `active` gauge also carries sub/load state, and `running`
/// carries the image name.
pub fn observations(s: &Snapshot) -> Vec<Obs> {
let b = |v: bool| if v { 1.0 } else { 0.0 };
let mut out = Vec::new();
for u in &s.units {
let mut active_attrs = unit_id(&u.name);
active_attrs.push(("systemd.sub_state", u.sub_state.clone()));
active_attrs.push(("systemd.load_state", u.load_state.clone()));
out.push(Obs {
metric: "systemd.unit.active",
value: b(u.active_state == "active"),
attrs: unit_attrs(u),
attrs: active_attrs,
});
out.push(Obs {
metric: "systemd.unit.failed",
value: b(u.active_state == "failed"),
attrs: unit_attrs(u),
attrs: unit_id(&u.name),
});
if let Some(n) = s.restarts.get(&u.name) {
out.push(Obs {
metric: "systemd.unit.restarts",
value: f64::from(*n),
attrs: unit_attrs(u),
attrs: unit_id(&u.name),
});
}
}
for t in &s.timers {
// Absolute unix time of the next elapse; unscheduled timers are omitted.
// Unscheduled timers are omitted.
if let Some(next) = t.next_elapse {
out.push(Obs {
metric: "systemd.timer.next_elapse_seconds",
metric: "systemd.timer.next_elapse_timestamp_seconds",
value: next.as_second() as f64,
attrs: vec![
("systemd.unit", t.name.clone()),
("systemd.sub_state", t.active_state.clone()),
],
attrs: unit_id(&t.name),
});
}
}
for c in &s.containers {
let mut running_attrs = container_id(c);
running_attrs.push(("container.image.name", c.image.clone()));
out.push(Obs {
metric: "docker.container.running",
value: b(c.state == "running"),
attrs: container_attrs(c),
attrs: running_attrs,
});
if let Some(h) = &c.health {
out.push(Obs {
metric: "docker.container.healthy",
value: b(h == "healthy"),
attrs: container_attrs(c),
attrs: container_id(c),
});
}
if let Some(up) = c.uptime_seconds {
out.push(Obs {
metric: "docker.container.uptime_seconds",
value: up as f64,
attrs: container_attrs(c),
attrs: container_id(c),
});
}
if let Some(r) = c.restart_count {
out.push(Obs {
metric: "docker.container.restarts",
value: r as f64,
attrs: container_attrs(c),
attrs: container_id(c),
});
}
}
out
}
async fn collect(state: &AppState) -> Snapshot {
/// Poll the sources. A section whose call fails or times out keeps its value
/// from `prev` instead of being emptied.
async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot {
let cfg = &state.config.otel.metrics;
let mut snap = Snapshot::default();
let mut snap = prev.clone();
match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_units(None)).await {
Ok(Ok(units)) => {
@@ -171,24 +177,38 @@ async fn collect(state: &AppState) -> Snapshot {
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_timers failed"),
Err(_) => tracing::debug!("otel metrics: list_timers timed out"),
}
// Restart counts: bounded concurrency and an overall budget. Entries not
// refreshed in time keep their previous value; vanished units are dropped.
let services: Vec<String> = snap
.units
.iter()
.filter(|u| u.name.ends_with(".service") && u.load_state == "loaded")
.map(|u| u.name.clone())
.collect();
for name in services {
if state.shutdown.is_cancelled() {
break;
}
if let Ok(Ok(st)) =
tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await
{
if let Some(n) = st.n_restarts {
snap.restarts.insert(name, n);
let mut restarts: HashMap<String, u32> = services
.iter()
.filter_map(|n| prev.restarts.get(n).map(|v| (n.clone(), *v)))
.collect();
let fetch = futures::stream::iter(services)
.map(|name| async move {
let r = tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await;
(name, r)
})
.buffer_unordered(RESTARTS_CONCURRENCY)
.for_each(|(name, r)| {
if let Ok(Ok(st)) = r {
if let Some(n) = st.n_restarts {
restarts.insert(name, n);
}
}
}
std::future::ready(())
});
if tokio::time::timeout(RESTARTS_BUDGET, fetch).await.is_err() {
tracing::debug!("otel metrics: restart polling hit its time budget");
}
snap.restarts = restarts;
match tokio::time::timeout(CALL_TIMEOUT, state.docker.ps(true)).await {
Ok(Ok(cs)) => {
snap.containers = cs
@@ -204,20 +224,22 @@ async fn collect(state: &AppState) -> Snapshot {
/// Register the gauges and start the poll loop.
pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>> {
let snapshot: Arc<RwLock<Arc<Snapshot>>> = Arc::default();
// Observations are computed once per poll; callbacks only filter.
let current: Arc<RwLock<Arc<Vec<Obs>>>> = Arc::default();
let meter = provider.meter("skyg-linux-tracker");
for name in METRIC_NAMES {
let snap = Arc::clone(&snapshot);
for (name, unit) in METRICS {
let cur = Arc::clone(&current);
// Gauges live as long as the provider; the instrument handle is not needed.
let _ = meter
.f64_observable_gauge(name)
.with_unit(unit)
.with_callback(move |o| {
let cur = snap.read().map(|g| Arc::clone(&g)).unwrap_or_default();
for ob in observations(&cur).into_iter().filter(|x| x.metric == name) {
let obs = cur.read().map(|g| Arc::clone(&g)).unwrap_or_default();
for ob in obs.iter().filter(|x| x.metric == name) {
let attrs: Vec<KeyValue> = ob
.attrs
.into_iter()
.map(|(k, v)| KeyValue::new(k, v))
.iter()
.map(|(k, v)| KeyValue::new(*k, v.clone()))
.collect();
o.observe(ob.value, &attrs);
}
@@ -229,17 +251,18 @@ pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>
let h = tokio::spawn(async move {
let mut tick = tokio::time::interval(interval);
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
let mut snap = Snapshot::default();
loop {
tokio::select! {
_ = state.shutdown.cancelled() => break,
_ = tick.tick() => {}
}
let snap = tokio::select! {
snap = tokio::select! {
_ = state.shutdown.cancelled() => break,
s = collect(&state) => s,
s = collect(&state, &snap) => s,
};
if let Ok(mut g) = snapshot.write() {
*g = Arc::new(snap);
if let Ok(mut g) = current.write() {
*g = Arc::new(observations(&snap));
}
}
});
@@ -359,7 +382,7 @@ mod tests {
let o = observations(&s);
let timers: Vec<_> = o
.iter()
.filter(|x| x.metric == "systemd.timer.next_elapse_seconds")
.filter(|x| x.metric == "systemd.timer.next_elapse_timestamp_seconds")
.collect();
assert_eq!(timers.len(), 1);
assert_eq!(timers[0].value, 1_792_065_600.0);
@@ -374,7 +397,12 @@ mod tests {
.unwrap();
assert_eq!(h.value, 0.0);
assert_eq!(h.attrs[0], ("container.name", "web".to_string()));
assert_eq!(h.attrs[2], ("container.image.name", "nginx:1".to_string()));
assert_eq!(h.attrs.len(), 1);
let r = o
.iter()
.find(|x| x.metric == "docker.container.running")
.unwrap();
assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string()));
}
#[tokio::test]
@@ -383,13 +411,13 @@ mod tests {
cfg.otel.metrics.units = vec!["__nomatch__*".into()];
cfg.otel.metrics.containers = vec!["__nomatch__*".into()];
let st = fake_state(cfg);
let snap = collect(&st).await;
let snap = collect(&st, &Snapshot::default()).await;
assert!(snap.units.is_empty());
assert!(snap.timers.is_empty());
assert!(snap.containers.is_empty());
let st = fake_state(Config::default());
let snap = collect(&st).await;
let snap = collect(&st, &Snapshot::default()).await;
assert!(snap
.units
.iter()