diff --git a/plan/workstreams/WS-G-otel.md b/plan/workstreams/WS-G-otel.md index 475bf42..2686d4c 100644 --- a/plan/workstreams/WS-G-otel.md +++ b/plan/workstreams/WS-G-otel.md @@ -53,4 +53,6 @@ Tick these in [`TODO.md`](../../TODO.md). ## Log - 2026-10-15: Card created. +- 2026-10-15: Review fixes F1-F8. Restart polling uses buffer_unordered(8) with a 15s budget; failed sections keep the previous snapshot; series carry identity attributes only; timer metric renamed `systemd.timer.next_elapse_timestamp_seconds`; units added; observations computed once per poll. New containers start from StartedAt, docker lines deduped on reconnect, own unit/identifier excluded from the journal follow (by name; cursor not used). `TelemetryGuard::shutdown().await` added; Drop is best-effort. +- P3 notes: (1) call `guard.shutdown().await` after awaiting the exporter handles. (2) devenv: add `processes.otelcol.exec = "otelcol-contrib --config dev/otelcol.yaml"` to devenv.nix (not done here, P0 owner). (3) Cargo.toml needs `reqwest-blocking-client` on opentelemetry-otlp for http/protobuf; the exporter-build test is strict and fails without it. (4) RUST_LOG/EnvFilter deferred. - 2026-10-15: Implemented G1-G7 on `ws-g/otel`. Notes: no glob crate, so a small `*`/`?` matcher lives in `telemetry/mod.rs`. `DockerSource` has no events stream, so the log reconciler polls `ps(false)` every 5s (30s while Docker is unavailable) instead of subscribing to `events`. No `EnvFilter` (feature not enabled in Cargo.toml): level is fixed at INFO. `systemd.timer.next_elapse_seconds` exports absolute unix seconds. `systemd.unit.restarts` costs one `unit_status` per in-scope loaded service per interval. HTTP endpoints get `/v1/` appended unless already present. Live otelcol smoke test not run (no collector binary on PATH). diff --git a/src/telemetry/logs_export.rs b/src/telemetry/logs_export.rs index 0e1aead..aa5fcbf 100644 --- a/src/telemetry/logs_export.rs +++ b/src/telemetry/logs_export.rs @@ -130,7 +130,31 @@ pub fn spawn(state: AppState, provider: &SdkLoggerProvider) -> Vec Vec { + let mut v = vec![service_name.to_owned(), "skyg-linux-tracker".to_owned()]; + if let Some(exe) = std::env::current_exe() + .ok() + .and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())) + { + v.push(exe); + } + v +} + +pub fn is_own_entry(e: &JournalEntry, own: &[String]) -> bool { + e.identifier.as_ref().is_some_and(|i| own.contains(i)) + || e.unit.as_ref().is_some_and(|u| { + own.iter() + .any(|n| u == n || u.strip_suffix(".service") == Some(n.as_str())) + }) +} + async fn journal_task(state: AppState, logger: SdkLogger, unit: Option) { + let own = own_names(&state.config.otel.service_name); let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX); let mut last: Option = None; let max_lines = state.config.limits.max_lines; @@ -155,6 +179,9 @@ async fn journal_task(state: AppState, logger: SdkLogger, unit: Option) e = rx.recv() => e, }; let Some(e) = e else { break }; + if is_own_entry(&e, &own) { + continue; + } if last.is_some_and(|l| e.timestamp <= l) { continue; } @@ -210,6 +237,8 @@ async fn docker_reconciler(state: AppState, logger: SdkLogger) { let allow = state.config.otel.logs.docker_containers.clone(); let mut tasks: HashMap)> = HashMap::new(); let mut warned = false; + let mut failures = 0u32; + let mut first_pass = true; loop { let mut delay = RECONCILE_INTERVAL; let ps = tokio::select! { @@ -219,6 +248,9 @@ async fn docker_reconciler(state: AppState, logger: SdkLogger) { match ps { Ok(cs) => { warned = false; + failures = 0; + let started: HashMap> = + cs.iter().map(|c| (c.name.clone(), c.started_at)).collect(); let running: Vec = cs.into_iter().map(|c| c.name).collect(); let tracked: Vec = tasks.keys().cloned().collect(); let (start, stop) = reconcile(&running, &tracked, &allow); @@ -230,21 +262,35 @@ async fn docker_reconciler(state: AppState, logger: SdkLogger) { } for name in start { let tok = state.shutdown.child_token(); + // Containers present at startup: only new lines. Containers that + // appear later: from their StartedAt so startup output is kept. + let from = if first_pass { + None + } else { + started.get(&name).copied().flatten() + }; let h = tokio::spawn(container_task( state.clone(), logger.clone(), name.clone(), + from, tok.clone(), )); tasks.insert(name, (tok, h)); } + first_pass = false; } Err(e) => { - if !warned { - tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly"); - warned = true; + failures += 1; + // Transient errors keep the normal cadence; only a persistent + // outage switches to the slow retry. + if failures >= 3 { + if !warned { + tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly"); + warned = true; + } + delay = DOCKER_SLOW_RETRY; } - delay = DOCKER_SLOW_RETRY; } } if !sleep_or_shutdown(&state.shutdown, delay).await { @@ -261,13 +307,23 @@ async fn container_task( state: AppState, logger: SdkLogger, name: String, + from: Option, token: CancellationToken, ) { let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX); - let mut since: Option = Some(Timestamp::now()); + // `from == None`: container was already running at startup, so only new lines. + // docker `tail=0` returns nothing regardless of `since`, so a backlog needs + // a real tail. + let mut since: Option = Some(from.unwrap_or_else(Timestamp::now)); + let mut tail = if from.is_some() { + state.config.limits.max_tail + } else { + 0 + }; + let mut last_emitted: Option = None; loop { let opts = LogOptions { - tail: Some(0), + tail: Some(tail), since, }; let rx = tokio::select! { @@ -285,6 +341,11 @@ async fn container_task( let Some(l) = l else { break }; got = true; if let Some(ts) = l.ts { + // Dedupe lines replayed after a reconnect. + if last_emitted.is_some_and(|le| ts <= le) { + continue; + } + last_emitted = Some(ts); since = Some(ts); } emit(&logger, docker_record(&name, &l)); @@ -297,6 +358,10 @@ async fn container_task( tracing::debug!(error = %e, container = %name, "otel logs: container follow failed"); } } + // After the first connection, reconnects replay from the last seen line. + if last_emitted.is_some() { + tail = state.config.limits.max_tail; + } if !sleep_or_shutdown(&token, backoff.next_delay()).await { return; } @@ -320,6 +385,18 @@ mod tests { } } + #[test] + fn own_entries_are_excluded() { + let own = vec!["skyg-linux-tracker".to_string()]; + let mut e = entry(None); + assert!(!is_own_entry(&e, &own)); + e.identifier = Some("skyg-linux-tracker".into()); + assert!(is_own_entry(&e, &own)); + e.identifier = Some("slinuxm".into()); + e.unit = Some("skyg-linux-tracker.service".into()); + assert!(is_own_entry(&e, &own)); + } + #[test] fn severity_mapping() { use Priority::*; diff --git a/src/telemetry/metrics_export.rs b/src/telemetry/metrics_export.rs index 4b182c3..025f000 100644 --- a/src/telemetry/metrics_export.rs +++ b/src/telemetry/metrics_export.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; use std::time::Duration; +use futures::StreamExt; use opentelemetry::metrics::MeterProvider as _; use opentelemetry::KeyValue; use opentelemetry_sdk::metrics::SdkMeterProvider; @@ -21,17 +22,23 @@ use crate::state::AppState; /// Per-call bound so a hung backend cannot stall the poll loop. const CALL_TIMEOUT: Duration = Duration::from_secs(10); -pub const METRIC_NAMES: [&str; 8] = [ - "systemd.unit.active", - "systemd.unit.failed", - "systemd.unit.restarts", - "systemd.timer.next_elapse_seconds", - "docker.container.running", - "docker.container.healthy", - "docker.container.uptime_seconds", - "docker.container.restarts", +/// Metric name and UCUM unit. +pub const METRICS: [(&str, &str); 8] = [ + ("systemd.unit.active", "1"), + ("systemd.unit.failed", "1"), + ("systemd.unit.restarts", "{restart}"), + // Absolute unix timestamp of the next elapse, not a duration. + ("systemd.timer.next_elapse_timestamp_seconds", "s"), + ("docker.container.running", "1"), + ("docker.container.healthy", "1"), + ("docker.container.uptime_seconds", "s"), + ("docker.container.restarts", "{restart}"), ]; +/// Overall budget for the per-unit restart-count polling. +const RESTARTS_BUDGET: Duration = Duration::from_secs(15); +const RESTARTS_CONCURRENCY: usize = 8; + /// Latest poll result, already filtered by the allowlists. #[derive(Debug, Default, Clone)] pub struct Snapshot { @@ -64,92 +71,91 @@ pub fn container_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool { cfg.containers.is_empty() || any_glob(&cfg.containers, name) } -fn unit_attrs(u: &UnitSummary) -> Vec<(&'static str, String)> { - vec![ - ("systemd.unit", u.name.clone()), - ("systemd.sub_state", u.sub_state.clone()), - ("systemd.load_state", u.load_state.clone()), - ] +fn unit_id(name: &str) -> Vec<(&'static str, String)> { + vec![("systemd.unit", name.to_owned())] } -fn container_attrs(c: &ContainerSummary) -> Vec<(&'static str, String)> { - vec![ - ("container.name", c.name.clone()), - ("container.id", c.id.clone()), - ("container.image.name", c.image.clone()), - ] +fn container_id(c: &ContainerSummary) -> Vec<(&'static str, String)> { + vec![("container.name", c.name.clone())] } -/// Map a snapshot to gauge observations. +/// Map a snapshot to gauge observations. Series carry identity attributes +/// only (`systemd.unit` / `container.name`) so state changes do not churn +/// series. The `active` gauge also carries sub/load state, and `running` +/// carries the image name. pub fn observations(s: &Snapshot) -> Vec { let b = |v: bool| if v { 1.0 } else { 0.0 }; let mut out = Vec::new(); for u in &s.units { + let mut active_attrs = unit_id(&u.name); + active_attrs.push(("systemd.sub_state", u.sub_state.clone())); + active_attrs.push(("systemd.load_state", u.load_state.clone())); out.push(Obs { metric: "systemd.unit.active", value: b(u.active_state == "active"), - attrs: unit_attrs(u), + attrs: active_attrs, }); out.push(Obs { metric: "systemd.unit.failed", value: b(u.active_state == "failed"), - attrs: unit_attrs(u), + attrs: unit_id(&u.name), }); if let Some(n) = s.restarts.get(&u.name) { out.push(Obs { metric: "systemd.unit.restarts", value: f64::from(*n), - attrs: unit_attrs(u), + attrs: unit_id(&u.name), }); } } for t in &s.timers { - // Absolute unix time of the next elapse; unscheduled timers are omitted. + // Unscheduled timers are omitted. if let Some(next) = t.next_elapse { out.push(Obs { - metric: "systemd.timer.next_elapse_seconds", + metric: "systemd.timer.next_elapse_timestamp_seconds", value: next.as_second() as f64, - attrs: vec![ - ("systemd.unit", t.name.clone()), - ("systemd.sub_state", t.active_state.clone()), - ], + attrs: unit_id(&t.name), }); } } for c in &s.containers { + let mut running_attrs = container_id(c); + running_attrs.push(("container.image.name", c.image.clone())); out.push(Obs { metric: "docker.container.running", value: b(c.state == "running"), - attrs: container_attrs(c), + attrs: running_attrs, }); if let Some(h) = &c.health { out.push(Obs { metric: "docker.container.healthy", value: b(h == "healthy"), - attrs: container_attrs(c), + attrs: container_id(c), }); } if let Some(up) = c.uptime_seconds { out.push(Obs { metric: "docker.container.uptime_seconds", value: up as f64, - attrs: container_attrs(c), + attrs: container_id(c), }); } if let Some(r) = c.restart_count { out.push(Obs { metric: "docker.container.restarts", value: r as f64, - attrs: container_attrs(c), + attrs: container_id(c), }); } } out } -async fn collect(state: &AppState) -> Snapshot { +/// Poll the sources. A section whose call fails or times out keeps its value +/// from `prev` instead of being emptied. +async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot { let cfg = &state.config.otel.metrics; - let mut snap = Snapshot::default(); + let mut snap = prev.clone(); match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_units(None)).await { Ok(Ok(units)) => { @@ -171,24 +177,38 @@ async fn collect(state: &AppState) -> Snapshot { Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_timers failed"), Err(_) => tracing::debug!("otel metrics: list_timers timed out"), } + + // Restart counts: bounded concurrency and an overall budget. Entries not + // refreshed in time keep their previous value; vanished units are dropped. let services: Vec = snap .units .iter() .filter(|u| u.name.ends_with(".service") && u.load_state == "loaded") .map(|u| u.name.clone()) .collect(); - for name in services { - if state.shutdown.is_cancelled() { - break; - } - if let Ok(Ok(st)) = - tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await - { - if let Some(n) = st.n_restarts { - snap.restarts.insert(name, n); + let mut restarts: HashMap = services + .iter() + .filter_map(|n| prev.restarts.get(n).map(|v| (n.clone(), *v))) + .collect(); + let fetch = futures::stream::iter(services) + .map(|name| async move { + let r = tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await; + (name, r) + }) + .buffer_unordered(RESTARTS_CONCURRENCY) + .for_each(|(name, r)| { + if let Ok(Ok(st)) = r { + if let Some(n) = st.n_restarts { + restarts.insert(name, n); + } } - } + std::future::ready(()) + }); + if tokio::time::timeout(RESTARTS_BUDGET, fetch).await.is_err() { + tracing::debug!("otel metrics: restart polling hit its time budget"); } + snap.restarts = restarts; + match tokio::time::timeout(CALL_TIMEOUT, state.docker.ps(true)).await { Ok(Ok(cs)) => { snap.containers = cs @@ -204,20 +224,22 @@ async fn collect(state: &AppState) -> Snapshot { /// Register the gauges and start the poll loop. pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec> { - let snapshot: Arc>> = Arc::default(); + // Observations are computed once per poll; callbacks only filter. + let current: Arc>>> = Arc::default(); let meter = provider.meter("skyg-linux-tracker"); - for name in METRIC_NAMES { - let snap = Arc::clone(&snapshot); + for (name, unit) in METRICS { + let cur = Arc::clone(¤t); // Gauges live as long as the provider; the instrument handle is not needed. let _ = meter .f64_observable_gauge(name) + .with_unit(unit) .with_callback(move |o| { - let cur = snap.read().map(|g| Arc::clone(&g)).unwrap_or_default(); - for ob in observations(&cur).into_iter().filter(|x| x.metric == name) { + let obs = cur.read().map(|g| Arc::clone(&g)).unwrap_or_default(); + for ob in obs.iter().filter(|x| x.metric == name) { let attrs: Vec = ob .attrs - .into_iter() - .map(|(k, v)| KeyValue::new(k, v)) + .iter() + .map(|(k, v)| KeyValue::new(*k, v.clone())) .collect(); o.observe(ob.value, &attrs); } @@ -229,17 +251,18 @@ pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec let h = tokio::spawn(async move { let mut tick = tokio::time::interval(interval); tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut snap = Snapshot::default(); loop { tokio::select! { _ = state.shutdown.cancelled() => break, _ = tick.tick() => {} } - let snap = tokio::select! { + snap = tokio::select! { _ = state.shutdown.cancelled() => break, - s = collect(&state) => s, + s = collect(&state, &snap) => s, }; - if let Ok(mut g) = snapshot.write() { - *g = Arc::new(snap); + if let Ok(mut g) = current.write() { + *g = Arc::new(observations(&snap)); } } }); @@ -359,7 +382,7 @@ mod tests { let o = observations(&s); let timers: Vec<_> = o .iter() - .filter(|x| x.metric == "systemd.timer.next_elapse_seconds") + .filter(|x| x.metric == "systemd.timer.next_elapse_timestamp_seconds") .collect(); assert_eq!(timers.len(), 1); assert_eq!(timers[0].value, 1_792_065_600.0); @@ -374,7 +397,12 @@ mod tests { .unwrap(); assert_eq!(h.value, 0.0); assert_eq!(h.attrs[0], ("container.name", "web".to_string())); - assert_eq!(h.attrs[2], ("container.image.name", "nginx:1".to_string())); + assert_eq!(h.attrs.len(), 1); + let r = o + .iter() + .find(|x| x.metric == "docker.container.running") + .unwrap(); + assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string())); } #[tokio::test] @@ -383,13 +411,13 @@ mod tests { cfg.otel.metrics.units = vec!["__nomatch__*".into()]; cfg.otel.metrics.containers = vec!["__nomatch__*".into()]; let st = fake_state(cfg); - let snap = collect(&st).await; + let snap = collect(&st, &Snapshot::default()).await; assert!(snap.units.is_empty()); assert!(snap.timers.is_empty()); assert!(snap.containers.is_empty()); let st = fake_state(Config::default()); - let snap = collect(&st).await; + let snap = collect(&st, &Snapshot::default()).await; assert!(snap .units .iter() diff --git a/src/telemetry/mod.rs b/src/telemetry/mod.rs index 01ec08e..871bdba 100644 --- a/src/telemetry/mod.rs +++ b/src/telemetry/mod.rs @@ -49,29 +49,20 @@ pub struct TelemetryGuard { providers: Option, } -impl TelemetryGuard { - /// True when OTLP export is active. - pub fn is_enabled(&self) -> bool { - self.providers.is_some() - } -} - -impl Drop for TelemetryGuard { - fn drop(&mut self) { - let Some(p) = self.providers.take() else { - return; - }; - if let Err(e) = p.tracer.force_flush() { +impl Providers { + /// Blocking flush and shutdown, each provider bounded by `FLUSH_TIMEOUT`. + fn flush_and_shutdown(&self) { + if let Err(e) = self.tracer.force_flush() { eprintln!("otel: trace flush failed: {e}"); } - let _ = p.tracer.shutdown_with_timeout(FLUSH_TIMEOUT); - if let Some(m) = &p.meter { + let _ = self.tracer.shutdown_with_timeout(FLUSH_TIMEOUT); + if let Some(m) = &self.meter { if let Err(e) = m.force_flush() { eprintln!("otel: metrics flush failed: {e}"); } let _ = m.shutdown_with_timeout(FLUSH_TIMEOUT); } - if let Some(l) = &p.logger { + if let Some(l) = &self.logger { if let Err(e) = l.force_flush() { eprintln!("otel: logs flush failed: {e}"); } @@ -80,6 +71,40 @@ impl Drop for TelemetryGuard { } } +impl TelemetryGuard { + /// True when OTLP export is active. + pub fn is_enabled(&self) -> bool { + self.providers.is_some() + } + + /// Flush and shut down all providers without blocking the async runtime. + /// Call this (and await it) on graceful shutdown, after the exporter task + /// handles from `spawn_exporters` have finished. `Drop` is only a + /// best-effort fallback. + pub async fn shutdown(mut self) { + if let Some(p) = self.providers.take() { + let _ = tokio::task::spawn_blocking(move || p.flush_and_shutdown()).await; + } + } +} + +impl Drop for TelemetryGuard { + /// Best effort. Inside a tokio runtime the flush runs on a detached thread + /// (it may not finish before the process exits); outside one it runs inline. + fn drop(&mut self) { + let Some(p) = self.providers.take() else { + return; + }; + if tokio::runtime::Handle::try_current().is_ok() { + let _ = std::thread::Builder::new() + .name("otel-flush".into()) + .spawn(move || p.flush_and_shutdown()); + } else { + p.flush_and_shutdown(); + } + } +} + /// Install the tracing subscriber and, when `config.otel.enabled`, the OTLP /// providers. Works (stderr logging only) when OTel is disabled. pub fn init(config: &Config) -> Result { @@ -359,21 +384,9 @@ mod tests { ..Default::default() }; o.headers.insert("Authorization".into(), "Bearer x".into()); - let results = [ - build_span_exporter(&o).map(drop), - build_metric_exporter(&o).map(drop), - build_log_exporter(&o).map(drop), - ]; - for r in results { - match (protocol, r) { - (_, Ok(())) => {} - // Contract: HTTP needs the `reqwest-blocking-client` feature on - // opentelemetry-otlp (Cargo.toml, owned by P0). - (OtelProtocol::HttpProtobuf, Err(e)) - if format!("{e:?}").contains("http_client") => {} - (_, Err(e)) => panic!("exporter build failed: {e:?}"), - } - } + build_span_exporter(&o).unwrap(); + build_metric_exporter(&o).unwrap(); + build_log_exporter(&o).unwrap(); } }