From 2aeb9ae154c9c5f81cecf6f5419a52c01c381b49 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Thu, 15 Oct 2026 18:37:40 -0400 Subject: [PATCH 1/2] WS-G: OpenTelemetry export (traces, metrics, logs) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs --- TODO.md | 14 +- dev/otelcol.yaml | 25 ++ plan/workstreams/WS-G-otel.md | 3 +- src/telemetry/logs_export.rs | 379 +++++++++++++++++++++++++++++- src/telemetry/metrics_export.rs | 400 +++++++++++++++++++++++++++++++- src/telemetry/mod.rs | 379 ++++++++++++++++++++++++++++-- 6 files changed, 1175 insertions(+), 25 deletions(-) create mode 100644 dev/otelcol.yaml diff --git a/TODO.md b/TODO.md index 3edda80..b73e796 100644 --- a/TODO.md +++ b/TODO.md @@ -86,13 +86,13 @@ Design: [`plan.md`](plan.md) · Review fixes: [`plan/review.md`](plan/review.md) - [ ] **F9** Tests: tool list snapshot + each tool against fakes ### WS-G OTel · [card](plan/workstreams/WS-G-otel.md) -- [ ] **G1** `telemetry::init(cfg)`: OTLP grpc/http with headers. Resource has `service.name`, `service.version`, `host.name`. Returns a guard. -- [ ] **G2** `tracing` subscriber: fmt to stderr + `tracing-opentelemetry` layer when enabled -- [ ] **G3** Metrics poll loop feeds a snapshot. Observable gauges read it, so vanished series drop out. -- [ ] **G4** Metric set from `plan.md` with semconv attribute names, allowlists and the DEC-7 default scope -- [ ] **G5** Log export: journal follow + per-container docker follow, reconciled via the Docker events stream, with backoff -- [ ] **G6** Shutdown cancels tasks and flushes all providers -- [ ] **G7** `dev/otelcol.yaml` with a debug exporter, plus a devenv process to run it +- [x] **G1** `telemetry::init(cfg)`: OTLP grpc/http with headers. Resource has `service.name`, `service.version`, `host.name`. Returns a guard. +- [x] **G2** `tracing` subscriber: fmt to stderr + `tracing-opentelemetry` layer when enabled +- [x] **G3** Metrics poll loop feeds a snapshot. Observable gauges read it, so vanished series drop out. +- [x] **G4** Metric set from `plan.md` with semconv attribute names, allowlists and the DEC-7 default scope +- [x] **G5** Log export: journal follow + per-container docker follow, reconciled via the Docker events stream, with backoff +- [x] **G6** Shutdown cancels tasks and flushes all providers +- [x] **G7** `dev/otelcol.yaml` with a debug exporter, plus a devenv process to run it ## Phase 3 — Integration & hardening · [card](plan/workstreams/P3-integration.md) diff --git a/dev/otelcol.yaml b/dev/otelcol.yaml new file mode 100644 index 0000000..2841182 --- /dev/null +++ b/dev/otelcol.yaml @@ -0,0 +1,25 @@ +# Local OpenTelemetry collector for development (WS-G). +# Run: otelcol-contrib --config dev/otelcol.yaml (or `devenv up`) +receivers: + otlp: + protocols: + grpc: + endpoint: 127.0.0.1:4317 + http: + endpoint: 127.0.0.1:4318 + +exporters: + debug: + verbosity: detailed + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + metrics: + receivers: [otlp] + exporters: [debug] + logs: + receivers: [otlp] + exporters: [debug] diff --git a/plan/workstreams/WS-G-otel.md b/plan/workstreams/WS-G-otel.md index f206fe7..475bf42 100644 --- a/plan/workstreams/WS-G-otel.md +++ b/plan/workstreams/WS-G-otel.md @@ -2,7 +2,7 @@ | Field | Value | |---|---| -| Status | Backlog | +| Status | Review | | Owner | — | | Phase | 2 (parallel) | | Depends on | P0 to start (fakes, spike findings). A, B, C to finish. | @@ -53,3 +53,4 @@ Tick these in [`TODO.md`](../../TODO.md). ## Log - 2026-10-15: Card created. +- 2026-10-15: Implemented G1-G7 on `ws-g/otel`. Notes: no glob crate, so a small `*`/`?` matcher lives in `telemetry/mod.rs`. `DockerSource` has no events stream, so the log reconciler polls `ps(false)` every 5s (30s while Docker is unavailable) instead of subscribing to `events`. No `EnvFilter` (feature not enabled in Cargo.toml): level is fixed at INFO. `systemd.timer.next_elapse_seconds` exports absolute unix seconds. `systemd.unit.restarts` costs one `unit_status` per in-scope loaded service per interval. HTTP endpoints get `/v1/` appended unless already present. Live otelcol smoke test not run (no collector binary on PATH). diff --git a/src/telemetry/logs_export.rs b/src/telemetry/logs_export.rs index 671cca7..0e1aead 100644 --- a/src/telemetry/logs_export.rs +++ b/src/telemetry/logs_export.rs @@ -1,3 +1,378 @@ -//! Journal and docker log fan-out to OTel logs (WS-G) +//! Journal and docker log fan-out to OTel logs (WS-G). //! -//! Placeholder created in P0 so the owning workstream only edits its own files. +//! Journal: one `follow()` per configured unit, or one unfiltered follow when +//! `otel.logs.journal_units` is empty. Docker: a reconciler keeps one +//! `follow_logs` task per allowlisted running container. +//! +//! Deviation: `DockerSource` exposes no events stream, so the reconciler +//! polls `ps(false)` every few seconds instead of subscribing to `events`. + +use std::collections::HashMap; +use std::time::{Duration, SystemTime}; + +use jiff::Timestamp; +use opentelemetry::logs::{AnyValue, LogRecord as _, Logger as _, LoggerProvider as _, Severity}; +use opentelemetry_sdk::logs::{SdkLogger, SdkLoggerProvider}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +use super::{any_glob, Backoff}; +use crate::sources::{JournalEntry, JournalQuery, LogLine, LogOptions, LogStream, Priority}; +use crate::state::AppState; + +const BACKOFF_MIN: Duration = Duration::from_secs(1); +const BACKOFF_MAX: Duration = Duration::from_secs(60); +/// Retry interval while Docker is unavailable. +const DOCKER_SLOW_RETRY: Duration = Duration::from_secs(30); +const RECONCILE_INTERVAL: Duration = Duration::from_secs(5); + +/// Backend-neutral log record, built by pure mapping functions. +#[derive(Debug, Clone, PartialEq)] +pub struct LogRec { + pub severity: Severity, + pub severity_text: &'static str, + pub body: String, + pub timestamp: Option, + pub attrs: Vec<(&'static str, String)>, +} + +/// Syslog priority to OTel severity. +pub fn journal_severity(p: Option) -> (Severity, &'static str) { + match p { + Some(Priority::Emerg | Priority::Alert | Priority::Crit) => (Severity::Fatal, "FATAL"), + Some(Priority::Err) => (Severity::Error, "ERROR"), + Some(Priority::Warning) => (Severity::Warn, "WARN"), + Some(Priority::Notice | Priority::Info) | None => (Severity::Info, "INFO"), + Some(Priority::Debug) => (Severity::Debug, "DEBUG"), + } +} + +pub fn journal_record(e: &JournalEntry, unit_hint: Option<&str>) -> LogRec { + let (severity, severity_text) = journal_severity(e.priority); + let mut attrs = Vec::new(); + if let Some(u) = e.unit.as_deref().or(unit_hint) { + attrs.push(("systemd.unit", u.to_owned())); + } + if let Some(i) = &e.identifier { + attrs.push(("log.syslog.identifier", i.clone())); + } + if let Some(p) = e.pid { + attrs.push(("process.pid", p.to_string())); + } + LogRec { + severity, + severity_text, + body: e.message.clone(), + timestamp: Some(SystemTime::from(e.timestamp)), + attrs, + } +} + +/// stderr maps to WARN, stdout to INFO. +pub fn docker_record(container: &str, l: &LogLine) -> LogRec { + let (severity, severity_text) = match l.stream { + LogStream::Stderr => (Severity::Warn, "WARN"), + LogStream::Stdout => (Severity::Info, "INFO"), + }; + LogRec { + severity, + severity_text, + body: l.message.clone(), + timestamp: l.ts.map(SystemTime::from), + attrs: vec![ + ("container.name", container.to_owned()), + ( + "log.iostream", + match l.stream { + LogStream::Stderr => "stderr", + LogStream::Stdout => "stdout", + } + .to_owned(), + ), + ], + } +} + +fn emit(logger: &SdkLogger, r: LogRec) { + let mut rec = logger.create_log_record(); + rec.set_severity_number(r.severity); + rec.set_severity_text(r.severity_text); + rec.set_body(AnyValue::String(r.body.into())); + if let Some(ts) = r.timestamp { + rec.set_timestamp(ts); + } + rec.set_observed_timestamp(SystemTime::now()); + for (k, v) in r.attrs { + rec.add_attribute(k, AnyValue::String(v.into())); + } + logger.emit(rec); +} + +/// Start the journal tasks and the docker reconciler. +pub fn spawn(state: AppState, provider: &SdkLoggerProvider) -> Vec> { + let logger = provider.logger("skyg-linux-tracker"); + let mut handles = Vec::new(); + + let units = state.config.otel.logs.journal_units.clone(); + let targets: Vec> = if units.is_empty() { + vec![None] + } else { + units.into_iter().map(Some).collect() + }; + for unit in targets { + handles.push(tokio::spawn(journal_task( + state.clone(), + logger.clone(), + unit, + ))); + } + handles.push(tokio::spawn(docker_reconciler(state, logger))); + handles +} + +async fn journal_task(state: AppState, logger: SdkLogger, unit: Option) { + let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX); + let mut last: Option = None; + let max_lines = state.config.limits.max_lines; + loop { + let q = JournalQuery { + unit: unit.clone(), + priority: None, + // No backlog on first start; catch up from the last seen entry afterwards. + lines: Some(if last.is_some() { max_lines } else { 0 }), + since: last, + }; + let rx = tokio::select! { + _ = state.shutdown.cancelled() => return, + r = state.journal.follow(&q) => r, + }; + match rx { + Ok(mut rx) => { + let mut got = false; + loop { + let e = tokio::select! { + _ = state.shutdown.cancelled() => return, + e = rx.recv() => e, + }; + let Some(e) = e else { break }; + if last.is_some_and(|l| e.timestamp <= l) { + continue; + } + got = true; + last = Some(e.timestamp); + emit(&logger, journal_record(&e, unit.as_deref())); + } + if got { + backoff.reset(); + } + } + Err(e) => tracing::warn!(error = %e, unit = ?unit, "otel logs: journal follow failed"), + } + if !sleep_or_shutdown(&state.shutdown, backoff.next_delay()).await { + return; + } + } +} + +/// Returns false when shutdown fired during the sleep. +async fn sleep_or_shutdown(shutdown: &CancellationToken, d: Duration) -> bool { + tokio::select! { + _ = shutdown.cancelled() => false, + _ = tokio::time::sleep(d) => true, + } +} + +/// Pure reconcile step: which running containers to start and which tracked +/// ones to stop. +pub fn reconcile( + running: &[String], + tracked: &[String], + allow: &[String], +) -> (Vec, Vec) { + let wanted: Vec<&String> = running + .iter() + .filter(|n| allow.is_empty() || any_glob(allow, n)) + .collect(); + let start = wanted + .iter() + .filter(|n| !tracked.contains(n)) + .map(|n| (*n).clone()) + .collect(); + let stop = tracked + .iter() + .filter(|t| !wanted.contains(t)) + .cloned() + .collect(); + (start, stop) +} + +async fn docker_reconciler(state: AppState, logger: SdkLogger) { + let allow = state.config.otel.logs.docker_containers.clone(); + let mut tasks: HashMap)> = HashMap::new(); + let mut warned = false; + loop { + let mut delay = RECONCILE_INTERVAL; + let ps = tokio::select! { + _ = state.shutdown.cancelled() => break, + r = state.docker.ps(false) => r, + }; + match ps { + Ok(cs) => { + warned = false; + let running: Vec = cs.into_iter().map(|c| c.name).collect(); + let tracked: Vec = tasks.keys().cloned().collect(); + let (start, stop) = reconcile(&running, &tracked, &allow); + for name in stop { + if let Some((tok, h)) = tasks.remove(&name) { + tok.cancel(); + h.abort(); + } + } + for name in start { + let tok = state.shutdown.child_token(); + let h = tokio::spawn(container_task( + state.clone(), + logger.clone(), + name.clone(), + tok.clone(), + )); + tasks.insert(name, (tok, h)); + } + } + Err(e) => { + if !warned { + tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly"); + warned = true; + } + delay = DOCKER_SLOW_RETRY; + } + } + if !sleep_or_shutdown(&state.shutdown, delay).await { + break; + } + } + for (_, (tok, h)) in tasks { + tok.cancel(); + h.abort(); + } +} + +async fn container_task( + state: AppState, + logger: SdkLogger, + name: String, + token: CancellationToken, +) { + let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX); + let mut since: Option = Some(Timestamp::now()); + loop { + let opts = LogOptions { + tail: Some(0), + since, + }; + let rx = tokio::select! { + _ = token.cancelled() => return, + r = state.docker.follow_logs(&name, &opts) => r, + }; + match rx { + Ok(mut rx) => { + let mut got = false; + loop { + let l = tokio::select! { + _ = token.cancelled() => return, + l = rx.recv() => l, + }; + let Some(l) = l else { break }; + got = true; + if let Some(ts) = l.ts { + since = Some(ts); + } + emit(&logger, docker_record(&name, &l)); + } + if got { + backoff.reset(); + } + } + Err(e) => { + tracing::debug!(error = %e, container = %name, "otel logs: container follow failed"); + } + } + if !sleep_or_shutdown(&token, backoff.next_delay()).await { + return; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(p: Option) -> JournalEntry { + JournalEntry { + timestamp: "2026-10-15T12:00:00Z".parse().unwrap(), + priority: p, + unit: Some("sshd.service".into()), + identifier: Some("sshd".into()), + pid: Some(7), + hostname: None, + message: "hello".into(), + cursor: None, + } + } + + #[test] + fn severity_mapping() { + use Priority::*; + let sev = |p| journal_severity(Some(p)).0; + assert_eq!(sev(Emerg), Severity::Fatal); + assert_eq!(sev(Alert), Severity::Fatal); + assert_eq!(sev(Crit), Severity::Fatal); + assert_eq!(sev(Err), Severity::Error); + assert_eq!(sev(Warning), Severity::Warn); + assert_eq!(sev(Notice), Severity::Info); + assert_eq!(sev(Info), Severity::Info); + assert_eq!(sev(Debug), Severity::Debug); + assert_eq!(journal_severity(None).0, Severity::Info); + } + + #[test] + fn journal_attrs() { + let r = journal_record(&entry(Some(Priority::Err)), None); + assert_eq!(r.severity_text, "ERROR"); + assert_eq!(r.body, "hello"); + assert!(r + .attrs + .contains(&("systemd.unit", "sshd.service".to_string()))); + let mut e = entry(None); + e.unit = None; + let r = journal_record(&e, Some("x.service")); + assert!(r.attrs.contains(&("systemd.unit", "x.service".to_string()))); + } + + #[test] + fn docker_streams() { + let l = |s| LogLine { + stream: s, + ts: None, + message: "m".into(), + }; + let e = docker_record("web", &l(LogStream::Stderr)); + assert_eq!(e.severity, Severity::Warn); + assert!(e.attrs.contains(&("container.name", "web".to_string()))); + assert_eq!( + docker_record("web", &l(LogStream::Stdout)).severity, + Severity::Info + ); + } + + #[test] + fn reconcile_starts_and_stops() { + let s = |v: &[&str]| v.iter().map(|x| x.to_string()).collect::>(); + let (start, stop) = reconcile(&s(&["a", "b"]), &s(&["b", "c"]), &[]); + assert_eq!(start, s(&["a"])); + assert_eq!(stop, s(&["c"])); + let (start, stop) = reconcile(&s(&["web-1", "db"]), &[], &s(&["web-*"])); + assert_eq!(start, s(&["web-1"])); + assert!(stop.is_empty()); + } +} diff --git a/src/telemetry/metrics_export.rs b/src/telemetry/metrics_export.rs index d7b1243..4b182c3 100644 --- a/src/telemetry/metrics_export.rs +++ b/src/telemetry/metrics_export.rs @@ -1,3 +1,399 @@ -//! Metrics poll loop and observable gauges (WS-G) +//! Metrics poll loop and observable gauges (WS-G). //! -//! Placeholder created in P0 so the owning workstream only edits its own files. +//! A background task polls the sources every `metrics_interval_secs` and +//! replaces a shared snapshot. Observable gauge callbacks read the latest +//! snapshot, so units or containers that vanish stop being reported. + +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use opentelemetry::metrics::MeterProvider as _; +use opentelemetry::KeyValue; +use opentelemetry_sdk::metrics::SdkMeterProvider; +use tokio::task::JoinHandle; + +use super::any_glob; +use crate::config::OtelMetricsConfig; +use crate::sources::{ContainerSummary, TimerInfo, UnitSummary}; +use crate::state::AppState; + +/// Per-call bound so a hung backend cannot stall the poll loop. +const CALL_TIMEOUT: Duration = Duration::from_secs(10); + +pub const METRIC_NAMES: [&str; 8] = [ + "systemd.unit.active", + "systemd.unit.failed", + "systemd.unit.restarts", + "systemd.timer.next_elapse_seconds", + "docker.container.running", + "docker.container.healthy", + "docker.container.uptime_seconds", + "docker.container.restarts", +]; + +/// Latest poll result, already filtered by the allowlists. +#[derive(Debug, Default, Clone)] +pub struct Snapshot { + pub units: Vec, + /// Automatic restart count per unit name (services only). + pub restarts: HashMap, + pub timers: Vec, + pub containers: Vec, +} + +/// One gauge data point. +#[derive(Debug, Clone, PartialEq)] +pub struct Obs { + pub metric: &'static str, + pub value: f64, + pub attrs: Vec<(&'static str, String)>, +} + +/// DEC-7: empty list means `*.service` and `*.timer`; otherwise a glob allowlist. +pub fn unit_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool { + if cfg.units.is_empty() { + name.ends_with(".service") || name.ends_with(".timer") + } else { + any_glob(&cfg.units, name) + } +} + +/// Empty list means all containers. +pub fn container_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool { + cfg.containers.is_empty() || any_glob(&cfg.containers, name) +} + +fn unit_attrs(u: &UnitSummary) -> Vec<(&'static str, String)> { + vec![ + ("systemd.unit", u.name.clone()), + ("systemd.sub_state", u.sub_state.clone()), + ("systemd.load_state", u.load_state.clone()), + ] +} + +fn container_attrs(c: &ContainerSummary) -> Vec<(&'static str, String)> { + vec![ + ("container.name", c.name.clone()), + ("container.id", c.id.clone()), + ("container.image.name", c.image.clone()), + ] +} + +/// Map a snapshot to gauge observations. +pub fn observations(s: &Snapshot) -> Vec { + let b = |v: bool| if v { 1.0 } else { 0.0 }; + let mut out = Vec::new(); + for u in &s.units { + out.push(Obs { + metric: "systemd.unit.active", + value: b(u.active_state == "active"), + attrs: unit_attrs(u), + }); + out.push(Obs { + metric: "systemd.unit.failed", + value: b(u.active_state == "failed"), + attrs: unit_attrs(u), + }); + if let Some(n) = s.restarts.get(&u.name) { + out.push(Obs { + metric: "systemd.unit.restarts", + value: f64::from(*n), + attrs: unit_attrs(u), + }); + } + } + for t in &s.timers { + // Absolute unix time of the next elapse; unscheduled timers are omitted. + if let Some(next) = t.next_elapse { + out.push(Obs { + metric: "systemd.timer.next_elapse_seconds", + value: next.as_second() as f64, + attrs: vec![ + ("systemd.unit", t.name.clone()), + ("systemd.sub_state", t.active_state.clone()), + ], + }); + } + } + for c in &s.containers { + out.push(Obs { + metric: "docker.container.running", + value: b(c.state == "running"), + attrs: container_attrs(c), + }); + if let Some(h) = &c.health { + out.push(Obs { + metric: "docker.container.healthy", + value: b(h == "healthy"), + attrs: container_attrs(c), + }); + } + if let Some(up) = c.uptime_seconds { + out.push(Obs { + metric: "docker.container.uptime_seconds", + value: up as f64, + attrs: container_attrs(c), + }); + } + if let Some(r) = c.restart_count { + out.push(Obs { + metric: "docker.container.restarts", + value: r as f64, + attrs: container_attrs(c), + }); + } + } + out +} + +async fn collect(state: &AppState) -> Snapshot { + let cfg = &state.config.otel.metrics; + let mut snap = Snapshot::default(); + + match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_units(None)).await { + Ok(Ok(units)) => { + snap.units = units + .into_iter() + .filter(|u| unit_in_scope(cfg, &u.name)) + .collect(); + } + Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_units failed"), + Err(_) => tracing::debug!("otel metrics: list_units timed out"), + } + match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_timers()).await { + Ok(Ok(timers)) => { + snap.timers = timers + .into_iter() + .filter(|t| unit_in_scope(cfg, &t.name)) + .collect(); + } + Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_timers failed"), + Err(_) => tracing::debug!("otel metrics: list_timers timed out"), + } + let services: Vec = snap + .units + .iter() + .filter(|u| u.name.ends_with(".service") && u.load_state == "loaded") + .map(|u| u.name.clone()) + .collect(); + for name in services { + if state.shutdown.is_cancelled() { + break; + } + if let Ok(Ok(st)) = + tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await + { + if let Some(n) = st.n_restarts { + snap.restarts.insert(name, n); + } + } + } + match tokio::time::timeout(CALL_TIMEOUT, state.docker.ps(true)).await { + Ok(Ok(cs)) => { + snap.containers = cs + .into_iter() + .filter(|c| container_in_scope(cfg, &c.name)) + .collect(); + } + Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: docker ps failed"), + Err(_) => tracing::debug!("otel metrics: docker ps timed out"), + } + snap +} + +/// Register the gauges and start the poll loop. +pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec> { + let snapshot: Arc>> = Arc::default(); + let meter = provider.meter("skyg-linux-tracker"); + for name in METRIC_NAMES { + let snap = Arc::clone(&snapshot); + // Gauges live as long as the provider; the instrument handle is not needed. + let _ = meter + .f64_observable_gauge(name) + .with_callback(move |o| { + let cur = snap.read().map(|g| Arc::clone(&g)).unwrap_or_default(); + for ob in observations(&cur).into_iter().filter(|x| x.metric == name) { + let attrs: Vec = ob + .attrs + .into_iter() + .map(|(k, v)| KeyValue::new(k, v)) + .collect(); + o.observe(ob.value, &attrs); + } + }) + .build(); + } + + let interval = Duration::from_secs(state.config.otel.metrics_interval_secs.max(1)); + let h = tokio::spawn(async move { + let mut tick = tokio::time::interval(interval); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + _ = state.shutdown.cancelled() => break, + _ = tick.tick() => {} + } + let snap = tokio::select! { + _ = state.shutdown.cancelled() => break, + s = collect(&state) => s, + }; + if let Ok(mut g) = snapshot.write() { + *g = Arc::new(snap); + } + } + }); + vec![h] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + use crate::sources::fake::fake_state; + + fn unit(name: &str, active: &str) -> UnitSummary { + UnitSummary { + name: name.into(), + description: String::new(), + load_state: "loaded".into(), + active_state: active.into(), + sub_state: "running".into(), + } + } + + #[test] + fn default_scope_is_service_and_timer() { + let c = OtelMetricsConfig::default(); + assert!(unit_in_scope(&c, "a.service")); + assert!(unit_in_scope(&c, "a.timer")); + assert!(!unit_in_scope(&c, "a.socket")); + assert!(!unit_in_scope(&c, "a.mount")); + assert!(container_in_scope(&c, "anything")); + } + + #[test] + fn allowlist_replaces_default() { + let c = OtelMetricsConfig { + units: vec!["*.socket".into(), "ssh*".into()], + containers: vec!["web-*".into()], + }; + assert!(unit_in_scope(&c, "dbus.socket")); + assert!(unit_in_scope(&c, "sshd.service")); + assert!(!unit_in_scope(&c, "cron.service")); + assert!(container_in_scope(&c, "web-1")); + assert!(!container_in_scope(&c, "db")); + } + + #[test] + fn maps_units_and_restarts() { + let mut s = Snapshot { + units: vec![unit("a.service", "active"), unit("b.service", "failed")], + ..Default::default() + }; + s.restarts.insert("b.service".into(), 3); + let o = observations(&s); + let get = |m: &str, unit: &str| { + o.iter() + .find(|x| x.metric == m && x.attrs[0].1 == unit) + .map(|x| x.value) + }; + assert_eq!(get("systemd.unit.active", "a.service"), Some(1.0)); + assert_eq!(get("systemd.unit.failed", "a.service"), Some(0.0)); + assert_eq!(get("systemd.unit.active", "b.service"), Some(0.0)); + assert_eq!(get("systemd.unit.failed", "b.service"), Some(1.0)); + assert_eq!(get("systemd.unit.restarts", "b.service"), Some(3.0)); + assert_eq!(get("systemd.unit.restarts", "a.service"), None); + let a = o + .iter() + .find(|x| x.metric == "systemd.unit.active") + .unwrap(); + let keys: Vec<_> = a.attrs.iter().map(|(k, _)| *k).collect(); + assert_eq!( + keys, + ["systemd.unit", "systemd.sub_state", "systemd.load_state"] + ); + } + + #[test] + fn maps_timers_and_containers() { + let c = ContainerSummary { + id: "abc".into(), + name: "web".into(), + image: "nginx:1".into(), + state: "running".into(), + health: Some("unhealthy".into()), + created: None, + started_at: None, + uptime_seconds: Some(42), + restart_count: Some(2), + }; + let nohealth = ContainerSummary { + health: None, + uptime_seconds: None, + restart_count: None, + state: "exited".into(), + name: "job".into(), + ..c.clone() + }; + let s = Snapshot { + timers: vec![ + TimerInfo { + name: "t.timer".into(), + unit: "t.service".into(), + active_state: "active".into(), + next_elapse: Some("2026-10-15T12:00:00Z".parse().unwrap()), + last_trigger: None, + }, + TimerInfo { + name: "u.timer".into(), + unit: "u.service".into(), + active_state: "inactive".into(), + next_elapse: None, + last_trigger: None, + }, + ], + containers: vec![c, nohealth], + ..Default::default() + }; + let o = observations(&s); + let timers: Vec<_> = o + .iter() + .filter(|x| x.metric == "systemd.timer.next_elapse_seconds") + .collect(); + assert_eq!(timers.len(), 1); + assert_eq!(timers[0].value, 1_792_065_600.0); + let of = |m: &str| o.iter().filter(|x| x.metric == m).count(); + assert_eq!(of("docker.container.running"), 2); + assert_eq!(of("docker.container.healthy"), 1); + assert_eq!(of("docker.container.uptime_seconds"), 1); + assert_eq!(of("docker.container.restarts"), 1); + let h = o + .iter() + .find(|x| x.metric == "docker.container.healthy") + .unwrap(); + assert_eq!(h.value, 0.0); + assert_eq!(h.attrs[0], ("container.name", "web".to_string())); + assert_eq!(h.attrs[2], ("container.image.name", "nginx:1".to_string())); + } + + #[tokio::test] + async fn collect_applies_scope_with_fakes() { + let mut cfg = Config::default(); + cfg.otel.metrics.units = vec!["__nomatch__*".into()]; + cfg.otel.metrics.containers = vec!["__nomatch__*".into()]; + let st = fake_state(cfg); + let snap = collect(&st).await; + assert!(snap.units.is_empty()); + assert!(snap.timers.is_empty()); + assert!(snap.containers.is_empty()); + + let st = fake_state(Config::default()); + let snap = collect(&st).await; + assert!(snap + .units + .iter() + .all(|u| u.name.ends_with(".service") || u.name.ends_with(".timer"))); + assert!(!snap.units.is_empty()); + } +} diff --git a/src/telemetry/mod.rs b/src/telemetry/mod.rs index 7ce6333..01ec08e 100644 --- a/src/telemetry/mod.rs +++ b/src/telemetry/mod.rs @@ -1,32 +1,385 @@ //! OpenTelemetry export (WS-G owns `src/telemetry/`). //! -//! STUB: signatures are final. Pinned `opentelemetry-otlp` 0.33.1: use -//! `WithExportConfig::{with_endpoint, with_protocol}` plus -//! `WithHttpConfig::with_headers` (HashMap) or -//! `WithTonicConfig::with_metadata` (tonic `MetadataMap`). +//! One exporter config (`otel.endpoint`, `otel.protocol`, `otel.headers`) +//! feeds traces, metrics and logs. The app's own `tracing` events go to +//! stderr (and to OTel traces); they are never routed to the OTel logs +//! pipeline, which carries host logs only. pub mod logs_export; pub mod metrics_export; -use anyhow::Result; -use tokio::task::JoinHandle; +use std::collections::HashMap; +use std::sync::OnceLock; +use std::time::Duration; -use crate::config::Config; +use anyhow::{Context, Result}; +use opentelemetry::trace::TracerProvider as _; +use opentelemetry::KeyValue; +use opentelemetry_otlp::{ + LogExporter, MetricExporter, Protocol, SpanExporter, WithExportConfig, WithHttpConfig, + WithTonicConfig, +}; +use opentelemetry_sdk::logs::SdkLoggerProvider; +use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; +use opentelemetry_sdk::trace::SdkTracerProvider; +use opentelemetry_sdk::Resource; +use tokio::task::JoinHandle; +use tracing_subscriber::filter::LevelFilter; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; + +use crate::config::{Config, OtelConfig, OtelProtocol}; use crate::state::AppState; +/// Upper bound for shutting down each provider. +const FLUSH_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Clone)] +struct Providers { + tracer: SdkTracerProvider, + meter: Option, + logger: Option, +} + +/// Providers shared with `spawn_exporters`. Set once by `init` when enabled. +static PROVIDERS: OnceLock = OnceLock::new(); + /// Flushes and shuts down providers on drop. pub struct TelemetryGuard { - _private: (), + providers: Option, +} + +impl TelemetryGuard { + /// True when OTLP export is active. + pub fn is_enabled(&self) -> bool { + self.providers.is_some() + } +} + +impl Drop for TelemetryGuard { + fn drop(&mut self) { + let Some(p) = self.providers.take() else { + return; + }; + if let Err(e) = p.tracer.force_flush() { + eprintln!("otel: trace flush failed: {e}"); + } + let _ = p.tracer.shutdown_with_timeout(FLUSH_TIMEOUT); + if let Some(m) = &p.meter { + if let Err(e) = m.force_flush() { + eprintln!("otel: metrics flush failed: {e}"); + } + let _ = m.shutdown_with_timeout(FLUSH_TIMEOUT); + } + if let Some(l) = &p.logger { + if let Err(e) = l.force_flush() { + eprintln!("otel: logs flush failed: {e}"); + } + let _ = l.shutdown_with_timeout(FLUSH_TIMEOUT); + } + } } /// Install the tracing subscriber and, when `config.otel.enabled`, the OTLP -/// providers. Must work (stderr logging only) when OTel is disabled. -pub fn init(_config: &Config) -> Result { - todo!("WS-G G1/G2") +/// providers. Works (stderr logging only) when OTel is disabled. +pub fn init(config: &Config) -> Result { + let otel = &config.otel; + + if !otel.enabled { + let _ = tracing_subscriber::registry() + .with(LevelFilter::INFO) + .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) + .try_init(); + return Ok(TelemetryGuard { providers: None }); + } + + let resource = build_resource(otel); + let tracer_provider = SdkTracerProvider::builder() + .with_batch_exporter(build_span_exporter(otel)?) + .with_resource(resource.clone()) + .build(); + let meter = if otel.export_unit_metrics { + let reader = PeriodicReader::builder(build_metric_exporter(otel)?) + .with_interval(Duration::from_secs(otel.metrics_interval_secs.max(1))) + .build(); + Some( + SdkMeterProvider::builder() + .with_reader(reader) + .with_resource(resource.clone()) + .build(), + ) + } else { + None + }; + let logger = if otel.export_logs { + Some( + SdkLoggerProvider::builder() + .with_batch_exporter(build_log_exporter(otel)?) + .with_resource(resource) + .build(), + ) + } else { + None + }; + + let tracer = tracer_provider.tracer(otel.service_name.clone()); + let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer); + let _ = tracing_subscriber::registry() + .with(LevelFilter::INFO) + .with(otel_layer) + .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) + .try_init(); + + let providers = Providers { + tracer: tracer_provider, + meter, + logger, + }; + let _ = PROVIDERS.set(providers.clone()); + Ok(TelemetryGuard { + providers: Some(providers), + }) } /// Start the metrics poll loop and log fan-out tasks. All tasks stop on /// `state.shutdown`. Returns no handles when export is disabled. -pub fn spawn_exporters(_state: AppState) -> Vec> { - todo!("WS-G G3/G5") +pub fn spawn_exporters(state: AppState) -> Vec> { + let mut handles = Vec::new(); + if !state.config.otel.enabled { + return handles; + } + let Some(p) = PROVIDERS.get() else { + return handles; + }; + if let Some(meter) = &p.meter { + handles.extend(metrics_export::spawn(state.clone(), meter)); + } + if let Some(logger) = &p.logger { + handles.extend(logs_export::spawn(state, logger)); + } + handles +} + +fn hostname() -> String { + for path in ["/proc/sys/kernel/hostname", "/etc/hostname"] { + if let Ok(s) = std::fs::read_to_string(path) { + let s = s.trim(); + if !s.is_empty() { + return s.to_owned(); + } + } + } + "unknown".to_owned() +} + +fn build_resource(otel: &OtelConfig) -> Resource { + Resource::builder() + .with_service_name(otel.service_name.clone()) + .with_attribute(KeyValue::new( + "service.version", + env!("CARGO_PKG_VERSION").to_owned(), + )) + .with_attribute(KeyValue::new("host.name", hostname())) + .build() +} + +/// Endpoint for one signal. gRPC uses the base URL. HTTP appends +/// `/v1/` unless the configured endpoint already ends with it. +fn signal_endpoint(otel: &OtelConfig, signal: &str) -> String { + match otel.protocol { + OtelProtocol::Grpc => otel.endpoint.clone(), + OtelProtocol::HttpProtobuf => { + let base = otel.endpoint.trim_end_matches('/'); + if base.ends_with(&format!("/v1/{signal}")) { + base.to_owned() + } else { + format!("{base}/v1/{signal}") + } + } + } +} + +fn header_map(otel: &OtelConfig) -> HashMap { + otel.headers + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect() +} + +fn metadata(otel: &OtelConfig) -> Result { + use opentelemetry_otlp::tonic_types::metadata::MetadataMap; + let mut m = MetadataMap::new(); + for (k, v) in &otel.headers { + // Metadata keys must be `'static`; the config lives for the whole process + // and `init` runs once, so leaking the few header names is fine. + let key: &'static str = Box::leak(k.to_ascii_lowercase().into_boxed_str()); + let val = v + .parse() + .with_context(|| format!("invalid value for otel header '{k}'"))?; + m.insert(key, val); + } + Ok(m) +} + +macro_rules! build_exporter { + ($ty:ident, $otel:expr, $signal:expr) => {{ + let otel: &OtelConfig = $otel; + let ep = signal_endpoint(otel, $signal); + let res = match otel.protocol { + OtelProtocol::Grpc => $ty::builder() + .with_tonic() + .with_endpoint(ep) + .with_metadata(metadata(otel)?) + .build(), + OtelProtocol::HttpProtobuf => $ty::builder() + .with_http() + .with_protocol(Protocol::HttpBinary) + .with_endpoint(ep) + .with_headers(header_map(otel)) + .build(), + }; + res.with_context(|| format!("building OTLP {} exporter", $signal)) + }}; +} + +fn build_span_exporter(otel: &OtelConfig) -> Result { + build_exporter!(SpanExporter, otel, "traces") +} + +fn build_metric_exporter(otel: &OtelConfig) -> Result { + build_exporter!(MetricExporter, otel, "metrics") +} + +fn build_log_exporter(otel: &OtelConfig) -> Result { + build_exporter!(LogExporter, otel, "logs") +} + +/// Shell-style glob match supporting `*` and `?`. +pub(crate) fn glob_match(pattern: &str, text: &str) -> bool { + let p: Vec = pattern.chars().collect(); + let t: Vec = text.chars().collect(); + let (mut pi, mut ti) = (0, 0); + let (mut star, mut mark) = (None, 0); + while ti < t.len() { + if pi < p.len() && (p[pi] == '?' || p[pi] == t[ti]) { + pi += 1; + ti += 1; + } else if pi < p.len() && p[pi] == '*' { + star = Some(pi); + mark = ti; + pi += 1; + } else if let Some(s) = star { + pi = s + 1; + mark += 1; + ti = mark; + } else { + return false; + } + } + while pi < p.len() && p[pi] == '*' { + pi += 1; + } + pi == p.len() +} + +pub(crate) fn any_glob(patterns: &[String], text: &str) -> bool { + patterns.iter().any(|p| glob_match(p, text)) +} + +/// Exponential backoff for restarting failed export tasks. +#[derive(Debug, Clone)] +pub(crate) struct Backoff { + cur: Duration, + min: Duration, + max: Duration, +} + +impl Backoff { + pub(crate) fn new(min: Duration, max: Duration) -> Self { + Self { cur: min, min, max } + } + + pub(crate) fn next_delay(&mut self) -> Duration { + let d = self.cur; + self.cur = (self.cur * 2).min(self.max); + d + } + + pub(crate) fn reset(&mut self) { + self.cur = self.min; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn glob() { + assert!(glob_match("*.service", "sshd.service")); + assert!(!glob_match("*.service", "sshd.timer")); + assert!(glob_match("web-?", "web-1")); + assert!(!glob_match("web-?", "web-12")); + assert!(glob_match("a*b*c", "aXXbYYc")); + assert!(!glob_match("a*b*c", "aXXbYY")); + assert!(glob_match("*", "")); + assert!(glob_match("exact", "exact")); + assert!(!glob_match("exact", "exactly")); + } + + #[test] + fn backoff_doubles_and_caps() { + let mut b = Backoff::new(Duration::from_secs(1), Duration::from_secs(5)); + let v: Vec = (0..5).map(|_| b.next_delay().as_secs()).collect(); + assert_eq!(v, [1, 2, 4, 5, 5]); + b.reset(); + assert_eq!(b.next_delay().as_secs(), 1); + } + + #[test] + fn http_endpoint_gets_signal_path() { + let mut o = OtelConfig { + protocol: OtelProtocol::HttpProtobuf, + endpoint: "http://localhost:4318/".into(), + ..Default::default() + }; + assert_eq!(signal_endpoint(&o, "logs"), "http://localhost:4318/v1/logs"); + o.endpoint = "http://c:4318/v1/logs".into(); + assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs"); + o.protocol = OtelProtocol::Grpc; + assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs"); + } + + #[tokio::test] + async fn exporters_build_for_both_protocols() { + for protocol in [OtelProtocol::Grpc, OtelProtocol::HttpProtobuf] { + let mut o = OtelConfig { + protocol, + endpoint: "http://127.0.0.1:1".into(), + ..Default::default() + }; + o.headers.insert("Authorization".into(), "Bearer x".into()); + let results = [ + build_span_exporter(&o).map(drop), + build_metric_exporter(&o).map(drop), + build_log_exporter(&o).map(drop), + ]; + for r in results { + match (protocol, r) { + (_, Ok(())) => {} + // Contract: HTTP needs the `reqwest-blocking-client` feature on + // opentelemetry-otlp (Cargo.toml, owned by P0). + (OtelProtocol::HttpProtobuf, Err(e)) + if format!("{e:?}").contains("http_client") => {} + (_, Err(e)) => panic!("exporter build failed: {e:?}"), + } + } + } + } + + #[test] + fn disabled_init_is_noop_guard() { + let g = init(&Config::default()).unwrap(); + assert!(!g.is_enabled()); + } } From 1eb7ffce1961b67b6dab943e4767134479dc269b Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Thu, 15 Oct 2026 18:43:21 -0400 Subject: [PATCH 2/2] WS-G: apply review fixes F1-F8 Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs --- plan/workstreams/WS-G-otel.md | 2 + src/telemetry/logs_export.rs | 89 +++++++++++++++++-- src/telemetry/metrics_export.rs | 152 +++++++++++++++++++------------- src/telemetry/mod.rs | 75 +++++++++------- 4 files changed, 219 insertions(+), 99 deletions(-) diff --git a/plan/workstreams/WS-G-otel.md b/plan/workstreams/WS-G-otel.md index 475bf42..2686d4c 100644 --- a/plan/workstreams/WS-G-otel.md +++ b/plan/workstreams/WS-G-otel.md @@ -53,4 +53,6 @@ Tick these in [`TODO.md`](../../TODO.md). ## Log - 2026-10-15: Card created. +- 2026-10-15: Review fixes F1-F8. Restart polling uses buffer_unordered(8) with a 15s budget; failed sections keep the previous snapshot; series carry identity attributes only; timer metric renamed `systemd.timer.next_elapse_timestamp_seconds`; units added; observations computed once per poll. New containers start from StartedAt, docker lines deduped on reconnect, own unit/identifier excluded from the journal follow (by name; cursor not used). `TelemetryGuard::shutdown().await` added; Drop is best-effort. +- P3 notes: (1) call `guard.shutdown().await` after awaiting the exporter handles. (2) devenv: add `processes.otelcol.exec = "otelcol-contrib --config dev/otelcol.yaml"` to devenv.nix (not done here, P0 owner). (3) Cargo.toml needs `reqwest-blocking-client` on opentelemetry-otlp for http/protobuf; the exporter-build test is strict and fails without it. (4) RUST_LOG/EnvFilter deferred. - 2026-10-15: Implemented G1-G7 on `ws-g/otel`. Notes: no glob crate, so a small `*`/`?` matcher lives in `telemetry/mod.rs`. `DockerSource` has no events stream, so the log reconciler polls `ps(false)` every 5s (30s while Docker is unavailable) instead of subscribing to `events`. No `EnvFilter` (feature not enabled in Cargo.toml): level is fixed at INFO. `systemd.timer.next_elapse_seconds` exports absolute unix seconds. `systemd.unit.restarts` costs one `unit_status` per in-scope loaded service per interval. HTTP endpoints get `/v1/` appended unless already present. Live otelcol smoke test not run (no collector binary on PATH). diff --git a/src/telemetry/logs_export.rs b/src/telemetry/logs_export.rs index 0e1aead..aa5fcbf 100644 --- a/src/telemetry/logs_export.rs +++ b/src/telemetry/logs_export.rs @@ -130,7 +130,31 @@ pub fn spawn(state: AppState, provider: &SdkLoggerProvider) -> Vec Vec { + let mut v = vec![service_name.to_owned(), "skyg-linux-tracker".to_owned()]; + if let Some(exe) = std::env::current_exe() + .ok() + .and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())) + { + v.push(exe); + } + v +} + +pub fn is_own_entry(e: &JournalEntry, own: &[String]) -> bool { + e.identifier.as_ref().is_some_and(|i| own.contains(i)) + || e.unit.as_ref().is_some_and(|u| { + own.iter() + .any(|n| u == n || u.strip_suffix(".service") == Some(n.as_str())) + }) +} + async fn journal_task(state: AppState, logger: SdkLogger, unit: Option) { + let own = own_names(&state.config.otel.service_name); let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX); let mut last: Option = None; let max_lines = state.config.limits.max_lines; @@ -155,6 +179,9 @@ async fn journal_task(state: AppState, logger: SdkLogger, unit: Option) e = rx.recv() => e, }; let Some(e) = e else { break }; + if is_own_entry(&e, &own) { + continue; + } if last.is_some_and(|l| e.timestamp <= l) { continue; } @@ -210,6 +237,8 @@ async fn docker_reconciler(state: AppState, logger: SdkLogger) { let allow = state.config.otel.logs.docker_containers.clone(); let mut tasks: HashMap)> = HashMap::new(); let mut warned = false; + let mut failures = 0u32; + let mut first_pass = true; loop { let mut delay = RECONCILE_INTERVAL; let ps = tokio::select! { @@ -219,6 +248,9 @@ async fn docker_reconciler(state: AppState, logger: SdkLogger) { match ps { Ok(cs) => { warned = false; + failures = 0; + let started: HashMap> = + cs.iter().map(|c| (c.name.clone(), c.started_at)).collect(); let running: Vec = cs.into_iter().map(|c| c.name).collect(); let tracked: Vec = tasks.keys().cloned().collect(); let (start, stop) = reconcile(&running, &tracked, &allow); @@ -230,21 +262,35 @@ async fn docker_reconciler(state: AppState, logger: SdkLogger) { } for name in start { let tok = state.shutdown.child_token(); + // Containers present at startup: only new lines. Containers that + // appear later: from their StartedAt so startup output is kept. + let from = if first_pass { + None + } else { + started.get(&name).copied().flatten() + }; let h = tokio::spawn(container_task( state.clone(), logger.clone(), name.clone(), + from, tok.clone(), )); tasks.insert(name, (tok, h)); } + first_pass = false; } Err(e) => { - if !warned { - tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly"); - warned = true; + failures += 1; + // Transient errors keep the normal cadence; only a persistent + // outage switches to the slow retry. + if failures >= 3 { + if !warned { + tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly"); + warned = true; + } + delay = DOCKER_SLOW_RETRY; } - delay = DOCKER_SLOW_RETRY; } } if !sleep_or_shutdown(&state.shutdown, delay).await { @@ -261,13 +307,23 @@ async fn container_task( state: AppState, logger: SdkLogger, name: String, + from: Option, token: CancellationToken, ) { let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX); - let mut since: Option = Some(Timestamp::now()); + // `from == None`: container was already running at startup, so only new lines. + // docker `tail=0` returns nothing regardless of `since`, so a backlog needs + // a real tail. + let mut since: Option = Some(from.unwrap_or_else(Timestamp::now)); + let mut tail = if from.is_some() { + state.config.limits.max_tail + } else { + 0 + }; + let mut last_emitted: Option = None; loop { let opts = LogOptions { - tail: Some(0), + tail: Some(tail), since, }; let rx = tokio::select! { @@ -285,6 +341,11 @@ async fn container_task( let Some(l) = l else { break }; got = true; if let Some(ts) = l.ts { + // Dedupe lines replayed after a reconnect. + if last_emitted.is_some_and(|le| ts <= le) { + continue; + } + last_emitted = Some(ts); since = Some(ts); } emit(&logger, docker_record(&name, &l)); @@ -297,6 +358,10 @@ async fn container_task( tracing::debug!(error = %e, container = %name, "otel logs: container follow failed"); } } + // After the first connection, reconnects replay from the last seen line. + if last_emitted.is_some() { + tail = state.config.limits.max_tail; + } if !sleep_or_shutdown(&token, backoff.next_delay()).await { return; } @@ -320,6 +385,18 @@ mod tests { } } + #[test] + fn own_entries_are_excluded() { + let own = vec!["skyg-linux-tracker".to_string()]; + let mut e = entry(None); + assert!(!is_own_entry(&e, &own)); + e.identifier = Some("skyg-linux-tracker".into()); + assert!(is_own_entry(&e, &own)); + e.identifier = Some("slinuxm".into()); + e.unit = Some("skyg-linux-tracker.service".into()); + assert!(is_own_entry(&e, &own)); + } + #[test] fn severity_mapping() { use Priority::*; diff --git a/src/telemetry/metrics_export.rs b/src/telemetry/metrics_export.rs index 4b182c3..025f000 100644 --- a/src/telemetry/metrics_export.rs +++ b/src/telemetry/metrics_export.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; use std::time::Duration; +use futures::StreamExt; use opentelemetry::metrics::MeterProvider as _; use opentelemetry::KeyValue; use opentelemetry_sdk::metrics::SdkMeterProvider; @@ -21,17 +22,23 @@ use crate::state::AppState; /// Per-call bound so a hung backend cannot stall the poll loop. const CALL_TIMEOUT: Duration = Duration::from_secs(10); -pub const METRIC_NAMES: [&str; 8] = [ - "systemd.unit.active", - "systemd.unit.failed", - "systemd.unit.restarts", - "systemd.timer.next_elapse_seconds", - "docker.container.running", - "docker.container.healthy", - "docker.container.uptime_seconds", - "docker.container.restarts", +/// Metric name and UCUM unit. +pub const METRICS: [(&str, &str); 8] = [ + ("systemd.unit.active", "1"), + ("systemd.unit.failed", "1"), + ("systemd.unit.restarts", "{restart}"), + // Absolute unix timestamp of the next elapse, not a duration. + ("systemd.timer.next_elapse_timestamp_seconds", "s"), + ("docker.container.running", "1"), + ("docker.container.healthy", "1"), + ("docker.container.uptime_seconds", "s"), + ("docker.container.restarts", "{restart}"), ]; +/// Overall budget for the per-unit restart-count polling. +const RESTARTS_BUDGET: Duration = Duration::from_secs(15); +const RESTARTS_CONCURRENCY: usize = 8; + /// Latest poll result, already filtered by the allowlists. #[derive(Debug, Default, Clone)] pub struct Snapshot { @@ -64,92 +71,91 @@ pub fn container_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool { cfg.containers.is_empty() || any_glob(&cfg.containers, name) } -fn unit_attrs(u: &UnitSummary) -> Vec<(&'static str, String)> { - vec![ - ("systemd.unit", u.name.clone()), - ("systemd.sub_state", u.sub_state.clone()), - ("systemd.load_state", u.load_state.clone()), - ] +fn unit_id(name: &str) -> Vec<(&'static str, String)> { + vec![("systemd.unit", name.to_owned())] } -fn container_attrs(c: &ContainerSummary) -> Vec<(&'static str, String)> { - vec![ - ("container.name", c.name.clone()), - ("container.id", c.id.clone()), - ("container.image.name", c.image.clone()), - ] +fn container_id(c: &ContainerSummary) -> Vec<(&'static str, String)> { + vec![("container.name", c.name.clone())] } -/// Map a snapshot to gauge observations. +/// Map a snapshot to gauge observations. Series carry identity attributes +/// only (`systemd.unit` / `container.name`) so state changes do not churn +/// series. The `active` gauge also carries sub/load state, and `running` +/// carries the image name. pub fn observations(s: &Snapshot) -> Vec { let b = |v: bool| if v { 1.0 } else { 0.0 }; let mut out = Vec::new(); for u in &s.units { + let mut active_attrs = unit_id(&u.name); + active_attrs.push(("systemd.sub_state", u.sub_state.clone())); + active_attrs.push(("systemd.load_state", u.load_state.clone())); out.push(Obs { metric: "systemd.unit.active", value: b(u.active_state == "active"), - attrs: unit_attrs(u), + attrs: active_attrs, }); out.push(Obs { metric: "systemd.unit.failed", value: b(u.active_state == "failed"), - attrs: unit_attrs(u), + attrs: unit_id(&u.name), }); if let Some(n) = s.restarts.get(&u.name) { out.push(Obs { metric: "systemd.unit.restarts", value: f64::from(*n), - attrs: unit_attrs(u), + attrs: unit_id(&u.name), }); } } for t in &s.timers { - // Absolute unix time of the next elapse; unscheduled timers are omitted. + // Unscheduled timers are omitted. if let Some(next) = t.next_elapse { out.push(Obs { - metric: "systemd.timer.next_elapse_seconds", + metric: "systemd.timer.next_elapse_timestamp_seconds", value: next.as_second() as f64, - attrs: vec![ - ("systemd.unit", t.name.clone()), - ("systemd.sub_state", t.active_state.clone()), - ], + attrs: unit_id(&t.name), }); } } for c in &s.containers { + let mut running_attrs = container_id(c); + running_attrs.push(("container.image.name", c.image.clone())); out.push(Obs { metric: "docker.container.running", value: b(c.state == "running"), - attrs: container_attrs(c), + attrs: running_attrs, }); if let Some(h) = &c.health { out.push(Obs { metric: "docker.container.healthy", value: b(h == "healthy"), - attrs: container_attrs(c), + attrs: container_id(c), }); } if let Some(up) = c.uptime_seconds { out.push(Obs { metric: "docker.container.uptime_seconds", value: up as f64, - attrs: container_attrs(c), + attrs: container_id(c), }); } if let Some(r) = c.restart_count { out.push(Obs { metric: "docker.container.restarts", value: r as f64, - attrs: container_attrs(c), + attrs: container_id(c), }); } } out } -async fn collect(state: &AppState) -> Snapshot { +/// Poll the sources. A section whose call fails or times out keeps its value +/// from `prev` instead of being emptied. +async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot { let cfg = &state.config.otel.metrics; - let mut snap = Snapshot::default(); + let mut snap = prev.clone(); match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_units(None)).await { Ok(Ok(units)) => { @@ -171,24 +177,38 @@ async fn collect(state: &AppState) -> Snapshot { Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_timers failed"), Err(_) => tracing::debug!("otel metrics: list_timers timed out"), } + + // Restart counts: bounded concurrency and an overall budget. Entries not + // refreshed in time keep their previous value; vanished units are dropped. let services: Vec = snap .units .iter() .filter(|u| u.name.ends_with(".service") && u.load_state == "loaded") .map(|u| u.name.clone()) .collect(); - for name in services { - if state.shutdown.is_cancelled() { - break; - } - if let Ok(Ok(st)) = - tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await - { - if let Some(n) = st.n_restarts { - snap.restarts.insert(name, n); + let mut restarts: HashMap = services + .iter() + .filter_map(|n| prev.restarts.get(n).map(|v| (n.clone(), *v))) + .collect(); + let fetch = futures::stream::iter(services) + .map(|name| async move { + let r = tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await; + (name, r) + }) + .buffer_unordered(RESTARTS_CONCURRENCY) + .for_each(|(name, r)| { + if let Ok(Ok(st)) = r { + if let Some(n) = st.n_restarts { + restarts.insert(name, n); + } } - } + std::future::ready(()) + }); + if tokio::time::timeout(RESTARTS_BUDGET, fetch).await.is_err() { + tracing::debug!("otel metrics: restart polling hit its time budget"); } + snap.restarts = restarts; + match tokio::time::timeout(CALL_TIMEOUT, state.docker.ps(true)).await { Ok(Ok(cs)) => { snap.containers = cs @@ -204,20 +224,22 @@ async fn collect(state: &AppState) -> Snapshot { /// Register the gauges and start the poll loop. pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec> { - let snapshot: Arc>> = Arc::default(); + // Observations are computed once per poll; callbacks only filter. + let current: Arc>>> = Arc::default(); let meter = provider.meter("skyg-linux-tracker"); - for name in METRIC_NAMES { - let snap = Arc::clone(&snapshot); + for (name, unit) in METRICS { + let cur = Arc::clone(¤t); // Gauges live as long as the provider; the instrument handle is not needed. let _ = meter .f64_observable_gauge(name) + .with_unit(unit) .with_callback(move |o| { - let cur = snap.read().map(|g| Arc::clone(&g)).unwrap_or_default(); - for ob in observations(&cur).into_iter().filter(|x| x.metric == name) { + let obs = cur.read().map(|g| Arc::clone(&g)).unwrap_or_default(); + for ob in obs.iter().filter(|x| x.metric == name) { let attrs: Vec = ob .attrs - .into_iter() - .map(|(k, v)| KeyValue::new(k, v)) + .iter() + .map(|(k, v)| KeyValue::new(*k, v.clone())) .collect(); o.observe(ob.value, &attrs); } @@ -229,17 +251,18 @@ pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec let h = tokio::spawn(async move { let mut tick = tokio::time::interval(interval); tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut snap = Snapshot::default(); loop { tokio::select! { _ = state.shutdown.cancelled() => break, _ = tick.tick() => {} } - let snap = tokio::select! { + snap = tokio::select! { _ = state.shutdown.cancelled() => break, - s = collect(&state) => s, + s = collect(&state, &snap) => s, }; - if let Ok(mut g) = snapshot.write() { - *g = Arc::new(snap); + if let Ok(mut g) = current.write() { + *g = Arc::new(observations(&snap)); } } }); @@ -359,7 +382,7 @@ mod tests { let o = observations(&s); let timers: Vec<_> = o .iter() - .filter(|x| x.metric == "systemd.timer.next_elapse_seconds") + .filter(|x| x.metric == "systemd.timer.next_elapse_timestamp_seconds") .collect(); assert_eq!(timers.len(), 1); assert_eq!(timers[0].value, 1_792_065_600.0); @@ -374,7 +397,12 @@ mod tests { .unwrap(); assert_eq!(h.value, 0.0); assert_eq!(h.attrs[0], ("container.name", "web".to_string())); - assert_eq!(h.attrs[2], ("container.image.name", "nginx:1".to_string())); + assert_eq!(h.attrs.len(), 1); + let r = o + .iter() + .find(|x| x.metric == "docker.container.running") + .unwrap(); + assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string())); } #[tokio::test] @@ -383,13 +411,13 @@ mod tests { cfg.otel.metrics.units = vec!["__nomatch__*".into()]; cfg.otel.metrics.containers = vec!["__nomatch__*".into()]; let st = fake_state(cfg); - let snap = collect(&st).await; + let snap = collect(&st, &Snapshot::default()).await; assert!(snap.units.is_empty()); assert!(snap.timers.is_empty()); assert!(snap.containers.is_empty()); let st = fake_state(Config::default()); - let snap = collect(&st).await; + let snap = collect(&st, &Snapshot::default()).await; assert!(snap .units .iter() diff --git a/src/telemetry/mod.rs b/src/telemetry/mod.rs index 01ec08e..871bdba 100644 --- a/src/telemetry/mod.rs +++ b/src/telemetry/mod.rs @@ -49,29 +49,20 @@ pub struct TelemetryGuard { providers: Option, } -impl TelemetryGuard { - /// True when OTLP export is active. - pub fn is_enabled(&self) -> bool { - self.providers.is_some() - } -} - -impl Drop for TelemetryGuard { - fn drop(&mut self) { - let Some(p) = self.providers.take() else { - return; - }; - if let Err(e) = p.tracer.force_flush() { +impl Providers { + /// Blocking flush and shutdown, each provider bounded by `FLUSH_TIMEOUT`. + fn flush_and_shutdown(&self) { + if let Err(e) = self.tracer.force_flush() { eprintln!("otel: trace flush failed: {e}"); } - let _ = p.tracer.shutdown_with_timeout(FLUSH_TIMEOUT); - if let Some(m) = &p.meter { + let _ = self.tracer.shutdown_with_timeout(FLUSH_TIMEOUT); + if let Some(m) = &self.meter { if let Err(e) = m.force_flush() { eprintln!("otel: metrics flush failed: {e}"); } let _ = m.shutdown_with_timeout(FLUSH_TIMEOUT); } - if let Some(l) = &p.logger { + if let Some(l) = &self.logger { if let Err(e) = l.force_flush() { eprintln!("otel: logs flush failed: {e}"); } @@ -80,6 +71,40 @@ impl Drop for TelemetryGuard { } } +impl TelemetryGuard { + /// True when OTLP export is active. + pub fn is_enabled(&self) -> bool { + self.providers.is_some() + } + + /// Flush and shut down all providers without blocking the async runtime. + /// Call this (and await it) on graceful shutdown, after the exporter task + /// handles from `spawn_exporters` have finished. `Drop` is only a + /// best-effort fallback. + pub async fn shutdown(mut self) { + if let Some(p) = self.providers.take() { + let _ = tokio::task::spawn_blocking(move || p.flush_and_shutdown()).await; + } + } +} + +impl Drop for TelemetryGuard { + /// Best effort. Inside a tokio runtime the flush runs on a detached thread + /// (it may not finish before the process exits); outside one it runs inline. + fn drop(&mut self) { + let Some(p) = self.providers.take() else { + return; + }; + if tokio::runtime::Handle::try_current().is_ok() { + let _ = std::thread::Builder::new() + .name("otel-flush".into()) + .spawn(move || p.flush_and_shutdown()); + } else { + p.flush_and_shutdown(); + } + } +} + /// Install the tracing subscriber and, when `config.otel.enabled`, the OTLP /// providers. Works (stderr logging only) when OTel is disabled. pub fn init(config: &Config) -> Result { @@ -359,21 +384,9 @@ mod tests { ..Default::default() }; o.headers.insert("Authorization".into(), "Bearer x".into()); - let results = [ - build_span_exporter(&o).map(drop), - build_metric_exporter(&o).map(drop), - build_log_exporter(&o).map(drop), - ]; - for r in results { - match (protocol, r) { - (_, Ok(())) => {} - // Contract: HTTP needs the `reqwest-blocking-client` feature on - // opentelemetry-otlp (Cargo.toml, owned by P0). - (OtelProtocol::HttpProtobuf, Err(e)) - if format!("{e:?}").contains("http_client") => {} - (_, Err(e)) => panic!("exporter build failed: {e:?}"), - } - } + build_span_exporter(&o).unwrap(); + build_metric_exporter(&o).unwrap(); + build_log_exporter(&o).unwrap(); } }