//! Journal and docker log fan-out to OTel logs. //! //! Journal: one `follow()` per configured unit, or one unfiltered follow when //! `otel.logs.journal_units` is empty. Docker: a reconciler keeps one //! `follow_logs` task per allowlisted running container. //! //! Deviation: `DockerSource` exposes no events stream, so the reconciler //! polls `ps(false)` every few seconds instead of subscribing to `events`. use std::collections::HashMap; use std::time::{Duration, SystemTime}; use jiff::Timestamp; use opentelemetry::logs::{AnyValue, LogRecord as _, Logger as _, LoggerProvider as _, Severity}; use opentelemetry_sdk::logs::{SdkLogger, SdkLoggerProvider}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use super::{any_glob, Backoff}; use crate::sources::{JournalEntry, JournalQuery, LogLine, LogOptions, LogStream, Priority}; use crate::state::AppState; const BACKOFF_MIN: Duration = Duration::from_secs(1); const BACKOFF_MAX: Duration = Duration::from_secs(60); /// Retry interval while Docker is unavailable. const DOCKER_SLOW_RETRY: Duration = Duration::from_secs(30); const RECONCILE_INTERVAL: Duration = Duration::from_secs(5); /// Backend-neutral log record, built by pure mapping functions. #[derive(Debug, Clone, PartialEq)] pub struct LogRec { pub severity: Severity, pub severity_text: &'static str, pub body: String, pub timestamp: Option, pub attrs: Vec<(&'static str, String)>, } /// Syslog priority to OTel severity. pub fn journal_severity(p: Option) -> (Severity, &'static str) { match p { Some(Priority::Emerg | Priority::Alert | Priority::Crit) => (Severity::Fatal, "FATAL"), Some(Priority::Err) => (Severity::Error, "ERROR"), Some(Priority::Warning) => (Severity::Warn, "WARN"), Some(Priority::Notice | Priority::Info) | None => (Severity::Info, "INFO"), Some(Priority::Debug) => (Severity::Debug, "DEBUG"), } } pub fn journal_record(e: &JournalEntry, unit_hint: Option<&str>) -> LogRec { let (severity, severity_text) = journal_severity(e.priority); let mut attrs = Vec::new(); if let Some(u) = e.unit.as_deref().or(unit_hint) { attrs.push(("systemd.unit", u.to_owned())); } if let Some(i) = &e.identifier { attrs.push(("log.syslog.identifier", i.clone())); } if let Some(p) = e.pid { attrs.push(("process.pid", p.to_string())); } LogRec { severity, severity_text, body: e.message.clone(), timestamp: Some(SystemTime::from(e.timestamp)), attrs, } } /// stderr maps to WARN, stdout to INFO. pub fn docker_record(container: &str, l: &LogLine) -> LogRec { let (severity, severity_text) = match l.stream { LogStream::Stderr => (Severity::Warn, "WARN"), LogStream::Stdout => (Severity::Info, "INFO"), }; LogRec { severity, severity_text, body: l.message.clone(), timestamp: l.ts.map(SystemTime::from), attrs: vec![ ("container.name", container.to_owned()), ( "log.iostream", match l.stream { LogStream::Stderr => "stderr", LogStream::Stdout => "stdout", } .to_owned(), ), ], } } fn emit(logger: &SdkLogger, r: LogRec) { let mut rec = logger.create_log_record(); rec.set_severity_number(r.severity); rec.set_severity_text(r.severity_text); rec.set_body(AnyValue::String(r.body.into())); if let Some(ts) = r.timestamp { rec.set_timestamp(ts); } rec.set_observed_timestamp(SystemTime::now()); for (k, v) in r.attrs { rec.add_attribute(k, AnyValue::String(v.into())); } logger.emit(rec); } pub fn spawn(state: AppState, provider: &SdkLoggerProvider) -> Vec> { let logger = provider.logger("skyg-linux-tracker"); let mut handles = Vec::new(); let units = state.config.otel.logs.journal_units.clone(); let targets: Vec> = if units.is_empty() { vec![None] } else { units.into_iter().map(Some).collect() }; for unit in targets { handles.push(tokio::spawn(journal_task( state.clone(), logger.clone(), unit, ))); } handles.push(tokio::spawn(docker_reconciler(state, logger))); handles } /// Names identifying this process in the journal (unit and syslog identifier). /// Used to keep the app's own log lines out of the unfiltered follow, which /// would otherwise feed back into itself. Limitation: matching is by name; the /// journal cursor is not used. pub fn own_names(service_name: &str) -> Vec { let mut v = vec![service_name.to_owned(), "skyg-linux-tracker".to_owned()]; if let Some(exe) = std::env::current_exe() .ok() .and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())) { v.push(exe); } v } pub fn is_own_entry(e: &JournalEntry, own: &[String]) -> bool { e.identifier.as_ref().is_some_and(|i| own.contains(i)) || e.unit.as_ref().is_some_and(|u| { own.iter() .any(|n| u == n || u.strip_suffix(".service") == Some(n.as_str())) }) } async fn journal_task(state: AppState, logger: SdkLogger, unit: Option) { let own = own_names(&state.config.otel.service_name); let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX); let mut last: Option = None; let max_lines = state.config.limits.max_lines; loop { let q = JournalQuery { unit: unit.clone(), priority: None, // No backlog on first start; catch up from the last seen entry afterwards. lines: Some(if last.is_some() { max_lines } else { 0 }), since: last, }; let rx = tokio::select! { _ = state.shutdown.cancelled() => return, r = state.journal.follow(&q) => r, }; match rx { Ok(mut rx) => { let mut got = false; loop { let e = tokio::select! { _ = state.shutdown.cancelled() => return, e = rx.recv() => e, }; let Some(e) = e else { break }; if is_own_entry(&e, &own) { continue; } if last.is_some_and(|l| e.timestamp <= l) { continue; } got = true; last = Some(e.timestamp); emit(&logger, journal_record(&e, unit.as_deref())); } if got { backoff.reset(); } } Err(e) => tracing::warn!(error = %e, unit = ?unit, "otel logs: journal follow failed"), } if !sleep_or_shutdown(&state.shutdown, backoff.next_delay()).await { return; } } } /// Returns false when shutdown fired during the sleep. async fn sleep_or_shutdown(shutdown: &CancellationToken, d: Duration) -> bool { tokio::select! { _ = shutdown.cancelled() => false, _ = tokio::time::sleep(d) => true, } } /// Pure reconcile step: which running containers to start and which tracked /// ones to stop. pub fn reconcile( running: &[String], tracked: &[String], allow: &[String], ) -> (Vec, Vec) { let wanted: Vec<&String> = running .iter() .filter(|n| allow.is_empty() || any_glob(allow, n)) .collect(); let start = wanted .iter() .filter(|n| !tracked.contains(n)) .map(|n| (*n).clone()) .collect(); let stop = tracked .iter() .filter(|t| !wanted.contains(t)) .cloned() .collect(); (start, stop) } async fn docker_reconciler(state: AppState, logger: SdkLogger) { let allow = state.config.otel.logs.docker_containers.clone(); let mut tasks: HashMap)> = HashMap::new(); let mut warned = false; let mut failures = 0u32; let mut first_pass = true; loop { let mut delay = RECONCILE_INTERVAL; let ps = tokio::select! { _ = state.shutdown.cancelled() => break, r = state.docker.ps(false) => r, }; match ps { Ok(cs) => { warned = false; failures = 0; let started: HashMap> = cs.iter().map(|c| (c.name.clone(), c.started_at)).collect(); let running: Vec = cs.into_iter().map(|c| c.name).collect(); let tracked: Vec = tasks.keys().cloned().collect(); let (start, stop) = reconcile(&running, &tracked, &allow); for name in stop { if let Some((tok, h)) = tasks.remove(&name) { tok.cancel(); h.abort(); } } for name in start { let tok = state.shutdown.child_token(); // Containers present at startup: only new lines. Containers that // appear later: from their StartedAt so startup output is kept. let from = if first_pass { None } else { started.get(&name).copied().flatten() }; let h = tokio::spawn(container_task( state.clone(), logger.clone(), name.clone(), from, tok.clone(), )); tasks.insert(name, (tok, h)); } first_pass = false; } Err(e) => { failures += 1; // Transient errors keep the normal cadence; only a persistent // outage switches to the slow retry. if failures >= 3 { if !warned { tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly"); warned = true; } delay = DOCKER_SLOW_RETRY; } } } if !sleep_or_shutdown(&state.shutdown, delay).await { break; } } for (_, (tok, h)) in tasks { tok.cancel(); h.abort(); } } async fn container_task( state: AppState, logger: SdkLogger, name: String, from: Option, token: CancellationToken, ) { let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX); // `from == None`: container was already running at startup, so only new lines. // docker `tail=0` returns nothing regardless of `since`, so a backlog needs // a real tail. let mut since: Option = Some(from.unwrap_or_else(Timestamp::now)); let mut tail = if from.is_some() { state.config.limits.max_tail } else { 0 }; let mut last_emitted: Option = None; loop { let opts = LogOptions { tail: Some(tail), since, }; let rx = tokio::select! { _ = token.cancelled() => return, r = state.docker.follow_logs(&name, &opts) => r, }; match rx { Ok(mut rx) => { let mut got = false; loop { let l = tokio::select! { _ = token.cancelled() => return, l = rx.recv() => l, }; let Some(l) = l else { break }; got = true; if let Some(ts) = l.ts { // Dedupe lines replayed after a reconnect. if last_emitted.is_some_and(|le| ts <= le) { continue; } last_emitted = Some(ts); since = Some(ts); } emit(&logger, docker_record(&name, &l)); } if got { backoff.reset(); } } Err(e) => { tracing::debug!(error = %e, container = %name, "otel logs: container follow failed"); } } // After the first connection, reconnects replay from the last seen line. if last_emitted.is_some() { tail = state.config.limits.max_tail; } if !sleep_or_shutdown(&token, backoff.next_delay()).await { return; } } } #[cfg(test)] mod tests { use super::*; fn entry(p: Option) -> JournalEntry { JournalEntry { timestamp: "2026-10-15T12:00:00Z".parse().unwrap(), priority: p, unit: Some("sshd.service".into()), identifier: Some("sshd".into()), pid: Some(7), hostname: None, message: "hello".into(), cursor: None, } } #[test] fn own_entries_are_excluded() { let own = vec!["skyg-linux-tracker".to_string()]; let mut e = entry(None); assert!(!is_own_entry(&e, &own)); e.identifier = Some("skyg-linux-tracker".into()); assert!(is_own_entry(&e, &own)); e.identifier = Some("slinuxm".into()); e.unit = Some("skyg-linux-tracker.service".into()); assert!(is_own_entry(&e, &own)); } #[test] fn severity_mapping() { use Priority::*; let sev = |p| journal_severity(Some(p)).0; assert_eq!(sev(Emerg), Severity::Fatal); assert_eq!(sev(Alert), Severity::Fatal); assert_eq!(sev(Crit), Severity::Fatal); assert_eq!(sev(Err), Severity::Error); assert_eq!(sev(Warning), Severity::Warn); assert_eq!(sev(Notice), Severity::Info); assert_eq!(sev(Info), Severity::Info); assert_eq!(sev(Debug), Severity::Debug); assert_eq!(journal_severity(None).0, Severity::Info); } #[test] fn journal_attrs() { let r = journal_record(&entry(Some(Priority::Err)), None); assert_eq!(r.severity_text, "ERROR"); assert_eq!(r.body, "hello"); assert!(r .attrs .contains(&("systemd.unit", "sshd.service".to_string()))); let mut e = entry(None); e.unit = None; let r = journal_record(&e, Some("x.service")); assert!(r.attrs.contains(&("systemd.unit", "x.service".to_string()))); } #[test] fn docker_streams() { let l = |s| LogLine { stream: s, ts: None, message: "m".into(), }; let e = docker_record("web", &l(LogStream::Stderr)); assert_eq!(e.severity, Severity::Warn); assert!(e.attrs.contains(&("container.name", "web".to_string()))); assert_eq!( docker_record("web", &l(LogStream::Stdout)).severity, Severity::Info ); } #[test] fn reconcile_starts_and_stops() { let s = |v: &[&str]| v.iter().map(|x| x.to_string()).collect::>(); let (start, stop) = reconcile(&s(&["a", "b"]), &s(&["b", "c"]), &[]); assert_eq!(start, s(&["a"])); assert_eq!(stop, s(&["c"])); let (start, stop) = reconcile(&s(&["web-1", "db"]), &[], &s(&["web-*"])); assert_eq!(start, s(&["web-1"])); assert!(stop.is_empty()); } }