Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
455 lines
15 KiB
Rust
455 lines
15 KiB
Rust
//! Journal and docker log fan-out to OTel logs.
|
|
//!
|
|
//! Journal: one `follow()` per configured unit, or one unfiltered follow when
|
|
//! `otel.logs.journal_units` is empty. Docker: a reconciler keeps one
|
|
//! `follow_logs` task per allowlisted running container.
|
|
//!
|
|
//! Deviation: `DockerSource` exposes no events stream, so the reconciler
|
|
//! polls `ps(false)` every few seconds instead of subscribing to `events`.
|
|
|
|
use std::collections::HashMap;
|
|
use std::time::{Duration, SystemTime};
|
|
|
|
use jiff::Timestamp;
|
|
use opentelemetry::logs::{AnyValue, LogRecord as _, Logger as _, LoggerProvider as _, Severity};
|
|
use opentelemetry_sdk::logs::{SdkLogger, SdkLoggerProvider};
|
|
use tokio::task::JoinHandle;
|
|
use tokio_util::sync::CancellationToken;
|
|
|
|
use super::{any_glob, Backoff};
|
|
use crate::sources::{JournalEntry, JournalQuery, LogLine, LogOptions, LogStream, Priority};
|
|
use crate::state::AppState;
|
|
|
|
const BACKOFF_MIN: Duration = Duration::from_secs(1);
|
|
const BACKOFF_MAX: Duration = Duration::from_secs(60);
|
|
/// Retry interval while Docker is unavailable.
|
|
const DOCKER_SLOW_RETRY: Duration = Duration::from_secs(30);
|
|
const RECONCILE_INTERVAL: Duration = Duration::from_secs(5);
|
|
|
|
/// Backend-neutral log record, built by pure mapping functions.
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub struct LogRec {
|
|
pub severity: Severity,
|
|
pub severity_text: &'static str,
|
|
pub body: String,
|
|
pub timestamp: Option<SystemTime>,
|
|
pub attrs: Vec<(&'static str, String)>,
|
|
}
|
|
|
|
/// Syslog priority to OTel severity.
|
|
pub fn journal_severity(p: Option<Priority>) -> (Severity, &'static str) {
|
|
match p {
|
|
Some(Priority::Emerg | Priority::Alert | Priority::Crit) => (Severity::Fatal, "FATAL"),
|
|
Some(Priority::Err) => (Severity::Error, "ERROR"),
|
|
Some(Priority::Warning) => (Severity::Warn, "WARN"),
|
|
Some(Priority::Notice | Priority::Info) | None => (Severity::Info, "INFO"),
|
|
Some(Priority::Debug) => (Severity::Debug, "DEBUG"),
|
|
}
|
|
}
|
|
|
|
pub fn journal_record(e: &JournalEntry, unit_hint: Option<&str>) -> LogRec {
|
|
let (severity, severity_text) = journal_severity(e.priority);
|
|
let mut attrs = Vec::new();
|
|
if let Some(u) = e.unit.as_deref().or(unit_hint) {
|
|
attrs.push(("systemd.unit", u.to_owned()));
|
|
}
|
|
if let Some(i) = &e.identifier {
|
|
attrs.push(("log.syslog.identifier", i.clone()));
|
|
}
|
|
if let Some(p) = e.pid {
|
|
attrs.push(("process.pid", p.to_string()));
|
|
}
|
|
LogRec {
|
|
severity,
|
|
severity_text,
|
|
body: e.message.clone(),
|
|
timestamp: Some(SystemTime::from(e.timestamp)),
|
|
attrs,
|
|
}
|
|
}
|
|
|
|
/// stderr maps to WARN, stdout to INFO.
|
|
pub fn docker_record(container: &str, l: &LogLine) -> LogRec {
|
|
let (severity, severity_text) = match l.stream {
|
|
LogStream::Stderr => (Severity::Warn, "WARN"),
|
|
LogStream::Stdout => (Severity::Info, "INFO"),
|
|
};
|
|
LogRec {
|
|
severity,
|
|
severity_text,
|
|
body: l.message.clone(),
|
|
timestamp: l.ts.map(SystemTime::from),
|
|
attrs: vec![
|
|
("container.name", container.to_owned()),
|
|
(
|
|
"log.iostream",
|
|
match l.stream {
|
|
LogStream::Stderr => "stderr",
|
|
LogStream::Stdout => "stdout",
|
|
}
|
|
.to_owned(),
|
|
),
|
|
],
|
|
}
|
|
}
|
|
|
|
fn emit(logger: &SdkLogger, r: LogRec) {
|
|
let mut rec = logger.create_log_record();
|
|
rec.set_severity_number(r.severity);
|
|
rec.set_severity_text(r.severity_text);
|
|
rec.set_body(AnyValue::String(r.body.into()));
|
|
if let Some(ts) = r.timestamp {
|
|
rec.set_timestamp(ts);
|
|
}
|
|
rec.set_observed_timestamp(SystemTime::now());
|
|
for (k, v) in r.attrs {
|
|
rec.add_attribute(k, AnyValue::String(v.into()));
|
|
}
|
|
logger.emit(rec);
|
|
}
|
|
|
|
pub fn spawn(state: AppState, provider: &SdkLoggerProvider) -> Vec<JoinHandle<()>> {
|
|
let logger = provider.logger("skyg-linux-tracker");
|
|
let mut handles = Vec::new();
|
|
|
|
let units = state.config.otel.logs.journal_units.clone();
|
|
let targets: Vec<Option<String>> = if units.is_empty() {
|
|
vec![None]
|
|
} else {
|
|
units.into_iter().map(Some).collect()
|
|
};
|
|
for unit in targets {
|
|
handles.push(tokio::spawn(journal_task(
|
|
state.clone(),
|
|
logger.clone(),
|
|
unit,
|
|
)));
|
|
}
|
|
handles.push(tokio::spawn(docker_reconciler(state, logger)));
|
|
handles
|
|
}
|
|
|
|
/// Names identifying this process in the journal (unit and syslog identifier).
|
|
/// Used to keep the app's own log lines out of the unfiltered follow, which
|
|
/// would otherwise feed back into itself. Limitation: matching is by name; the
|
|
/// journal cursor is not used.
|
|
pub fn own_names(service_name: &str) -> Vec<String> {
|
|
let mut v = vec![service_name.to_owned(), "skyg-linux-tracker".to_owned()];
|
|
if let Some(exe) = std::env::current_exe()
|
|
.ok()
|
|
.and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned()))
|
|
{
|
|
v.push(exe);
|
|
}
|
|
v
|
|
}
|
|
|
|
pub fn is_own_entry(e: &JournalEntry, own: &[String]) -> bool {
|
|
e.identifier.as_ref().is_some_and(|i| own.contains(i))
|
|
|| e.unit.as_ref().is_some_and(|u| {
|
|
own.iter()
|
|
.any(|n| u == n || u.strip_suffix(".service") == Some(n.as_str()))
|
|
})
|
|
}
|
|
|
|
async fn journal_task(state: AppState, logger: SdkLogger, unit: Option<String>) {
|
|
let own = own_names(&state.config.otel.service_name);
|
|
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
|
|
let mut last: Option<Timestamp> = None;
|
|
let max_lines = state.config.limits.max_lines;
|
|
loop {
|
|
let q = JournalQuery {
|
|
unit: unit.clone(),
|
|
priority: None,
|
|
// No backlog on first start; catch up from the last seen entry afterwards.
|
|
lines: Some(if last.is_some() { max_lines } else { 0 }),
|
|
since: last,
|
|
};
|
|
let rx = tokio::select! {
|
|
_ = state.shutdown.cancelled() => return,
|
|
r = state.journal.follow(&q) => r,
|
|
};
|
|
match rx {
|
|
Ok(mut rx) => {
|
|
let mut got = false;
|
|
loop {
|
|
let e = tokio::select! {
|
|
_ = state.shutdown.cancelled() => return,
|
|
e = rx.recv() => e,
|
|
};
|
|
let Some(e) = e else { break };
|
|
if is_own_entry(&e, &own) {
|
|
continue;
|
|
}
|
|
if last.is_some_and(|l| e.timestamp <= l) {
|
|
continue;
|
|
}
|
|
got = true;
|
|
last = Some(e.timestamp);
|
|
emit(&logger, journal_record(&e, unit.as_deref()));
|
|
}
|
|
if got {
|
|
backoff.reset();
|
|
}
|
|
}
|
|
Err(e) => tracing::warn!(error = %e, unit = ?unit, "otel logs: journal follow failed"),
|
|
}
|
|
if !sleep_or_shutdown(&state.shutdown, backoff.next_delay()).await {
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Returns false when shutdown fired during the sleep.
|
|
async fn sleep_or_shutdown(shutdown: &CancellationToken, d: Duration) -> bool {
|
|
tokio::select! {
|
|
_ = shutdown.cancelled() => false,
|
|
_ = tokio::time::sleep(d) => true,
|
|
}
|
|
}
|
|
|
|
/// Pure reconcile step: which running containers to start and which tracked
|
|
/// ones to stop.
|
|
pub fn reconcile(
|
|
running: &[String],
|
|
tracked: &[String],
|
|
allow: &[String],
|
|
) -> (Vec<String>, Vec<String>) {
|
|
let wanted: Vec<&String> = running
|
|
.iter()
|
|
.filter(|n| allow.is_empty() || any_glob(allow, n))
|
|
.collect();
|
|
let start = wanted
|
|
.iter()
|
|
.filter(|n| !tracked.contains(n))
|
|
.map(|n| (*n).clone())
|
|
.collect();
|
|
let stop = tracked
|
|
.iter()
|
|
.filter(|t| !wanted.contains(t))
|
|
.cloned()
|
|
.collect();
|
|
(start, stop)
|
|
}
|
|
|
|
async fn docker_reconciler(state: AppState, logger: SdkLogger) {
|
|
let allow = state.config.otel.logs.docker_containers.clone();
|
|
let mut tasks: HashMap<String, (CancellationToken, JoinHandle<()>)> = HashMap::new();
|
|
let mut warned = false;
|
|
let mut failures = 0u32;
|
|
let mut first_pass = true;
|
|
loop {
|
|
let mut delay = RECONCILE_INTERVAL;
|
|
let ps = tokio::select! {
|
|
_ = state.shutdown.cancelled() => break,
|
|
r = state.docker.ps(false) => r,
|
|
};
|
|
match ps {
|
|
Ok(cs) => {
|
|
warned = false;
|
|
failures = 0;
|
|
let started: HashMap<String, Option<Timestamp>> =
|
|
cs.iter().map(|c| (c.name.clone(), c.started_at)).collect();
|
|
let running: Vec<String> = cs.into_iter().map(|c| c.name).collect();
|
|
let tracked: Vec<String> = tasks.keys().cloned().collect();
|
|
let (start, stop) = reconcile(&running, &tracked, &allow);
|
|
for name in stop {
|
|
if let Some((tok, h)) = tasks.remove(&name) {
|
|
tok.cancel();
|
|
h.abort();
|
|
}
|
|
}
|
|
for name in start {
|
|
let tok = state.shutdown.child_token();
|
|
// Containers present at startup: only new lines. Containers that
|
|
// appear later: from their StartedAt so startup output is kept.
|
|
let from = if first_pass {
|
|
None
|
|
} else {
|
|
started.get(&name).copied().flatten()
|
|
};
|
|
let h = tokio::spawn(container_task(
|
|
state.clone(),
|
|
logger.clone(),
|
|
name.clone(),
|
|
from,
|
|
tok.clone(),
|
|
));
|
|
tasks.insert(name, (tok, h));
|
|
}
|
|
first_pass = false;
|
|
}
|
|
Err(e) => {
|
|
failures += 1;
|
|
// Transient errors keep the normal cadence; only a persistent
|
|
// outage switches to the slow retry.
|
|
if failures >= 3 {
|
|
if !warned {
|
|
tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly");
|
|
warned = true;
|
|
}
|
|
delay = DOCKER_SLOW_RETRY;
|
|
}
|
|
}
|
|
}
|
|
if !sleep_or_shutdown(&state.shutdown, delay).await {
|
|
break;
|
|
}
|
|
}
|
|
for (_, (tok, h)) in tasks {
|
|
tok.cancel();
|
|
h.abort();
|
|
}
|
|
}
|
|
|
|
async fn container_task(
|
|
state: AppState,
|
|
logger: SdkLogger,
|
|
name: String,
|
|
from: Option<Timestamp>,
|
|
token: CancellationToken,
|
|
) {
|
|
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
|
|
// `from == None`: container was already running at startup, so only new lines.
|
|
// docker `tail=0` returns nothing regardless of `since`, so a backlog needs
|
|
// a real tail.
|
|
let mut since: Option<Timestamp> = Some(from.unwrap_or_else(Timestamp::now));
|
|
let mut tail = if from.is_some() {
|
|
state.config.limits.max_tail
|
|
} else {
|
|
0
|
|
};
|
|
let mut last_emitted: Option<Timestamp> = None;
|
|
loop {
|
|
let opts = LogOptions {
|
|
tail: Some(tail),
|
|
since,
|
|
};
|
|
let rx = tokio::select! {
|
|
_ = token.cancelled() => return,
|
|
r = state.docker.follow_logs(&name, &opts) => r,
|
|
};
|
|
match rx {
|
|
Ok(mut rx) => {
|
|
let mut got = false;
|
|
loop {
|
|
let l = tokio::select! {
|
|
_ = token.cancelled() => return,
|
|
l = rx.recv() => l,
|
|
};
|
|
let Some(l) = l else { break };
|
|
got = true;
|
|
if let Some(ts) = l.ts {
|
|
// Dedupe lines replayed after a reconnect.
|
|
if last_emitted.is_some_and(|le| ts <= le) {
|
|
continue;
|
|
}
|
|
last_emitted = Some(ts);
|
|
since = Some(ts);
|
|
}
|
|
emit(&logger, docker_record(&name, &l));
|
|
}
|
|
if got {
|
|
backoff.reset();
|
|
}
|
|
}
|
|
Err(e) => {
|
|
tracing::debug!(error = %e, container = %name, "otel logs: container follow failed");
|
|
}
|
|
}
|
|
// After the first connection, reconnects replay from the last seen line.
|
|
if last_emitted.is_some() {
|
|
tail = state.config.limits.max_tail;
|
|
}
|
|
if !sleep_or_shutdown(&token, backoff.next_delay()).await {
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn entry(p: Option<Priority>) -> JournalEntry {
|
|
JournalEntry {
|
|
timestamp: "2026-10-15T12:00:00Z".parse().unwrap(),
|
|
priority: p,
|
|
unit: Some("sshd.service".into()),
|
|
identifier: Some("sshd".into()),
|
|
pid: Some(7),
|
|
hostname: None,
|
|
message: "hello".into(),
|
|
cursor: None,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn own_entries_are_excluded() {
|
|
let own = vec!["skyg-linux-tracker".to_string()];
|
|
let mut e = entry(None);
|
|
assert!(!is_own_entry(&e, &own));
|
|
e.identifier = Some("skyg-linux-tracker".into());
|
|
assert!(is_own_entry(&e, &own));
|
|
e.identifier = Some("slinuxm".into());
|
|
e.unit = Some("skyg-linux-tracker.service".into());
|
|
assert!(is_own_entry(&e, &own));
|
|
}
|
|
|
|
#[test]
|
|
fn severity_mapping() {
|
|
use Priority::*;
|
|
let sev = |p| journal_severity(Some(p)).0;
|
|
assert_eq!(sev(Emerg), Severity::Fatal);
|
|
assert_eq!(sev(Alert), Severity::Fatal);
|
|
assert_eq!(sev(Crit), Severity::Fatal);
|
|
assert_eq!(sev(Err), Severity::Error);
|
|
assert_eq!(sev(Warning), Severity::Warn);
|
|
assert_eq!(sev(Notice), Severity::Info);
|
|
assert_eq!(sev(Info), Severity::Info);
|
|
assert_eq!(sev(Debug), Severity::Debug);
|
|
assert_eq!(journal_severity(None).0, Severity::Info);
|
|
}
|
|
|
|
#[test]
|
|
fn journal_attrs() {
|
|
let r = journal_record(&entry(Some(Priority::Err)), None);
|
|
assert_eq!(r.severity_text, "ERROR");
|
|
assert_eq!(r.body, "hello");
|
|
assert!(r
|
|
.attrs
|
|
.contains(&("systemd.unit", "sshd.service".to_string())));
|
|
let mut e = entry(None);
|
|
e.unit = None;
|
|
let r = journal_record(&e, Some("x.service"));
|
|
assert!(r.attrs.contains(&("systemd.unit", "x.service".to_string())));
|
|
}
|
|
|
|
#[test]
|
|
fn docker_streams() {
|
|
let l = |s| LogLine {
|
|
stream: s,
|
|
ts: None,
|
|
message: "m".into(),
|
|
};
|
|
let e = docker_record("web", &l(LogStream::Stderr));
|
|
assert_eq!(e.severity, Severity::Warn);
|
|
assert!(e.attrs.contains(&("container.name", "web".to_string())));
|
|
assert_eq!(
|
|
docker_record("web", &l(LogStream::Stdout)).severity,
|
|
Severity::Info
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn reconcile_starts_and_stops() {
|
|
let s = |v: &[&str]| v.iter().map(|x| x.to_string()).collect::<Vec<_>>();
|
|
let (start, stop) = reconcile(&s(&["a", "b"]), &s(&["b", "c"]), &[]);
|
|
assert_eq!(start, s(&["a"]));
|
|
assert_eq!(stop, s(&["c"]));
|
|
let (start, stop) = reconcile(&s(&["web-1", "db"]), &[], &s(&["web-*"]));
|
|
assert_eq!(start, s(&["web-1"]));
|
|
assert!(stop.is_empty());
|
|
}
|
|
}
|