WS-G: OpenTelemetry export (traces, metrics, logs)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
ariandClaude Sonnet 5.5 committed 2026-10-15 18:37:40 -04:00
1 parent 2d57fb1a78
commit 2aeb9ae154
6 files changed
+1175 -25

No files matched your search

+377 -2
View File
@@ -1,3 +1,378 @@
//! Journal and docker log fan-out to OTel logs (WS-G)
//! Journal and docker log fan-out to OTel logs (WS-G).
//!
//! Placeholder created in P0 so the owning workstream only edits its own files.
//! Journal: one `follow()` per configured unit, or one unfiltered follow when
//! `otel.logs.journal_units` is empty. Docker: a reconciler keeps one
//! `follow_logs` task per allowlisted running container.
//!
//! Deviation: `DockerSource` exposes no events stream, so the reconciler
//! polls `ps(false)` every few seconds instead of subscribing to `events`.
use std::collections::HashMap;
use std::time::{Duration, SystemTime};
use jiff::Timestamp;
use opentelemetry::logs::{AnyValue, LogRecord as _, Logger as _, LoggerProvider as _, Severity};
use opentelemetry_sdk::logs::{SdkLogger, SdkLoggerProvider};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use super::{any_glob, Backoff};
use crate::sources::{JournalEntry, JournalQuery, LogLine, LogOptions, LogStream, Priority};
use crate::state::AppState;
const BACKOFF_MIN: Duration = Duration::from_secs(1);
const BACKOFF_MAX: Duration = Duration::from_secs(60);
/// Retry interval while Docker is unavailable.
const DOCKER_SLOW_RETRY: Duration = Duration::from_secs(30);
const RECONCILE_INTERVAL: Duration = Duration::from_secs(5);
/// Backend-neutral log record, built by pure mapping functions.
#[derive(Debug, Clone, PartialEq)]
pub struct LogRec {
pub severity: Severity,
pub severity_text: &'static str,
pub body: String,
pub timestamp: Option<SystemTime>,
pub attrs: Vec<(&'static str, String)>,
}
/// Syslog priority to OTel severity.
pub fn journal_severity(p: Option<Priority>) -> (Severity, &'static str) {
match p {
Some(Priority::Emerg | Priority::Alert | Priority::Crit) => (Severity::Fatal, "FATAL"),
Some(Priority::Err) => (Severity::Error, "ERROR"),
Some(Priority::Warning) => (Severity::Warn, "WARN"),
Some(Priority::Notice | Priority::Info) | None => (Severity::Info, "INFO"),
Some(Priority::Debug) => (Severity::Debug, "DEBUG"),
}
}
pub fn journal_record(e: &JournalEntry, unit_hint: Option<&str>) -> LogRec {
let (severity, severity_text) = journal_severity(e.priority);
let mut attrs = Vec::new();
if let Some(u) = e.unit.as_deref().or(unit_hint) {
attrs.push(("systemd.unit", u.to_owned()));
}
if let Some(i) = &e.identifier {
attrs.push(("log.syslog.identifier", i.clone()));
}
if let Some(p) = e.pid {
attrs.push(("process.pid", p.to_string()));
}
LogRec {
severity,
severity_text,
body: e.message.clone(),
timestamp: Some(SystemTime::from(e.timestamp)),
attrs,
}
}
/// stderr maps to WARN, stdout to INFO.
pub fn docker_record(container: &str, l: &LogLine) -> LogRec {
let (severity, severity_text) = match l.stream {
LogStream::Stderr => (Severity::Warn, "WARN"),
LogStream::Stdout => (Severity::Info, "INFO"),
};
LogRec {
severity,
severity_text,
body: l.message.clone(),
timestamp: l.ts.map(SystemTime::from),
attrs: vec![
("container.name", container.to_owned()),
(
"log.iostream",
match l.stream {
LogStream::Stderr => "stderr",
LogStream::Stdout => "stdout",
}
.to_owned(),
),
],
}
}
fn emit(logger: &SdkLogger, r: LogRec) {
let mut rec = logger.create_log_record();
rec.set_severity_number(r.severity);
rec.set_severity_text(r.severity_text);
rec.set_body(AnyValue::String(r.body.into()));
if let Some(ts) = r.timestamp {
rec.set_timestamp(ts);
}
rec.set_observed_timestamp(SystemTime::now());
for (k, v) in r.attrs {
rec.add_attribute(k, AnyValue::String(v.into()));
}
logger.emit(rec);
}
/// Start the journal tasks and the docker reconciler.
pub fn spawn(state: AppState, provider: &SdkLoggerProvider) -> Vec<JoinHandle<()>> {
let logger = provider.logger("skyg-linux-tracker");
let mut handles = Vec::new();
let units = state.config.otel.logs.journal_units.clone();
let targets: Vec<Option<String>> = if units.is_empty() {
vec![None]
} else {
units.into_iter().map(Some).collect()
};
for unit in targets {
handles.push(tokio::spawn(journal_task(
state.clone(),
logger.clone(),
unit,
)));
}
handles.push(tokio::spawn(docker_reconciler(state, logger)));
handles
}
async fn journal_task(state: AppState, logger: SdkLogger, unit: Option<String>) {
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
let mut last: Option<Timestamp> = None;
let max_lines = state.config.limits.max_lines;
loop {
let q = JournalQuery {
unit: unit.clone(),
priority: None,
// No backlog on first start; catch up from the last seen entry afterwards.
lines: Some(if last.is_some() { max_lines } else { 0 }),
since: last,
};
let rx = tokio::select! {
_ = state.shutdown.cancelled() => return,
r = state.journal.follow(&q) => r,
};
match rx {
Ok(mut rx) => {
let mut got = false;
loop {
let e = tokio::select! {
_ = state.shutdown.cancelled() => return,
e = rx.recv() => e,
};
let Some(e) = e else { break };
if last.is_some_and(|l| e.timestamp <= l) {
continue;
}
got = true;
last = Some(e.timestamp);
emit(&logger, journal_record(&e, unit.as_deref()));
}
if got {
backoff.reset();
}
}
Err(e) => tracing::warn!(error = %e, unit = ?unit, "otel logs: journal follow failed"),
}
if !sleep_or_shutdown(&state.shutdown, backoff.next_delay()).await {
return;
}
}
}
/// Returns false when shutdown fired during the sleep.
async fn sleep_or_shutdown(shutdown: &CancellationToken, d: Duration) -> bool {
tokio::select! {
_ = shutdown.cancelled() => false,
_ = tokio::time::sleep(d) => true,
}
}
/// Pure reconcile step: which running containers to start and which tracked
/// ones to stop.
pub fn reconcile(
running: &[String],
tracked: &[String],
allow: &[String],
) -> (Vec<String>, Vec<String>) {
let wanted: Vec<&String> = running
.iter()
.filter(|n| allow.is_empty() || any_glob(allow, n))
.collect();
let start = wanted
.iter()
.filter(|n| !tracked.contains(n))
.map(|n| (*n).clone())
.collect();
let stop = tracked
.iter()
.filter(|t| !wanted.contains(t))
.cloned()
.collect();
(start, stop)
}
async fn docker_reconciler(state: AppState, logger: SdkLogger) {
let allow = state.config.otel.logs.docker_containers.clone();
let mut tasks: HashMap<String, (CancellationToken, JoinHandle<()>)> = HashMap::new();
let mut warned = false;
loop {
let mut delay = RECONCILE_INTERVAL;
let ps = tokio::select! {
_ = state.shutdown.cancelled() => break,
r = state.docker.ps(false) => r,
};
match ps {
Ok(cs) => {
warned = false;
let running: Vec<String> = cs.into_iter().map(|c| c.name).collect();
let tracked: Vec<String> = tasks.keys().cloned().collect();
let (start, stop) = reconcile(&running, &tracked, &allow);
for name in stop {
if let Some((tok, h)) = tasks.remove(&name) {
tok.cancel();
h.abort();
}
}
for name in start {
let tok = state.shutdown.child_token();
let h = tokio::spawn(container_task(
state.clone(),
logger.clone(),
name.clone(),
tok.clone(),
));
tasks.insert(name, (tok, h));
}
}
Err(e) => {
if !warned {
tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly");
warned = true;
}
delay = DOCKER_SLOW_RETRY;
}
}
if !sleep_or_shutdown(&state.shutdown, delay).await {
break;
}
}
for (_, (tok, h)) in tasks {
tok.cancel();
h.abort();
}
}
async fn container_task(
state: AppState,
logger: SdkLogger,
name: String,
token: CancellationToken,
) {
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
let mut since: Option<Timestamp> = Some(Timestamp::now());
loop {
let opts = LogOptions {
tail: Some(0),
since,
};
let rx = tokio::select! {
_ = token.cancelled() => return,
r = state.docker.follow_logs(&name, &opts) => r,
};
match rx {
Ok(mut rx) => {
let mut got = false;
loop {
let l = tokio::select! {
_ = token.cancelled() => return,
l = rx.recv() => l,
};
let Some(l) = l else { break };
got = true;
if let Some(ts) = l.ts {
since = Some(ts);
}
emit(&logger, docker_record(&name, &l));
}
if got {
backoff.reset();
}
}
Err(e) => {
tracing::debug!(error = %e, container = %name, "otel logs: container follow failed");
}
}
if !sleep_or_shutdown(&token, backoff.next_delay()).await {
return;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn entry(p: Option<Priority>) -> JournalEntry {
JournalEntry {
timestamp: "2026-10-15T12:00:00Z".parse().unwrap(),
priority: p,
unit: Some("sshd.service".into()),
identifier: Some("sshd".into()),
pid: Some(7),
hostname: None,
message: "hello".into(),
cursor: None,
}
}
#[test]
fn severity_mapping() {
use Priority::*;
let sev = |p| journal_severity(Some(p)).0;
assert_eq!(sev(Emerg), Severity::Fatal);
assert_eq!(sev(Alert), Severity::Fatal);
assert_eq!(sev(Crit), Severity::Fatal);
assert_eq!(sev(Err), Severity::Error);
assert_eq!(sev(Warning), Severity::Warn);
assert_eq!(sev(Notice), Severity::Info);
assert_eq!(sev(Info), Severity::Info);
assert_eq!(sev(Debug), Severity::Debug);
assert_eq!(journal_severity(None).0, Severity::Info);
}
#[test]
fn journal_attrs() {
let r = journal_record(&entry(Some(Priority::Err)), None);
assert_eq!(r.severity_text, "ERROR");
assert_eq!(r.body, "hello");
assert!(r
.attrs
.contains(&("systemd.unit", "sshd.service".to_string())));
let mut e = entry(None);
e.unit = None;
let r = journal_record(&e, Some("x.service"));
assert!(r.attrs.contains(&("systemd.unit", "x.service".to_string())));
}
#[test]
fn docker_streams() {
let l = |s| LogLine {
stream: s,
ts: None,
message: "m".into(),
};
let e = docker_record("web", &l(LogStream::Stderr));
assert_eq!(e.severity, Severity::Warn);
assert!(e.attrs.contains(&("container.name", "web".to_string())));
assert_eq!(
docker_record("web", &l(LogStream::Stdout)).severity,
Severity::Info
);
}
#[test]
fn reconcile_starts_and_stops() {
let s = |v: &[&str]| v.iter().map(|x| x.to_string()).collect::<Vec<_>>();
let (start, stop) = reconcile(&s(&["a", "b"]), &s(&["b", "c"]), &[]);
assert_eq!(start, s(&["a"]));
assert_eq!(stop, s(&["c"]));
let (start, stop) = reconcile(&s(&["web-1", "db"]), &[], &s(&["web-*"]));
assert_eq!(start, s(&["web-1"]));
assert!(stop.is_empty());
}
}