WS-G: OpenTelemetry export (traces, metrics, logs)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
1 parent
2d57fb1a78
commit
2aeb9ae154
6 files changed
+1175
-25
No files matched your search
@@ -1,3 +1,378 @@
|
||||
//! Journal and docker log fan-out to OTel logs (WS-G)
|
||||
//! Journal and docker log fan-out to OTel logs (WS-G).
|
||||
//!
|
||||
//! Placeholder created in P0 so the owning workstream only edits its own files.
|
||||
//! Journal: one `follow()` per configured unit, or one unfiltered follow when
|
||||
//! `otel.logs.journal_units` is empty. Docker: a reconciler keeps one
|
||||
//! `follow_logs` task per allowlisted running container.
|
||||
//!
|
||||
//! Deviation: `DockerSource` exposes no events stream, so the reconciler
|
||||
//! polls `ps(false)` every few seconds instead of subscribing to `events`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use jiff::Timestamp;
|
||||
use opentelemetry::logs::{AnyValue, LogRecord as _, Logger as _, LoggerProvider as _, Severity};
|
||||
use opentelemetry_sdk::logs::{SdkLogger, SdkLoggerProvider};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::{any_glob, Backoff};
|
||||
use crate::sources::{JournalEntry, JournalQuery, LogLine, LogOptions, LogStream, Priority};
|
||||
use crate::state::AppState;
|
||||
|
||||
const BACKOFF_MIN: Duration = Duration::from_secs(1);
|
||||
const BACKOFF_MAX: Duration = Duration::from_secs(60);
|
||||
/// Retry interval while Docker is unavailable.
|
||||
const DOCKER_SLOW_RETRY: Duration = Duration::from_secs(30);
|
||||
const RECONCILE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Backend-neutral log record, built by pure mapping functions.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct LogRec {
|
||||
pub severity: Severity,
|
||||
pub severity_text: &'static str,
|
||||
pub body: String,
|
||||
pub timestamp: Option<SystemTime>,
|
||||
pub attrs: Vec<(&'static str, String)>,
|
||||
}
|
||||
|
||||
/// Syslog priority to OTel severity.
|
||||
pub fn journal_severity(p: Option<Priority>) -> (Severity, &'static str) {
|
||||
match p {
|
||||
Some(Priority::Emerg | Priority::Alert | Priority::Crit) => (Severity::Fatal, "FATAL"),
|
||||
Some(Priority::Err) => (Severity::Error, "ERROR"),
|
||||
Some(Priority::Warning) => (Severity::Warn, "WARN"),
|
||||
Some(Priority::Notice | Priority::Info) | None => (Severity::Info, "INFO"),
|
||||
Some(Priority::Debug) => (Severity::Debug, "DEBUG"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn journal_record(e: &JournalEntry, unit_hint: Option<&str>) -> LogRec {
|
||||
let (severity, severity_text) = journal_severity(e.priority);
|
||||
let mut attrs = Vec::new();
|
||||
if let Some(u) = e.unit.as_deref().or(unit_hint) {
|
||||
attrs.push(("systemd.unit", u.to_owned()));
|
||||
}
|
||||
if let Some(i) = &e.identifier {
|
||||
attrs.push(("log.syslog.identifier", i.clone()));
|
||||
}
|
||||
if let Some(p) = e.pid {
|
||||
attrs.push(("process.pid", p.to_string()));
|
||||
}
|
||||
LogRec {
|
||||
severity,
|
||||
severity_text,
|
||||
body: e.message.clone(),
|
||||
timestamp: Some(SystemTime::from(e.timestamp)),
|
||||
attrs,
|
||||
}
|
||||
}
|
||||
|
||||
/// stderr maps to WARN, stdout to INFO.
|
||||
pub fn docker_record(container: &str, l: &LogLine) -> LogRec {
|
||||
let (severity, severity_text) = match l.stream {
|
||||
LogStream::Stderr => (Severity::Warn, "WARN"),
|
||||
LogStream::Stdout => (Severity::Info, "INFO"),
|
||||
};
|
||||
LogRec {
|
||||
severity,
|
||||
severity_text,
|
||||
body: l.message.clone(),
|
||||
timestamp: l.ts.map(SystemTime::from),
|
||||
attrs: vec![
|
||||
("container.name", container.to_owned()),
|
||||
(
|
||||
"log.iostream",
|
||||
match l.stream {
|
||||
LogStream::Stderr => "stderr",
|
||||
LogStream::Stdout => "stdout",
|
||||
}
|
||||
.to_owned(),
|
||||
),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn emit(logger: &SdkLogger, r: LogRec) {
|
||||
let mut rec = logger.create_log_record();
|
||||
rec.set_severity_number(r.severity);
|
||||
rec.set_severity_text(r.severity_text);
|
||||
rec.set_body(AnyValue::String(r.body.into()));
|
||||
if let Some(ts) = r.timestamp {
|
||||
rec.set_timestamp(ts);
|
||||
}
|
||||
rec.set_observed_timestamp(SystemTime::now());
|
||||
for (k, v) in r.attrs {
|
||||
rec.add_attribute(k, AnyValue::String(v.into()));
|
||||
}
|
||||
logger.emit(rec);
|
||||
}
|
||||
|
||||
/// Start the journal tasks and the docker reconciler.
|
||||
pub fn spawn(state: AppState, provider: &SdkLoggerProvider) -> Vec<JoinHandle<()>> {
|
||||
let logger = provider.logger("skyg-linux-tracker");
|
||||
let mut handles = Vec::new();
|
||||
|
||||
let units = state.config.otel.logs.journal_units.clone();
|
||||
let targets: Vec<Option<String>> = if units.is_empty() {
|
||||
vec![None]
|
||||
} else {
|
||||
units.into_iter().map(Some).collect()
|
||||
};
|
||||
for unit in targets {
|
||||
handles.push(tokio::spawn(journal_task(
|
||||
state.clone(),
|
||||
logger.clone(),
|
||||
unit,
|
||||
)));
|
||||
}
|
||||
handles.push(tokio::spawn(docker_reconciler(state, logger)));
|
||||
handles
|
||||
}
|
||||
|
||||
async fn journal_task(state: AppState, logger: SdkLogger, unit: Option<String>) {
|
||||
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
|
||||
let mut last: Option<Timestamp> = None;
|
||||
let max_lines = state.config.limits.max_lines;
|
||||
loop {
|
||||
let q = JournalQuery {
|
||||
unit: unit.clone(),
|
||||
priority: None,
|
||||
// No backlog on first start; catch up from the last seen entry afterwards.
|
||||
lines: Some(if last.is_some() { max_lines } else { 0 }),
|
||||
since: last,
|
||||
};
|
||||
let rx = tokio::select! {
|
||||
_ = state.shutdown.cancelled() => return,
|
||||
r = state.journal.follow(&q) => r,
|
||||
};
|
||||
match rx {
|
||||
Ok(mut rx) => {
|
||||
let mut got = false;
|
||||
loop {
|
||||
let e = tokio::select! {
|
||||
_ = state.shutdown.cancelled() => return,
|
||||
e = rx.recv() => e,
|
||||
};
|
||||
let Some(e) = e else { break };
|
||||
if last.is_some_and(|l| e.timestamp <= l) {
|
||||
continue;
|
||||
}
|
||||
got = true;
|
||||
last = Some(e.timestamp);
|
||||
emit(&logger, journal_record(&e, unit.as_deref()));
|
||||
}
|
||||
if got {
|
||||
backoff.reset();
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!(error = %e, unit = ?unit, "otel logs: journal follow failed"),
|
||||
}
|
||||
if !sleep_or_shutdown(&state.shutdown, backoff.next_delay()).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns false when shutdown fired during the sleep.
|
||||
async fn sleep_or_shutdown(shutdown: &CancellationToken, d: Duration) -> bool {
|
||||
tokio::select! {
|
||||
_ = shutdown.cancelled() => false,
|
||||
_ = tokio::time::sleep(d) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure reconcile step: which running containers to start and which tracked
|
||||
/// ones to stop.
|
||||
pub fn reconcile(
|
||||
running: &[String],
|
||||
tracked: &[String],
|
||||
allow: &[String],
|
||||
) -> (Vec<String>, Vec<String>) {
|
||||
let wanted: Vec<&String> = running
|
||||
.iter()
|
||||
.filter(|n| allow.is_empty() || any_glob(allow, n))
|
||||
.collect();
|
||||
let start = wanted
|
||||
.iter()
|
||||
.filter(|n| !tracked.contains(n))
|
||||
.map(|n| (*n).clone())
|
||||
.collect();
|
||||
let stop = tracked
|
||||
.iter()
|
||||
.filter(|t| !wanted.contains(t))
|
||||
.cloned()
|
||||
.collect();
|
||||
(start, stop)
|
||||
}
|
||||
|
||||
async fn docker_reconciler(state: AppState, logger: SdkLogger) {
|
||||
let allow = state.config.otel.logs.docker_containers.clone();
|
||||
let mut tasks: HashMap<String, (CancellationToken, JoinHandle<()>)> = HashMap::new();
|
||||
let mut warned = false;
|
||||
loop {
|
||||
let mut delay = RECONCILE_INTERVAL;
|
||||
let ps = tokio::select! {
|
||||
_ = state.shutdown.cancelled() => break,
|
||||
r = state.docker.ps(false) => r,
|
||||
};
|
||||
match ps {
|
||||
Ok(cs) => {
|
||||
warned = false;
|
||||
let running: Vec<String> = cs.into_iter().map(|c| c.name).collect();
|
||||
let tracked: Vec<String> = tasks.keys().cloned().collect();
|
||||
let (start, stop) = reconcile(&running, &tracked, &allow);
|
||||
for name in stop {
|
||||
if let Some((tok, h)) = tasks.remove(&name) {
|
||||
tok.cancel();
|
||||
h.abort();
|
||||
}
|
||||
}
|
||||
for name in start {
|
||||
let tok = state.shutdown.child_token();
|
||||
let h = tokio::spawn(container_task(
|
||||
state.clone(),
|
||||
logger.clone(),
|
||||
name.clone(),
|
||||
tok.clone(),
|
||||
));
|
||||
tasks.insert(name, (tok, h));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if !warned {
|
||||
tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly");
|
||||
warned = true;
|
||||
}
|
||||
delay = DOCKER_SLOW_RETRY;
|
||||
}
|
||||
}
|
||||
if !sleep_or_shutdown(&state.shutdown, delay).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
for (_, (tok, h)) in tasks {
|
||||
tok.cancel();
|
||||
h.abort();
|
||||
}
|
||||
}
|
||||
|
||||
async fn container_task(
|
||||
state: AppState,
|
||||
logger: SdkLogger,
|
||||
name: String,
|
||||
token: CancellationToken,
|
||||
) {
|
||||
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
|
||||
let mut since: Option<Timestamp> = Some(Timestamp::now());
|
||||
loop {
|
||||
let opts = LogOptions {
|
||||
tail: Some(0),
|
||||
since,
|
||||
};
|
||||
let rx = tokio::select! {
|
||||
_ = token.cancelled() => return,
|
||||
r = state.docker.follow_logs(&name, &opts) => r,
|
||||
};
|
||||
match rx {
|
||||
Ok(mut rx) => {
|
||||
let mut got = false;
|
||||
loop {
|
||||
let l = tokio::select! {
|
||||
_ = token.cancelled() => return,
|
||||
l = rx.recv() => l,
|
||||
};
|
||||
let Some(l) = l else { break };
|
||||
got = true;
|
||||
if let Some(ts) = l.ts {
|
||||
since = Some(ts);
|
||||
}
|
||||
emit(&logger, docker_record(&name, &l));
|
||||
}
|
||||
if got {
|
||||
backoff.reset();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(error = %e, container = %name, "otel logs: container follow failed");
|
||||
}
|
||||
}
|
||||
if !sleep_or_shutdown(&token, backoff.next_delay()).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn entry(p: Option<Priority>) -> JournalEntry {
|
||||
JournalEntry {
|
||||
timestamp: "2026-10-15T12:00:00Z".parse().unwrap(),
|
||||
priority: p,
|
||||
unit: Some("sshd.service".into()),
|
||||
identifier: Some("sshd".into()),
|
||||
pid: Some(7),
|
||||
hostname: None,
|
||||
message: "hello".into(),
|
||||
cursor: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn severity_mapping() {
|
||||
use Priority::*;
|
||||
let sev = |p| journal_severity(Some(p)).0;
|
||||
assert_eq!(sev(Emerg), Severity::Fatal);
|
||||
assert_eq!(sev(Alert), Severity::Fatal);
|
||||
assert_eq!(sev(Crit), Severity::Fatal);
|
||||
assert_eq!(sev(Err), Severity::Error);
|
||||
assert_eq!(sev(Warning), Severity::Warn);
|
||||
assert_eq!(sev(Notice), Severity::Info);
|
||||
assert_eq!(sev(Info), Severity::Info);
|
||||
assert_eq!(sev(Debug), Severity::Debug);
|
||||
assert_eq!(journal_severity(None).0, Severity::Info);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn journal_attrs() {
|
||||
let r = journal_record(&entry(Some(Priority::Err)), None);
|
||||
assert_eq!(r.severity_text, "ERROR");
|
||||
assert_eq!(r.body, "hello");
|
||||
assert!(r
|
||||
.attrs
|
||||
.contains(&("systemd.unit", "sshd.service".to_string())));
|
||||
let mut e = entry(None);
|
||||
e.unit = None;
|
||||
let r = journal_record(&e, Some("x.service"));
|
||||
assert!(r.attrs.contains(&("systemd.unit", "x.service".to_string())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn docker_streams() {
|
||||
let l = |s| LogLine {
|
||||
stream: s,
|
||||
ts: None,
|
||||
message: "m".into(),
|
||||
};
|
||||
let e = docker_record("web", &l(LogStream::Stderr));
|
||||
assert_eq!(e.severity, Severity::Warn);
|
||||
assert!(e.attrs.contains(&("container.name", "web".to_string())));
|
||||
assert_eq!(
|
||||
docker_record("web", &l(LogStream::Stdout)).severity,
|
||||
Severity::Info
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconcile_starts_and_stops() {
|
||||
let s = |v: &[&str]| v.iter().map(|x| x.to_string()).collect::<Vec<_>>();
|
||||
let (start, stop) = reconcile(&s(&["a", "b"]), &s(&["b", "c"]), &[]);
|
||||
assert_eq!(start, s(&["a"]));
|
||||
assert_eq!(stop, s(&["c"]));
|
||||
let (start, stop) = reconcile(&s(&["web-1", "db"]), &[], &s(&["web-*"]));
|
||||
assert_eq!(start, s(&["web-1"]));
|
||||
assert!(stop.is_empty());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user