WS-G: OpenTelemetry export (traces, metrics, logs)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
ariandClaude Sonnet 5.5 committed 2026-10-15 18:37:40 -04:00
1 parent 2d57fb1a78
commit 2aeb9ae154
6 files changed
+1175 -25

No files matched your search

+377 -2
View File
@@ -1,3 +1,378 @@
//! Journal and docker log fan-out to OTel logs (WS-G)
//! Journal and docker log fan-out to OTel logs (WS-G).
//!
//! Placeholder created in P0 so the owning workstream only edits its own files.
//! Journal: one `follow()` per configured unit, or one unfiltered follow when
//! `otel.logs.journal_units` is empty. Docker: a reconciler keeps one
//! `follow_logs` task per allowlisted running container.
//!
//! Deviation: `DockerSource` exposes no events stream, so the reconciler
//! polls `ps(false)` every few seconds instead of subscribing to `events`.
use std::collections::HashMap;
use std::time::{Duration, SystemTime};
use jiff::Timestamp;
use opentelemetry::logs::{AnyValue, LogRecord as _, Logger as _, LoggerProvider as _, Severity};
use opentelemetry_sdk::logs::{SdkLogger, SdkLoggerProvider};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use super::{any_glob, Backoff};
use crate::sources::{JournalEntry, JournalQuery, LogLine, LogOptions, LogStream, Priority};
use crate::state::AppState;
const BACKOFF_MIN: Duration = Duration::from_secs(1);
const BACKOFF_MAX: Duration = Duration::from_secs(60);
/// Retry interval while Docker is unavailable.
const DOCKER_SLOW_RETRY: Duration = Duration::from_secs(30);
const RECONCILE_INTERVAL: Duration = Duration::from_secs(5);
/// Backend-neutral log record, built by pure mapping functions.
#[derive(Debug, Clone, PartialEq)]
pub struct LogRec {
pub severity: Severity,
pub severity_text: &'static str,
pub body: String,
pub timestamp: Option<SystemTime>,
pub attrs: Vec<(&'static str, String)>,
}
/// Syslog priority to OTel severity.
pub fn journal_severity(p: Option<Priority>) -> (Severity, &'static str) {
match p {
Some(Priority::Emerg | Priority::Alert | Priority::Crit) => (Severity::Fatal, "FATAL"),
Some(Priority::Err) => (Severity::Error, "ERROR"),
Some(Priority::Warning) => (Severity::Warn, "WARN"),
Some(Priority::Notice | Priority::Info) | None => (Severity::Info, "INFO"),
Some(Priority::Debug) => (Severity::Debug, "DEBUG"),
}
}
pub fn journal_record(e: &JournalEntry, unit_hint: Option<&str>) -> LogRec {
let (severity, severity_text) = journal_severity(e.priority);
let mut attrs = Vec::new();
if let Some(u) = e.unit.as_deref().or(unit_hint) {
attrs.push(("systemd.unit", u.to_owned()));
}
if let Some(i) = &e.identifier {
attrs.push(("log.syslog.identifier", i.clone()));
}
if let Some(p) = e.pid {
attrs.push(("process.pid", p.to_string()));
}
LogRec {
severity,
severity_text,
body: e.message.clone(),
timestamp: Some(SystemTime::from(e.timestamp)),
attrs,
}
}
/// stderr maps to WARN, stdout to INFO.
pub fn docker_record(container: &str, l: &LogLine) -> LogRec {
let (severity, severity_text) = match l.stream {
LogStream::Stderr => (Severity::Warn, "WARN"),
LogStream::Stdout => (Severity::Info, "INFO"),
};
LogRec {
severity,
severity_text,
body: l.message.clone(),
timestamp: l.ts.map(SystemTime::from),
attrs: vec![
("container.name", container.to_owned()),
(
"log.iostream",
match l.stream {
LogStream::Stderr => "stderr",
LogStream::Stdout => "stdout",
}
.to_owned(),
),
],
}
}
fn emit(logger: &SdkLogger, r: LogRec) {
let mut rec = logger.create_log_record();
rec.set_severity_number(r.severity);
rec.set_severity_text(r.severity_text);
rec.set_body(AnyValue::String(r.body.into()));
if let Some(ts) = r.timestamp {
rec.set_timestamp(ts);
}
rec.set_observed_timestamp(SystemTime::now());
for (k, v) in r.attrs {
rec.add_attribute(k, AnyValue::String(v.into()));
}
logger.emit(rec);
}
/// Start the journal tasks and the docker reconciler.
pub fn spawn(state: AppState, provider: &SdkLoggerProvider) -> Vec<JoinHandle<()>> {
let logger = provider.logger("skyg-linux-tracker");
let mut handles = Vec::new();
let units = state.config.otel.logs.journal_units.clone();
let targets: Vec<Option<String>> = if units.is_empty() {
vec![None]
} else {
units.into_iter().map(Some).collect()
};
for unit in targets {
handles.push(tokio::spawn(journal_task(
state.clone(),
logger.clone(),
unit,
)));
}
handles.push(tokio::spawn(docker_reconciler(state, logger)));
handles
}
async fn journal_task(state: AppState, logger: SdkLogger, unit: Option<String>) {
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
let mut last: Option<Timestamp> = None;
let max_lines = state.config.limits.max_lines;
loop {
let q = JournalQuery {
unit: unit.clone(),
priority: None,
// No backlog on first start; catch up from the last seen entry afterwards.
lines: Some(if last.is_some() { max_lines } else { 0 }),
since: last,
};
let rx = tokio::select! {
_ = state.shutdown.cancelled() => return,
r = state.journal.follow(&q) => r,
};
match rx {
Ok(mut rx) => {
let mut got = false;
loop {
let e = tokio::select! {
_ = state.shutdown.cancelled() => return,
e = rx.recv() => e,
};
let Some(e) = e else { break };
if last.is_some_and(|l| e.timestamp <= l) {
continue;
}
got = true;
last = Some(e.timestamp);
emit(&logger, journal_record(&e, unit.as_deref()));
}
if got {
backoff.reset();
}
}
Err(e) => tracing::warn!(error = %e, unit = ?unit, "otel logs: journal follow failed"),
}
if !sleep_or_shutdown(&state.shutdown, backoff.next_delay()).await {
return;
}
}
}
/// Returns false when shutdown fired during the sleep.
async fn sleep_or_shutdown(shutdown: &CancellationToken, d: Duration) -> bool {
tokio::select! {
_ = shutdown.cancelled() => false,
_ = tokio::time::sleep(d) => true,
}
}
/// Pure reconcile step: which running containers to start and which tracked
/// ones to stop.
pub fn reconcile(
running: &[String],
tracked: &[String],
allow: &[String],
) -> (Vec<String>, Vec<String>) {
let wanted: Vec<&String> = running
.iter()
.filter(|n| allow.is_empty() || any_glob(allow, n))
.collect();
let start = wanted
.iter()
.filter(|n| !tracked.contains(n))
.map(|n| (*n).clone())
.collect();
let stop = tracked
.iter()
.filter(|t| !wanted.contains(t))
.cloned()
.collect();
(start, stop)
}
async fn docker_reconciler(state: AppState, logger: SdkLogger) {
let allow = state.config.otel.logs.docker_containers.clone();
let mut tasks: HashMap<String, (CancellationToken, JoinHandle<()>)> = HashMap::new();
let mut warned = false;
loop {
let mut delay = RECONCILE_INTERVAL;
let ps = tokio::select! {
_ = state.shutdown.cancelled() => break,
r = state.docker.ps(false) => r,
};
match ps {
Ok(cs) => {
warned = false;
let running: Vec<String> = cs.into_iter().map(|c| c.name).collect();
let tracked: Vec<String> = tasks.keys().cloned().collect();
let (start, stop) = reconcile(&running, &tracked, &allow);
for name in stop {
if let Some((tok, h)) = tasks.remove(&name) {
tok.cancel();
h.abort();
}
}
for name in start {
let tok = state.shutdown.child_token();
let h = tokio::spawn(container_task(
state.clone(),
logger.clone(),
name.clone(),
tok.clone(),
));
tasks.insert(name, (tok, h));
}
}
Err(e) => {
if !warned {
tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly");
warned = true;
}
delay = DOCKER_SLOW_RETRY;
}
}
if !sleep_or_shutdown(&state.shutdown, delay).await {
break;
}
}
for (_, (tok, h)) in tasks {
tok.cancel();
h.abort();
}
}
async fn container_task(
state: AppState,
logger: SdkLogger,
name: String,
token: CancellationToken,
) {
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
let mut since: Option<Timestamp> = Some(Timestamp::now());
loop {
let opts = LogOptions {
tail: Some(0),
since,
};
let rx = tokio::select! {
_ = token.cancelled() => return,
r = state.docker.follow_logs(&name, &opts) => r,
};
match rx {
Ok(mut rx) => {
let mut got = false;
loop {
let l = tokio::select! {
_ = token.cancelled() => return,
l = rx.recv() => l,
};
let Some(l) = l else { break };
got = true;
if let Some(ts) = l.ts {
since = Some(ts);
}
emit(&logger, docker_record(&name, &l));
}
if got {
backoff.reset();
}
}
Err(e) => {
tracing::debug!(error = %e, container = %name, "otel logs: container follow failed");
}
}
if !sleep_or_shutdown(&token, backoff.next_delay()).await {
return;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn entry(p: Option<Priority>) -> JournalEntry {
JournalEntry {
timestamp: "2026-10-15T12:00:00Z".parse().unwrap(),
priority: p,
unit: Some("sshd.service".into()),
identifier: Some("sshd".into()),
pid: Some(7),
hostname: None,
message: "hello".into(),
cursor: None,
}
}
#[test]
fn severity_mapping() {
use Priority::*;
let sev = |p| journal_severity(Some(p)).0;
assert_eq!(sev(Emerg), Severity::Fatal);
assert_eq!(sev(Alert), Severity::Fatal);
assert_eq!(sev(Crit), Severity::Fatal);
assert_eq!(sev(Err), Severity::Error);
assert_eq!(sev(Warning), Severity::Warn);
assert_eq!(sev(Notice), Severity::Info);
assert_eq!(sev(Info), Severity::Info);
assert_eq!(sev(Debug), Severity::Debug);
assert_eq!(journal_severity(None).0, Severity::Info);
}
#[test]
fn journal_attrs() {
let r = journal_record(&entry(Some(Priority::Err)), None);
assert_eq!(r.severity_text, "ERROR");
assert_eq!(r.body, "hello");
assert!(r
.attrs
.contains(&("systemd.unit", "sshd.service".to_string())));
let mut e = entry(None);
e.unit = None;
let r = journal_record(&e, Some("x.service"));
assert!(r.attrs.contains(&("systemd.unit", "x.service".to_string())));
}
#[test]
fn docker_streams() {
let l = |s| LogLine {
stream: s,
ts: None,
message: "m".into(),
};
let e = docker_record("web", &l(LogStream::Stderr));
assert_eq!(e.severity, Severity::Warn);
assert!(e.attrs.contains(&("container.name", "web".to_string())));
assert_eq!(
docker_record("web", &l(LogStream::Stdout)).severity,
Severity::Info
);
}
#[test]
fn reconcile_starts_and_stops() {
let s = |v: &[&str]| v.iter().map(|x| x.to_string()).collect::<Vec<_>>();
let (start, stop) = reconcile(&s(&["a", "b"]), &s(&["b", "c"]), &[]);
assert_eq!(start, s(&["a"]));
assert_eq!(stop, s(&["c"]));
let (start, stop) = reconcile(&s(&["web-1", "db"]), &[], &s(&["web-*"]));
assert_eq!(start, s(&["web-1"]));
assert!(stop.is_empty());
}
}
+398 -2
View File
@@ -1,3 +1,399 @@
//! Metrics poll loop and observable gauges (WS-G)
//! Metrics poll loop and observable gauges (WS-G).
//!
//! Placeholder created in P0 so the owning workstream only edits its own files.
//! A background task polls the sources every `metrics_interval_secs` and
//! replaces a shared snapshot. Observable gauge callbacks read the latest
//! snapshot, so units or containers that vanish stop being reported.
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::time::Duration;
use opentelemetry::metrics::MeterProvider as _;
use opentelemetry::KeyValue;
use opentelemetry_sdk::metrics::SdkMeterProvider;
use tokio::task::JoinHandle;
use super::any_glob;
use crate::config::OtelMetricsConfig;
use crate::sources::{ContainerSummary, TimerInfo, UnitSummary};
use crate::state::AppState;
/// Per-call bound so a hung backend cannot stall the poll loop.
const CALL_TIMEOUT: Duration = Duration::from_secs(10);
pub const METRIC_NAMES: [&str; 8] = [
"systemd.unit.active",
"systemd.unit.failed",
"systemd.unit.restarts",
"systemd.timer.next_elapse_seconds",
"docker.container.running",
"docker.container.healthy",
"docker.container.uptime_seconds",
"docker.container.restarts",
];
/// Latest poll result, already filtered by the allowlists.
#[derive(Debug, Default, Clone)]
pub struct Snapshot {
pub units: Vec<UnitSummary>,
/// Automatic restart count per unit name (services only).
pub restarts: HashMap<String, u32>,
pub timers: Vec<TimerInfo>,
pub containers: Vec<ContainerSummary>,
}
/// One gauge data point.
#[derive(Debug, Clone, PartialEq)]
pub struct Obs {
pub metric: &'static str,
pub value: f64,
pub attrs: Vec<(&'static str, String)>,
}
/// DEC-7: empty list means `*.service` and `*.timer`; otherwise a glob allowlist.
pub fn unit_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool {
if cfg.units.is_empty() {
name.ends_with(".service") || name.ends_with(".timer")
} else {
any_glob(&cfg.units, name)
}
}
/// Empty list means all containers.
pub fn container_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool {
cfg.containers.is_empty() || any_glob(&cfg.containers, name)
}
fn unit_attrs(u: &UnitSummary) -> Vec<(&'static str, String)> {
vec![
("systemd.unit", u.name.clone()),
("systemd.sub_state", u.sub_state.clone()),
("systemd.load_state", u.load_state.clone()),
]
}
fn container_attrs(c: &ContainerSummary) -> Vec<(&'static str, String)> {
vec![
("container.name", c.name.clone()),
("container.id", c.id.clone()),
("container.image.name", c.image.clone()),
]
}
/// Map a snapshot to gauge observations.
pub fn observations(s: &Snapshot) -> Vec<Obs> {
let b = |v: bool| if v { 1.0 } else { 0.0 };
let mut out = Vec::new();
for u in &s.units {
out.push(Obs {
metric: "systemd.unit.active",
value: b(u.active_state == "active"),
attrs: unit_attrs(u),
});
out.push(Obs {
metric: "systemd.unit.failed",
value: b(u.active_state == "failed"),
attrs: unit_attrs(u),
});
if let Some(n) = s.restarts.get(&u.name) {
out.push(Obs {
metric: "systemd.unit.restarts",
value: f64::from(*n),
attrs: unit_attrs(u),
});
}
}
for t in &s.timers {
// Absolute unix time of the next elapse; unscheduled timers are omitted.
if let Some(next) = t.next_elapse {
out.push(Obs {
metric: "systemd.timer.next_elapse_seconds",
value: next.as_second() as f64,
attrs: vec![
("systemd.unit", t.name.clone()),
("systemd.sub_state", t.active_state.clone()),
],
});
}
}
for c in &s.containers {
out.push(Obs {
metric: "docker.container.running",
value: b(c.state == "running"),
attrs: container_attrs(c),
});
if let Some(h) = &c.health {
out.push(Obs {
metric: "docker.container.healthy",
value: b(h == "healthy"),
attrs: container_attrs(c),
});
}
if let Some(up) = c.uptime_seconds {
out.push(Obs {
metric: "docker.container.uptime_seconds",
value: up as f64,
attrs: container_attrs(c),
});
}
if let Some(r) = c.restart_count {
out.push(Obs {
metric: "docker.container.restarts",
value: r as f64,
attrs: container_attrs(c),
});
}
}
out
}
async fn collect(state: &AppState) -> Snapshot {
let cfg = &state.config.otel.metrics;
let mut snap = Snapshot::default();
match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_units(None)).await {
Ok(Ok(units)) => {
snap.units = units
.into_iter()
.filter(|u| unit_in_scope(cfg, &u.name))
.collect();
}
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_units failed"),
Err(_) => tracing::debug!("otel metrics: list_units timed out"),
}
match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_timers()).await {
Ok(Ok(timers)) => {
snap.timers = timers
.into_iter()
.filter(|t| unit_in_scope(cfg, &t.name))
.collect();
}
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_timers failed"),
Err(_) => tracing::debug!("otel metrics: list_timers timed out"),
}
let services: Vec<String> = snap
.units
.iter()
.filter(|u| u.name.ends_with(".service") && u.load_state == "loaded")
.map(|u| u.name.clone())
.collect();
for name in services {
if state.shutdown.is_cancelled() {
break;
}
if let Ok(Ok(st)) =
tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await
{
if let Some(n) = st.n_restarts {
snap.restarts.insert(name, n);
}
}
}
match tokio::time::timeout(CALL_TIMEOUT, state.docker.ps(true)).await {
Ok(Ok(cs)) => {
snap.containers = cs
.into_iter()
.filter(|c| container_in_scope(cfg, &c.name))
.collect();
}
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: docker ps failed"),
Err(_) => tracing::debug!("otel metrics: docker ps timed out"),
}
snap
}
/// Register the gauges and start the poll loop.
pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>> {
let snapshot: Arc<RwLock<Arc<Snapshot>>> = Arc::default();
let meter = provider.meter("skyg-linux-tracker");
for name in METRIC_NAMES {
let snap = Arc::clone(&snapshot);
// Gauges live as long as the provider; the instrument handle is not needed.
let _ = meter
.f64_observable_gauge(name)
.with_callback(move |o| {
let cur = snap.read().map(|g| Arc::clone(&g)).unwrap_or_default();
for ob in observations(&cur).into_iter().filter(|x| x.metric == name) {
let attrs: Vec<KeyValue> = ob
.attrs
.into_iter()
.map(|(k, v)| KeyValue::new(k, v))
.collect();
o.observe(ob.value, &attrs);
}
})
.build();
}
let interval = Duration::from_secs(state.config.otel.metrics_interval_secs.max(1));
let h = tokio::spawn(async move {
let mut tick = tokio::time::interval(interval);
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
_ = state.shutdown.cancelled() => break,
_ = tick.tick() => {}
}
let snap = tokio::select! {
_ = state.shutdown.cancelled() => break,
s = collect(&state) => s,
};
if let Ok(mut g) = snapshot.write() {
*g = Arc::new(snap);
}
}
});
vec![h]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::Config;
use crate::sources::fake::fake_state;
fn unit(name: &str, active: &str) -> UnitSummary {
UnitSummary {
name: name.into(),
description: String::new(),
load_state: "loaded".into(),
active_state: active.into(),
sub_state: "running".into(),
}
}
#[test]
fn default_scope_is_service_and_timer() {
let c = OtelMetricsConfig::default();
assert!(unit_in_scope(&c, "a.service"));
assert!(unit_in_scope(&c, "a.timer"));
assert!(!unit_in_scope(&c, "a.socket"));
assert!(!unit_in_scope(&c, "a.mount"));
assert!(container_in_scope(&c, "anything"));
}
#[test]
fn allowlist_replaces_default() {
let c = OtelMetricsConfig {
units: vec!["*.socket".into(), "ssh*".into()],
containers: vec!["web-*".into()],
};
assert!(unit_in_scope(&c, "dbus.socket"));
assert!(unit_in_scope(&c, "sshd.service"));
assert!(!unit_in_scope(&c, "cron.service"));
assert!(container_in_scope(&c, "web-1"));
assert!(!container_in_scope(&c, "db"));
}
#[test]
fn maps_units_and_restarts() {
let mut s = Snapshot {
units: vec![unit("a.service", "active"), unit("b.service", "failed")],
..Default::default()
};
s.restarts.insert("b.service".into(), 3);
let o = observations(&s);
let get = |m: &str, unit: &str| {
o.iter()
.find(|x| x.metric == m && x.attrs[0].1 == unit)
.map(|x| x.value)
};
assert_eq!(get("systemd.unit.active", "a.service"), Some(1.0));
assert_eq!(get("systemd.unit.failed", "a.service"), Some(0.0));
assert_eq!(get("systemd.unit.active", "b.service"), Some(0.0));
assert_eq!(get("systemd.unit.failed", "b.service"), Some(1.0));
assert_eq!(get("systemd.unit.restarts", "b.service"), Some(3.0));
assert_eq!(get("systemd.unit.restarts", "a.service"), None);
let a = o
.iter()
.find(|x| x.metric == "systemd.unit.active")
.unwrap();
let keys: Vec<_> = a.attrs.iter().map(|(k, _)| *k).collect();
assert_eq!(
keys,
["systemd.unit", "systemd.sub_state", "systemd.load_state"]
);
}
#[test]
fn maps_timers_and_containers() {
let c = ContainerSummary {
id: "abc".into(),
name: "web".into(),
image: "nginx:1".into(),
state: "running".into(),
health: Some("unhealthy".into()),
created: None,
started_at: None,
uptime_seconds: Some(42),
restart_count: Some(2),
};
let nohealth = ContainerSummary {
health: None,
uptime_seconds: None,
restart_count: None,
state: "exited".into(),
name: "job".into(),
..c.clone()
};
let s = Snapshot {
timers: vec![
TimerInfo {
name: "t.timer".into(),
unit: "t.service".into(),
active_state: "active".into(),
next_elapse: Some("2026-10-15T12:00:00Z".parse().unwrap()),
last_trigger: None,
},
TimerInfo {
name: "u.timer".into(),
unit: "u.service".into(),
active_state: "inactive".into(),
next_elapse: None,
last_trigger: None,
},
],
containers: vec![c, nohealth],
..Default::default()
};
let o = observations(&s);
let timers: Vec<_> = o
.iter()
.filter(|x| x.metric == "systemd.timer.next_elapse_seconds")
.collect();
assert_eq!(timers.len(), 1);
assert_eq!(timers[0].value, 1_792_065_600.0);
let of = |m: &str| o.iter().filter(|x| x.metric == m).count();
assert_eq!(of("docker.container.running"), 2);
assert_eq!(of("docker.container.healthy"), 1);
assert_eq!(of("docker.container.uptime_seconds"), 1);
assert_eq!(of("docker.container.restarts"), 1);
let h = o
.iter()
.find(|x| x.metric == "docker.container.healthy")
.unwrap();
assert_eq!(h.value, 0.0);
assert_eq!(h.attrs[0], ("container.name", "web".to_string()));
assert_eq!(h.attrs[2], ("container.image.name", "nginx:1".to_string()));
}
#[tokio::test]
async fn collect_applies_scope_with_fakes() {
let mut cfg = Config::default();
cfg.otel.metrics.units = vec!["__nomatch__*".into()];
cfg.otel.metrics.containers = vec!["__nomatch__*".into()];
let st = fake_state(cfg);
let snap = collect(&st).await;
assert!(snap.units.is_empty());
assert!(snap.timers.is_empty());
assert!(snap.containers.is_empty());
let st = fake_state(Config::default());
let snap = collect(&st).await;
assert!(snap
.units
.iter()
.all(|u| u.name.ends_with(".service") || u.name.ends_with(".timer")));
assert!(!snap.units.is_empty());
}
}
+366 -13
View File
@@ -1,32 +1,385 @@
//! OpenTelemetry export (WS-G owns `src/telemetry/`).
//!
//! STUB: signatures are final. Pinned `opentelemetry-otlp` 0.33.1: use
//! `WithExportConfig::{with_endpoint, with_protocol}` plus
//! `WithHttpConfig::with_headers` (HashMap) or
//! `WithTonicConfig::with_metadata` (tonic `MetadataMap`).
//! One exporter config (`otel.endpoint`, `otel.protocol`, `otel.headers`)
//! feeds traces, metrics and logs. The app's own `tracing` events go to
//! stderr (and to OTel traces); they are never routed to the OTel logs
//! pipeline, which carries host logs only.
pub mod logs_export;
pub mod metrics_export;
use anyhow::Result;
use tokio::task::JoinHandle;
use std::collections::HashMap;
use std::sync::OnceLock;
use std::time::Duration;
use crate::config::Config;
use anyhow::{Context, Result};
use opentelemetry::trace::TracerProvider as _;
use opentelemetry::KeyValue;
use opentelemetry_otlp::{
LogExporter, MetricExporter, Protocol, SpanExporter, WithExportConfig, WithHttpConfig,
WithTonicConfig,
};
use opentelemetry_sdk::logs::SdkLoggerProvider;
use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
use opentelemetry_sdk::trace::SdkTracerProvider;
use opentelemetry_sdk::Resource;
use tokio::task::JoinHandle;
use tracing_subscriber::filter::LevelFilter;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use crate::config::{Config, OtelConfig, OtelProtocol};
use crate::state::AppState;
/// Upper bound for shutting down each provider.
const FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Clone)]
struct Providers {
tracer: SdkTracerProvider,
meter: Option<SdkMeterProvider>,
logger: Option<SdkLoggerProvider>,
}
/// Providers shared with `spawn_exporters`. Set once by `init` when enabled.
static PROVIDERS: OnceLock<Providers> = OnceLock::new();
/// Flushes and shuts down providers on drop.
pub struct TelemetryGuard {
_private: (),
providers: Option<Providers>,
}
impl TelemetryGuard {
/// True when OTLP export is active.
pub fn is_enabled(&self) -> bool {
self.providers.is_some()
}
}
impl Drop for TelemetryGuard {
fn drop(&mut self) {
let Some(p) = self.providers.take() else {
return;
};
if let Err(e) = p.tracer.force_flush() {
eprintln!("otel: trace flush failed: {e}");
}
let _ = p.tracer.shutdown_with_timeout(FLUSH_TIMEOUT);
if let Some(m) = &p.meter {
if let Err(e) = m.force_flush() {
eprintln!("otel: metrics flush failed: {e}");
}
let _ = m.shutdown_with_timeout(FLUSH_TIMEOUT);
}
if let Some(l) = &p.logger {
if let Err(e) = l.force_flush() {
eprintln!("otel: logs flush failed: {e}");
}
let _ = l.shutdown_with_timeout(FLUSH_TIMEOUT);
}
}
}
/// Install the tracing subscriber and, when `config.otel.enabled`, the OTLP
/// providers. Must work (stderr logging only) when OTel is disabled.
pub fn init(_config: &Config) -> Result<TelemetryGuard> {
todo!("WS-G G1/G2")
/// providers. Works (stderr logging only) when OTel is disabled.
pub fn init(config: &Config) -> Result<TelemetryGuard> {
let otel = &config.otel;
if !otel.enabled {
let _ = tracing_subscriber::registry()
.with(LevelFilter::INFO)
.with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
.try_init();
return Ok(TelemetryGuard { providers: None });
}
let resource = build_resource(otel);
let tracer_provider = SdkTracerProvider::builder()
.with_batch_exporter(build_span_exporter(otel)?)
.with_resource(resource.clone())
.build();
let meter = if otel.export_unit_metrics {
let reader = PeriodicReader::builder(build_metric_exporter(otel)?)
.with_interval(Duration::from_secs(otel.metrics_interval_secs.max(1)))
.build();
Some(
SdkMeterProvider::builder()
.with_reader(reader)
.with_resource(resource.clone())
.build(),
)
} else {
None
};
let logger = if otel.export_logs {
Some(
SdkLoggerProvider::builder()
.with_batch_exporter(build_log_exporter(otel)?)
.with_resource(resource)
.build(),
)
} else {
None
};
let tracer = tracer_provider.tracer(otel.service_name.clone());
let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
let _ = tracing_subscriber::registry()
.with(LevelFilter::INFO)
.with(otel_layer)
.with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
.try_init();
let providers = Providers {
tracer: tracer_provider,
meter,
logger,
};
let _ = PROVIDERS.set(providers.clone());
Ok(TelemetryGuard {
providers: Some(providers),
})
}
/// Start the metrics poll loop and log fan-out tasks. All tasks stop on
/// `state.shutdown`. Returns no handles when export is disabled.
pub fn spawn_exporters(_state: AppState) -> Vec<JoinHandle<()>> {
todo!("WS-G G3/G5")
pub fn spawn_exporters(state: AppState) -> Vec<JoinHandle<()>> {
let mut handles = Vec::new();
if !state.config.otel.enabled {
return handles;
}
let Some(p) = PROVIDERS.get() else {
return handles;
};
if let Some(meter) = &p.meter {
handles.extend(metrics_export::spawn(state.clone(), meter));
}
if let Some(logger) = &p.logger {
handles.extend(logs_export::spawn(state, logger));
}
handles
}
fn hostname() -> String {
for path in ["/proc/sys/kernel/hostname", "/etc/hostname"] {
if let Ok(s) = std::fs::read_to_string(path) {
let s = s.trim();
if !s.is_empty() {
return s.to_owned();
}
}
}
"unknown".to_owned()
}
fn build_resource(otel: &OtelConfig) -> Resource {
Resource::builder()
.with_service_name(otel.service_name.clone())
.with_attribute(KeyValue::new(
"service.version",
env!("CARGO_PKG_VERSION").to_owned(),
))
.with_attribute(KeyValue::new("host.name", hostname()))
.build()
}
/// Endpoint for one signal. gRPC uses the base URL. HTTP appends
/// `/v1/<signal>` unless the configured endpoint already ends with it.
fn signal_endpoint(otel: &OtelConfig, signal: &str) -> String {
match otel.protocol {
OtelProtocol::Grpc => otel.endpoint.clone(),
OtelProtocol::HttpProtobuf => {
let base = otel.endpoint.trim_end_matches('/');
if base.ends_with(&format!("/v1/{signal}")) {
base.to_owned()
} else {
format!("{base}/v1/{signal}")
}
}
}
}
fn header_map(otel: &OtelConfig) -> HashMap<String, String> {
otel.headers
.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect()
}
fn metadata(otel: &OtelConfig) -> Result<opentelemetry_otlp::tonic_types::metadata::MetadataMap> {
use opentelemetry_otlp::tonic_types::metadata::MetadataMap;
let mut m = MetadataMap::new();
for (k, v) in &otel.headers {
// Metadata keys must be `'static`; the config lives for the whole process
// and `init` runs once, so leaking the few header names is fine.
let key: &'static str = Box::leak(k.to_ascii_lowercase().into_boxed_str());
let val = v
.parse()
.with_context(|| format!("invalid value for otel header '{k}'"))?;
m.insert(key, val);
}
Ok(m)
}
macro_rules! build_exporter {
($ty:ident, $otel:expr, $signal:expr) => {{
let otel: &OtelConfig = $otel;
let ep = signal_endpoint(otel, $signal);
let res = match otel.protocol {
OtelProtocol::Grpc => $ty::builder()
.with_tonic()
.with_endpoint(ep)
.with_metadata(metadata(otel)?)
.build(),
OtelProtocol::HttpProtobuf => $ty::builder()
.with_http()
.with_protocol(Protocol::HttpBinary)
.with_endpoint(ep)
.with_headers(header_map(otel))
.build(),
};
res.with_context(|| format!("building OTLP {} exporter", $signal))
}};
}
fn build_span_exporter(otel: &OtelConfig) -> Result<SpanExporter> {
build_exporter!(SpanExporter, otel, "traces")
}
fn build_metric_exporter(otel: &OtelConfig) -> Result<MetricExporter> {
build_exporter!(MetricExporter, otel, "metrics")
}
fn build_log_exporter(otel: &OtelConfig) -> Result<LogExporter> {
build_exporter!(LogExporter, otel, "logs")
}
/// Shell-style glob match supporting `*` and `?`.
pub(crate) fn glob_match(pattern: &str, text: &str) -> bool {
let p: Vec<char> = pattern.chars().collect();
let t: Vec<char> = text.chars().collect();
let (mut pi, mut ti) = (0, 0);
let (mut star, mut mark) = (None, 0);
while ti < t.len() {
if pi < p.len() && (p[pi] == '?' || p[pi] == t[ti]) {
pi += 1;
ti += 1;
} else if pi < p.len() && p[pi] == '*' {
star = Some(pi);
mark = ti;
pi += 1;
} else if let Some(s) = star {
pi = s + 1;
mark += 1;
ti = mark;
} else {
return false;
}
}
while pi < p.len() && p[pi] == '*' {
pi += 1;
}
pi == p.len()
}
pub(crate) fn any_glob(patterns: &[String], text: &str) -> bool {
patterns.iter().any(|p| glob_match(p, text))
}
/// Exponential backoff for restarting failed export tasks.
#[derive(Debug, Clone)]
pub(crate) struct Backoff {
cur: Duration,
min: Duration,
max: Duration,
}
impl Backoff {
pub(crate) fn new(min: Duration, max: Duration) -> Self {
Self { cur: min, min, max }
}
pub(crate) fn next_delay(&mut self) -> Duration {
let d = self.cur;
self.cur = (self.cur * 2).min(self.max);
d
}
pub(crate) fn reset(&mut self) {
self.cur = self.min;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn glob() {
assert!(glob_match("*.service", "sshd.service"));
assert!(!glob_match("*.service", "sshd.timer"));
assert!(glob_match("web-?", "web-1"));
assert!(!glob_match("web-?", "web-12"));
assert!(glob_match("a*b*c", "aXXbYYc"));
assert!(!glob_match("a*b*c", "aXXbYY"));
assert!(glob_match("*", ""));
assert!(glob_match("exact", "exact"));
assert!(!glob_match("exact", "exactly"));
}
#[test]
fn backoff_doubles_and_caps() {
let mut b = Backoff::new(Duration::from_secs(1), Duration::from_secs(5));
let v: Vec<u64> = (0..5).map(|_| b.next_delay().as_secs()).collect();
assert_eq!(v, [1, 2, 4, 5, 5]);
b.reset();
assert_eq!(b.next_delay().as_secs(), 1);
}
#[test]
fn http_endpoint_gets_signal_path() {
let mut o = OtelConfig {
protocol: OtelProtocol::HttpProtobuf,
endpoint: "http://localhost:4318/".into(),
..Default::default()
};
assert_eq!(signal_endpoint(&o, "logs"), "http://localhost:4318/v1/logs");
o.endpoint = "http://c:4318/v1/logs".into();
assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs");
o.protocol = OtelProtocol::Grpc;
assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs");
}
#[tokio::test]
async fn exporters_build_for_both_protocols() {
for protocol in [OtelProtocol::Grpc, OtelProtocol::HttpProtobuf] {
let mut o = OtelConfig {
protocol,
endpoint: "http://127.0.0.1:1".into(),
..Default::default()
};
o.headers.insert("Authorization".into(), "Bearer x".into());
let results = [
build_span_exporter(&o).map(drop),
build_metric_exporter(&o).map(drop),
build_log_exporter(&o).map(drop),
];
for r in results {
match (protocol, r) {
(_, Ok(())) => {}
// Contract: HTTP needs the `reqwest-blocking-client` feature on
// opentelemetry-otlp (Cargo.toml, owned by P0).
(OtelProtocol::HttpProtobuf, Err(e))
if format!("{e:?}").contains("http_client") => {}
(_, Err(e)) => panic!("exporter build failed: {e:?}"),
}
}
}
}
#[test]
fn disabled_init_is_noop_guard() {
let g = init(&Config::default()).unwrap();
assert!(!g.is_enabled());
}
}