//! OpenTelemetry export (WS-G owns `src/telemetry/`). //! //! One exporter config (`otel.endpoint`, `otel.protocol`, `otel.headers`) //! feeds traces, metrics and logs. The app's own `tracing` events go to //! stderr (and to OTel traces); they are never routed to the OTel logs //! pipeline, which carries host logs only. pub mod logs_export; pub mod metrics_export; use std::collections::HashMap; use std::sync::OnceLock; use std::time::Duration; use anyhow::{Context, Result}; use opentelemetry::trace::TracerProvider as _; use opentelemetry::KeyValue; use opentelemetry_otlp::{ LogExporter, MetricExporter, Protocol, SpanExporter, WithExportConfig, WithHttpConfig, WithTonicConfig, }; use opentelemetry_sdk::logs::SdkLoggerProvider; use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; use opentelemetry_sdk::trace::SdkTracerProvider; use opentelemetry_sdk::Resource; use tokio::task::JoinHandle; use tracing_subscriber::filter::LevelFilter; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use crate::config::{Config, OtelConfig, OtelProtocol}; use crate::state::AppState; /// Upper bound for shutting down each provider. const FLUSH_TIMEOUT: Duration = Duration::from_secs(5); #[derive(Clone)] struct Providers { tracer: SdkTracerProvider, meter: Option, logger: Option, } /// Providers shared with `spawn_exporters`. Set once by `init` when enabled. static PROVIDERS: OnceLock = OnceLock::new(); /// Flushes and shuts down providers on drop. pub struct TelemetryGuard { providers: Option, } impl TelemetryGuard { /// True when OTLP export is active. pub fn is_enabled(&self) -> bool { self.providers.is_some() } } impl Drop for TelemetryGuard { fn drop(&mut self) { let Some(p) = self.providers.take() else { return; }; if let Err(e) = p.tracer.force_flush() { eprintln!("otel: trace flush failed: {e}"); } let _ = p.tracer.shutdown_with_timeout(FLUSH_TIMEOUT); if let Some(m) = &p.meter { if let Err(e) = m.force_flush() { eprintln!("otel: metrics flush failed: {e}"); } let _ = m.shutdown_with_timeout(FLUSH_TIMEOUT); } if let Some(l) = &p.logger { if let Err(e) = l.force_flush() { eprintln!("otel: logs flush failed: {e}"); } let _ = l.shutdown_with_timeout(FLUSH_TIMEOUT); } } } /// Install the tracing subscriber and, when `config.otel.enabled`, the OTLP /// providers. Works (stderr logging only) when OTel is disabled. pub fn init(config: &Config) -> Result { let otel = &config.otel; if !otel.enabled { let _ = tracing_subscriber::registry() .with(LevelFilter::INFO) .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) .try_init(); return Ok(TelemetryGuard { providers: None }); } let resource = build_resource(otel); let tracer_provider = SdkTracerProvider::builder() .with_batch_exporter(build_span_exporter(otel)?) .with_resource(resource.clone()) .build(); let meter = if otel.export_unit_metrics { let reader = PeriodicReader::builder(build_metric_exporter(otel)?) .with_interval(Duration::from_secs(otel.metrics_interval_secs.max(1))) .build(); Some( SdkMeterProvider::builder() .with_reader(reader) .with_resource(resource.clone()) .build(), ) } else { None }; let logger = if otel.export_logs { Some( SdkLoggerProvider::builder() .with_batch_exporter(build_log_exporter(otel)?) .with_resource(resource) .build(), ) } else { None }; let tracer = tracer_provider.tracer(otel.service_name.clone()); let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer); let _ = tracing_subscriber::registry() .with(LevelFilter::INFO) .with(otel_layer) .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) .try_init(); let providers = Providers { tracer: tracer_provider, meter, logger, }; let _ = PROVIDERS.set(providers.clone()); Ok(TelemetryGuard { providers: Some(providers), }) } /// Start the metrics poll loop and log fan-out tasks. All tasks stop on /// `state.shutdown`. Returns no handles when export is disabled. pub fn spawn_exporters(state: AppState) -> Vec> { let mut handles = Vec::new(); if !state.config.otel.enabled { return handles; } let Some(p) = PROVIDERS.get() else { return handles; }; if let Some(meter) = &p.meter { handles.extend(metrics_export::spawn(state.clone(), meter)); } if let Some(logger) = &p.logger { handles.extend(logs_export::spawn(state, logger)); } handles } fn hostname() -> String { for path in ["/proc/sys/kernel/hostname", "/etc/hostname"] { if let Ok(s) = std::fs::read_to_string(path) { let s = s.trim(); if !s.is_empty() { return s.to_owned(); } } } "unknown".to_owned() } fn build_resource(otel: &OtelConfig) -> Resource { Resource::builder() .with_service_name(otel.service_name.clone()) .with_attribute(KeyValue::new( "service.version", env!("CARGO_PKG_VERSION").to_owned(), )) .with_attribute(KeyValue::new("host.name", hostname())) .build() } /// Endpoint for one signal. gRPC uses the base URL. HTTP appends /// `/v1/` unless the configured endpoint already ends with it. fn signal_endpoint(otel: &OtelConfig, signal: &str) -> String { match otel.protocol { OtelProtocol::Grpc => otel.endpoint.clone(), OtelProtocol::HttpProtobuf => { let base = otel.endpoint.trim_end_matches('/'); if base.ends_with(&format!("/v1/{signal}")) { base.to_owned() } else { format!("{base}/v1/{signal}") } } } } fn header_map(otel: &OtelConfig) -> HashMap { otel.headers .iter() .map(|(k, v)| (k.clone(), v.clone())) .collect() } fn metadata(otel: &OtelConfig) -> Result { use opentelemetry_otlp::tonic_types::metadata::MetadataMap; let mut m = MetadataMap::new(); for (k, v) in &otel.headers { // Metadata keys must be `'static`; the config lives for the whole process // and `init` runs once, so leaking the few header names is fine. let key: &'static str = Box::leak(k.to_ascii_lowercase().into_boxed_str()); let val = v .parse() .with_context(|| format!("invalid value for otel header '{k}'"))?; m.insert(key, val); } Ok(m) } macro_rules! build_exporter { ($ty:ident, $otel:expr, $signal:expr) => {{ let otel: &OtelConfig = $otel; let ep = signal_endpoint(otel, $signal); let res = match otel.protocol { OtelProtocol::Grpc => $ty::builder() .with_tonic() .with_endpoint(ep) .with_metadata(metadata(otel)?) .build(), OtelProtocol::HttpProtobuf => $ty::builder() .with_http() .with_protocol(Protocol::HttpBinary) .with_endpoint(ep) .with_headers(header_map(otel)) .build(), }; res.with_context(|| format!("building OTLP {} exporter", $signal)) }}; } fn build_span_exporter(otel: &OtelConfig) -> Result { build_exporter!(SpanExporter, otel, "traces") } fn build_metric_exporter(otel: &OtelConfig) -> Result { build_exporter!(MetricExporter, otel, "metrics") } fn build_log_exporter(otel: &OtelConfig) -> Result { build_exporter!(LogExporter, otel, "logs") } /// Shell-style glob match supporting `*` and `?`. pub(crate) fn glob_match(pattern: &str, text: &str) -> bool { let p: Vec = pattern.chars().collect(); let t: Vec = text.chars().collect(); let (mut pi, mut ti) = (0, 0); let (mut star, mut mark) = (None, 0); while ti < t.len() { if pi < p.len() && (p[pi] == '?' || p[pi] == t[ti]) { pi += 1; ti += 1; } else if pi < p.len() && p[pi] == '*' { star = Some(pi); mark = ti; pi += 1; } else if let Some(s) = star { pi = s + 1; mark += 1; ti = mark; } else { return false; } } while pi < p.len() && p[pi] == '*' { pi += 1; } pi == p.len() } pub(crate) fn any_glob(patterns: &[String], text: &str) -> bool { patterns.iter().any(|p| glob_match(p, text)) } /// Exponential backoff for restarting failed export tasks. #[derive(Debug, Clone)] pub(crate) struct Backoff { cur: Duration, min: Duration, max: Duration, } impl Backoff { pub(crate) fn new(min: Duration, max: Duration) -> Self { Self { cur: min, min, max } } pub(crate) fn next_delay(&mut self) -> Duration { let d = self.cur; self.cur = (self.cur * 2).min(self.max); d } pub(crate) fn reset(&mut self) { self.cur = self.min; } } #[cfg(test)] mod tests { use super::*; #[test] fn glob() { assert!(glob_match("*.service", "sshd.service")); assert!(!glob_match("*.service", "sshd.timer")); assert!(glob_match("web-?", "web-1")); assert!(!glob_match("web-?", "web-12")); assert!(glob_match("a*b*c", "aXXbYYc")); assert!(!glob_match("a*b*c", "aXXbYY")); assert!(glob_match("*", "")); assert!(glob_match("exact", "exact")); assert!(!glob_match("exact", "exactly")); } #[test] fn backoff_doubles_and_caps() { let mut b = Backoff::new(Duration::from_secs(1), Duration::from_secs(5)); let v: Vec = (0..5).map(|_| b.next_delay().as_secs()).collect(); assert_eq!(v, [1, 2, 4, 5, 5]); b.reset(); assert_eq!(b.next_delay().as_secs(), 1); } #[test] fn http_endpoint_gets_signal_path() { let mut o = OtelConfig { protocol: OtelProtocol::HttpProtobuf, endpoint: "http://localhost:4318/".into(), ..Default::default() }; assert_eq!(signal_endpoint(&o, "logs"), "http://localhost:4318/v1/logs"); o.endpoint = "http://c:4318/v1/logs".into(); assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs"); o.protocol = OtelProtocol::Grpc; assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs"); } #[tokio::test] async fn exporters_build_for_both_protocols() { for protocol in [OtelProtocol::Grpc, OtelProtocol::HttpProtobuf] { let mut o = OtelConfig { protocol, endpoint: "http://127.0.0.1:1".into(), ..Default::default() }; o.headers.insert("Authorization".into(), "Bearer x".into()); let results = [ build_span_exporter(&o).map(drop), build_metric_exporter(&o).map(drop), build_log_exporter(&o).map(drop), ]; for r in results { match (protocol, r) { (_, Ok(())) => {} // Contract: HTTP needs the `reqwest-blocking-client` feature on // opentelemetry-otlp (Cargo.toml, owned by P0). (OtelProtocol::HttpProtobuf, Err(e)) if format!("{e:?}").contains("http_client") => {} (_, Err(e)) => panic!("exporter build failed: {e:?}"), } } } } #[test] fn disabled_init_is_noop_guard() { let g = init(&Config::default()).unwrap(); assert!(!g.is_enabled()); } }