WS-G: OpenTelemetry export (traces, metrics, logs)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
ariandClaude Sonnet 5.5 committed 2026-10-15 18:37:40 -04:00
1 parent 2d57fb1a78
commit 2aeb9ae154
6 files changed
+1175 -25

No files matched your search

+366 -13
View File
@@ -1,32 +1,385 @@
//! OpenTelemetry export (WS-G owns `src/telemetry/`).
//!
//! STUB: signatures are final. Pinned `opentelemetry-otlp` 0.33.1: use
//! `WithExportConfig::{with_endpoint, with_protocol}` plus
//! `WithHttpConfig::with_headers` (HashMap) or
//! `WithTonicConfig::with_metadata` (tonic `MetadataMap`).
//! One exporter config (`otel.endpoint`, `otel.protocol`, `otel.headers`)
//! feeds traces, metrics and logs. The app's own `tracing` events go to
//! stderr (and to OTel traces); they are never routed to the OTel logs
//! pipeline, which carries host logs only.
pub mod logs_export;
pub mod metrics_export;
use anyhow::Result;
use tokio::task::JoinHandle;
use std::collections::HashMap;
use std::sync::OnceLock;
use std::time::Duration;
use crate::config::Config;
use anyhow::{Context, Result};
use opentelemetry::trace::TracerProvider as _;
use opentelemetry::KeyValue;
use opentelemetry_otlp::{
LogExporter, MetricExporter, Protocol, SpanExporter, WithExportConfig, WithHttpConfig,
WithTonicConfig,
};
use opentelemetry_sdk::logs::SdkLoggerProvider;
use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
use opentelemetry_sdk::trace::SdkTracerProvider;
use opentelemetry_sdk::Resource;
use tokio::task::JoinHandle;
use tracing_subscriber::filter::LevelFilter;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use crate::config::{Config, OtelConfig, OtelProtocol};
use crate::state::AppState;
/// Upper bound for shutting down each provider.
const FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Clone)]
struct Providers {
tracer: SdkTracerProvider,
meter: Option<SdkMeterProvider>,
logger: Option<SdkLoggerProvider>,
}
/// Providers shared with `spawn_exporters`. Set once by `init` when enabled.
static PROVIDERS: OnceLock<Providers> = OnceLock::new();
/// Flushes and shuts down providers on drop.
pub struct TelemetryGuard {
_private: (),
providers: Option<Providers>,
}
impl TelemetryGuard {
/// True when OTLP export is active.
pub fn is_enabled(&self) -> bool {
self.providers.is_some()
}
}
impl Drop for TelemetryGuard {
fn drop(&mut self) {
let Some(p) = self.providers.take() else {
return;
};
if let Err(e) = p.tracer.force_flush() {
eprintln!("otel: trace flush failed: {e}");
}
let _ = p.tracer.shutdown_with_timeout(FLUSH_TIMEOUT);
if let Some(m) = &p.meter {
if let Err(e) = m.force_flush() {
eprintln!("otel: metrics flush failed: {e}");
}
let _ = m.shutdown_with_timeout(FLUSH_TIMEOUT);
}
if let Some(l) = &p.logger {
if let Err(e) = l.force_flush() {
eprintln!("otel: logs flush failed: {e}");
}
let _ = l.shutdown_with_timeout(FLUSH_TIMEOUT);
}
}
}
/// Install the tracing subscriber and, when `config.otel.enabled`, the OTLP
/// providers. Must work (stderr logging only) when OTel is disabled.
pub fn init(_config: &Config) -> Result<TelemetryGuard> {
todo!("WS-G G1/G2")
/// providers. Works (stderr logging only) when OTel is disabled.
pub fn init(config: &Config) -> Result<TelemetryGuard> {
let otel = &config.otel;
if !otel.enabled {
let _ = tracing_subscriber::registry()
.with(LevelFilter::INFO)
.with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
.try_init();
return Ok(TelemetryGuard { providers: None });
}
let resource = build_resource(otel);
let tracer_provider = SdkTracerProvider::builder()
.with_batch_exporter(build_span_exporter(otel)?)
.with_resource(resource.clone())
.build();
let meter = if otel.export_unit_metrics {
let reader = PeriodicReader::builder(build_metric_exporter(otel)?)
.with_interval(Duration::from_secs(otel.metrics_interval_secs.max(1)))
.build();
Some(
SdkMeterProvider::builder()
.with_reader(reader)
.with_resource(resource.clone())
.build(),
)
} else {
None
};
let logger = if otel.export_logs {
Some(
SdkLoggerProvider::builder()
.with_batch_exporter(build_log_exporter(otel)?)
.with_resource(resource)
.build(),
)
} else {
None
};
let tracer = tracer_provider.tracer(otel.service_name.clone());
let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
let _ = tracing_subscriber::registry()
.with(LevelFilter::INFO)
.with(otel_layer)
.with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
.try_init();
let providers = Providers {
tracer: tracer_provider,
meter,
logger,
};
let _ = PROVIDERS.set(providers.clone());
Ok(TelemetryGuard {
providers: Some(providers),
})
}
/// Start the metrics poll loop and log fan-out tasks. All tasks stop on
/// `state.shutdown`. Returns no handles when export is disabled.
pub fn spawn_exporters(_state: AppState) -> Vec<JoinHandle<()>> {
todo!("WS-G G3/G5")
pub fn spawn_exporters(state: AppState) -> Vec<JoinHandle<()>> {
let mut handles = Vec::new();
if !state.config.otel.enabled {
return handles;
}
let Some(p) = PROVIDERS.get() else {
return handles;
};
if let Some(meter) = &p.meter {
handles.extend(metrics_export::spawn(state.clone(), meter));
}
if let Some(logger) = &p.logger {
handles.extend(logs_export::spawn(state, logger));
}
handles
}
fn hostname() -> String {
for path in ["/proc/sys/kernel/hostname", "/etc/hostname"] {
if let Ok(s) = std::fs::read_to_string(path) {
let s = s.trim();
if !s.is_empty() {
return s.to_owned();
}
}
}
"unknown".to_owned()
}
fn build_resource(otel: &OtelConfig) -> Resource {
Resource::builder()
.with_service_name(otel.service_name.clone())
.with_attribute(KeyValue::new(
"service.version",
env!("CARGO_PKG_VERSION").to_owned(),
))
.with_attribute(KeyValue::new("host.name", hostname()))
.build()
}
/// Endpoint for one signal. gRPC uses the base URL. HTTP appends
/// `/v1/<signal>` unless the configured endpoint already ends with it.
fn signal_endpoint(otel: &OtelConfig, signal: &str) -> String {
match otel.protocol {
OtelProtocol::Grpc => otel.endpoint.clone(),
OtelProtocol::HttpProtobuf => {
let base = otel.endpoint.trim_end_matches('/');
if base.ends_with(&format!("/v1/{signal}")) {
base.to_owned()
} else {
format!("{base}/v1/{signal}")
}
}
}
}
fn header_map(otel: &OtelConfig) -> HashMap<String, String> {
otel.headers
.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect()
}
fn metadata(otel: &OtelConfig) -> Result<opentelemetry_otlp::tonic_types::metadata::MetadataMap> {
use opentelemetry_otlp::tonic_types::metadata::MetadataMap;
let mut m = MetadataMap::new();
for (k, v) in &otel.headers {
// Metadata keys must be `'static`; the config lives for the whole process
// and `init` runs once, so leaking the few header names is fine.
let key: &'static str = Box::leak(k.to_ascii_lowercase().into_boxed_str());
let val = v
.parse()
.with_context(|| format!("invalid value for otel header '{k}'"))?;
m.insert(key, val);
}
Ok(m)
}
macro_rules! build_exporter {
($ty:ident, $otel:expr, $signal:expr) => {{
let otel: &OtelConfig = $otel;
let ep = signal_endpoint(otel, $signal);
let res = match otel.protocol {
OtelProtocol::Grpc => $ty::builder()
.with_tonic()
.with_endpoint(ep)
.with_metadata(metadata(otel)?)
.build(),
OtelProtocol::HttpProtobuf => $ty::builder()
.with_http()
.with_protocol(Protocol::HttpBinary)
.with_endpoint(ep)
.with_headers(header_map(otel))
.build(),
};
res.with_context(|| format!("building OTLP {} exporter", $signal))
}};
}
fn build_span_exporter(otel: &OtelConfig) -> Result<SpanExporter> {
build_exporter!(SpanExporter, otel, "traces")
}
fn build_metric_exporter(otel: &OtelConfig) -> Result<MetricExporter> {
build_exporter!(MetricExporter, otel, "metrics")
}
fn build_log_exporter(otel: &OtelConfig) -> Result<LogExporter> {
build_exporter!(LogExporter, otel, "logs")
}
/// Shell-style glob match supporting `*` and `?`.
pub(crate) fn glob_match(pattern: &str, text: &str) -> bool {
let p: Vec<char> = pattern.chars().collect();
let t: Vec<char> = text.chars().collect();
let (mut pi, mut ti) = (0, 0);
let (mut star, mut mark) = (None, 0);
while ti < t.len() {
if pi < p.len() && (p[pi] == '?' || p[pi] == t[ti]) {
pi += 1;
ti += 1;
} else if pi < p.len() && p[pi] == '*' {
star = Some(pi);
mark = ti;
pi += 1;
} else if let Some(s) = star {
pi = s + 1;
mark += 1;
ti = mark;
} else {
return false;
}
}
while pi < p.len() && p[pi] == '*' {
pi += 1;
}
pi == p.len()
}
pub(crate) fn any_glob(patterns: &[String], text: &str) -> bool {
patterns.iter().any(|p| glob_match(p, text))
}
/// Exponential backoff for restarting failed export tasks.
#[derive(Debug, Clone)]
pub(crate) struct Backoff {
cur: Duration,
min: Duration,
max: Duration,
}
impl Backoff {
pub(crate) fn new(min: Duration, max: Duration) -> Self {
Self { cur: min, min, max }
}
pub(crate) fn next_delay(&mut self) -> Duration {
let d = self.cur;
self.cur = (self.cur * 2).min(self.max);
d
}
pub(crate) fn reset(&mut self) {
self.cur = self.min;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn glob() {
assert!(glob_match("*.service", "sshd.service"));
assert!(!glob_match("*.service", "sshd.timer"));
assert!(glob_match("web-?", "web-1"));
assert!(!glob_match("web-?", "web-12"));
assert!(glob_match("a*b*c", "aXXbYYc"));
assert!(!glob_match("a*b*c", "aXXbYY"));
assert!(glob_match("*", ""));
assert!(glob_match("exact", "exact"));
assert!(!glob_match("exact", "exactly"));
}
#[test]
fn backoff_doubles_and_caps() {
let mut b = Backoff::new(Duration::from_secs(1), Duration::from_secs(5));
let v: Vec<u64> = (0..5).map(|_| b.next_delay().as_secs()).collect();
assert_eq!(v, [1, 2, 4, 5, 5]);
b.reset();
assert_eq!(b.next_delay().as_secs(), 1);
}
#[test]
fn http_endpoint_gets_signal_path() {
let mut o = OtelConfig {
protocol: OtelProtocol::HttpProtobuf,
endpoint: "http://localhost:4318/".into(),
..Default::default()
};
assert_eq!(signal_endpoint(&o, "logs"), "http://localhost:4318/v1/logs");
o.endpoint = "http://c:4318/v1/logs".into();
assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs");
o.protocol = OtelProtocol::Grpc;
assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs");
}
#[tokio::test]
async fn exporters_build_for_both_protocols() {
for protocol in [OtelProtocol::Grpc, OtelProtocol::HttpProtobuf] {
let mut o = OtelConfig {
protocol,
endpoint: "http://127.0.0.1:1".into(),
..Default::default()
};
o.headers.insert("Authorization".into(), "Bearer x".into());
let results = [
build_span_exporter(&o).map(drop),
build_metric_exporter(&o).map(drop),
build_log_exporter(&o).map(drop),
];
for r in results {
match (protocol, r) {
(_, Ok(())) => {}
// Contract: HTTP needs the `reqwest-blocking-client` feature on
// opentelemetry-otlp (Cargo.toml, owned by P0).
(OtelProtocol::HttpProtobuf, Err(e))
if format!("{e:?}").contains("http_client") => {}
(_, Err(e)) => panic!("exporter build failed: {e:?}"),
}
}
}
}
#[test]
fn disabled_init_is_noop_guard() {
let g = init(&Config::default()).unwrap();
assert!(!g.is_enabled());
}
}