Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
399 lines
13 KiB
Rust
399 lines
13 KiB
Rust
//! OpenTelemetry export (WS-G owns `src/telemetry/`).
|
|
//!
|
|
//! One exporter config (`otel.endpoint`, `otel.protocol`, `otel.headers`)
|
|
//! feeds traces, metrics and logs. The app's own `tracing` events go to
|
|
//! stderr (and to OTel traces); they are never routed to the OTel logs
|
|
//! pipeline, which carries host logs only.
|
|
|
|
pub mod logs_export;
|
|
pub mod metrics_export;
|
|
|
|
use std::collections::HashMap;
|
|
use std::sync::OnceLock;
|
|
use std::time::Duration;
|
|
|
|
use anyhow::{Context, Result};
|
|
use opentelemetry::trace::TracerProvider as _;
|
|
use opentelemetry::KeyValue;
|
|
use opentelemetry_otlp::{
|
|
LogExporter, MetricExporter, Protocol, SpanExporter, WithExportConfig, WithHttpConfig,
|
|
WithTonicConfig,
|
|
};
|
|
use opentelemetry_sdk::logs::SdkLoggerProvider;
|
|
use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
|
|
use opentelemetry_sdk::trace::SdkTracerProvider;
|
|
use opentelemetry_sdk::Resource;
|
|
use tokio::task::JoinHandle;
|
|
use tracing_subscriber::filter::LevelFilter;
|
|
use tracing_subscriber::layer::SubscriberExt;
|
|
use tracing_subscriber::util::SubscriberInitExt;
|
|
|
|
use crate::config::{Config, OtelConfig, OtelProtocol};
|
|
use crate::state::AppState;
|
|
|
|
/// Upper bound for shutting down each provider.
|
|
const FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
|
|
|
|
#[derive(Clone)]
|
|
struct Providers {
|
|
tracer: SdkTracerProvider,
|
|
meter: Option<SdkMeterProvider>,
|
|
logger: Option<SdkLoggerProvider>,
|
|
}
|
|
|
|
/// Providers shared with `spawn_exporters`. Set once by `init` when enabled.
|
|
static PROVIDERS: OnceLock<Providers> = OnceLock::new();
|
|
|
|
/// Flushes and shuts down providers on drop.
|
|
pub struct TelemetryGuard {
|
|
providers: Option<Providers>,
|
|
}
|
|
|
|
impl Providers {
|
|
/// Blocking flush and shutdown, each provider bounded by `FLUSH_TIMEOUT`.
|
|
fn flush_and_shutdown(&self) {
|
|
if let Err(e) = self.tracer.force_flush() {
|
|
eprintln!("otel: trace flush failed: {e}");
|
|
}
|
|
let _ = self.tracer.shutdown_with_timeout(FLUSH_TIMEOUT);
|
|
if let Some(m) = &self.meter {
|
|
if let Err(e) = m.force_flush() {
|
|
eprintln!("otel: metrics flush failed: {e}");
|
|
}
|
|
let _ = m.shutdown_with_timeout(FLUSH_TIMEOUT);
|
|
}
|
|
if let Some(l) = &self.logger {
|
|
if let Err(e) = l.force_flush() {
|
|
eprintln!("otel: logs flush failed: {e}");
|
|
}
|
|
let _ = l.shutdown_with_timeout(FLUSH_TIMEOUT);
|
|
}
|
|
}
|
|
}
|
|
|
|
impl TelemetryGuard {
|
|
/// True when OTLP export is active.
|
|
pub fn is_enabled(&self) -> bool {
|
|
self.providers.is_some()
|
|
}
|
|
|
|
/// Flush and shut down all providers without blocking the async runtime.
|
|
/// Call this (and await it) on graceful shutdown, after the exporter task
|
|
/// handles from `spawn_exporters` have finished. `Drop` is only a
|
|
/// best-effort fallback.
|
|
pub async fn shutdown(mut self) {
|
|
if let Some(p) = self.providers.take() {
|
|
let _ = tokio::task::spawn_blocking(move || p.flush_and_shutdown()).await;
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Drop for TelemetryGuard {
|
|
/// Best effort. Inside a tokio runtime the flush runs on a detached thread
|
|
/// (it may not finish before the process exits); outside one it runs inline.
|
|
fn drop(&mut self) {
|
|
let Some(p) = self.providers.take() else {
|
|
return;
|
|
};
|
|
if tokio::runtime::Handle::try_current().is_ok() {
|
|
let _ = std::thread::Builder::new()
|
|
.name("otel-flush".into())
|
|
.spawn(move || p.flush_and_shutdown());
|
|
} else {
|
|
p.flush_and_shutdown();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Install the tracing subscriber and, when `config.otel.enabled`, the OTLP
|
|
/// providers. Works (stderr logging only) when OTel is disabled.
|
|
pub fn init(config: &Config) -> Result<TelemetryGuard> {
|
|
let otel = &config.otel;
|
|
|
|
if !otel.enabled {
|
|
let _ = tracing_subscriber::registry()
|
|
.with(LevelFilter::INFO)
|
|
.with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
|
|
.try_init();
|
|
return Ok(TelemetryGuard { providers: None });
|
|
}
|
|
|
|
let resource = build_resource(otel);
|
|
let tracer_provider = SdkTracerProvider::builder()
|
|
.with_batch_exporter(build_span_exporter(otel)?)
|
|
.with_resource(resource.clone())
|
|
.build();
|
|
let meter = if otel.export_unit_metrics {
|
|
let reader = PeriodicReader::builder(build_metric_exporter(otel)?)
|
|
.with_interval(Duration::from_secs(otel.metrics_interval_secs.max(1)))
|
|
.build();
|
|
Some(
|
|
SdkMeterProvider::builder()
|
|
.with_reader(reader)
|
|
.with_resource(resource.clone())
|
|
.build(),
|
|
)
|
|
} else {
|
|
None
|
|
};
|
|
let logger = if otel.export_logs {
|
|
Some(
|
|
SdkLoggerProvider::builder()
|
|
.with_batch_exporter(build_log_exporter(otel)?)
|
|
.with_resource(resource)
|
|
.build(),
|
|
)
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let tracer = tracer_provider.tracer(otel.service_name.clone());
|
|
let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
|
|
let _ = tracing_subscriber::registry()
|
|
.with(LevelFilter::INFO)
|
|
.with(otel_layer)
|
|
.with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
|
|
.try_init();
|
|
|
|
let providers = Providers {
|
|
tracer: tracer_provider,
|
|
meter,
|
|
logger,
|
|
};
|
|
let _ = PROVIDERS.set(providers.clone());
|
|
Ok(TelemetryGuard {
|
|
providers: Some(providers),
|
|
})
|
|
}
|
|
|
|
/// Start the metrics poll loop and log fan-out tasks. All tasks stop on
|
|
/// `state.shutdown`. Returns no handles when export is disabled.
|
|
pub fn spawn_exporters(state: AppState) -> Vec<JoinHandle<()>> {
|
|
let mut handles = Vec::new();
|
|
if !state.config.otel.enabled {
|
|
return handles;
|
|
}
|
|
let Some(p) = PROVIDERS.get() else {
|
|
return handles;
|
|
};
|
|
if let Some(meter) = &p.meter {
|
|
handles.extend(metrics_export::spawn(state.clone(), meter));
|
|
}
|
|
if let Some(logger) = &p.logger {
|
|
handles.extend(logs_export::spawn(state, logger));
|
|
}
|
|
handles
|
|
}
|
|
|
|
fn hostname() -> String {
|
|
for path in ["/proc/sys/kernel/hostname", "/etc/hostname"] {
|
|
if let Ok(s) = std::fs::read_to_string(path) {
|
|
let s = s.trim();
|
|
if !s.is_empty() {
|
|
return s.to_owned();
|
|
}
|
|
}
|
|
}
|
|
"unknown".to_owned()
|
|
}
|
|
|
|
fn build_resource(otel: &OtelConfig) -> Resource {
|
|
Resource::builder()
|
|
.with_service_name(otel.service_name.clone())
|
|
.with_attribute(KeyValue::new(
|
|
"service.version",
|
|
env!("CARGO_PKG_VERSION").to_owned(),
|
|
))
|
|
.with_attribute(KeyValue::new("host.name", hostname()))
|
|
.build()
|
|
}
|
|
|
|
/// Endpoint for one signal. gRPC uses the base URL. HTTP appends
|
|
/// `/v1/<signal>` unless the configured endpoint already ends with it.
|
|
fn signal_endpoint(otel: &OtelConfig, signal: &str) -> String {
|
|
match otel.protocol {
|
|
OtelProtocol::Grpc => otel.endpoint.clone(),
|
|
OtelProtocol::HttpProtobuf => {
|
|
let base = otel.endpoint.trim_end_matches('/');
|
|
if base.ends_with(&format!("/v1/{signal}")) {
|
|
base.to_owned()
|
|
} else {
|
|
format!("{base}/v1/{signal}")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn header_map(otel: &OtelConfig) -> HashMap<String, String> {
|
|
otel.headers
|
|
.iter()
|
|
.map(|(k, v)| (k.clone(), v.clone()))
|
|
.collect()
|
|
}
|
|
|
|
fn metadata(otel: &OtelConfig) -> Result<opentelemetry_otlp::tonic_types::metadata::MetadataMap> {
|
|
use opentelemetry_otlp::tonic_types::metadata::MetadataMap;
|
|
let mut m = MetadataMap::new();
|
|
for (k, v) in &otel.headers {
|
|
// Metadata keys must be `'static`; the config lives for the whole process
|
|
// and `init` runs once, so leaking the few header names is fine.
|
|
let key: &'static str = Box::leak(k.to_ascii_lowercase().into_boxed_str());
|
|
let val = v
|
|
.parse()
|
|
.with_context(|| format!("invalid value for otel header '{k}'"))?;
|
|
m.insert(key, val);
|
|
}
|
|
Ok(m)
|
|
}
|
|
|
|
macro_rules! build_exporter {
|
|
($ty:ident, $otel:expr, $signal:expr) => {{
|
|
let otel: &OtelConfig = $otel;
|
|
let ep = signal_endpoint(otel, $signal);
|
|
let res = match otel.protocol {
|
|
OtelProtocol::Grpc => $ty::builder()
|
|
.with_tonic()
|
|
.with_endpoint(ep)
|
|
.with_metadata(metadata(otel)?)
|
|
.build(),
|
|
OtelProtocol::HttpProtobuf => $ty::builder()
|
|
.with_http()
|
|
.with_protocol(Protocol::HttpBinary)
|
|
.with_endpoint(ep)
|
|
.with_headers(header_map(otel))
|
|
.build(),
|
|
};
|
|
res.with_context(|| format!("building OTLP {} exporter", $signal))
|
|
}};
|
|
}
|
|
|
|
fn build_span_exporter(otel: &OtelConfig) -> Result<SpanExporter> {
|
|
build_exporter!(SpanExporter, otel, "traces")
|
|
}
|
|
|
|
fn build_metric_exporter(otel: &OtelConfig) -> Result<MetricExporter> {
|
|
build_exporter!(MetricExporter, otel, "metrics")
|
|
}
|
|
|
|
fn build_log_exporter(otel: &OtelConfig) -> Result<LogExporter> {
|
|
build_exporter!(LogExporter, otel, "logs")
|
|
}
|
|
|
|
/// Shell-style glob match supporting `*` and `?`.
|
|
pub(crate) fn glob_match(pattern: &str, text: &str) -> bool {
|
|
let p: Vec<char> = pattern.chars().collect();
|
|
let t: Vec<char> = text.chars().collect();
|
|
let (mut pi, mut ti) = (0, 0);
|
|
let (mut star, mut mark) = (None, 0);
|
|
while ti < t.len() {
|
|
if pi < p.len() && (p[pi] == '?' || p[pi] == t[ti]) {
|
|
pi += 1;
|
|
ti += 1;
|
|
} else if pi < p.len() && p[pi] == '*' {
|
|
star = Some(pi);
|
|
mark = ti;
|
|
pi += 1;
|
|
} else if let Some(s) = star {
|
|
pi = s + 1;
|
|
mark += 1;
|
|
ti = mark;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
while pi < p.len() && p[pi] == '*' {
|
|
pi += 1;
|
|
}
|
|
pi == p.len()
|
|
}
|
|
|
|
pub(crate) fn any_glob(patterns: &[String], text: &str) -> bool {
|
|
patterns.iter().any(|p| glob_match(p, text))
|
|
}
|
|
|
|
/// Exponential backoff for restarting failed export tasks.
|
|
#[derive(Debug, Clone)]
|
|
pub(crate) struct Backoff {
|
|
cur: Duration,
|
|
min: Duration,
|
|
max: Duration,
|
|
}
|
|
|
|
impl Backoff {
|
|
pub(crate) fn new(min: Duration, max: Duration) -> Self {
|
|
Self { cur: min, min, max }
|
|
}
|
|
|
|
pub(crate) fn next_delay(&mut self) -> Duration {
|
|
let d = self.cur;
|
|
self.cur = (self.cur * 2).min(self.max);
|
|
d
|
|
}
|
|
|
|
pub(crate) fn reset(&mut self) {
|
|
self.cur = self.min;
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn glob() {
|
|
assert!(glob_match("*.service", "sshd.service"));
|
|
assert!(!glob_match("*.service", "sshd.timer"));
|
|
assert!(glob_match("web-?", "web-1"));
|
|
assert!(!glob_match("web-?", "web-12"));
|
|
assert!(glob_match("a*b*c", "aXXbYYc"));
|
|
assert!(!glob_match("a*b*c", "aXXbYY"));
|
|
assert!(glob_match("*", ""));
|
|
assert!(glob_match("exact", "exact"));
|
|
assert!(!glob_match("exact", "exactly"));
|
|
}
|
|
|
|
#[test]
|
|
fn backoff_doubles_and_caps() {
|
|
let mut b = Backoff::new(Duration::from_secs(1), Duration::from_secs(5));
|
|
let v: Vec<u64> = (0..5).map(|_| b.next_delay().as_secs()).collect();
|
|
assert_eq!(v, [1, 2, 4, 5, 5]);
|
|
b.reset();
|
|
assert_eq!(b.next_delay().as_secs(), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn http_endpoint_gets_signal_path() {
|
|
let mut o = OtelConfig {
|
|
protocol: OtelProtocol::HttpProtobuf,
|
|
endpoint: "http://localhost:4318/".into(),
|
|
..Default::default()
|
|
};
|
|
assert_eq!(signal_endpoint(&o, "logs"), "http://localhost:4318/v1/logs");
|
|
o.endpoint = "http://c:4318/v1/logs".into();
|
|
assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs");
|
|
o.protocol = OtelProtocol::Grpc;
|
|
assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn exporters_build_for_both_protocols() {
|
|
for protocol in [OtelProtocol::Grpc, OtelProtocol::HttpProtobuf] {
|
|
let mut o = OtelConfig {
|
|
protocol,
|
|
endpoint: "http://127.0.0.1:1".into(),
|
|
..Default::default()
|
|
};
|
|
o.headers.insert("Authorization".into(), "Bearer x".into());
|
|
build_span_exporter(&o).unwrap();
|
|
build_metric_exporter(&o).unwrap();
|
|
build_log_exporter(&o).unwrap();
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn disabled_init_is_noop_guard() {
|
|
let g = init(&Config::default()).unwrap();
|
|
assert!(!g.is_enabled());
|
|
}
|
|
}
|