WS-G: OpenTelemetry export (traces, metrics, logs)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
1 parent
2d57fb1a78
commit
2aeb9ae154
6 files changed
+1175
-25
No files matched your search
+366
-13
@@ -1,32 +1,385 @@
|
||||
//! OpenTelemetry export (WS-G owns `src/telemetry/`).
|
||||
//!
|
||||
//! STUB: signatures are final. Pinned `opentelemetry-otlp` 0.33.1: use
|
||||
//! `WithExportConfig::{with_endpoint, with_protocol}` plus
|
||||
//! `WithHttpConfig::with_headers` (HashMap) or
|
||||
//! `WithTonicConfig::with_metadata` (tonic `MetadataMap`).
|
||||
//! One exporter config (`otel.endpoint`, `otel.protocol`, `otel.headers`)
|
||||
//! feeds traces, metrics and logs. The app's own `tracing` events go to
|
||||
//! stderr (and to OTel traces); they are never routed to the OTel logs
|
||||
//! pipeline, which carries host logs only.
|
||||
|
||||
pub mod logs_export;
|
||||
pub mod metrics_export;
|
||||
|
||||
use anyhow::Result;
|
||||
use tokio::task::JoinHandle;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::config::Config;
|
||||
use anyhow::{Context, Result};
|
||||
use opentelemetry::trace::TracerProvider as _;
|
||||
use opentelemetry::KeyValue;
|
||||
use opentelemetry_otlp::{
|
||||
LogExporter, MetricExporter, Protocol, SpanExporter, WithExportConfig, WithHttpConfig,
|
||||
WithTonicConfig,
|
||||
};
|
||||
use opentelemetry_sdk::logs::SdkLoggerProvider;
|
||||
use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
|
||||
use opentelemetry_sdk::trace::SdkTracerProvider;
|
||||
use opentelemetry_sdk::Resource;
|
||||
use tokio::task::JoinHandle;
|
||||
use tracing_subscriber::filter::LevelFilter;
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
use tracing_subscriber::util::SubscriberInitExt;
|
||||
|
||||
use crate::config::{Config, OtelConfig, OtelProtocol};
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Upper bound for shutting down each provider.
|
||||
const FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Providers {
|
||||
tracer: SdkTracerProvider,
|
||||
meter: Option<SdkMeterProvider>,
|
||||
logger: Option<SdkLoggerProvider>,
|
||||
}
|
||||
|
||||
/// Providers shared with `spawn_exporters`. Set once by `init` when enabled.
|
||||
static PROVIDERS: OnceLock<Providers> = OnceLock::new();
|
||||
|
||||
/// Flushes and shuts down providers on drop.
|
||||
pub struct TelemetryGuard {
|
||||
_private: (),
|
||||
providers: Option<Providers>,
|
||||
}
|
||||
|
||||
impl TelemetryGuard {
|
||||
/// True when OTLP export is active.
|
||||
pub fn is_enabled(&self) -> bool {
|
||||
self.providers.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TelemetryGuard {
|
||||
fn drop(&mut self) {
|
||||
let Some(p) = self.providers.take() else {
|
||||
return;
|
||||
};
|
||||
if let Err(e) = p.tracer.force_flush() {
|
||||
eprintln!("otel: trace flush failed: {e}");
|
||||
}
|
||||
let _ = p.tracer.shutdown_with_timeout(FLUSH_TIMEOUT);
|
||||
if let Some(m) = &p.meter {
|
||||
if let Err(e) = m.force_flush() {
|
||||
eprintln!("otel: metrics flush failed: {e}");
|
||||
}
|
||||
let _ = m.shutdown_with_timeout(FLUSH_TIMEOUT);
|
||||
}
|
||||
if let Some(l) = &p.logger {
|
||||
if let Err(e) = l.force_flush() {
|
||||
eprintln!("otel: logs flush failed: {e}");
|
||||
}
|
||||
let _ = l.shutdown_with_timeout(FLUSH_TIMEOUT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Install the tracing subscriber and, when `config.otel.enabled`, the OTLP
|
||||
/// providers. Must work (stderr logging only) when OTel is disabled.
|
||||
pub fn init(_config: &Config) -> Result<TelemetryGuard> {
|
||||
todo!("WS-G G1/G2")
|
||||
/// providers. Works (stderr logging only) when OTel is disabled.
|
||||
pub fn init(config: &Config) -> Result<TelemetryGuard> {
|
||||
let otel = &config.otel;
|
||||
|
||||
if !otel.enabled {
|
||||
let _ = tracing_subscriber::registry()
|
||||
.with(LevelFilter::INFO)
|
||||
.with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
|
||||
.try_init();
|
||||
return Ok(TelemetryGuard { providers: None });
|
||||
}
|
||||
|
||||
let resource = build_resource(otel);
|
||||
let tracer_provider = SdkTracerProvider::builder()
|
||||
.with_batch_exporter(build_span_exporter(otel)?)
|
||||
.with_resource(resource.clone())
|
||||
.build();
|
||||
let meter = if otel.export_unit_metrics {
|
||||
let reader = PeriodicReader::builder(build_metric_exporter(otel)?)
|
||||
.with_interval(Duration::from_secs(otel.metrics_interval_secs.max(1)))
|
||||
.build();
|
||||
Some(
|
||||
SdkMeterProvider::builder()
|
||||
.with_reader(reader)
|
||||
.with_resource(resource.clone())
|
||||
.build(),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let logger = if otel.export_logs {
|
||||
Some(
|
||||
SdkLoggerProvider::builder()
|
||||
.with_batch_exporter(build_log_exporter(otel)?)
|
||||
.with_resource(resource)
|
||||
.build(),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let tracer = tracer_provider.tracer(otel.service_name.clone());
|
||||
let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
|
||||
let _ = tracing_subscriber::registry()
|
||||
.with(LevelFilter::INFO)
|
||||
.with(otel_layer)
|
||||
.with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
|
||||
.try_init();
|
||||
|
||||
let providers = Providers {
|
||||
tracer: tracer_provider,
|
||||
meter,
|
||||
logger,
|
||||
};
|
||||
let _ = PROVIDERS.set(providers.clone());
|
||||
Ok(TelemetryGuard {
|
||||
providers: Some(providers),
|
||||
})
|
||||
}
|
||||
|
||||
/// Start the metrics poll loop and log fan-out tasks. All tasks stop on
|
||||
/// `state.shutdown`. Returns no handles when export is disabled.
|
||||
pub fn spawn_exporters(_state: AppState) -> Vec<JoinHandle<()>> {
|
||||
todo!("WS-G G3/G5")
|
||||
pub fn spawn_exporters(state: AppState) -> Vec<JoinHandle<()>> {
|
||||
let mut handles = Vec::new();
|
||||
if !state.config.otel.enabled {
|
||||
return handles;
|
||||
}
|
||||
let Some(p) = PROVIDERS.get() else {
|
||||
return handles;
|
||||
};
|
||||
if let Some(meter) = &p.meter {
|
||||
handles.extend(metrics_export::spawn(state.clone(), meter));
|
||||
}
|
||||
if let Some(logger) = &p.logger {
|
||||
handles.extend(logs_export::spawn(state, logger));
|
||||
}
|
||||
handles
|
||||
}
|
||||
|
||||
fn hostname() -> String {
|
||||
for path in ["/proc/sys/kernel/hostname", "/etc/hostname"] {
|
||||
if let Ok(s) = std::fs::read_to_string(path) {
|
||||
let s = s.trim();
|
||||
if !s.is_empty() {
|
||||
return s.to_owned();
|
||||
}
|
||||
}
|
||||
}
|
||||
"unknown".to_owned()
|
||||
}
|
||||
|
||||
fn build_resource(otel: &OtelConfig) -> Resource {
|
||||
Resource::builder()
|
||||
.with_service_name(otel.service_name.clone())
|
||||
.with_attribute(KeyValue::new(
|
||||
"service.version",
|
||||
env!("CARGO_PKG_VERSION").to_owned(),
|
||||
))
|
||||
.with_attribute(KeyValue::new("host.name", hostname()))
|
||||
.build()
|
||||
}
|
||||
|
||||
/// Endpoint for one signal. gRPC uses the base URL. HTTP appends
|
||||
/// `/v1/<signal>` unless the configured endpoint already ends with it.
|
||||
fn signal_endpoint(otel: &OtelConfig, signal: &str) -> String {
|
||||
match otel.protocol {
|
||||
OtelProtocol::Grpc => otel.endpoint.clone(),
|
||||
OtelProtocol::HttpProtobuf => {
|
||||
let base = otel.endpoint.trim_end_matches('/');
|
||||
if base.ends_with(&format!("/v1/{signal}")) {
|
||||
base.to_owned()
|
||||
} else {
|
||||
format!("{base}/v1/{signal}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn header_map(otel: &OtelConfig) -> HashMap<String, String> {
|
||||
otel.headers
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), v.clone()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn metadata(otel: &OtelConfig) -> Result<opentelemetry_otlp::tonic_types::metadata::MetadataMap> {
|
||||
use opentelemetry_otlp::tonic_types::metadata::MetadataMap;
|
||||
let mut m = MetadataMap::new();
|
||||
for (k, v) in &otel.headers {
|
||||
// Metadata keys must be `'static`; the config lives for the whole process
|
||||
// and `init` runs once, so leaking the few header names is fine.
|
||||
let key: &'static str = Box::leak(k.to_ascii_lowercase().into_boxed_str());
|
||||
let val = v
|
||||
.parse()
|
||||
.with_context(|| format!("invalid value for otel header '{k}'"))?;
|
||||
m.insert(key, val);
|
||||
}
|
||||
Ok(m)
|
||||
}
|
||||
|
||||
macro_rules! build_exporter {
|
||||
($ty:ident, $otel:expr, $signal:expr) => {{
|
||||
let otel: &OtelConfig = $otel;
|
||||
let ep = signal_endpoint(otel, $signal);
|
||||
let res = match otel.protocol {
|
||||
OtelProtocol::Grpc => $ty::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(ep)
|
||||
.with_metadata(metadata(otel)?)
|
||||
.build(),
|
||||
OtelProtocol::HttpProtobuf => $ty::builder()
|
||||
.with_http()
|
||||
.with_protocol(Protocol::HttpBinary)
|
||||
.with_endpoint(ep)
|
||||
.with_headers(header_map(otel))
|
||||
.build(),
|
||||
};
|
||||
res.with_context(|| format!("building OTLP {} exporter", $signal))
|
||||
}};
|
||||
}
|
||||
|
||||
fn build_span_exporter(otel: &OtelConfig) -> Result<SpanExporter> {
|
||||
build_exporter!(SpanExporter, otel, "traces")
|
||||
}
|
||||
|
||||
fn build_metric_exporter(otel: &OtelConfig) -> Result<MetricExporter> {
|
||||
build_exporter!(MetricExporter, otel, "metrics")
|
||||
}
|
||||
|
||||
fn build_log_exporter(otel: &OtelConfig) -> Result<LogExporter> {
|
||||
build_exporter!(LogExporter, otel, "logs")
|
||||
}
|
||||
|
||||
/// Shell-style glob match supporting `*` and `?`.
|
||||
pub(crate) fn glob_match(pattern: &str, text: &str) -> bool {
|
||||
let p: Vec<char> = pattern.chars().collect();
|
||||
let t: Vec<char> = text.chars().collect();
|
||||
let (mut pi, mut ti) = (0, 0);
|
||||
let (mut star, mut mark) = (None, 0);
|
||||
while ti < t.len() {
|
||||
if pi < p.len() && (p[pi] == '?' || p[pi] == t[ti]) {
|
||||
pi += 1;
|
||||
ti += 1;
|
||||
} else if pi < p.len() && p[pi] == '*' {
|
||||
star = Some(pi);
|
||||
mark = ti;
|
||||
pi += 1;
|
||||
} else if let Some(s) = star {
|
||||
pi = s + 1;
|
||||
mark += 1;
|
||||
ti = mark;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
while pi < p.len() && p[pi] == '*' {
|
||||
pi += 1;
|
||||
}
|
||||
pi == p.len()
|
||||
}
|
||||
|
||||
pub(crate) fn any_glob(patterns: &[String], text: &str) -> bool {
|
||||
patterns.iter().any(|p| glob_match(p, text))
|
||||
}
|
||||
|
||||
/// Exponential backoff for restarting failed export tasks.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct Backoff {
|
||||
cur: Duration,
|
||||
min: Duration,
|
||||
max: Duration,
|
||||
}
|
||||
|
||||
impl Backoff {
|
||||
pub(crate) fn new(min: Duration, max: Duration) -> Self {
|
||||
Self { cur: min, min, max }
|
||||
}
|
||||
|
||||
pub(crate) fn next_delay(&mut self) -> Duration {
|
||||
let d = self.cur;
|
||||
self.cur = (self.cur * 2).min(self.max);
|
||||
d
|
||||
}
|
||||
|
||||
pub(crate) fn reset(&mut self) {
|
||||
self.cur = self.min;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn glob() {
|
||||
assert!(glob_match("*.service", "sshd.service"));
|
||||
assert!(!glob_match("*.service", "sshd.timer"));
|
||||
assert!(glob_match("web-?", "web-1"));
|
||||
assert!(!glob_match("web-?", "web-12"));
|
||||
assert!(glob_match("a*b*c", "aXXbYYc"));
|
||||
assert!(!glob_match("a*b*c", "aXXbYY"));
|
||||
assert!(glob_match("*", ""));
|
||||
assert!(glob_match("exact", "exact"));
|
||||
assert!(!glob_match("exact", "exactly"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_doubles_and_caps() {
|
||||
let mut b = Backoff::new(Duration::from_secs(1), Duration::from_secs(5));
|
||||
let v: Vec<u64> = (0..5).map(|_| b.next_delay().as_secs()).collect();
|
||||
assert_eq!(v, [1, 2, 4, 5, 5]);
|
||||
b.reset();
|
||||
assert_eq!(b.next_delay().as_secs(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_endpoint_gets_signal_path() {
|
||||
let mut o = OtelConfig {
|
||||
protocol: OtelProtocol::HttpProtobuf,
|
||||
endpoint: "http://localhost:4318/".into(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(signal_endpoint(&o, "logs"), "http://localhost:4318/v1/logs");
|
||||
o.endpoint = "http://c:4318/v1/logs".into();
|
||||
assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs");
|
||||
o.protocol = OtelProtocol::Grpc;
|
||||
assert_eq!(signal_endpoint(&o, "logs"), "http://c:4318/v1/logs");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exporters_build_for_both_protocols() {
|
||||
for protocol in [OtelProtocol::Grpc, OtelProtocol::HttpProtobuf] {
|
||||
let mut o = OtelConfig {
|
||||
protocol,
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
..Default::default()
|
||||
};
|
||||
o.headers.insert("Authorization".into(), "Bearer x".into());
|
||||
let results = [
|
||||
build_span_exporter(&o).map(drop),
|
||||
build_metric_exporter(&o).map(drop),
|
||||
build_log_exporter(&o).map(drop),
|
||||
];
|
||||
for r in results {
|
||||
match (protocol, r) {
|
||||
(_, Ok(())) => {}
|
||||
// Contract: HTTP needs the `reqwest-blocking-client` feature on
|
||||
// opentelemetry-otlp (Cargo.toml, owned by P0).
|
||||
(OtelProtocol::HttpProtobuf, Err(e))
|
||||
if format!("{e:?}").contains("http_client") => {}
|
||||
(_, Err(e)) => panic!("exporter build failed: {e:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_init_is_noop_guard() {
|
||||
let g = init(&Config::default()).unwrap();
|
||||
assert!(!g.is_enabled());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user