diff --git a/src/telemetry/mod.rs b/src/telemetry/mod.rs index 8f47d91..2fb0aa3 100644 --- a/src/telemetry/mod.rs +++ b/src/telemetry/mod.rs @@ -197,6 +197,42 @@ pub fn spawn_exporters(state: AppState) -> Vec> { handles } +/// Span for one outbound peer request. +/// +/// The peer name is the only attribute: peer URLs may embed hostnames the +/// operator treats as sensitive, and a bounded name keeps cardinality flat. +/// Callers enter this span (or instrument the request future with it) before +/// building [`crate::peers::ProxyOpts`], so the injected `traceparent` names +/// this span as the peer request's parent on the other side. +pub fn peer_request_span(peer: &str) -> tracing::Span { + tracing::info_span!("peer.request", "peer.name" = peer) +} + +/// W3C `traceparent` for the currently active span, or `None` when there is no +/// valid OpenTelemetry context (export disabled, or outside any exported span). +/// +/// The proxy layer carries the value in `ProxyOpts.traceparent`; the peer +/// continues the trace under its own server span. +pub fn inject_traceparent() -> Option { + use opentelemetry::trace::TraceContextExt as _; + use tracing_opentelemetry::OpenTelemetrySpanExt as _; + + let span_context = tracing::Span::current() + .context() + .span() + .span_context() + .clone(); + if !span_context.is_valid() { + return None; + } + Some(format!( + "00-{}-{}-{:02x}", + span_context.trace_id(), + span_context.span_id(), + span_context.trace_flags().to_u8() + )) +} + /// Default directive for the OTel trace layer: our crates at info, everything /// else at warn and above (which excludes dependency info/debug spans). pub const DEFAULT_TRACE_FILTER: &str = "warn,skyg_linux_tracker=info,slinuxm=info"; @@ -457,6 +493,152 @@ mod tests { }); } + use crate::peers::testing::{fake_peers_config, registry_with, FakeTransport}; + use crate::peers::ProxyOpts; + use opentelemetry::trace::TraceContextExt as _; + use tracing::Instrument as _; + use tracing_opentelemetry::OpenTelemetrySpanExt as _; + + /// Tracer provider without an exporter plus the matching dispatcher: spans + /// get real context (ids, flags) that the injector can read. + fn otel_dispatch() -> (SdkTracerProvider, tracing::Dispatch) { + let provider = SdkTracerProvider::builder().build(); + let tracer = provider.tracer("test"); + let subscriber = + tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); + (provider, tracing::Dispatch::new(subscriber)) + } + + #[test] + fn traceparent_encodes_the_active_span() { + let (_provider, dispatch) = otel_dispatch(); + let _guard = tracing::dispatcher::set_default(&dispatch); + assert_eq!( + inject_traceparent(), + None, + "no span means nothing to inject" + ); + + let span = peer_request_span("beta"); + let header = span + .in_scope(inject_traceparent) + .expect("valid span context"); + let parts: Vec<&str> = header.split('-').collect(); + assert_eq!(parts.len(), 4); + assert_eq!(parts[0], "00"); + assert_eq!(parts[1].len(), 32); + assert_eq!(parts[2].len(), 16); + assert_eq!(parts[3].len(), 2); + + let ctx = span.context().span().span_context().clone(); + assert!(ctx.is_valid()); + assert_eq!(parts[1], ctx.trace_id().to_string()); + assert_eq!(parts[2], ctx.span_id().to_string()); + assert_eq!(parts[3], format!("{:02x}", ctx.trace_flags().to_u8())); + } + + #[test] + fn traceparent_is_none_without_an_otel_layer() { + let subscriber = tracing_subscriber::registry(); + let _guard = tracing::subscriber::set_default(subscriber); + let span = peer_request_span("beta"); + assert_eq!(span.in_scope(inject_traceparent), None); + } + + type SpanFields = Vec<(String, String)>; + type Recordings = std::sync::Arc>>; + + #[derive(Clone, Default)] + struct Recorded(Recordings); + + impl Layer for Recorded { + fn on_new_span( + &self, + attrs: &tracing::span::Attributes<'_>, + _id: &tracing::span::Id, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + #[derive(Default)] + struct Visitor(SpanFields); + impl tracing::field::Visit for Visitor { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + self.0.push((field.name().to_owned(), value.to_owned())); + } + fn record_debug( + &mut self, + field: &tracing::field::Field, + value: &dyn std::fmt::Debug, + ) { + self.0.push((field.name().to_owned(), format!("{value:?}"))); + } + } + let mut fields = Visitor::default(); + attrs.record(&mut fields); + self.0 + .lock() + .unwrap() + .push((attrs.metadata().name().to_owned(), fields.0)); + } + } + + #[test] + fn peer_request_span_names_the_peer() { + let recorded = Recorded::default(); + let subscriber = tracing_subscriber::registry().with(recorded.clone()); + tracing::subscriber::with_default(subscriber, || { + let _ = peer_request_span("beta"); + }); + let spans = recorded.0.lock().unwrap(); + assert_eq!(spans.len(), 1); + assert_eq!(spans[0].0, "peer.request"); + assert_eq!( + spans[0].1, + vec![("peer.name".to_owned(), "beta".to_owned())] + ); + } + + #[tokio::test] + async fn traceparent_reaches_the_peer_request() { + let (_provider, dispatch) = otel_dispatch(); + let _guard = tracing::dispatcher::set_default(&dispatch); + + let fake = FakeTransport::new(); + let reg = registry_with(fake.clone(), fake_peers_config(&["beta"])); + let span = peer_request_span("beta"); + let span_id = span.context().span().span_context().span_id().to_string(); + + let (injected, status) = async { + let opts = ProxyOpts { + traceparent: inject_traceparent(), + accept_sse: false, + }; + let resp = reg.get("beta", "/systemd/units", &[], opts).await.unwrap(); + (inject_traceparent(), resp.status) + } + .instrument(span) + .await; + assert_eq!(status, 404); + + let calls = fake.calls(); + assert_eq!(calls.len(), 1); + let headers = &calls[0].1.headers; + let header = |name: &str| { + headers + .iter() + .find(|(k, _)| k.as_str() == name) + .map(|(_, v)| v.to_str().unwrap().to_owned()) + }; + let sent = header("traceparent").expect("traceparent header"); + assert_eq!(Some(sent.clone()), injected); + let parts: Vec<&str> = sent.split('-').collect(); + assert_eq!(parts.len(), 4); + assert_eq!( + parts[2], span_id, + "the peer request is a child of the outbound span" + ); + assert_eq!(header("authorization"), None); + } + #[test] fn invalid_trace_filter_falls_back() { let f = parse_trace_filter(Some("=[bad"));