E: E2 outbound peer spans and traceparent injection helpers
This commit is contained in:
1 parent
eee253b162
commit
5cb2c43e39
1 file changed
+182
@@ -197,6 +197,42 @@ pub fn spawn_exporters(state: AppState) -> Vec<JoinHandle<()>> {
|
||||
handles
|
||||
}
|
||||
|
||||
/// Span for one outbound peer request.
|
||||
///
|
||||
/// The peer name is the only attribute: peer URLs may embed hostnames the
|
||||
/// operator treats as sensitive, and a bounded name keeps cardinality flat.
|
||||
/// Callers enter this span (or instrument the request future with it) before
|
||||
/// building [`crate::peers::ProxyOpts`], so the injected `traceparent` names
|
||||
/// this span as the peer request's parent on the other side.
|
||||
pub fn peer_request_span(peer: &str) -> tracing::Span {
|
||||
tracing::info_span!("peer.request", "peer.name" = peer)
|
||||
}
|
||||
|
||||
/// W3C `traceparent` for the currently active span, or `None` when there is no
|
||||
/// valid OpenTelemetry context (export disabled, or outside any exported span).
|
||||
///
|
||||
/// The proxy layer carries the value in `ProxyOpts.traceparent`; the peer
|
||||
/// continues the trace under its own server span.
|
||||
pub fn inject_traceparent() -> Option<String> {
|
||||
use opentelemetry::trace::TraceContextExt as _;
|
||||
use tracing_opentelemetry::OpenTelemetrySpanExt as _;
|
||||
|
||||
let span_context = tracing::Span::current()
|
||||
.context()
|
||||
.span()
|
||||
.span_context()
|
||||
.clone();
|
||||
if !span_context.is_valid() {
|
||||
return None;
|
||||
}
|
||||
Some(format!(
|
||||
"00-{}-{}-{:02x}",
|
||||
span_context.trace_id(),
|
||||
span_context.span_id(),
|
||||
span_context.trace_flags().to_u8()
|
||||
))
|
||||
}
|
||||
|
||||
/// Default directive for the OTel trace layer: our crates at info, everything
|
||||
/// else at warn and above (which excludes dependency info/debug spans).
|
||||
pub const DEFAULT_TRACE_FILTER: &str = "warn,skyg_linux_tracker=info,slinuxm=info";
|
||||
@@ -457,6 +493,152 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
use crate::peers::testing::{fake_peers_config, registry_with, FakeTransport};
|
||||
use crate::peers::ProxyOpts;
|
||||
use opentelemetry::trace::TraceContextExt as _;
|
||||
use tracing::Instrument as _;
|
||||
use tracing_opentelemetry::OpenTelemetrySpanExt as _;
|
||||
|
||||
/// Tracer provider without an exporter plus the matching dispatcher: spans
|
||||
/// get real context (ids, flags) that the injector can read.
|
||||
fn otel_dispatch() -> (SdkTracerProvider, tracing::Dispatch) {
|
||||
let provider = SdkTracerProvider::builder().build();
|
||||
let tracer = provider.tracer("test");
|
||||
let subscriber =
|
||||
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||
(provider, tracing::Dispatch::new(subscriber))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn traceparent_encodes_the_active_span() {
|
||||
let (_provider, dispatch) = otel_dispatch();
|
||||
let _guard = tracing::dispatcher::set_default(&dispatch);
|
||||
assert_eq!(
|
||||
inject_traceparent(),
|
||||
None,
|
||||
"no span means nothing to inject"
|
||||
);
|
||||
|
||||
let span = peer_request_span("beta");
|
||||
let header = span
|
||||
.in_scope(inject_traceparent)
|
||||
.expect("valid span context");
|
||||
let parts: Vec<&str> = header.split('-').collect();
|
||||
assert_eq!(parts.len(), 4);
|
||||
assert_eq!(parts[0], "00");
|
||||
assert_eq!(parts[1].len(), 32);
|
||||
assert_eq!(parts[2].len(), 16);
|
||||
assert_eq!(parts[3].len(), 2);
|
||||
|
||||
let ctx = span.context().span().span_context().clone();
|
||||
assert!(ctx.is_valid());
|
||||
assert_eq!(parts[1], ctx.trace_id().to_string());
|
||||
assert_eq!(parts[2], ctx.span_id().to_string());
|
||||
assert_eq!(parts[3], format!("{:02x}", ctx.trace_flags().to_u8()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn traceparent_is_none_without_an_otel_layer() {
|
||||
let subscriber = tracing_subscriber::registry();
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
let span = peer_request_span("beta");
|
||||
assert_eq!(span.in_scope(inject_traceparent), None);
|
||||
}
|
||||
|
||||
type SpanFields = Vec<(String, String)>;
|
||||
type Recordings = std::sync::Arc<std::sync::Mutex<Vec<(String, SpanFields)>>>;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct Recorded(Recordings);
|
||||
|
||||
impl<S: tracing::Subscriber> Layer<S> for Recorded {
|
||||
fn on_new_span(
|
||||
&self,
|
||||
attrs: &tracing::span::Attributes<'_>,
|
||||
_id: &tracing::span::Id,
|
||||
_ctx: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
#[derive(Default)]
|
||||
struct Visitor(SpanFields);
|
||||
impl tracing::field::Visit for Visitor {
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
self.0.push((field.name().to_owned(), value.to_owned()));
|
||||
}
|
||||
fn record_debug(
|
||||
&mut self,
|
||||
field: &tracing::field::Field,
|
||||
value: &dyn std::fmt::Debug,
|
||||
) {
|
||||
self.0.push((field.name().to_owned(), format!("{value:?}")));
|
||||
}
|
||||
}
|
||||
let mut fields = Visitor::default();
|
||||
attrs.record(&mut fields);
|
||||
self.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((attrs.metadata().name().to_owned(), fields.0));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_request_span_names_the_peer() {
|
||||
let recorded = Recorded::default();
|
||||
let subscriber = tracing_subscriber::registry().with(recorded.clone());
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
let _ = peer_request_span("beta");
|
||||
});
|
||||
let spans = recorded.0.lock().unwrap();
|
||||
assert_eq!(spans.len(), 1);
|
||||
assert_eq!(spans[0].0, "peer.request");
|
||||
assert_eq!(
|
||||
spans[0].1,
|
||||
vec![("peer.name".to_owned(), "beta".to_owned())]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn traceparent_reaches_the_peer_request() {
|
||||
let (_provider, dispatch) = otel_dispatch();
|
||||
let _guard = tracing::dispatcher::set_default(&dispatch);
|
||||
|
||||
let fake = FakeTransport::new();
|
||||
let reg = registry_with(fake.clone(), fake_peers_config(&["beta"]));
|
||||
let span = peer_request_span("beta");
|
||||
let span_id = span.context().span().span_context().span_id().to_string();
|
||||
|
||||
let (injected, status) = async {
|
||||
let opts = ProxyOpts {
|
||||
traceparent: inject_traceparent(),
|
||||
accept_sse: false,
|
||||
};
|
||||
let resp = reg.get("beta", "/systemd/units", &[], opts).await.unwrap();
|
||||
(inject_traceparent(), resp.status)
|
||||
}
|
||||
.instrument(span)
|
||||
.await;
|
||||
assert_eq!(status, 404);
|
||||
|
||||
let calls = fake.calls();
|
||||
assert_eq!(calls.len(), 1);
|
||||
let headers = &calls[0].1.headers;
|
||||
let header = |name: &str| {
|
||||
headers
|
||||
.iter()
|
||||
.find(|(k, _)| k.as_str() == name)
|
||||
.map(|(_, v)| v.to_str().unwrap().to_owned())
|
||||
};
|
||||
let sent = header("traceparent").expect("traceparent header");
|
||||
assert_eq!(Some(sent.clone()), injected);
|
||||
let parts: Vec<&str> = sent.split('-').collect();
|
||||
assert_eq!(parts.len(), 4);
|
||||
assert_eq!(
|
||||
parts[2], span_id,
|
||||
"the peer request is a child of the outbound span"
|
||||
);
|
||||
assert_eq!(header("authorization"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_trace_filter_falls_back() {
|
||||
let f = parse_trace_filter(Some("=[bad"));
|
||||
|
||||
Reference in new issue
Block a user