WS-G: per-layer filters so only app spans feed the trace exporter
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
1 parent
3a0084ac3a
commit
c69745e579
1 file changed
+82
-6
+82
-6
@@ -24,9 +24,10 @@ use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
|
|||||||
use opentelemetry_sdk::trace::SdkTracerProvider;
|
use opentelemetry_sdk::trace::SdkTracerProvider;
|
||||||
use opentelemetry_sdk::Resource;
|
use opentelemetry_sdk::Resource;
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
use tracing_subscriber::filter::LevelFilter;
|
use tracing_subscriber::filter::EnvFilter;
|
||||||
use tracing_subscriber::layer::SubscriberExt;
|
use tracing_subscriber::layer::SubscriberExt;
|
||||||
use tracing_subscriber::util::SubscriberInitExt;
|
use tracing_subscriber::util::SubscriberInitExt;
|
||||||
|
use tracing_subscriber::Layer;
|
||||||
|
|
||||||
use crate::config::{Config, OtelConfig, OtelProtocol};
|
use crate::config::{Config, OtelConfig, OtelProtocol};
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
@@ -112,8 +113,11 @@ pub fn init(config: &Config) -> Result<TelemetryGuard> {
|
|||||||
|
|
||||||
if !otel.enabled {
|
if !otel.enabled {
|
||||||
let _ = tracing_subscriber::registry()
|
let _ = tracing_subscriber::registry()
|
||||||
.with(LevelFilter::INFO)
|
.with(
|
||||||
.with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
|
tracing_subscriber::fmt::layer()
|
||||||
|
.with_writer(std::io::stderr)
|
||||||
|
.with_filter(fmt_filter()),
|
||||||
|
)
|
||||||
.try_init();
|
.try_init();
|
||||||
return Ok(TelemetryGuard { providers: None });
|
return Ok(TelemetryGuard { providers: None });
|
||||||
}
|
}
|
||||||
@@ -148,11 +152,19 @@ pub fn init(config: &Config) -> Result<TelemetryGuard> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let tracer = tracer_provider.tracer(otel.service_name.clone());
|
let tracer = tracer_provider.tracer(otel.service_name.clone());
|
||||||
let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
|
// Only this app's spans feed the trace exporter. Dependency spans (zbus,
|
||||||
|
// hyper, tonic, reqwest, h2) must never reach it, which also prevents
|
||||||
|
// export-of-export feedback.
|
||||||
|
let otel_layer = tracing_opentelemetry::layer()
|
||||||
|
.with_tracer(tracer)
|
||||||
|
.with_filter(trace_filter());
|
||||||
let _ = tracing_subscriber::registry()
|
let _ = tracing_subscriber::registry()
|
||||||
.with(LevelFilter::INFO)
|
|
||||||
.with(otel_layer)
|
.with(otel_layer)
|
||||||
.with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
|
.with(
|
||||||
|
tracing_subscriber::fmt::layer()
|
||||||
|
.with_writer(std::io::stderr)
|
||||||
|
.with_filter(fmt_filter()),
|
||||||
|
)
|
||||||
.try_init();
|
.try_init();
|
||||||
|
|
||||||
let providers = Providers {
|
let providers = Providers {
|
||||||
@@ -185,6 +197,27 @@ pub fn spawn_exporters(state: AppState) -> Vec<JoinHandle<()>> {
|
|||||||
handles
|
handles
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Default directive for the OTel trace layer: our crates at info, everything
|
||||||
|
/// else at warn and above (which excludes dependency info/debug spans).
|
||||||
|
pub const DEFAULT_TRACE_FILTER: &str = "warn,skyg_linux_tracker=info,slinuxm=info";
|
||||||
|
/// Env var overriding the OTel trace layer directive.
|
||||||
|
pub const TRACE_FILTER_ENV: &str = "SKYG_OTEL_TRACE_FILTER";
|
||||||
|
|
||||||
|
/// stderr logging honours `RUST_LOG`, defaulting to `info`.
|
||||||
|
fn fmt_filter() -> EnvFilter {
|
||||||
|
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse a trace directive, falling back to the default when invalid.
|
||||||
|
fn parse_trace_filter(spec: Option<&str>) -> EnvFilter {
|
||||||
|
spec.and_then(|s| EnvFilter::try_new(s).ok())
|
||||||
|
.unwrap_or_else(|| EnvFilter::new(DEFAULT_TRACE_FILTER))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn trace_filter() -> EnvFilter {
|
||||||
|
parse_trace_filter(std::env::var(TRACE_FILTER_ENV).ok().as_deref())
|
||||||
|
}
|
||||||
|
|
||||||
fn hostname() -> String {
|
fn hostname() -> String {
|
||||||
for path in ["/proc/sys/kernel/hostname", "/etc/hostname"] {
|
for path in ["/proc/sys/kernel/hostname", "/etc/hostname"] {
|
||||||
if let Ok(s) = std::fs::read_to_string(path) {
|
if let Ok(s) = std::fs::read_to_string(path) {
|
||||||
@@ -390,6 +423,49 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn trace_filter_excludes_dependency_spans() {
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
struct Count(Arc<AtomicUsize>);
|
||||||
|
impl<S: tracing::Subscriber> Layer<S> for Count {
|
||||||
|
fn on_new_span(
|
||||||
|
&self,
|
||||||
|
_: &tracing::span::Attributes<'_>,
|
||||||
|
_: &tracing::span::Id,
|
||||||
|
_: tracing_subscriber::layer::Context<'_, S>,
|
||||||
|
) {
|
||||||
|
self.0.fetch_add(1, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let n = Arc::new(AtomicUsize::new(0));
|
||||||
|
let sub = tracing_subscriber::registry()
|
||||||
|
.with(Count(n.clone()).with_filter(parse_trace_filter(None)));
|
||||||
|
tracing::subscriber::with_default(sub, || {
|
||||||
|
tracing::info_span!(target: "zbus::connection", "dep").in_scope(|| {});
|
||||||
|
tracing::info_span!(target: "h2::codec", "dep").in_scope(|| {});
|
||||||
|
tracing::info_span!(target: "tonic::transport", "dep").in_scope(|| {});
|
||||||
|
tracing::info_span!(target: "reqwest::connect", "dep").in_scope(|| {});
|
||||||
|
assert_eq!(n.load(Ordering::SeqCst), 0);
|
||||||
|
tracing::info_span!(target: "skyg_linux_tracker::api", "ours").in_scope(|| {});
|
||||||
|
assert_eq!(n.load(Ordering::SeqCst), 1);
|
||||||
|
tracing::info_span!(target: "hyper_util::client", "dep").in_scope(|| {});
|
||||||
|
tracing::debug_span!(target: "skyg_linux_tracker::api", "debug").in_scope(|| {});
|
||||||
|
assert_eq!(n.load(Ordering::SeqCst), 1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn invalid_trace_filter_falls_back() {
|
||||||
|
let f = parse_trace_filter(Some("=[bad"));
|
||||||
|
assert_eq!(
|
||||||
|
f.to_string(),
|
||||||
|
EnvFilter::new(DEFAULT_TRACE_FILTER).to_string()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn disabled_init_is_noop_guard() {
|
fn disabled_init_is_noop_guard() {
|
||||||
let g = init(&Config::default()).unwrap();
|
let g = init(&Config::default()).unwrap();
|
||||||
|
|||||||
Reference in new issue
Block a user