WS-G: apply review fixes F1-F8
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
1 parent
2aeb9ae154
commit
1eb7ffce19
4 files changed
+219
-99
No files matched your search
@@ -53,4 +53,6 @@ Tick these in [`TODO.md`](../../TODO.md).
|
|||||||
## Log
|
## Log
|
||||||
|
|
||||||
- 2026-10-15: Card created.
|
- 2026-10-15: Card created.
|
||||||
|
- 2026-10-15: Review fixes F1-F8. Restart polling uses buffer_unordered(8) with a 15s budget; failed sections keep the previous snapshot; series carry identity attributes only; timer metric renamed `systemd.timer.next_elapse_timestamp_seconds`; units added; observations computed once per poll. New containers start from StartedAt, docker lines deduped on reconnect, own unit/identifier excluded from the journal follow (by name; cursor not used). `TelemetryGuard::shutdown().await` added; Drop is best-effort.
|
||||||
|
- P3 notes: (1) call `guard.shutdown().await` after awaiting the exporter handles. (2) devenv: add `processes.otelcol.exec = "otelcol-contrib --config dev/otelcol.yaml"` to devenv.nix (not done here, P0 owner). (3) Cargo.toml needs `reqwest-blocking-client` on opentelemetry-otlp for http/protobuf; the exporter-build test is strict and fails without it. (4) RUST_LOG/EnvFilter deferred.
|
||||||
- 2026-10-15: Implemented G1-G7 on `ws-g/otel`. Notes: no glob crate, so a small `*`/`?` matcher lives in `telemetry/mod.rs`. `DockerSource` has no events stream, so the log reconciler polls `ps(false)` every 5s (30s while Docker is unavailable) instead of subscribing to `events`. No `EnvFilter` (feature not enabled in Cargo.toml): level is fixed at INFO. `systemd.timer.next_elapse_seconds` exports absolute unix seconds. `systemd.unit.restarts` costs one `unit_status` per in-scope loaded service per interval. HTTP endpoints get `/v1/<signal>` appended unless already present. Live otelcol smoke test not run (no collector binary on PATH).
|
- 2026-10-15: Implemented G1-G7 on `ws-g/otel`. Notes: no glob crate, so a small `*`/`?` matcher lives in `telemetry/mod.rs`. `DockerSource` has no events stream, so the log reconciler polls `ps(false)` every 5s (30s while Docker is unavailable) instead of subscribing to `events`. No `EnvFilter` (feature not enabled in Cargo.toml): level is fixed at INFO. `systemd.timer.next_elapse_seconds` exports absolute unix seconds. `systemd.unit.restarts` costs one `unit_status` per in-scope loaded service per interval. HTTP endpoints get `/v1/<signal>` appended unless already present. Live otelcol smoke test not run (no collector binary on PATH).
|
||||||
@@ -130,7 +130,31 @@ pub fn spawn(state: AppState, provider: &SdkLoggerProvider) -> Vec<JoinHandle<()
|
|||||||
handles
|
handles
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Names identifying this process in the journal (unit and syslog identifier).
|
||||||
|
/// Used to keep the app's own log lines out of the unfiltered follow, which
|
||||||
|
/// would otherwise feed back into itself. Limitation: matching is by name; the
|
||||||
|
/// journal cursor is not used.
|
||||||
|
pub fn own_names(service_name: &str) -> Vec<String> {
|
||||||
|
let mut v = vec![service_name.to_owned(), "skyg-linux-tracker".to_owned()];
|
||||||
|
if let Some(exe) = std::env::current_exe()
|
||||||
|
.ok()
|
||||||
|
.and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned()))
|
||||||
|
{
|
||||||
|
v.push(exe);
|
||||||
|
}
|
||||||
|
v
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_own_entry(e: &JournalEntry, own: &[String]) -> bool {
|
||||||
|
e.identifier.as_ref().is_some_and(|i| own.contains(i))
|
||||||
|
|| e.unit.as_ref().is_some_and(|u| {
|
||||||
|
own.iter()
|
||||||
|
.any(|n| u == n || u.strip_suffix(".service") == Some(n.as_str()))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn journal_task(state: AppState, logger: SdkLogger, unit: Option<String>) {
|
async fn journal_task(state: AppState, logger: SdkLogger, unit: Option<String>) {
|
||||||
|
let own = own_names(&state.config.otel.service_name);
|
||||||
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
|
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
|
||||||
let mut last: Option<Timestamp> = None;
|
let mut last: Option<Timestamp> = None;
|
||||||
let max_lines = state.config.limits.max_lines;
|
let max_lines = state.config.limits.max_lines;
|
||||||
@@ -155,6 +179,9 @@ async fn journal_task(state: AppState, logger: SdkLogger, unit: Option<String>)
|
|||||||
e = rx.recv() => e,
|
e = rx.recv() => e,
|
||||||
};
|
};
|
||||||
let Some(e) = e else { break };
|
let Some(e) = e else { break };
|
||||||
|
if is_own_entry(&e, &own) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if last.is_some_and(|l| e.timestamp <= l) {
|
if last.is_some_and(|l| e.timestamp <= l) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -210,6 +237,8 @@ async fn docker_reconciler(state: AppState, logger: SdkLogger) {
|
|||||||
let allow = state.config.otel.logs.docker_containers.clone();
|
let allow = state.config.otel.logs.docker_containers.clone();
|
||||||
let mut tasks: HashMap<String, (CancellationToken, JoinHandle<()>)> = HashMap::new();
|
let mut tasks: HashMap<String, (CancellationToken, JoinHandle<()>)> = HashMap::new();
|
||||||
let mut warned = false;
|
let mut warned = false;
|
||||||
|
let mut failures = 0u32;
|
||||||
|
let mut first_pass = true;
|
||||||
loop {
|
loop {
|
||||||
let mut delay = RECONCILE_INTERVAL;
|
let mut delay = RECONCILE_INTERVAL;
|
||||||
let ps = tokio::select! {
|
let ps = tokio::select! {
|
||||||
@@ -219,6 +248,9 @@ async fn docker_reconciler(state: AppState, logger: SdkLogger) {
|
|||||||
match ps {
|
match ps {
|
||||||
Ok(cs) => {
|
Ok(cs) => {
|
||||||
warned = false;
|
warned = false;
|
||||||
|
failures = 0;
|
||||||
|
let started: HashMap<String, Option<Timestamp>> =
|
||||||
|
cs.iter().map(|c| (c.name.clone(), c.started_at)).collect();
|
||||||
let running: Vec<String> = cs.into_iter().map(|c| c.name).collect();
|
let running: Vec<String> = cs.into_iter().map(|c| c.name).collect();
|
||||||
let tracked: Vec<String> = tasks.keys().cloned().collect();
|
let tracked: Vec<String> = tasks.keys().cloned().collect();
|
||||||
let (start, stop) = reconcile(&running, &tracked, &allow);
|
let (start, stop) = reconcile(&running, &tracked, &allow);
|
||||||
@@ -230,21 +262,35 @@ async fn docker_reconciler(state: AppState, logger: SdkLogger) {
|
|||||||
}
|
}
|
||||||
for name in start {
|
for name in start {
|
||||||
let tok = state.shutdown.child_token();
|
let tok = state.shutdown.child_token();
|
||||||
|
// Containers present at startup: only new lines. Containers that
|
||||||
|
// appear later: from their StartedAt so startup output is kept.
|
||||||
|
let from = if first_pass {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
started.get(&name).copied().flatten()
|
||||||
|
};
|
||||||
let h = tokio::spawn(container_task(
|
let h = tokio::spawn(container_task(
|
||||||
state.clone(),
|
state.clone(),
|
||||||
logger.clone(),
|
logger.clone(),
|
||||||
name.clone(),
|
name.clone(),
|
||||||
|
from,
|
||||||
tok.clone(),
|
tok.clone(),
|
||||||
));
|
));
|
||||||
tasks.insert(name, (tok, h));
|
tasks.insert(name, (tok, h));
|
||||||
}
|
}
|
||||||
|
first_pass = false;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
if !warned {
|
failures += 1;
|
||||||
tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly");
|
// Transient errors keep the normal cadence; only a persistent
|
||||||
warned = true;
|
// outage switches to the slow retry.
|
||||||
|
if failures >= 3 {
|
||||||
|
if !warned {
|
||||||
|
tracing::warn!(error = %e, "otel logs: docker unavailable, retrying slowly");
|
||||||
|
warned = true;
|
||||||
|
}
|
||||||
|
delay = DOCKER_SLOW_RETRY;
|
||||||
}
|
}
|
||||||
delay = DOCKER_SLOW_RETRY;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !sleep_or_shutdown(&state.shutdown, delay).await {
|
if !sleep_or_shutdown(&state.shutdown, delay).await {
|
||||||
@@ -261,13 +307,23 @@ async fn container_task(
|
|||||||
state: AppState,
|
state: AppState,
|
||||||
logger: SdkLogger,
|
logger: SdkLogger,
|
||||||
name: String,
|
name: String,
|
||||||
|
from: Option<Timestamp>,
|
||||||
token: CancellationToken,
|
token: CancellationToken,
|
||||||
) {
|
) {
|
||||||
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
|
let mut backoff = Backoff::new(BACKOFF_MIN, BACKOFF_MAX);
|
||||||
let mut since: Option<Timestamp> = Some(Timestamp::now());
|
// `from == None`: container was already running at startup, so only new lines.
|
||||||
|
// docker `tail=0` returns nothing regardless of `since`, so a backlog needs
|
||||||
|
// a real tail.
|
||||||
|
let mut since: Option<Timestamp> = Some(from.unwrap_or_else(Timestamp::now));
|
||||||
|
let mut tail = if from.is_some() {
|
||||||
|
state.config.limits.max_tail
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
let mut last_emitted: Option<Timestamp> = None;
|
||||||
loop {
|
loop {
|
||||||
let opts = LogOptions {
|
let opts = LogOptions {
|
||||||
tail: Some(0),
|
tail: Some(tail),
|
||||||
since,
|
since,
|
||||||
};
|
};
|
||||||
let rx = tokio::select! {
|
let rx = tokio::select! {
|
||||||
@@ -285,6 +341,11 @@ async fn container_task(
|
|||||||
let Some(l) = l else { break };
|
let Some(l) = l else { break };
|
||||||
got = true;
|
got = true;
|
||||||
if let Some(ts) = l.ts {
|
if let Some(ts) = l.ts {
|
||||||
|
// Dedupe lines replayed after a reconnect.
|
||||||
|
if last_emitted.is_some_and(|le| ts <= le) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
last_emitted = Some(ts);
|
||||||
since = Some(ts);
|
since = Some(ts);
|
||||||
}
|
}
|
||||||
emit(&logger, docker_record(&name, &l));
|
emit(&logger, docker_record(&name, &l));
|
||||||
@@ -297,6 +358,10 @@ async fn container_task(
|
|||||||
tracing::debug!(error = %e, container = %name, "otel logs: container follow failed");
|
tracing::debug!(error = %e, container = %name, "otel logs: container follow failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// After the first connection, reconnects replay from the last seen line.
|
||||||
|
if last_emitted.is_some() {
|
||||||
|
tail = state.config.limits.max_tail;
|
||||||
|
}
|
||||||
if !sleep_or_shutdown(&token, backoff.next_delay()).await {
|
if !sleep_or_shutdown(&token, backoff.next_delay()).await {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -320,6 +385,18 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn own_entries_are_excluded() {
|
||||||
|
let own = vec!["skyg-linux-tracker".to_string()];
|
||||||
|
let mut e = entry(None);
|
||||||
|
assert!(!is_own_entry(&e, &own));
|
||||||
|
e.identifier = Some("skyg-linux-tracker".into());
|
||||||
|
assert!(is_own_entry(&e, &own));
|
||||||
|
e.identifier = Some("slinuxm".into());
|
||||||
|
e.unit = Some("skyg-linux-tracker.service".into());
|
||||||
|
assert!(is_own_entry(&e, &own));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn severity_mapping() {
|
fn severity_mapping() {
|
||||||
use Priority::*;
|
use Priority::*;
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ use std::collections::HashMap;
|
|||||||
use std::sync::{Arc, RwLock};
|
use std::sync::{Arc, RwLock};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use futures::StreamExt;
|
||||||
use opentelemetry::metrics::MeterProvider as _;
|
use opentelemetry::metrics::MeterProvider as _;
|
||||||
use opentelemetry::KeyValue;
|
use opentelemetry::KeyValue;
|
||||||
use opentelemetry_sdk::metrics::SdkMeterProvider;
|
use opentelemetry_sdk::metrics::SdkMeterProvider;
|
||||||
@@ -21,17 +22,23 @@ use crate::state::AppState;
|
|||||||
/// Per-call bound so a hung backend cannot stall the poll loop.
|
/// Per-call bound so a hung backend cannot stall the poll loop.
|
||||||
const CALL_TIMEOUT: Duration = Duration::from_secs(10);
|
const CALL_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
pub const METRIC_NAMES: [&str; 8] = [
|
/// Metric name and UCUM unit.
|
||||||
"systemd.unit.active",
|
pub const METRICS: [(&str, &str); 8] = [
|
||||||
"systemd.unit.failed",
|
("systemd.unit.active", "1"),
|
||||||
"systemd.unit.restarts",
|
("systemd.unit.failed", "1"),
|
||||||
"systemd.timer.next_elapse_seconds",
|
("systemd.unit.restarts", "{restart}"),
|
||||||
"docker.container.running",
|
// Absolute unix timestamp of the next elapse, not a duration.
|
||||||
"docker.container.healthy",
|
("systemd.timer.next_elapse_timestamp_seconds", "s"),
|
||||||
"docker.container.uptime_seconds",
|
("docker.container.running", "1"),
|
||||||
"docker.container.restarts",
|
("docker.container.healthy", "1"),
|
||||||
|
("docker.container.uptime_seconds", "s"),
|
||||||
|
("docker.container.restarts", "{restart}"),
|
||||||
];
|
];
|
||||||
|
|
||||||
|
/// Overall budget for the per-unit restart-count polling.
|
||||||
|
const RESTARTS_BUDGET: Duration = Duration::from_secs(15);
|
||||||
|
const RESTARTS_CONCURRENCY: usize = 8;
|
||||||
|
|
||||||
/// Latest poll result, already filtered by the allowlists.
|
/// Latest poll result, already filtered by the allowlists.
|
||||||
#[derive(Debug, Default, Clone)]
|
#[derive(Debug, Default, Clone)]
|
||||||
pub struct Snapshot {
|
pub struct Snapshot {
|
||||||
@@ -64,92 +71,91 @@ pub fn container_in_scope(cfg: &OtelMetricsConfig, name: &str) -> bool {
|
|||||||
cfg.containers.is_empty() || any_glob(&cfg.containers, name)
|
cfg.containers.is_empty() || any_glob(&cfg.containers, name)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn unit_attrs(u: &UnitSummary) -> Vec<(&'static str, String)> {
|
fn unit_id(name: &str) -> Vec<(&'static str, String)> {
|
||||||
vec![
|
vec![("systemd.unit", name.to_owned())]
|
||||||
("systemd.unit", u.name.clone()),
|
|
||||||
("systemd.sub_state", u.sub_state.clone()),
|
|
||||||
("systemd.load_state", u.load_state.clone()),
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn container_attrs(c: &ContainerSummary) -> Vec<(&'static str, String)> {
|
fn container_id(c: &ContainerSummary) -> Vec<(&'static str, String)> {
|
||||||
vec![
|
vec![("container.name", c.name.clone())]
|
||||||
("container.name", c.name.clone()),
|
|
||||||
("container.id", c.id.clone()),
|
|
||||||
("container.image.name", c.image.clone()),
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Map a snapshot to gauge observations.
|
/// Map a snapshot to gauge observations. Series carry identity attributes
|
||||||
|
/// only (`systemd.unit` / `container.name`) so state changes do not churn
|
||||||
|
/// series. The `active` gauge also carries sub/load state, and `running`
|
||||||
|
/// carries the image name.
|
||||||
pub fn observations(s: &Snapshot) -> Vec<Obs> {
|
pub fn observations(s: &Snapshot) -> Vec<Obs> {
|
||||||
let b = |v: bool| if v { 1.0 } else { 0.0 };
|
let b = |v: bool| if v { 1.0 } else { 0.0 };
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
for u in &s.units {
|
for u in &s.units {
|
||||||
|
let mut active_attrs = unit_id(&u.name);
|
||||||
|
active_attrs.push(("systemd.sub_state", u.sub_state.clone()));
|
||||||
|
active_attrs.push(("systemd.load_state", u.load_state.clone()));
|
||||||
out.push(Obs {
|
out.push(Obs {
|
||||||
metric: "systemd.unit.active",
|
metric: "systemd.unit.active",
|
||||||
value: b(u.active_state == "active"),
|
value: b(u.active_state == "active"),
|
||||||
attrs: unit_attrs(u),
|
attrs: active_attrs,
|
||||||
});
|
});
|
||||||
out.push(Obs {
|
out.push(Obs {
|
||||||
metric: "systemd.unit.failed",
|
metric: "systemd.unit.failed",
|
||||||
value: b(u.active_state == "failed"),
|
value: b(u.active_state == "failed"),
|
||||||
attrs: unit_attrs(u),
|
attrs: unit_id(&u.name),
|
||||||
});
|
});
|
||||||
if let Some(n) = s.restarts.get(&u.name) {
|
if let Some(n) = s.restarts.get(&u.name) {
|
||||||
out.push(Obs {
|
out.push(Obs {
|
||||||
metric: "systemd.unit.restarts",
|
metric: "systemd.unit.restarts",
|
||||||
value: f64::from(*n),
|
value: f64::from(*n),
|
||||||
attrs: unit_attrs(u),
|
attrs: unit_id(&u.name),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for t in &s.timers {
|
for t in &s.timers {
|
||||||
// Absolute unix time of the next elapse; unscheduled timers are omitted.
|
// Unscheduled timers are omitted.
|
||||||
if let Some(next) = t.next_elapse {
|
if let Some(next) = t.next_elapse {
|
||||||
out.push(Obs {
|
out.push(Obs {
|
||||||
metric: "systemd.timer.next_elapse_seconds",
|
metric: "systemd.timer.next_elapse_timestamp_seconds",
|
||||||
value: next.as_second() as f64,
|
value: next.as_second() as f64,
|
||||||
attrs: vec![
|
attrs: unit_id(&t.name),
|
||||||
("systemd.unit", t.name.clone()),
|
|
||||||
("systemd.sub_state", t.active_state.clone()),
|
|
||||||
],
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for c in &s.containers {
|
for c in &s.containers {
|
||||||
|
let mut running_attrs = container_id(c);
|
||||||
|
running_attrs.push(("container.image.name", c.image.clone()));
|
||||||
out.push(Obs {
|
out.push(Obs {
|
||||||
metric: "docker.container.running",
|
metric: "docker.container.running",
|
||||||
value: b(c.state == "running"),
|
value: b(c.state == "running"),
|
||||||
attrs: container_attrs(c),
|
attrs: running_attrs,
|
||||||
});
|
});
|
||||||
if let Some(h) = &c.health {
|
if let Some(h) = &c.health {
|
||||||
out.push(Obs {
|
out.push(Obs {
|
||||||
metric: "docker.container.healthy",
|
metric: "docker.container.healthy",
|
||||||
value: b(h == "healthy"),
|
value: b(h == "healthy"),
|
||||||
attrs: container_attrs(c),
|
attrs: container_id(c),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if let Some(up) = c.uptime_seconds {
|
if let Some(up) = c.uptime_seconds {
|
||||||
out.push(Obs {
|
out.push(Obs {
|
||||||
metric: "docker.container.uptime_seconds",
|
metric: "docker.container.uptime_seconds",
|
||||||
value: up as f64,
|
value: up as f64,
|
||||||
attrs: container_attrs(c),
|
attrs: container_id(c),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if let Some(r) = c.restart_count {
|
if let Some(r) = c.restart_count {
|
||||||
out.push(Obs {
|
out.push(Obs {
|
||||||
metric: "docker.container.restarts",
|
metric: "docker.container.restarts",
|
||||||
value: r as f64,
|
value: r as f64,
|
||||||
attrs: container_attrs(c),
|
attrs: container_id(c),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn collect(state: &AppState) -> Snapshot {
|
/// Poll the sources. A section whose call fails or times out keeps its value
|
||||||
|
/// from `prev` instead of being emptied.
|
||||||
|
async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot {
|
||||||
let cfg = &state.config.otel.metrics;
|
let cfg = &state.config.otel.metrics;
|
||||||
let mut snap = Snapshot::default();
|
let mut snap = prev.clone();
|
||||||
|
|
||||||
match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_units(None)).await {
|
match tokio::time::timeout(CALL_TIMEOUT, state.systemd.list_units(None)).await {
|
||||||
Ok(Ok(units)) => {
|
Ok(Ok(units)) => {
|
||||||
@@ -171,24 +177,38 @@ async fn collect(state: &AppState) -> Snapshot {
|
|||||||
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_timers failed"),
|
Ok(Err(e)) => tracing::debug!(error = %e, "otel metrics: list_timers failed"),
|
||||||
Err(_) => tracing::debug!("otel metrics: list_timers timed out"),
|
Err(_) => tracing::debug!("otel metrics: list_timers timed out"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Restart counts: bounded concurrency and an overall budget. Entries not
|
||||||
|
// refreshed in time keep their previous value; vanished units are dropped.
|
||||||
let services: Vec<String> = snap
|
let services: Vec<String> = snap
|
||||||
.units
|
.units
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|u| u.name.ends_with(".service") && u.load_state == "loaded")
|
.filter(|u| u.name.ends_with(".service") && u.load_state == "loaded")
|
||||||
.map(|u| u.name.clone())
|
.map(|u| u.name.clone())
|
||||||
.collect();
|
.collect();
|
||||||
for name in services {
|
let mut restarts: HashMap<String, u32> = services
|
||||||
if state.shutdown.is_cancelled() {
|
.iter()
|
||||||
break;
|
.filter_map(|n| prev.restarts.get(n).map(|v| (n.clone(), *v)))
|
||||||
}
|
.collect();
|
||||||
if let Ok(Ok(st)) =
|
let fetch = futures::stream::iter(services)
|
||||||
tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await
|
.map(|name| async move {
|
||||||
{
|
let r = tokio::time::timeout(CALL_TIMEOUT, state.systemd.unit_status(&name)).await;
|
||||||
if let Some(n) = st.n_restarts {
|
(name, r)
|
||||||
snap.restarts.insert(name, n);
|
})
|
||||||
|
.buffer_unordered(RESTARTS_CONCURRENCY)
|
||||||
|
.for_each(|(name, r)| {
|
||||||
|
if let Ok(Ok(st)) = r {
|
||||||
|
if let Some(n) = st.n_restarts {
|
||||||
|
restarts.insert(name, n);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
std::future::ready(())
|
||||||
|
});
|
||||||
|
if tokio::time::timeout(RESTARTS_BUDGET, fetch).await.is_err() {
|
||||||
|
tracing::debug!("otel metrics: restart polling hit its time budget");
|
||||||
}
|
}
|
||||||
|
snap.restarts = restarts;
|
||||||
|
|
||||||
match tokio::time::timeout(CALL_TIMEOUT, state.docker.ps(true)).await {
|
match tokio::time::timeout(CALL_TIMEOUT, state.docker.ps(true)).await {
|
||||||
Ok(Ok(cs)) => {
|
Ok(Ok(cs)) => {
|
||||||
snap.containers = cs
|
snap.containers = cs
|
||||||
@@ -204,20 +224,22 @@ async fn collect(state: &AppState) -> Snapshot {
|
|||||||
|
|
||||||
/// Register the gauges and start the poll loop.
|
/// Register the gauges and start the poll loop.
|
||||||
pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>> {
|
pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>> {
|
||||||
let snapshot: Arc<RwLock<Arc<Snapshot>>> = Arc::default();
|
// Observations are computed once per poll; callbacks only filter.
|
||||||
|
let current: Arc<RwLock<Arc<Vec<Obs>>>> = Arc::default();
|
||||||
let meter = provider.meter("skyg-linux-tracker");
|
let meter = provider.meter("skyg-linux-tracker");
|
||||||
for name in METRIC_NAMES {
|
for (name, unit) in METRICS {
|
||||||
let snap = Arc::clone(&snapshot);
|
let cur = Arc::clone(¤t);
|
||||||
// Gauges live as long as the provider; the instrument handle is not needed.
|
// Gauges live as long as the provider; the instrument handle is not needed.
|
||||||
let _ = meter
|
let _ = meter
|
||||||
.f64_observable_gauge(name)
|
.f64_observable_gauge(name)
|
||||||
|
.with_unit(unit)
|
||||||
.with_callback(move |o| {
|
.with_callback(move |o| {
|
||||||
let cur = snap.read().map(|g| Arc::clone(&g)).unwrap_or_default();
|
let obs = cur.read().map(|g| Arc::clone(&g)).unwrap_or_default();
|
||||||
for ob in observations(&cur).into_iter().filter(|x| x.metric == name) {
|
for ob in obs.iter().filter(|x| x.metric == name) {
|
||||||
let attrs: Vec<KeyValue> = ob
|
let attrs: Vec<KeyValue> = ob
|
||||||
.attrs
|
.attrs
|
||||||
.into_iter()
|
.iter()
|
||||||
.map(|(k, v)| KeyValue::new(k, v))
|
.map(|(k, v)| KeyValue::new(*k, v.clone()))
|
||||||
.collect();
|
.collect();
|
||||||
o.observe(ob.value, &attrs);
|
o.observe(ob.value, &attrs);
|
||||||
}
|
}
|
||||||
@@ -229,17 +251,18 @@ pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>
|
|||||||
let h = tokio::spawn(async move {
|
let h = tokio::spawn(async move {
|
||||||
let mut tick = tokio::time::interval(interval);
|
let mut tick = tokio::time::interval(interval);
|
||||||
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||||
|
let mut snap = Snapshot::default();
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = state.shutdown.cancelled() => break,
|
_ = state.shutdown.cancelled() => break,
|
||||||
_ = tick.tick() => {}
|
_ = tick.tick() => {}
|
||||||
}
|
}
|
||||||
let snap = tokio::select! {
|
snap = tokio::select! {
|
||||||
_ = state.shutdown.cancelled() => break,
|
_ = state.shutdown.cancelled() => break,
|
||||||
s = collect(&state) => s,
|
s = collect(&state, &snap) => s,
|
||||||
};
|
};
|
||||||
if let Ok(mut g) = snapshot.write() {
|
if let Ok(mut g) = current.write() {
|
||||||
*g = Arc::new(snap);
|
*g = Arc::new(observations(&snap));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -359,7 +382,7 @@ mod tests {
|
|||||||
let o = observations(&s);
|
let o = observations(&s);
|
||||||
let timers: Vec<_> = o
|
let timers: Vec<_> = o
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|x| x.metric == "systemd.timer.next_elapse_seconds")
|
.filter(|x| x.metric == "systemd.timer.next_elapse_timestamp_seconds")
|
||||||
.collect();
|
.collect();
|
||||||
assert_eq!(timers.len(), 1);
|
assert_eq!(timers.len(), 1);
|
||||||
assert_eq!(timers[0].value, 1_792_065_600.0);
|
assert_eq!(timers[0].value, 1_792_065_600.0);
|
||||||
@@ -374,7 +397,12 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(h.value, 0.0);
|
assert_eq!(h.value, 0.0);
|
||||||
assert_eq!(h.attrs[0], ("container.name", "web".to_string()));
|
assert_eq!(h.attrs[0], ("container.name", "web".to_string()));
|
||||||
assert_eq!(h.attrs[2], ("container.image.name", "nginx:1".to_string()));
|
assert_eq!(h.attrs.len(), 1);
|
||||||
|
let r = o
|
||||||
|
.iter()
|
||||||
|
.find(|x| x.metric == "docker.container.running")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -383,13 +411,13 @@ mod tests {
|
|||||||
cfg.otel.metrics.units = vec!["__nomatch__*".into()];
|
cfg.otel.metrics.units = vec!["__nomatch__*".into()];
|
||||||
cfg.otel.metrics.containers = vec!["__nomatch__*".into()];
|
cfg.otel.metrics.containers = vec!["__nomatch__*".into()];
|
||||||
let st = fake_state(cfg);
|
let st = fake_state(cfg);
|
||||||
let snap = collect(&st).await;
|
let snap = collect(&st, &Snapshot::default()).await;
|
||||||
assert!(snap.units.is_empty());
|
assert!(snap.units.is_empty());
|
||||||
assert!(snap.timers.is_empty());
|
assert!(snap.timers.is_empty());
|
||||||
assert!(snap.containers.is_empty());
|
assert!(snap.containers.is_empty());
|
||||||
|
|
||||||
let st = fake_state(Config::default());
|
let st = fake_state(Config::default());
|
||||||
let snap = collect(&st).await;
|
let snap = collect(&st, &Snapshot::default()).await;
|
||||||
assert!(snap
|
assert!(snap
|
||||||
.units
|
.units
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
+44
-31
@@ -49,29 +49,20 @@ pub struct TelemetryGuard {
|
|||||||
providers: Option<Providers>,
|
providers: Option<Providers>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TelemetryGuard {
|
impl Providers {
|
||||||
/// True when OTLP export is active.
|
/// Blocking flush and shutdown, each provider bounded by `FLUSH_TIMEOUT`.
|
||||||
pub fn is_enabled(&self) -> bool {
|
fn flush_and_shutdown(&self) {
|
||||||
self.providers.is_some()
|
if let Err(e) = self.tracer.force_flush() {
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for TelemetryGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
let Some(p) = self.providers.take() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
if let Err(e) = p.tracer.force_flush() {
|
|
||||||
eprintln!("otel: trace flush failed: {e}");
|
eprintln!("otel: trace flush failed: {e}");
|
||||||
}
|
}
|
||||||
let _ = p.tracer.shutdown_with_timeout(FLUSH_TIMEOUT);
|
let _ = self.tracer.shutdown_with_timeout(FLUSH_TIMEOUT);
|
||||||
if let Some(m) = &p.meter {
|
if let Some(m) = &self.meter {
|
||||||
if let Err(e) = m.force_flush() {
|
if let Err(e) = m.force_flush() {
|
||||||
eprintln!("otel: metrics flush failed: {e}");
|
eprintln!("otel: metrics flush failed: {e}");
|
||||||
}
|
}
|
||||||
let _ = m.shutdown_with_timeout(FLUSH_TIMEOUT);
|
let _ = m.shutdown_with_timeout(FLUSH_TIMEOUT);
|
||||||
}
|
}
|
||||||
if let Some(l) = &p.logger {
|
if let Some(l) = &self.logger {
|
||||||
if let Err(e) = l.force_flush() {
|
if let Err(e) = l.force_flush() {
|
||||||
eprintln!("otel: logs flush failed: {e}");
|
eprintln!("otel: logs flush failed: {e}");
|
||||||
}
|
}
|
||||||
@@ -80,6 +71,40 @@ impl Drop for TelemetryGuard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl TelemetryGuard {
|
||||||
|
/// True when OTLP export is active.
|
||||||
|
pub fn is_enabled(&self) -> bool {
|
||||||
|
self.providers.is_some()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Flush and shut down all providers without blocking the async runtime.
|
||||||
|
/// Call this (and await it) on graceful shutdown, after the exporter task
|
||||||
|
/// handles from `spawn_exporters` have finished. `Drop` is only a
|
||||||
|
/// best-effort fallback.
|
||||||
|
pub async fn shutdown(mut self) {
|
||||||
|
if let Some(p) = self.providers.take() {
|
||||||
|
let _ = tokio::task::spawn_blocking(move || p.flush_and_shutdown()).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for TelemetryGuard {
|
||||||
|
/// Best effort. Inside a tokio runtime the flush runs on a detached thread
|
||||||
|
/// (it may not finish before the process exits); outside one it runs inline.
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let Some(p) = self.providers.take() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if tokio::runtime::Handle::try_current().is_ok() {
|
||||||
|
let _ = std::thread::Builder::new()
|
||||||
|
.name("otel-flush".into())
|
||||||
|
.spawn(move || p.flush_and_shutdown());
|
||||||
|
} else {
|
||||||
|
p.flush_and_shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Install the tracing subscriber and, when `config.otel.enabled`, the OTLP
|
/// Install the tracing subscriber and, when `config.otel.enabled`, the OTLP
|
||||||
/// providers. Works (stderr logging only) when OTel is disabled.
|
/// providers. Works (stderr logging only) when OTel is disabled.
|
||||||
pub fn init(config: &Config) -> Result<TelemetryGuard> {
|
pub fn init(config: &Config) -> Result<TelemetryGuard> {
|
||||||
@@ -359,21 +384,9 @@ mod tests {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
o.headers.insert("Authorization".into(), "Bearer x".into());
|
o.headers.insert("Authorization".into(), "Bearer x".into());
|
||||||
let results = [
|
build_span_exporter(&o).unwrap();
|
||||||
build_span_exporter(&o).map(drop),
|
build_metric_exporter(&o).unwrap();
|
||||||
build_metric_exporter(&o).map(drop),
|
build_log_exporter(&o).unwrap();
|
||||||
build_log_exporter(&o).map(drop),
|
|
||||||
];
|
|
||||||
for r in results {
|
|
||||||
match (protocol, r) {
|
|
||||||
(_, Ok(())) => {}
|
|
||||||
// Contract: HTTP needs the `reqwest-blocking-client` feature on
|
|
||||||
// opentelemetry-otlp (Cargo.toml, owned by P0).
|
|
||||||
(OtelProtocol::HttpProtobuf, Err(e))
|
|
||||||
if format!("{e:?}").contains("http_client") => {}
|
|
||||||
(_, Err(e)) => panic!("exporter build failed: {e:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user