E: E1 peer health gauges from the registry snapshot
This commit is contained in:
1 parent
f0b6c29eac
commit
eee253b162
1 file changed
+425
-17
+425
-17
@@ -9,13 +9,14 @@ use std::sync::{Arc, RwLock};
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
use opentelemetry::metrics::MeterProvider as _;
|
use opentelemetry::metrics::{Meter, MeterProvider as _};
|
||||||
use opentelemetry::KeyValue;
|
use opentelemetry::KeyValue;
|
||||||
use opentelemetry_sdk::metrics::SdkMeterProvider;
|
use opentelemetry_sdk::metrics::SdkMeterProvider;
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
|
|
||||||
use super::any_glob;
|
use super::any_glob;
|
||||||
use crate::config::OtelMetricsConfig;
|
use crate::config::OtelMetricsConfig;
|
||||||
|
use crate::peers::types::{AuthState, PeerStatus};
|
||||||
use crate::sources::{ContainerSummary, TimerInfo, UnitSummary};
|
use crate::sources::{ContainerSummary, TimerInfo, UnitSummary};
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
|
|
||||||
@@ -35,6 +36,16 @@ pub const METRICS: [(&str, &str); 8] = [
|
|||||||
("docker.container.restarts", "{restart}"),
|
("docker.container.restarts", "{restart}"),
|
||||||
];
|
];
|
||||||
|
|
||||||
|
/// Peer health gauges. Registered only when the peers feature is on; names,
|
||||||
|
/// units and attributes are a contract with dashboards and alerts.
|
||||||
|
pub const PEER_METRICS: [(&str, &str); 4] = [
|
||||||
|
("peer.up", "1"),
|
||||||
|
("peer.api.auth_ok", "1"),
|
||||||
|
("peer.check.duration_seconds", "s"),
|
||||||
|
// Absolute unix timestamp of the last state change, not a duration.
|
||||||
|
("peer.last_change_timestamp_seconds", "s"),
|
||||||
|
];
|
||||||
|
|
||||||
/// Overall budget for the per-unit restart-count polling.
|
/// Overall budget for the per-unit restart-count polling.
|
||||||
const RESTARTS_BUDGET: Duration = Duration::from_secs(15);
|
const RESTARTS_BUDGET: Duration = Duration::from_secs(15);
|
||||||
const RESTARTS_CONCURRENCY: usize = 8;
|
const RESTARTS_CONCURRENCY: usize = 8;
|
||||||
@@ -151,6 +162,61 @@ pub fn observations(s: &Snapshot) -> Vec<Obs> {
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Map a registry snapshot to peer gauge observations.
|
||||||
|
///
|
||||||
|
/// Series carry `peer.name` and, per check, `peer.check` only: URLs can embed
|
||||||
|
/// hostnames an operator treats as sensitive, and a version attribute would
|
||||||
|
/// churn series on upgrade. A missing latency or `since` (no check yet, or a
|
||||||
|
/// failed one) yields no point rather than a stale or invented value.
|
||||||
|
///
|
||||||
|
/// Disabled peers are skipped: their checks never run, so a `0` would read as
|
||||||
|
/// "down" and page someone for a peer that was switched off deliberately.
|
||||||
|
pub fn peer_observations(peers: &[PeerStatus]) -> Vec<Obs> {
|
||||||
|
let b = |v: bool| if v { 1.0 } else { 0.0 };
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for p in peers.iter().filter(|p| p.enabled) {
|
||||||
|
let name = p.name.as_str();
|
||||||
|
let check_attrs = |check: &str| {
|
||||||
|
vec![
|
||||||
|
("peer.name", name.to_owned()),
|
||||||
|
("peer.check", check.to_owned()),
|
||||||
|
]
|
||||||
|
};
|
||||||
|
for (check, up, latency, since) in [
|
||||||
|
("server", p.server.up, p.server.latency_ms, p.server.since),
|
||||||
|
("api", p.api.up, p.api.latency_ms, p.api.since),
|
||||||
|
] {
|
||||||
|
out.push(Obs {
|
||||||
|
metric: "peer.up",
|
||||||
|
value: b(up),
|
||||||
|
attrs: check_attrs(check),
|
||||||
|
});
|
||||||
|
if let Some(ms) = latency {
|
||||||
|
out.push(Obs {
|
||||||
|
metric: "peer.check.duration_seconds",
|
||||||
|
value: ms as f64 / 1000.0,
|
||||||
|
attrs: check_attrs(check),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Some(ts) = since {
|
||||||
|
out.push(Obs {
|
||||||
|
metric: "peer.last_change_timestamp_seconds",
|
||||||
|
value: ts.as_second() as f64,
|
||||||
|
attrs: check_attrs(check),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// "server up + api up + auth_ok = 0" is the revoked-credential state,
|
||||||
|
// kept separate from an API outage so alerts can tell them apart.
|
||||||
|
out.push(Obs {
|
||||||
|
metric: "peer.api.auth_ok",
|
||||||
|
value: b(p.api.auth == AuthState::Ok),
|
||||||
|
attrs: vec![("peer.name", name.to_owned())],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
/// Poll the sources. A section whose call fails or times out keeps its value
|
/// Poll the sources. A section whose call fails or times out keeps its value
|
||||||
/// from `prev` instead of being emptied.
|
/// from `prev` instead of being emptied.
|
||||||
async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot {
|
async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot {
|
||||||
@@ -222,28 +288,52 @@ async fn collect(state: &AppState, prev: &Snapshot) -> Snapshot {
|
|||||||
snap
|
snap
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Register one observable f64 gauge fed by `obs`. The callback filters by
|
||||||
|
/// metric name, so a single observation list can feed several gauges.
|
||||||
|
fn register_gauge(
|
||||||
|
meter: &Meter,
|
||||||
|
name: &'static str,
|
||||||
|
unit: &'static str,
|
||||||
|
obs: impl Fn() -> Arc<Vec<Obs>> + Send + Sync + 'static,
|
||||||
|
) {
|
||||||
|
// Gauges live as long as the provider; the instrument handle is not needed.
|
||||||
|
let _ = meter
|
||||||
|
.f64_observable_gauge(name)
|
||||||
|
.with_unit(unit)
|
||||||
|
.with_callback(move |o| {
|
||||||
|
let obs = obs();
|
||||||
|
for ob in obs.iter().filter(|x| x.metric == name) {
|
||||||
|
let attrs: Vec<KeyValue> = ob
|
||||||
|
.attrs
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| KeyValue::new(*k, v.clone()))
|
||||||
|
.collect();
|
||||||
|
o.observe(ob.value, &attrs);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>> {
|
pub fn spawn(state: AppState, provider: &SdkMeterProvider) -> Vec<JoinHandle<()>> {
|
||||||
// Observations are computed once per poll; callbacks only filter.
|
// Observations are computed once per poll; callbacks only filter.
|
||||||
let current: Arc<RwLock<Arc<Vec<Obs>>>> = Arc::default();
|
let current: Arc<RwLock<Arc<Vec<Obs>>>> = Arc::default();
|
||||||
let meter = provider.meter("skyg-linux-tracker");
|
let meter = provider.meter("skyg-linux-tracker");
|
||||||
for (name, unit) in METRICS {
|
for (name, unit) in METRICS {
|
||||||
let cur = Arc::clone(¤t);
|
let cur = Arc::clone(¤t);
|
||||||
// Gauges live as long as the provider; the instrument handle is not needed.
|
register_gauge(&meter, name, unit, move || {
|
||||||
let _ = meter
|
cur.read().map(|g| Arc::clone(&g)).unwrap_or_default()
|
||||||
.f64_observable_gauge(name)
|
});
|
||||||
.with_unit(unit)
|
}
|
||||||
.with_callback(move |o| {
|
// Peer state already lives in memory, so peer gauges read the registry on
|
||||||
let obs = cur.read().map(|g| Arc::clone(&g)).unwrap_or_default();
|
// each collection instead of polling anything; a peer removed from config
|
||||||
for ob in obs.iter().filter(|x| x.metric == name) {
|
// therefore disappears from the next export.
|
||||||
let attrs: Vec<KeyValue> = ob
|
if let Some(peers) = state.peers.clone() {
|
||||||
.attrs
|
for (name, unit) in PEER_METRICS {
|
||||||
.iter()
|
let peers = Arc::clone(&peers);
|
||||||
.map(|(k, v)| KeyValue::new(*k, v.clone()))
|
register_gauge(&meter, name, unit, move || {
|
||||||
.collect();
|
Arc::new(peer_observations(&peers.snapshot()))
|
||||||
o.observe(ob.value, &attrs);
|
});
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.build();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let interval = Duration::from_secs(state.config.otel.metrics_interval_secs.max(1));
|
let interval = Duration::from_secs(state.config.otel.metrics_interval_secs.max(1));
|
||||||
@@ -404,6 +494,324 @@ mod tests {
|
|||||||
assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string()));
|
assert_eq!(r.attrs[1], ("container.image.name", "nginx:1".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn ts(s: &str) -> jiff::Timestamp {
|
||||||
|
s.parse().expect("valid timestamp")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn peer_status(name: &str, server_up: bool, api_up: bool, auth: AuthState) -> PeerStatus {
|
||||||
|
PeerStatus {
|
||||||
|
name: crate::peers::types::PeerName::parse(name).unwrap(),
|
||||||
|
enabled: true,
|
||||||
|
url: format!("http://{name}:1984"),
|
||||||
|
server: crate::peers::types::ServerStatus {
|
||||||
|
up: server_up,
|
||||||
|
since: Some(ts("2026-10-19T10:00:00Z")),
|
||||||
|
last_checked: Some(ts("2026-10-19T10:00:09Z")),
|
||||||
|
last_error: None,
|
||||||
|
latency_ms: Some(1500),
|
||||||
|
},
|
||||||
|
api: crate::peers::types::ApiStatus {
|
||||||
|
up: api_up,
|
||||||
|
since: Some(ts("2026-10-19T10:00:00Z")),
|
||||||
|
last_checked: Some(ts("2026-10-19T10:00:05Z")),
|
||||||
|
last_error: None,
|
||||||
|
latency_ms: Some(7),
|
||||||
|
auth,
|
||||||
|
peer_version: Some("0.1.0".into()),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn attrs(ob: &Obs) -> Vec<(&str, &str)> {
|
||||||
|
ob.attrs.iter().map(|(k, v)| (*k, v.as_str())).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn find<'a>(o: &'a [Obs], metric: &str, peer: &str, check: &str) -> Option<&'a Obs> {
|
||||||
|
o.iter().find(|x| {
|
||||||
|
x.metric == metric
|
||||||
|
&& x.attrs.iter().any(|(k, v)| *k == "peer.name" && v == peer)
|
||||||
|
&& x.attrs
|
||||||
|
.iter()
|
||||||
|
.any(|(k, v)| *k == "peer.check" && v == check)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn peer_metric_names_and_units_are_the_documented_contract() {
|
||||||
|
assert_eq!(
|
||||||
|
PEER_METRICS.map(|(name, _)| name),
|
||||||
|
[
|
||||||
|
"peer.up",
|
||||||
|
"peer.api.auth_ok",
|
||||||
|
"peer.check.duration_seconds",
|
||||||
|
"peer.last_change_timestamp_seconds",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(PEER_METRICS.map(|(_, unit)| unit), ["1", "1", "s", "s"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn maps_peer_status_to_gauges() {
|
||||||
|
let o = peer_observations(&[peer_status("beta", true, true, AuthState::Ok)]);
|
||||||
|
|
||||||
|
let up = find(&o, "peer.up", "beta", "server").unwrap();
|
||||||
|
assert_eq!(up.value, 1.0);
|
||||||
|
assert_eq!(attrs(up), [("peer.name", "beta"), ("peer.check", "server")]);
|
||||||
|
assert_eq!(find(&o, "peer.up", "beta", "api").unwrap().value, 1.0);
|
||||||
|
|
||||||
|
let auth = o.iter().find(|x| x.metric == "peer.api.auth_ok").unwrap();
|
||||||
|
assert_eq!(auth.value, 1.0);
|
||||||
|
assert_eq!(attrs(auth), [("peer.name", "beta")]);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
find(&o, "peer.check.duration_seconds", "beta", "server")
|
||||||
|
.unwrap()
|
||||||
|
.value,
|
||||||
|
1.5
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
find(&o, "peer.check.duration_seconds", "beta", "api")
|
||||||
|
.unwrap()
|
||||||
|
.value,
|
||||||
|
0.007
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
find(&o, "peer.last_change_timestamp_seconds", "beta", "api")
|
||||||
|
.unwrap()
|
||||||
|
.value,
|
||||||
|
ts("2026-10-19T10:00:00Z").as_second() as f64
|
||||||
|
);
|
||||||
|
|
||||||
|
// URLs and versions stay out of attributes: cardinality and hostnames
|
||||||
|
// the operator may treat as sensitive.
|
||||||
|
for ob in &o {
|
||||||
|
assert!(!ob.attrs.iter().any(|(_, v)| v.contains("http")));
|
||||||
|
assert!(!ob.attrs.iter().any(|(_, v)| v.contains("0.1.0")));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejected_credentials_are_visible_in_auth_ok() {
|
||||||
|
let o = peer_observations(&[peer_status("beta", true, true, AuthState::Rejected)]);
|
||||||
|
assert_eq!(find(&o, "peer.up", "beta", "api").unwrap().value, 1.0);
|
||||||
|
let auth = o.iter().find(|x| x.metric == "peer.api.auth_ok").unwrap();
|
||||||
|
assert_eq!(auth.value, 0.0);
|
||||||
|
|
||||||
|
let unknown = peer_observations(&[peer_status("beta", true, true, AuthState::Unknown)]);
|
||||||
|
let auth = unknown
|
||||||
|
.iter()
|
||||||
|
.find(|x| x.metric == "peer.api.auth_ok")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(auth.value, 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unchecked_peer_reports_up_only() {
|
||||||
|
let mut p = peer_status("beta", false, false, AuthState::Unknown);
|
||||||
|
p.server.latency_ms = None;
|
||||||
|
p.server.since = None;
|
||||||
|
p.api.latency_ms = None;
|
||||||
|
p.api.since = None;
|
||||||
|
let o = peer_observations(&[p]);
|
||||||
|
let metrics: Vec<&str> = o.iter().map(|x| x.metric).collect();
|
||||||
|
assert_eq!(metrics, ["peer.up", "peer.up", "peer.api.auth_ok"]);
|
||||||
|
assert!(o.iter().all(|x| x.value == 0.0));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn disabled_peer_is_not_reported() {
|
||||||
|
let mut p = peer_status("beta", true, true, AuthState::Ok);
|
||||||
|
p.enabled = false;
|
||||||
|
assert!(peer_observations(&[p]).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Collects exports and keeps the samples of the last one.
|
||||||
|
#[derive(Default, Clone)]
|
||||||
|
struct Capture(std::sync::Arc<std::sync::Mutex<Vec<Vec<Sample>>>>);
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
struct Sample {
|
||||||
|
metric: String,
|
||||||
|
unit: String,
|
||||||
|
value: f64,
|
||||||
|
attrs: Vec<(String, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Capture {
|
||||||
|
fn last(&self) -> Vec<Sample> {
|
||||||
|
self.0.lock().unwrap().last().cloned().unwrap_or_default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl opentelemetry_sdk::metrics::exporter::PushMetricExporter for Capture {
|
||||||
|
fn export(
|
||||||
|
&self,
|
||||||
|
metrics: &opentelemetry_sdk::metrics::data::ResourceMetrics,
|
||||||
|
) -> impl std::future::Future<Output = opentelemetry_sdk::error::OTelSdkResult> + Send
|
||||||
|
{
|
||||||
|
use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
|
||||||
|
let mut samples = Vec::new();
|
||||||
|
for scope in metrics.scope_metrics() {
|
||||||
|
for m in scope.metrics() {
|
||||||
|
let AggregatedMetrics::F64(MetricData::Gauge(g)) = m.data() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
for p in g.data_points() {
|
||||||
|
samples.push(Sample {
|
||||||
|
metric: m.name().to_owned(),
|
||||||
|
unit: m.unit().to_owned(),
|
||||||
|
value: p.value(),
|
||||||
|
attrs: p
|
||||||
|
.attributes()
|
||||||
|
.map(|kv| {
|
||||||
|
(kv.key.as_str().to_owned(), kv.value.as_str().into_owned())
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.0.lock().unwrap().push(samples);
|
||||||
|
std::future::ready(Ok(()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn force_flush(&self) -> opentelemetry_sdk::error::OTelSdkResult {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn shutdown_with_timeout(
|
||||||
|
&self,
|
||||||
|
_timeout: Duration,
|
||||||
|
) -> opentelemetry_sdk::error::OTelSdkResult {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn temporality(&self) -> opentelemetry_sdk::metrics::Temporality {
|
||||||
|
opentelemetry_sdk::metrics::Temporality::Cumulative
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Provider with a capture exporter; export happens on `force_flush`.
|
||||||
|
fn capture_provider() -> (SdkMeterProvider, Capture) {
|
||||||
|
let capture = Capture::default();
|
||||||
|
let reader = opentelemetry_sdk::metrics::PeriodicReader::builder(capture.clone())
|
||||||
|
.with_interval(Duration::from_secs(3600))
|
||||||
|
.build();
|
||||||
|
let provider = SdkMeterProvider::builder().with_reader(reader).build();
|
||||||
|
(provider, capture)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample<'a>(
|
||||||
|
s: &'a [Sample],
|
||||||
|
metric: &str,
|
||||||
|
peer: &str,
|
||||||
|
check: Option<&str>,
|
||||||
|
) -> Option<&'a Sample> {
|
||||||
|
s.iter().find(|x| {
|
||||||
|
x.metric == metric
|
||||||
|
&& x.attrs.iter().any(|(k, v)| k == "peer.name" && v == peer)
|
||||||
|
&& check.is_none_or(|c| x.attrs.iter().any(|(k, v)| k == "peer.check" && v == c))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn exported_peer_gauges_read_the_registry_snapshot() {
|
||||||
|
let reg = crate::peers::testing::static_registry(vec![
|
||||||
|
peer_status("beta", true, true, AuthState::Ok),
|
||||||
|
peer_status("gamma", true, false, AuthState::Unknown),
|
||||||
|
]);
|
||||||
|
let state = crate::peers::testing::fake_state_with_peers(Config::default(), reg.clone());
|
||||||
|
let (provider, capture) = capture_provider();
|
||||||
|
let handles = spawn(state.clone(), &provider);
|
||||||
|
provider.force_flush().unwrap();
|
||||||
|
|
||||||
|
let samples = capture.last();
|
||||||
|
let beta_up = sample(&samples, "peer.up", "beta", Some("server")).unwrap();
|
||||||
|
assert_eq!(beta_up.value, 1.0);
|
||||||
|
assert_eq!(beta_up.unit, "1");
|
||||||
|
assert_eq!(
|
||||||
|
sample(&samples, "peer.up", "gamma", Some("api"))
|
||||||
|
.unwrap()
|
||||||
|
.value,
|
||||||
|
0.0
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sample(&samples, "peer.api.auth_ok", "gamma", None)
|
||||||
|
.unwrap()
|
||||||
|
.value,
|
||||||
|
0.0
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sample(
|
||||||
|
&samples,
|
||||||
|
"peer.check.duration_seconds",
|
||||||
|
"beta",
|
||||||
|
Some("server")
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.value,
|
||||||
|
1.5
|
||||||
|
);
|
||||||
|
|
||||||
|
// Removing a peer from the snapshot drops its series; it is not cached.
|
||||||
|
reg.reload(crate::peers::testing::fake_peers_config(&["beta"]))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
provider.force_flush().unwrap();
|
||||||
|
let after = capture.last();
|
||||||
|
assert!(sample(&after, "peer.up", "beta", Some("server")).is_some());
|
||||||
|
assert!(sample(&after, "peer.up", "gamma", Some("server")).is_none());
|
||||||
|
|
||||||
|
state.shutdown.cancel();
|
||||||
|
for h in handles {
|
||||||
|
let _ = h.await;
|
||||||
|
}
|
||||||
|
provider.shutdown().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn collecting_peer_gauges_does_not_poll_the_transport() {
|
||||||
|
let fake = crate::peers::testing::FakeTransport::new();
|
||||||
|
let reg = crate::peers::testing::registry_with(
|
||||||
|
fake.clone(),
|
||||||
|
crate::peers::testing::fake_peers_config(&["beta"]),
|
||||||
|
);
|
||||||
|
let state = crate::peers::testing::fake_state_with_peers(Config::default(), reg);
|
||||||
|
let (provider, capture) = capture_provider();
|
||||||
|
let handles = spawn(state.clone(), &provider);
|
||||||
|
provider.force_flush().unwrap();
|
||||||
|
|
||||||
|
let samples = capture.last();
|
||||||
|
assert!(sample(&samples, "peer.up", "beta", Some("server")).is_some());
|
||||||
|
assert!(fake.calls().is_empty(), "gauges must not poll peers");
|
||||||
|
|
||||||
|
state.shutdown.cancel();
|
||||||
|
for h in handles {
|
||||||
|
let _ = h.await;
|
||||||
|
}
|
||||||
|
provider.shutdown().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn peerless_state_exports_no_peer_series() {
|
||||||
|
let state = fake_state(Config::default());
|
||||||
|
let (provider, capture) = capture_provider();
|
||||||
|
let handles = spawn(state.clone(), &provider);
|
||||||
|
provider.force_flush().unwrap();
|
||||||
|
|
||||||
|
let samples = capture.last();
|
||||||
|
assert!(
|
||||||
|
samples.iter().all(|s| !s.metric.starts_with("peer.")),
|
||||||
|
"no peer series without the feature: {samples:?}"
|
||||||
|
);
|
||||||
|
|
||||||
|
state.shutdown.cancel();
|
||||||
|
for h in handles {
|
||||||
|
let _ = h.await;
|
||||||
|
}
|
||||||
|
provider.shutdown().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn collect_applies_scope_with_fakes() {
|
async fn collect_applies_scope_with_fakes() {
|
||||||
let mut cfg = Config::default();
|
let mut cfg = Config::default();
|
||||||
|
|||||||
Reference in new issue
Block a user