From d94c2b3a1df6e750034021320e9c203d5deb5b8e Mon Sep 17 00:00:00 2001 From: ari Date: Mon, 19 Oct 2026 20:24:04 -0400 Subject: [PATCH] B: B2/B3 health probes, loops and the in-memory status model --- src/peers/health.rs | 520 +++++++++++++++++++++++++++++++++------- src/peers/registry.rs | 437 +++++++++++++++++++++++++++++---- tests/peers_registry.rs | 163 ++++++++++++- 3 files changed, 994 insertions(+), 126 deletions(-) diff --git a/src/peers/health.rs b/src/peers/health.rs index 265257f..3357dbd 100644 --- a/src/peers/health.rs +++ b/src/peers/health.rs @@ -1,47 +1,278 @@ -//! One-shot peer probes used by the diagnostic CLI. +//! Peer probes and health state. //! -//! The registry's periodic loops reuse the transport; this module only -//! classifies a single round so `slinuxm peer check` can report it. +//! One probe is cheap and bounded: a TCP connect for "server up" and an +//! authenticated `GET /api/v1/ping` for "API running". The registry schedules +//! them and stores the resulting status; `slinuxm peer check` calls the same +//! probes once. -use std::time::Duration; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; +use std::time::{Duration, Instant}; use jiff::Timestamp; +use serde_json::Value; +use tokio_util::sync::CancellationToken; use super::error::PeerError; -use super::transport::{PeerRequest, PeerTransport}; -use super::types::{ApiStatus, AuthState, PeerConfig, ServerStatus}; +use super::transport::{PeerBody, PeerRequest, PeerResponse, PeerTransport}; +use super::types::{ + ApiStatus, AuthState, PeerConfig, PeerName, PeerStatus, PeersConfig, ServerStatus, +}; + +const PING_PATH: &str = "/api/v1/ping"; +const HEALTHZ_PATH: &str = "/healthz"; fn millis(d: Duration) -> u64 { u64::try_from(d.as_millis()).unwrap_or(u64::MAX) } -fn failed_server(err: &PeerError, at: Timestamp) -> ServerStatus { - ServerStatus { - up: false, - since: None, - last_checked: Some(at), - last_error: Some(err.class_str()), - latency_ms: None, +/// Timing for one peer's loops, copied out of [`PeersConfig`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LoopConfig { + pub tcp_interval: Duration, + pub tcp_timeout: Duration, + pub api_interval: Duration, + pub api_timeout: Duration, + pub fail_threshold: u32, +} + +impl LoopConfig { + pub fn from_config(cfg: &PeersConfig) -> Self { + Self { + tcp_interval: Duration::from_secs(cfg.tcp_interval_secs.max(1)), + tcp_timeout: Duration::from_secs(cfg.tcp_timeout_secs.max(1)), + api_interval: Duration::from_secs(cfg.api_interval_secs.max(1)), + api_timeout: Duration::from_secs(cfg.api_timeout_secs.max(1)), + fail_threshold: cfg.fail_threshold.max(1), + } } } -fn failed_api(err: &PeerError, at: Timestamp) -> ApiStatus { - ApiStatus { - up: false, - since: None, - last_checked: Some(at), - last_error: Some(err.class_str()), - latency_ms: None, - auth: AuthState::Unknown, - peer_version: None, +/// Outcome of the authenticated API probe. +#[derive(Debug, Clone)] +pub struct ApiObservation { + /// The API answered as running, even if it rejected our credential. + pub ok: bool, + pub auth: AuthState, + pub version: Option, + pub latency: Option, + /// Class token, never peer text or a URL. + pub error: Option, +} + +impl ApiObservation { + fn down(err: &PeerError) -> Self { + Self { + ok: false, + auth: AuthState::Unknown, + version: None, + latency: None, + error: Some(err.class_str()), + } + } + + fn running(auth: AuthState, latency: Duration, version: Option) -> Self { + Self { + ok: true, + auth, + version, + latency: Some(latency), + error: (auth == AuthState::Rejected).then(|| "unauthorized".to_owned()), + } } } -/// One TCP connect and one authenticated `GET /api/v1/ping`. +/// Server-side state accumulated between checks. The `since` timestamp lives +/// here rather than in the serialized status so it survives a reload of an +/// unchanged peer. +#[derive(Debug, Clone)] +struct ServerTracker { + up: bool, + since: Option, + failures: u32, +} + +impl ServerTracker { + fn seeded(status: &ServerStatus) -> Self { + Self { + up: status.up, + since: status.since, + failures: 0, + } + } + + fn record( + &mut self, + result: Result, + now: Timestamp, + threshold: u32, + ) -> ServerStatus { + let first = self.since.is_none(); + let (last_error, latency) = match &result { + Ok(latency) => { + self.failures = 0; + if !self.up || first { + self.since = Some(now); + } + self.up = true; + (None, Some(millis(*latency))) + } + Err(err) => { + self.failures = self.failures.saturating_add(1); + if first && self.up { + self.since = Some(now); + } + if self.failures >= threshold && self.up { + self.up = false; + self.since = Some(now); + } + if first && self.since.is_none() { + self.since = Some(now); + } + (Some(err.class_str()), None) + } + }; + ServerStatus { + up: self.up, + since: self.since, + last_checked: Some(now), + last_error, + latency_ms: latency, + } + } +} + +#[derive(Debug, Clone)] +struct ApiTracker { + up: bool, + since: Option, + failures: u32, + auth: AuthState, +} + +impl ApiTracker { + fn seeded(status: &ApiStatus) -> Self { + Self { + up: status.up, + since: status.since, + failures: 0, + auth: status.auth, + } + } + + fn record(&mut self, observation: ApiObservation, now: Timestamp, threshold: u32) -> ApiStatus { + let first = self.since.is_none(); + if observation.ok { + self.failures = 0; + if !self.up || first { + self.since = Some(now); + } + self.up = true; + self.auth = observation.auth; + } else { + self.failures = self.failures.saturating_add(1); + if first { + self.since = Some(now); + } + if self.failures >= threshold && self.up { + self.up = false; + self.since = Some(now); + } + if !self.up { + self.auth = AuthState::Unknown; + } + } + ApiStatus { + up: self.up, + since: self.since, + last_checked: Some(now), + last_error: observation.error, + latency_ms: observation.latency.map(millis), + auth: self.auth, + peer_version: observation.version, + } + } +} + +/// TCP connect, returning the connect latency. +pub async fn probe_server( + transport: &dyn PeerTransport, + peer: &PeerConfig, + timeout: Duration, +) -> Result { + transport.tcp_check(peer, timeout).await +} + +/// Authenticated ping. A 401/403 means the API is up and our credential is +/// not; older peers without the route fall back to public `/healthz`. +pub async fn probe_api( + transport: &dyn PeerTransport, + peer: &PeerConfig, + timeout: Duration, +) -> ApiObservation { + let started = Instant::now(); + let req = PeerRequest { + path: PING_PATH.into(), + query: Vec::new(), + headers: Vec::new(), + accept: None, + }; + let resp = match transport.get(peer, req, timeout).await { + Ok(resp) => resp, + Err(PeerError::Rejected) => { + return ApiObservation::running(AuthState::Rejected, started.elapsed(), None) + } + Err(err) => return ApiObservation::down(&err), + }; + let latency = started.elapsed(); + match resp.status { + 200..=299 => ApiObservation::running(AuthState::Ok, latency, version_of(&resp)), + 401 | 403 => ApiObservation::running(AuthState::Rejected, latency, None), + 404 => probe_healthz(transport, peer, timeout, latency).await, + status => ApiObservation::down(&PeerError::Http(status)), + } +} + +async fn probe_healthz( + transport: &dyn PeerTransport, + peer: &PeerConfig, + timeout: Duration, + so_far: Duration, +) -> ApiObservation { + let req = PeerRequest { + path: HEALTHZ_PATH.into(), + query: Vec::new(), + headers: Vec::new(), + accept: None, + }; + match transport.get(peer, req, timeout).await { + Ok(resp) if (200..300).contains(&resp.status) => { + ApiObservation::running(AuthState::Unknown, so_far, version_of(&resp)) + } + Ok(resp) if resp.status == 401 || resp.status == 403 => { + ApiObservation::running(AuthState::Rejected, so_far, None) + } + Ok(resp) => ApiObservation::down(&PeerError::Http(resp.status)), + Err(PeerError::Rejected) => ApiObservation::running(AuthState::Rejected, so_far, None), + Err(err) => ApiObservation::down(&err), + } +} + +fn version_of(resp: &PeerResponse) -> Option { + let PeerBody::Bytes(body) = &resp.body else { + return None; + }; + let value: Value = serde_json::from_slice(body).ok()?; + value + .get("version") + .and_then(Value::as_str) + .map(str::to_owned) +} + +/// One TCP connect and one API probe, for the diagnostic CLI. /// -/// A peer that answers 401 is up with `auth = rejected`: the API is running -/// and our credential is the fault. Version and error classification beyond -/// that belong to the registry loops. +/// Both probes run concurrently so the caller waits for the slower one, not +/// the sum. The threshold is one: a single failure is reported as down. pub async fn check_once( transport: &dyn PeerTransport, peer: &PeerConfig, @@ -49,69 +280,93 @@ pub async fn check_once( api_timeout: Duration, ) -> (ServerStatus, ApiStatus) { let now = Timestamp::now(); - let server = match transport.tcp_check(peer, tcp_timeout).await { - Ok(latency) => ServerStatus { - up: true, - since: Some(now), - last_checked: Some(now), - last_error: None, - latency_ms: Some(millis(latency)), - }, - Err(err) => failed_server(&err, now), - }; - - let req = PeerRequest { - path: "/api/v1/ping".into(), - query: Vec::new(), - headers: Vec::new(), - accept: None, - }; - // The GET is not timed, so `latency_ms` stays empty. The probe that - // replaces this stub measures it; `peer check` prints that value. - let api = match transport.get(peer, req, api_timeout).await { - Ok(resp) if (200..300).contains(&resp.status) => ApiStatus { - up: true, - since: Some(now), - last_checked: Some(now), - last_error: None, - latency_ms: None, - auth: AuthState::Ok, - peer_version: None, - }, - Ok(resp) if resp.status == 401 || resp.status == 403 => ApiStatus { - up: true, - since: Some(now), - last_checked: Some(now), - last_error: Some("unauthorized".into()), - latency_ms: None, - auth: AuthState::Rejected, - peer_version: None, - }, - Ok(resp) => failed_api(&PeerError::Http(resp.status), now), - Err(PeerError::Rejected) => ApiStatus { - up: true, - since: Some(now), - last_checked: Some(now), - last_error: Some("unauthorized".into()), - latency_ms: None, - auth: AuthState::Rejected, - peer_version: None, - }, - Err(err) => failed_api(&err, now), - }; + let (server, api) = tokio::join!( + probe_server(transport, peer, tcp_timeout), + probe_api(transport, peer, api_timeout) + ); + let server = ServerTracker::seeded(&ServerStatus::default()).record(server, now, 1); + let api = ApiTracker::seeded(&ApiStatus::default()).record(api, now, 1); (server, api) } +/// Written by the loop after every probe. +pub trait StatusSink: Send + Sync { + fn apply_server(&self, name: &PeerName, status: ServerStatus); + fn apply_api(&self, name: &PeerName, status: ApiStatus); +} + +/// How long to wait before a peer's first check, so a restart of the +/// consolidator does not hit every peer at the same instant. +pub fn start_jitter(name: &PeerName, interval: Duration) -> Duration { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + name.hash(&mut hasher); + let window_ms = (u64::try_from(interval.as_millis()).unwrap_or(u64::MAX) / 4).max(1); + Duration::from_millis(hasher.finish() % window_ms) +} + +/// One loop per peer: both checks share a task so a peer's own slow check +/// cannot fan out into more tasks. A hung peer only delays its own next +/// round; every await is bounded by its timeout and cancellation. +pub async fn run_peer( + transport: Arc, + peer: PeerConfig, + cfg: LoopConfig, + initial: Option, + sink: Arc, + token: CancellationToken, +) { + let mut server = ServerTracker::seeded( + initial + .as_ref() + .map(|s| &s.server) + .unwrap_or(&ServerStatus::default()), + ); + let mut api = ApiTracker::seeded( + initial + .as_ref() + .map(|s| &s.api) + .unwrap_or(&ApiStatus::default()), + ); + let jitter = start_jitter(&peer.name, cfg.tcp_interval.min(cfg.api_interval)); + let first = tokio::time::Instant::now() + jitter; + let mut tcp_deadline = first; + let mut api_deadline = first; + + tokio::select! { + _ = token.cancelled() => return, + _ = tokio::time::sleep_until(first) => {} + } + + loop { + tokio::select! { + _ = token.cancelled() => break, + _ = tokio::time::sleep_until(tcp_deadline) => { + let result = probe_server(transport.as_ref(), &peer, cfg.tcp_timeout).await; + sink.apply_server(&peer.name, server.record(result, Timestamp::now(), cfg.fail_threshold)); + tcp_deadline = tokio::time::Instant::now() + cfg.tcp_interval; + } + _ = tokio::time::sleep_until(api_deadline) => { + let observation = probe_api(transport.as_ref(), &peer, cfg.api_timeout).await; + sink.apply_api(&peer.name, api.record(observation, Timestamp::now(), cfg.fail_threshold)); + api_deadline = tokio::time::Instant::now() + cfg.api_interval; + } + } + } +} + #[cfg(test)] mod tests { use super::*; - use crate::peers::error::PeerError; use crate::peers::testing::{fake_peers_config, FakeTransport}; + fn cfg() -> PeersConfig { + fake_peers_config(&["beta"]) + } + #[tokio::test] async fn healthy_peer() { let fake = FakeTransport::new(); - let cfg = fake_peers_config(&["beta"]); + let cfg = cfg(); let (server, api) = check_once( fake.as_ref(), &cfg.peer[0], @@ -123,6 +378,7 @@ mod tests { assert_eq!(server.latency_ms, Some(1)); assert!(api.up); assert_eq!(api.auth, AuthState::Ok); + assert_eq!(api.peer_version.as_deref(), Some("0.0.0-fake")); assert_eq!(fake.calls().len(), 1); } @@ -131,7 +387,7 @@ mod tests { let fake = FakeTransport::new(); fake.peer("beta") .respond_err("/api/v1/ping", PeerError::Rejected); - let cfg = fake_peers_config(&["beta"]); + let cfg = cfg(); let (_, api) = check_once( fake.as_ref(), &cfg.peer[0], @@ -144,13 +400,50 @@ mod tests { assert_eq!(api.last_error.as_deref(), Some("unauthorized")); } + #[tokio::test] + async fn a_401_status_is_rejected_not_down() { + let fake = FakeTransport::new(); + fake.peer("beta") + .respond("/api/v1/ping", 401, "application/json", "{}"); + let cfg = cfg(); + let (_, api) = check_once( + fake.as_ref(), + &cfg.peer[0], + Duration::from_secs(1), + Duration::from_secs(1), + ) + .await; + assert!(api.up); + assert_eq!(api.auth, AuthState::Rejected); + } + + #[tokio::test] + async fn older_peer_falls_back_to_healthz_with_unknown_auth() { + let fake = FakeTransport::new(); + fake.peer("beta") + .respond("/api/v1/ping", 404, "application/json", "{}") + .respond("/healthz", 200, "application/json", r#"{"status":"ok"}"#); + let cfg = cfg(); + let (_, api) = check_once( + fake.as_ref(), + &cfg.peer[0], + Duration::from_secs(1), + Duration::from_secs(1), + ) + .await; + assert!(api.up); + assert_eq!(api.auth, AuthState::Unknown); + let paths: Vec<_> = fake.calls().into_iter().map(|(_, r)| r.path).collect(); + assert_eq!(paths, ["/api/v1/ping", "/healthz"]); + } + #[tokio::test] async fn down_peer_reports_class() { let fake = FakeTransport::new(); fake.peer("beta").tcp(Err(PeerError::Timeout)); fake.peer("beta") .respond_err("/api/v1/ping", PeerError::Timeout); - let cfg = fake_peers_config(&["beta"]); + let cfg = cfg(); let (server, api) = check_once( fake.as_ref(), &cfg.peer[0], @@ -170,7 +463,7 @@ mod tests { let fake = FakeTransport::new(); fake.peer("beta") .respond_err("/api/v1/ping", PeerError::BadRequest(leaked.into())); - let cfg = fake_peers_config(&["beta"]); + let cfg = cfg(); let (_, api) = check_once( fake.as_ref(), &cfg.peer[0], @@ -183,4 +476,69 @@ mod tests { assert!(!stored.contains("token")); assert!(!stored.contains("http://")); } + + #[test] + fn threshold_needs_consecutive_failures_and_recovers_immediately() { + let mut server = ServerTracker::seeded(&ServerStatus::default()); + let t0 = Timestamp::now(); + let ok = Duration::from_millis(2); + let first = server.record(Ok(ok), t0, 2); + assert!(first.up); + assert_eq!(first.since, Some(t0)); + let since_up = first.since; + + let t1 = t0 + Duration::from_secs(1); + let still_up = server.record(Err(PeerError::Timeout), t1, 2); + assert!(still_up.up, "one failure below the threshold"); + assert_eq!(still_up.since, since_up); + assert_eq!(still_up.last_error.as_deref(), Some("timeout")); + + let t2 = t0 + Duration::from_secs(2); + let down = server.record(Err(PeerError::Timeout), t2, 2); + assert!(!down.up); + assert_eq!(down.since, Some(t2)); + + let t3 = t0 + Duration::from_secs(3); + let back = server.record(Ok(ok), t3, 2); + assert!(back.up, "recovery does not wait for a threshold"); + assert_eq!(back.since, Some(t3)); + } + + #[test] + fn first_failed_check_still_records_a_since() { + let mut server = ServerTracker::seeded(&ServerStatus::default()); + let t0 = Timestamp::now(); + let st = server.record(Err(PeerError::Unreachable), t0, 2); + assert!(!st.up); + assert_eq!(st.since, Some(t0)); + } + + #[test] + fn api_tracker_keeps_auth_when_up_and_clears_it_when_down() { + let mut api = ApiTracker::seeded(&ApiStatus::default()); + let t0 = Timestamp::now(); + let up = api.record( + ApiObservation::running(AuthState::Rejected, Duration::from_millis(3), None), + t0, + 2, + ); + assert!(up.up); + assert_eq!(up.auth, AuthState::Rejected); + assert_eq!(up.last_error.as_deref(), Some("unauthorized")); + + let t1 = t0 + Duration::from_secs(1); + let down = api.record(ApiObservation::down(&PeerError::Unreachable), t1, 1); + assert!(!down.up); + assert_eq!(down.auth, AuthState::Unknown); + assert_eq!(down.peer_version, None); + } + + #[test] + fn jitter_is_bounded_and_stable() { + let interval = Duration::from_secs(10); + let name = PeerName::parse("beta").unwrap(); + let j = start_jitter(&name, interval); + assert!(j < interval / 3, "{j:?}"); + assert_eq!(j, start_jitter(&name, interval)); + } } diff --git a/src/peers/registry.rs b/src/peers/registry.rs index e910d42..77849e7 100644 --- a/src/peers/registry.rs +++ b/src/peers/registry.rs @@ -2,11 +2,16 @@ //! side never awaits on a peer and is cheap enough for a metrics scrape. use std::collections::BTreeMap; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +use crate::config::{CliOverrides, Config}; use crate::error::AppError; use crate::peers::error::PeerConfigError; +use crate::peers::health::{self, LoopConfig, StatusSink}; use crate::peers::transport::{PeerResponse, PeerTransport}; use crate::peers::types::{ApiStatus, PeerConfig, PeerName, PeerStatus, PeersConfig, ServerStatus}; @@ -29,9 +34,39 @@ struct RegistryState { statuses: BTreeMap, } +/// Status storage shared with the loops; this is the only mutable state a +/// loop needs, so the registry can be dropped without leaking a task. +struct Shared { + state: RwLock, +} + +impl StatusSink for Shared { + fn apply_server(&self, name: &PeerName, status: ServerStatus) { + if let Some(entry) = lock_write(&self.state).statuses.get_mut(name) { + entry.server = status; + } + } + + fn apply_api(&self, name: &PeerName, status: ApiStatus) { + if let Some(entry) = lock_write(&self.state).statuses.get_mut(name) { + entry.api = status; + } + } +} + +struct LoopEntry { + peer: PeerConfig, + params: LoopConfig, + token: CancellationToken, + join: JoinHandle<()>, +} + pub struct PeerRegistry { transport: Arc, - state: RwLock, + shared: Arc, + loops: Mutex>, + loops_enabled: bool, + shutdown: CancellationToken, } fn initial_status(peer: &PeerConfig) -> PeerStatus { @@ -52,8 +87,22 @@ fn lock_write(lock: &RwLock) -> std::sync::RwLockWriteGuard<'_, T> { lock.write().unwrap_or_else(|e| e.into_inner()) } +fn lock_mutex(lock: &Mutex) -> std::sync::MutexGuard<'_, T> { + lock.lock().unwrap_or_else(|e| e.into_inner()) +} + impl PeerRegistry { + /// Registry without loops, for tests that drive the transport directly. pub(crate) fn new(config: PeersConfig, transport: Arc) -> Self { + Self::build(config, transport, CancellationToken::new(), false) + } + + fn build( + config: PeersConfig, + transport: Arc, + shutdown: CancellationToken, + loops_enabled: bool, + ) -> Self { let statuses = config .peer .iter() @@ -61,10 +110,15 @@ impl PeerRegistry { .collect(); Self { transport, - state: RwLock::new(RegistryState { - config: Arc::new(config), - statuses, + shared: Arc::new(Shared { + state: RwLock::new(RegistryState { + config: Arc::new(config), + statuses, + }), }), + loops: Mutex::new(BTreeMap::new()), + loops_enabled, + shutdown, } } @@ -73,40 +127,114 @@ impl PeerRegistry { config: PeersConfig, transport: Arc, ) -> Self { - Self { - transport, - state: RwLock::new(RegistryState { - config: Arc::new(config), - statuses: statuses.into_iter().map(|s| (s.name.clone(), s)).collect(), - }), + let reg = Self::build(config, transport, CancellationToken::new(), false); + lock_write(®.shared.state).statuses = + statuses.into_iter().map(|s| (s.name.clone(), s)).collect(); + reg + } + + /// Real registry for the server: builds the outbound transport and starts + /// one loop per enabled peer. `shutdown` is the process token; cancelling + /// it stops the loops. + pub fn start( + config: PeersConfig, + shutdown: CancellationToken, + ) -> Result, PeerConfigError> { + let transport = Arc::new(ReqwestTransport::new(&config)?); + Ok(Self::start_with_transport(config, transport, shutdown)) + } + + /// Loop-enabled registry over a supplied transport, for tests. + pub fn start_with_transport( + config: PeersConfig, + transport: Arc, + shutdown: CancellationToken, + ) -> Arc { + let registry = Arc::new(Self::build(config, transport, shutdown.child_token(), true)); + registry.spawn_all(); + registry + } + + fn spawn_all(&self) { + let peers: Vec = if self.config().enabled { + self.config() + .peer + .iter() + .filter(|peer| peer.enabled) + .cloned() + .collect() + } else { + Vec::new() + }; + for peer in peers { + self.spawn_peer(&peer); } } + fn spawn_peer(&self, peer: &PeerConfig) { + let params = LoopConfig::from_config(&self.config()); + let initial = self.status_internal(&peer.name); + let token = self.shutdown.child_token(); + let sink: Arc = self.shared.clone(); + let transport = self.transport.clone(); + let peer_copy = peer.clone(); + let name = peer.name.clone(); + let task_token = token.clone(); + let join = tokio::spawn(async move { + health::run_peer(transport, peer_copy, params, initial, sink, task_token).await; + }); + lock_mutex(&self.loops).insert( + name, + LoopEntry { + peer: peer.clone(), + params, + token, + join, + }, + ); + } + + fn stop_loop(&self, entry: LoopEntry) { + // Cancel, then let the task finish its bounded probe on its own. + entry.token.cancel(); + tokio::spawn(async move { + let _ = entry.join.await; + }); + } + /// Sorted by name; disabled peers included. pub fn snapshot(&self) -> Vec { - lock_read(&self.state).statuses.values().cloned().collect() + lock_read(&self.shared.state) + .statuses + .values() + .cloned() + .collect() } pub fn status(&self, name: &str) -> Result { let Ok(key) = PeerName::parse(name) else { return Err(AppError::NotFound(format!("peer {name}: unknown peer"))); }; - lock_read(&self.state) + lock_read(&self.shared.state) .statuses .get(&key) .cloned() .ok_or_else(|| AppError::NotFound(format!("peer {name}: unknown peer"))) } + fn status_internal(&self, name: &PeerName) -> Option { + lock_read(&self.shared.state).statuses.get(name).cloned() + } + pub fn contains(&self, name: &str) -> bool { let Ok(key) = PeerName::parse(name) else { return false; }; - lock_read(&self.state).statuses.contains_key(&key) + lock_read(&self.shared.state).statuses.contains_key(&key) } pub fn config(&self) -> Arc { - lock_read(&self.state).config.clone() + lock_read(&self.shared.state).config.clone() } /// Single proxy entry point for REST and MCP. @@ -119,7 +247,7 @@ impl PeerRegistry { ) -> Result { let key = PeerName::parse(name).ok(); let (peer, timeout) = { - let st = lock_read(&self.state); + let st = lock_read(&self.shared.state); let peer = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key).cloned()); (peer, Duration::from_secs(st.config.request_timeout_secs)) }; @@ -128,30 +256,114 @@ impl PeerRegistry { } /// Replace the peer set. Statuses of peers that survive a reload are kept - /// so `since` survives a config change. Validation arrives with the - /// config and secrets tasks; this body only swaps state. + /// so `since` survives a config change. Loops for new or changed peers are + /// started, loops for removed or disabled peers are cancelled. + /// + /// A bad `[peers]` block never reaches this method: the handler logs the + /// config error and keeps the old registry, as TLS reload does. pub async fn reload(&self, new: PeersConfig) -> Result<(), PeerConfigError> { - let mut st = lock_write(&self.state); - let old = std::mem::take(&mut st.statuses); - st.statuses = new - .peer - .iter() - .map(|peer| { - let mut status = old - .get(&peer.name) - .cloned() - .unwrap_or_else(|| initial_status(peer)); - status.enabled = peer.enabled; - status.url = peer.display_url(); - (peer.name.clone(), status) - }) - .collect(); - st.config = Arc::new(new); + let params = LoopConfig::from_config(&new); + { + let mut st = lock_write(&self.shared.state); + let old = std::mem::take(&mut st.statuses); + st.statuses = new + .peer + .iter() + .map(|peer| { + let mut status = old + .get(&peer.name) + .cloned() + .unwrap_or_else(|| initial_status(peer)); + status.enabled = peer.enabled; + status.url = peer.display_url(); + (peer.name.clone(), status) + }) + .collect(); + st.config = Arc::new(new.clone()); + } + if self.loops_enabled { + self.resync_loops(&new, params); + } Ok(()) } - /// Health loops stop on this; nothing is running yet. - pub async fn shutdown(&self) {} + fn resync_loops(&self, new: &PeersConfig, params: LoopConfig) { + let mut old = std::mem::take(&mut *lock_mutex(&self.loops)); + let mut kept = BTreeMap::new(); + for (name, entry) in std::mem::take(&mut old) { + let wanted = new + .enabled + .then(|| { + new.peer + .iter() + .find(|peer| peer.name == name && peer.enabled) + }) + .flatten(); + match wanted { + Some(peer) if entry.peer == *peer && entry.params == params => { + kept.insert(name, entry); + } + _ => self.stop_loop(entry), + } + } + *lock_mutex(&self.loops) = kept; + if new.enabled { + for peer in new.peer.iter().filter(|peer| peer.enabled) { + if !lock_mutex(&self.loops).contains_key(&peer.name) { + self.spawn_peer(peer); + } + } + } + } + + /// Stop every loop and wait (bounded) for the tasks to exit. + pub async fn shutdown(&self) { + self.shutdown.cancel(); + let joins: Vec> = std::mem::take(&mut *lock_mutex(&self.loops)) + .into_values() + .map(|entry| entry.join) + .collect(); + let join_all = async { + for join in joins { + let _ = join.await; + } + }; + if tokio::time::timeout(Duration::from_secs(5), join_all) + .await + .is_err() + { + tracing::warn!("peer health loops did not stop in time"); + } + } +} + +/// Everything the SIGHUP handler needs to re-read the `[peers]` block. +#[derive(Clone)] +pub struct PeersReloader { + registry: Arc, + path: Option, + overrides: CliOverrides, +} + +impl PeersReloader { + pub fn new( + registry: Arc, + path: Option, + overrides: CliOverrides, + ) -> Self { + Self { + registry, + path, + overrides, + } + } + + /// Re-read the whole config and apply only `[peers]`. A failure returns + /// the config error; the old registry stays in place. + pub async fn reload(&self) -> Result<(), PeerConfigError> { + let cfg = Config::load(self.path.as_deref(), &self.overrides)?; + self.registry.reload(cfg.peers).await + } } #[cfg(test)] @@ -159,6 +371,7 @@ mod tests { use super::*; use crate::peers::testing::{fake_peers_config, registry_with, static_registry, FakeTransport}; use crate::peers::transport::PeerBody; + use crate::peers::types::AuthState; fn status(name: &str, up: bool) -> PeerStatus { PeerStatus { @@ -184,7 +397,7 @@ mod tests { assert_eq!(names, ["beta", "gamma"]); let b = reg.status("beta").unwrap(); assert!(b.server.up); - assert_eq!(b.api.auth, crate::peers::types::AuthState::Unknown); + assert_eq!(b.api.auth, AuthState::Unknown); } #[test] @@ -247,7 +460,7 @@ mod tests { async fn reload_keeps_surviving_status_and_prunes_removed() { let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta", "gamma"])); { - let mut st = lock_write(®.state); + let mut st = lock_write(®.shared.state); let beta = st .statuses .get_mut(&PeerName::parse("beta").unwrap()) @@ -256,8 +469,7 @@ mod tests { } let mut new = fake_peers_config(&["beta", "delta"]); new.peer[0].enabled = false; - let reloaded: Result<(), PeerConfigError> = reg.reload(new).await; - reloaded.unwrap(); + reg.reload(new).await.unwrap(); let snap = reg.snapshot(); assert_eq!(snap.len(), 2); @@ -267,4 +479,149 @@ mod tests { assert_eq!(snap[1].name.as_str(), "delta"); assert!(!snap[1].server.up); } + + async fn wait_for bool>(reg: &PeerRegistry, f: F) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + loop { + if f(reg) { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "condition not met: {:?}", + reg.snapshot() + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + #[tokio::test] + async fn loops_mark_up_and_honour_the_failure_threshold() { + let fake = FakeTransport::new(); + let mut cfg = fake_peers_config(&["beta"]); + cfg.fail_threshold = 2; + let reg = PeerRegistry::start_with_transport(cfg, fake.clone(), CancellationToken::new()); + wait_for(®, |r| { + let s = r.status("beta").unwrap(); + s.server.up && s.api.up && s.api.auth == AuthState::Ok + }) + .await; + let up = reg.status("beta").unwrap(); + assert!(up.server.since.is_some()); + assert_eq!(up.api.peer_version.as_deref(), Some("0.0.0-fake")); + + fake.peer("beta").tcp(Err(crate::peers::PeerError::Timeout)); + fake.peer("beta") + .respond_err("/api/v1/ping", crate::peers::PeerError::Timeout); + // One failure each: still up with the error visible. + wait_for(®, |r| { + let s = r.status("beta").unwrap(); + s.server.last_error.as_deref() == Some("timeout") + && s.api.last_error.as_deref() == Some("timeout") + }) + .await; + assert!(reg.status("beta").unwrap().server.up); + + reg.shutdown().await; + let after = reg.status("beta").unwrap(); + assert!(after.server.up, "one failure below threshold = still up"); + } + + #[tokio::test] + async fn a_hung_peer_does_not_delay_another_peers_checks() { + let fake = FakeTransport::new(); + fake.peer("beta").delay(Duration::from_secs(2)); + fake.peer("gamma") + .respond("/api/v1/ping", 200, "application/json", "{}"); + let reg = PeerRegistry::start_with_transport( + fake_peers_config(&["beta", "gamma"]), + fake.clone(), + CancellationToken::new(), + ); + wait_for(®, |r| { + r.status("gamma").unwrap().api.last_checked.is_some() + }) + .await; + let gamma_checked = reg.status("gamma").unwrap().api.last_checked; + assert!(reg.status("beta").unwrap().api.last_checked.is_none()); + assert!(gamma_checked.is_some()); + reg.shutdown().await; + } + + #[test] + #[allow(clippy::result_large_err)] // figment::Jail closures return figment::Error + fn reloader_rereads_the_peers_block_and_keeps_the_old_set_on_error() { + use figment::Jail; + + let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"])); + Jail::expect_with(|jail| { + jail.create_file( + "config.toml", + r#" + [peers] + enabled = true + api_interval_secs = 2 + [[peers.peer]] + name = "gamma" + url = "http://gamma:1984" + token_file = "/run/credentials/skyg-linux-tracker.service/peer-gamma" + allow_plain_http = true + "#, + )?; + let reloader = PeersReloader::new( + reg.clone(), + Some(std::path::PathBuf::from("config.toml")), + CliOverrides::default(), + ); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(reloader.reload()) + .unwrap(); + let snap = reg.snapshot(); + assert_eq!(snap.len(), 1); + assert_eq!(snap[0].name.as_str(), "gamma"); + assert_eq!(reg.config().api_interval_secs, 2); + + let broken = PeersReloader::new( + reg.clone(), + Some(std::path::PathBuf::from("missing.toml")), + CliOverrides::default(), + ); + assert!(tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(broken.reload()) + .is_err()); + assert!(reg.contains("gamma"), "a bad reload keeps the old peers"); + Ok(()) + }); + } + + #[tokio::test] + async fn reload_moves_loops_to_the_new_peer_set() { + let fake = FakeTransport::new(); + let reg = PeerRegistry::start_with_transport( + fake_peers_config(&["beta"]), + fake.clone(), + CancellationToken::new(), + ); + wait_for(®, |r| r.status("beta").unwrap().server.up).await; + + reg.reload(fake_peers_config(&["beta", "gamma"])) + .await + .unwrap(); + wait_for(®, |r| r.status("gamma").unwrap().server.up).await; + assert!( + reg.status("beta").unwrap().server.up, + "kept peer keeps state" + ); + + reg.reload(fake_peers_config(&["gamma"])).await.unwrap(); + assert!(reg.status("beta").is_err()); + assert!(reg.contains("gamma")); + reg.shutdown().await; + } } diff --git a/tests/peers_registry.rs b/tests/peers_registry.rs index 0d36d93..af80026 100644 --- a/tests/peers_registry.rs +++ b/tests/peers_registry.rs @@ -1,17 +1,28 @@ -//! Peer client against a local HTTP listener. +//! Peer client against a local HTTP listener, then real health loops against +//! a second in-process `slinuxm` with fake sources. //! -//! The listener is not a full `slinuxm`; it only has to speak enough HTTP/1.1 -//! to prove the real transport. Later cases in this file drive a second -//! in-process server with fake sources. +//! The first cases use a hand-written listener: it only has to speak enough +//! HTTP/1.1 to prove the transport. The later cases run the assembled router +//! so the `PeerTransport`, probing and registry state are exercised together. +use std::path::Path; +use std::sync::Arc; use std::time::Duration; -use skyg_linux_tracker::peers::registry::ReqwestTransport; +use skyg_linux_tracker::auth::tokens; +use skyg_linux_tracker::config::Config; +use skyg_linux_tracker::peers::registry::{PeerRegistry, ReqwestTransport}; use skyg_linux_tracker::peers::transport::{PeerBody, PeerRequest, PeerTransport}; +use skyg_linux_tracker::peers::types::AuthState; +use skyg_linux_tracker::peers::types::PeerStatus; use skyg_linux_tracker::peers::types::{PeerConfig, PeerName, PeersConfig}; use skyg_linux_tracker::peers::PeerError; +use skyg_linux_tracker::server; +use skyg_linux_tracker::sources::fake::fake_state; +use sqlx::sqlite::SqlitePoolOptions; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; +use tokio_util::sync::CancellationToken; fn transport() -> ReqwestTransport { ReqwestTransport::new(&PeersConfig { @@ -131,3 +142,145 @@ async fn client_get_is_capped_and_does_not_follow_redirects() { assert!(matches!(err, PeerError::Oversize), "{err:?}"); let _ = std::fs::remove_file(&token); } + +/// A second in-process `slinuxm` (fake sources) acting as the peer. +struct RunningPeer { + port: u16, + shutdown: CancellationToken, + join: tokio::task::JoinHandle>, +} + +impl RunningPeer { + async fn stop(self) { + self.shutdown.cancel(); + let _ = self.join.await; + } +} + +async fn start_peer_at(port: u16, auth: bool) -> RunningPeer { + let listener = TcpListener::bind(("127.0.0.1", port)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let mut config = Config::default(); + config.server.port = port; + config.auth.enabled = auth; + let mut state = fake_state(config); + if auth { + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect("sqlite::memory:") + .await + .unwrap(); + skyg_linux_tracker::db::migrate(&pool).await.unwrap(); + let _ = tokens::create(&pool, "peer-consumer").await.unwrap(); + state.db = Some(pool); + } + let shutdown = state.shutdown.clone(); + let app = server::build_router(state); + let join = tokio::spawn(server::serve_listener( + listener, + None, + app, + shutdown.clone(), + server::Limits::default(), + )); + RunningPeer { + port, + shutdown, + join, + } +} + +fn registry_for(port: u16, token_path: &Path, shutdown: CancellationToken) -> Arc { + let mut cfg = skyg_linux_tracker::peers::testing::fake_peers_config(&["beta"]); + cfg.peer[0].url = url::Url::parse(&format!("http://127.0.0.1:{port}")).unwrap(); + cfg.peer[0].token_file = token_path.to_path_buf(); + PeerRegistry::start(cfg, shutdown).unwrap() +} + +async fn wait_for bool>(reg: &PeerRegistry, f: F) -> PeerStatus { + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + loop { + let status = reg.status("beta").unwrap(); + if f(&status) { + return status; + } + assert!( + tokio::time::Instant::now() < deadline, + "peer status stayed {:?}", + status + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } +} + +#[tokio::test] +async fn loops_observe_up_down_and_recovery() { + let token = token_file("anything\n"); + let peer = start_peer_at(0, false).await; + let port = peer.port; + let shutdown = CancellationToken::new(); + let reg = registry_for(port, &token, shutdown.clone()); + + let up = wait_for(®, |s| s.server.up && s.api.up).await; + assert_eq!(up.api.auth, AuthState::Ok); + let since_up = up.server.since.expect("first check records since"); + + // Port closed: both checks fail within their own timeout. + peer.stop().await; + let down = wait_for(®, |s| !s.server.up && !s.api.up).await; + assert!(down.server.last_error.is_some()); + assert_eq!(down.api.auth, AuthState::Unknown); + let since_down = down.server.since.expect("flip records since"); + assert_ne!(since_down, since_up); + + // Same port again: recovery is immediate, not threshold-bound. + let peer = start_peer_at(port, false).await; + let back = wait_for(®, |s| s.server.up && s.api.up).await; + assert_ne!(back.server.since, Some(since_down)); + reg.shutdown().await; + peer.stop().await; + let _ = std::fs::remove_file(&token); +} + +#[tokio::test] +async fn api_down_with_the_listener_still_accepting() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let hang = tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((sock, _)) = listener.accept().await { + held.push(sock); + if held.len() > 8 { + held.remove(0); + } + } + }); + + let token = token_file("anything\n"); + let shutdown = CancellationToken::new(); + let reg = registry_for(port, &token, shutdown.clone()); + let status = wait_for(®, |s| s.api.last_checked.is_some() && !s.api.up).await; + assert!(status.server.up, "TCP connect succeeds"); + assert!(!status.api.up, "no HTTP answer before the timeout"); + assert_eq!(status.api.last_error.as_deref(), Some("timeout")); + reg.shutdown().await; + hang.abort(); + let _ = std::fs::remove_file(&token); +} + +#[tokio::test] +async fn a_rejected_token_leaves_the_api_up_and_names_the_auth_state() { + let peer = start_peer_at(0, true).await; + let token = token_file("not-the-peer-token\n"); + let shutdown = CancellationToken::new(); + let reg = registry_for(peer.port, &token, shutdown.clone()); + + let status = wait_for(®, |s| s.api.last_checked.is_some()).await; + assert!(status.server.up); + assert!(status.api.up, "a 401 proves the API is running"); + assert_eq!(status.api.auth, AuthState::Rejected); + assert_eq!(status.api.last_error.as_deref(), Some("unauthorized")); + reg.shutdown().await; + peer.stop().await; + let _ = std::fs::remove_file(&token); +}