//! In-memory peer views. Health loops refresh the status entries; the read //! side never awaits on a peer and is cheap enough for a metrics scrape. use std::collections::BTreeMap; use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use crate::config::{CliOverrides, Config}; use crate::error::AppError; use crate::peers::error::PeerConfigError; use crate::peers::health::{self, LoopConfig, StatusSink}; use crate::peers::transport::{PeerResponse, PeerTransport}; use crate::peers::types::{ApiStatus, PeerConfig, PeerName, PeerStatus, PeersConfig, ServerStatus}; // `client.rs` lives next to this file. The peers module list does not declare it. #[path = "client.rs"] pub mod client; pub use client::ReqwestTransport; /// Options a surface passes down to the proxy layer. Trace context is a /// pre-built `traceparent` header value; the peer hop is always marked. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ProxyOpts { pub traceparent: Option, pub accept_sse: bool, } struct RegistryState { config: Arc, statuses: BTreeMap, } /// Status storage shared with the loops; this is the only mutable state a /// loop needs, so the registry can be dropped without leaking a task. struct Shared { state: RwLock, } impl StatusSink for Shared { fn apply_server(&self, name: &PeerName, status: ServerStatus) { if let Some(entry) = lock_write(&self.state).statuses.get_mut(name) { entry.server = status; } } fn apply_api(&self, name: &PeerName, status: ApiStatus) { if let Some(entry) = lock_write(&self.state).statuses.get_mut(name) { entry.api = status; } } } struct LoopEntry { peer: PeerConfig, params: LoopConfig, token: CancellationToken, join: JoinHandle<()>, } pub struct PeerRegistry { transport: Arc, shared: Arc, loops: Mutex>, loops_enabled: bool, shutdown: CancellationToken, } fn initial_status(peer: &PeerConfig) -> PeerStatus { PeerStatus { name: peer.name.clone(), enabled: peer.enabled, url: peer.display_url(), server: ServerStatus::default(), api: ApiStatus::default(), } } fn lock_read(lock: &RwLock) -> std::sync::RwLockReadGuard<'_, T> { lock.read().unwrap_or_else(|e| e.into_inner()) } fn lock_write(lock: &RwLock) -> std::sync::RwLockWriteGuard<'_, T> { lock.write().unwrap_or_else(|e| e.into_inner()) } fn lock_mutex(lock: &Mutex) -> std::sync::MutexGuard<'_, T> { lock.lock().unwrap_or_else(|e| e.into_inner()) } impl PeerRegistry { /// Registry without loops, for tests that drive the transport directly. pub(crate) fn new(config: PeersConfig, transport: Arc) -> Self { Self::build(config, transport, CancellationToken::new(), false) } fn build( config: PeersConfig, transport: Arc, shutdown: CancellationToken, loops_enabled: bool, ) -> Self { let statuses = config .peer .iter() .map(|peer| (peer.name.clone(), initial_status(peer))) .collect(); Self { transport, shared: Arc::new(Shared { state: RwLock::new(RegistryState { config: Arc::new(config), statuses, }), }), loops: Mutex::new(BTreeMap::new()), loops_enabled, shutdown, } } pub(crate) fn from_statuses( statuses: Vec, config: PeersConfig, transport: Arc, ) -> Self { let reg = Self::build(config, transport, CancellationToken::new(), false); lock_write(®.shared.state).statuses = statuses.into_iter().map(|s| (s.name.clone(), s)).collect(); reg } /// Real registry for the server: builds the outbound transport and starts /// one loop per enabled peer. `shutdown` is the process token; cancelling /// it stops the loops. pub fn start( config: PeersConfig, shutdown: CancellationToken, ) -> Result, PeerConfigError> { let transport = Arc::new(ReqwestTransport::new(&config)?); Ok(Self::start_with_transport(config, transport, shutdown)) } /// Loop-enabled registry over a supplied transport, for tests. pub fn start_with_transport( config: PeersConfig, transport: Arc, shutdown: CancellationToken, ) -> Arc { let registry = Arc::new(Self::build(config, transport, shutdown.child_token(), true)); registry.spawn_all(); registry } fn spawn_all(&self) { let peers: Vec = if self.config().enabled { self.config() .peer .iter() .filter(|peer| peer.enabled) .cloned() .collect() } else { Vec::new() }; for peer in peers { self.spawn_peer(&peer); } } fn spawn_peer(&self, peer: &PeerConfig) { let params = LoopConfig::from_config(&self.config()); let initial = self.status_internal(&peer.name); let token = self.shutdown.child_token(); let sink: Arc = self.shared.clone(); let transport = self.transport.clone(); let peer_copy = peer.clone(); let name = peer.name.clone(); let task_token = token.clone(); let join = tokio::spawn(async move { health::run_peer(transport, peer_copy, params, initial, sink, task_token).await; }); lock_mutex(&self.loops).insert( name, LoopEntry { peer: peer.clone(), params, token, join, }, ); } fn stop_loop(&self, entry: LoopEntry) { // Cancel, then let the task finish its bounded probe on its own. entry.token.cancel(); tokio::spawn(async move { let _ = entry.join.await; }); } /// Sorted by name; disabled peers included. pub fn snapshot(&self) -> Vec { lock_read(&self.shared.state) .statuses .values() .cloned() .collect() } pub fn status(&self, name: &str) -> Result { let Ok(key) = PeerName::parse(name) else { return Err(AppError::NotFound(format!("peer {name}: unknown peer"))); }; lock_read(&self.shared.state) .statuses .get(&key) .cloned() .ok_or_else(|| AppError::NotFound(format!("peer {name}: unknown peer"))) } fn status_internal(&self, name: &PeerName) -> Option { lock_read(&self.shared.state).statuses.get(name).cloned() } pub fn contains(&self, name: &str) -> bool { let Ok(key) = PeerName::parse(name) else { return false; }; lock_read(&self.shared.state).statuses.contains_key(&key) } pub fn config(&self) -> Arc { lock_read(&self.shared.state).config.clone() } /// Single proxy entry point for REST and MCP. pub async fn get( &self, name: &str, path: &str, query: &[(String, String)], opts: ProxyOpts, ) -> Result { let key = PeerName::parse(name).ok(); let (peer, timeout) = { let st = lock_read(&self.shared.state); let peer = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key).cloned()); (peer, Duration::from_secs(st.config.request_timeout_secs)) }; let peer = peer.ok_or_else(|| AppError::NotFound(format!("peer {name}: unknown peer")))?; crate::peers::proxy::get(self.transport.as_ref(), &peer, path, query, opts, timeout).await } /// Replace the peer set. Statuses of peers that survive a reload are kept /// so `since` survives a config change. Loops for new or changed peers are /// started, loops for removed or disabled peers are cancelled. /// /// A bad `[peers]` block never reaches this method: the handler logs the /// config error and keeps the old registry, as TLS reload does. pub async fn reload(&self, new: PeersConfig) -> Result<(), PeerConfigError> { let params = LoopConfig::from_config(&new); { let mut st = lock_write(&self.shared.state); let old = std::mem::take(&mut st.statuses); // Turning the feature off renders an empty list rather than a // stale snapshot; the routes themselves stay mounted until // restart because the router is built once. st.statuses = if new.enabled { new.peer .iter() .map(|peer| { let mut status = old .get(&peer.name) .cloned() .unwrap_or_else(|| initial_status(peer)); status.enabled = peer.enabled; status.url = peer.display_url(); (peer.name.clone(), status) }) .collect() } else { BTreeMap::new() }; st.config = Arc::new(new.clone()); } if self.loops_enabled { self.resync_loops(&new, params); } Ok(()) } fn resync_loops(&self, new: &PeersConfig, params: LoopConfig) { let mut old = std::mem::take(&mut *lock_mutex(&self.loops)); let mut kept = BTreeMap::new(); for (name, entry) in std::mem::take(&mut old) { let wanted = new .enabled .then(|| { new.peer .iter() .find(|peer| peer.name == name && peer.enabled) }) .flatten(); match wanted { Some(peer) if entry.peer == *peer && entry.params == params => { kept.insert(name, entry); } _ => self.stop_loop(entry), } } *lock_mutex(&self.loops) = kept; if new.enabled { for peer in new.peer.iter().filter(|peer| peer.enabled) { if !lock_mutex(&self.loops).contains_key(&peer.name) { self.spawn_peer(peer); } } } } /// Stop every loop and wait (bounded) for the tasks to exit. pub async fn shutdown(&self) { self.shutdown.cancel(); let joins: Vec> = std::mem::take(&mut *lock_mutex(&self.loops)) .into_values() .map(|entry| entry.join) .collect(); let join_all = async { for join in joins { let _ = join.await; } }; if tokio::time::timeout(Duration::from_secs(5), join_all) .await .is_err() { tracing::warn!("peer health loops did not stop in time"); } } } /// Everything the SIGHUP handler needs to re-read the `[peers]` block. #[derive(Clone)] pub struct PeersReloader { registry: Arc, path: Option, overrides: CliOverrides, } impl PeersReloader { pub fn new( registry: Arc, path: Option, overrides: CliOverrides, ) -> Self { Self { registry, path, overrides, } } /// Re-read the whole config and apply only `[peers]`. A failure returns /// the config error; the old registry stays in place. pub async fn reload(&self) -> Result<(), PeerConfigError> { let cfg = Config::load(self.path.as_deref(), &self.overrides)?; self.registry.reload(cfg.peers).await } } #[cfg(test)] mod tests { use super::*; use crate::peers::testing::{fake_peers_config, registry_with, static_registry, FakeTransport}; use crate::peers::transport::PeerBody; use crate::peers::types::AuthState; fn status(name: &str, up: bool) -> PeerStatus { PeerStatus { name: PeerName::parse(name).unwrap(), enabled: true, url: format!("http://{name}:1984"), server: ServerStatus { up, ..ServerStatus::default() }, api: ApiStatus::default(), } } #[test] fn snapshot_is_sorted_and_includes_initial_state() { let reg = static_registry(vec![status("gamma", false), status("beta", true)]); let names: Vec<_> = reg .snapshot() .into_iter() .map(|s| s.name.as_str().to_owned()) .collect(); assert_eq!(names, ["beta", "gamma"]); let b = reg.status("beta").unwrap(); assert!(b.server.up); assert_eq!(b.api.auth, AuthState::Unknown); } #[test] fn unknown_peer_status_is_not_found() { let reg = static_registry(vec![status("beta", true)]); let err = reg.status("nope").unwrap_err(); assert_eq!(err.code(), "not_found"); assert_eq!(err.message(), "peer nope: unknown peer"); assert!(reg.contains("beta")); assert!(!reg.contains("nope")); assert!(!reg.contains("Beta")); assert!(!reg.contains("-beta")); let err = reg.status("Beta").unwrap_err(); assert_eq!(err.code(), "not_found"); assert_eq!(err.message(), "peer Beta: unknown peer"); } #[test] fn config_is_shared() { let cfg = fake_peers_config(&["beta"]); let reg = registry_with(FakeTransport::new(), cfg); let got = reg.config(); assert!(got.enabled); assert_eq!(got.peer[0].name.as_str(), "beta"); assert_eq!(got.request_timeout_secs, 2); } #[tokio::test] async fn get_forwards_and_maps_errors() { let fake = FakeTransport::new(); fake.peer("beta") .respond("/systemd/units", 200, "application/json", "{\"ok\":true}"); let reg = registry_with(fake, fake_peers_config(&["beta"])); let resp = reg .get("beta", "/systemd/units", &[], ProxyOpts::default()) .await .unwrap(); assert_eq!(resp.status, 200); match resp.body { PeerBody::Bytes(b) => assert_eq!(&b[..], b"{\"ok\":true}"), PeerBody::Stream(_) => panic!("expected buffered body"), } let err = reg .get("nope", "/x", &[], ProxyOpts::default()) .await .unwrap_err(); assert_eq!(err.code(), "not_found"); assert_eq!(err.message(), "peer nope: unknown peer"); let err = reg .get("Beta", "/x", &[], ProxyOpts::default()) .await .unwrap_err(); assert_eq!(err.message(), "peer Beta: unknown peer"); } #[tokio::test] async fn reload_keeps_surviving_status_and_prunes_removed() { let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta", "gamma"])); { let mut st = lock_write(®.shared.state); let beta = st .statuses .get_mut(&PeerName::parse("beta").unwrap()) .unwrap(); beta.server.up = true; } let mut new = fake_peers_config(&["beta", "delta"]); new.peer[0].enabled = false; reg.reload(new).await.unwrap(); let snap = reg.snapshot(); assert_eq!(snap.len(), 2); assert_eq!(snap[0].name.as_str(), "beta"); assert!(snap[0].server.up, "health survives a reload"); assert!(!snap[0].enabled); assert_eq!(snap[1].name.as_str(), "delta"); assert!(!snap[1].server.up); } async fn wait_for bool>(reg: &PeerRegistry, f: F) { let deadline = tokio::time::Instant::now() + Duration::from_secs(5); loop { if f(reg) { return; } assert!( tokio::time::Instant::now() < deadline, "condition not met: {:?}", reg.snapshot() ); tokio::time::sleep(Duration::from_millis(10)).await; } } #[tokio::test] async fn loops_mark_up_and_honour_the_failure_threshold() { let fake = FakeTransport::new(); let mut cfg = fake_peers_config(&["beta"]); cfg.fail_threshold = 2; let reg = PeerRegistry::start_with_transport(cfg, fake.clone(), CancellationToken::new()); wait_for(®, |r| { let s = r.status("beta").unwrap(); s.server.up && s.api.up && s.api.auth == AuthState::Ok }) .await; let up = reg.status("beta").unwrap(); assert!(up.server.since.is_some()); assert_eq!(up.api.peer_version.as_deref(), Some("0.0.0-fake")); fake.peer("beta").tcp(Err(crate::peers::PeerError::Timeout)); fake.peer("beta") .respond_err("/api/v1/ping", crate::peers::PeerError::Timeout); // One failure each: still up with the error visible. wait_for(®, |r| { let s = r.status("beta").unwrap(); s.server.last_error.as_deref() == Some("timeout") && s.api.last_error.as_deref() == Some("timeout") }) .await; assert!(reg.status("beta").unwrap().server.up); reg.shutdown().await; let after = reg.status("beta").unwrap(); assert!(after.server.up, "one failure below threshold = still up"); } #[tokio::test] async fn a_hung_peer_does_not_delay_another_peers_checks() { let fake = FakeTransport::new(); fake.peer("beta").delay(Duration::from_secs(2)); fake.peer("gamma") .respond("/api/v1/ping", 200, "application/json", "{}"); let reg = PeerRegistry::start_with_transport( fake_peers_config(&["beta", "gamma"]), fake.clone(), CancellationToken::new(), ); wait_for(®, |r| { r.status("gamma").unwrap().api.last_checked.is_some() }) .await; let gamma_checked = reg.status("gamma").unwrap().api.last_checked; assert!(reg.status("beta").unwrap().api.last_checked.is_none()); assert!(gamma_checked.is_some()); reg.shutdown().await; } #[test] #[allow(clippy::result_large_err)] // figment::Jail closures return figment::Error fn reloader_rereads_the_peers_block_and_keeps_the_old_set_on_error() { use figment::Jail; let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"])); Jail::expect_with(|jail| { jail.create_file( "config.toml", r#" [peers] enabled = true api_interval_secs = 2 [[peers.peer]] name = "gamma" url = "http://gamma:1984" token_file = "/run/credentials/skyg-linux-tracker.service/peer-gamma" allow_plain_http = true "#, )?; let reloader = PeersReloader::new( reg.clone(), Some(std::path::PathBuf::from("config.toml")), CliOverrides::default(), ); tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() .block_on(reloader.reload()) .unwrap(); let snap = reg.snapshot(); assert_eq!(snap.len(), 1); assert_eq!(snap[0].name.as_str(), "gamma"); assert_eq!(reg.config().api_interval_secs, 2); let broken = PeersReloader::new( reg.clone(), Some(std::path::PathBuf::from("missing.toml")), CliOverrides::default(), ); assert!(tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() .block_on(broken.reload()) .is_err()); assert!(reg.contains("gamma"), "a bad reload keeps the old peers"); Ok(()) }); } #[tokio::test] async fn reload_disabling_the_feature_empties_the_snapshot() { let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"])); let mut new = fake_peers_config(&["beta"]); new.enabled = false; reg.reload(new).await.unwrap(); assert!(reg.snapshot().is_empty()); assert!(reg.status("beta").is_err()); assert!(!reg.config().enabled); } #[tokio::test] async fn reload_moves_loops_to_the_new_peer_set() { let fake = FakeTransport::new(); let reg = PeerRegistry::start_with_transport( fake_peers_config(&["beta"]), fake.clone(), CancellationToken::new(), ); wait_for(®, |r| r.status("beta").unwrap().server.up).await; reg.reload(fake_peers_config(&["beta", "gamma"])) .await .unwrap(); wait_for(®, |r| r.status("gamma").unwrap().server.up).await; assert!( reg.status("beta").unwrap().server.up, "kept peer keeps state" ); reg.reload(fake_peers_config(&["gamma"])).await.unwrap(); assert!(reg.status("beta").is_err()); assert!(reg.contains("gamma")); reg.shutdown().await; } }