B: B2/B3 health probes, loops and the in-memory status model

This commit is contained in:
ari committed 2026-10-19 20:55:25 -04:00
1 parent bd5b21bcc4
commit d94c2b3a1d
3 files changed
+994 -126

No files matched your search

+397 -40
View File
@@ -2,11 +2,16 @@
//! side never awaits on a peer and is cheap enough for a metrics scrape.
use std::collections::BTreeMap;
use std::sync::{Arc, RwLock};
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use crate::config::{CliOverrides, Config};
use crate::error::AppError;
use crate::peers::error::PeerConfigError;
use crate::peers::health::{self, LoopConfig, StatusSink};
use crate::peers::transport::{PeerResponse, PeerTransport};
use crate::peers::types::{ApiStatus, PeerConfig, PeerName, PeerStatus, PeersConfig, ServerStatus};
@@ -29,9 +34,39 @@ struct RegistryState {
statuses: BTreeMap<PeerName, PeerStatus>,
}
/// Status storage shared with the loops; this is the only mutable state a
/// loop needs, so the registry can be dropped without leaking a task.
struct Shared {
state: RwLock<RegistryState>,
}
impl StatusSink for Shared {
fn apply_server(&self, name: &PeerName, status: ServerStatus) {
if let Some(entry) = lock_write(&self.state).statuses.get_mut(name) {
entry.server = status;
}
}
fn apply_api(&self, name: &PeerName, status: ApiStatus) {
if let Some(entry) = lock_write(&self.state).statuses.get_mut(name) {
entry.api = status;
}
}
}
struct LoopEntry {
peer: PeerConfig,
params: LoopConfig,
token: CancellationToken,
join: JoinHandle<()>,
}
pub struct PeerRegistry {
transport: Arc<dyn PeerTransport>,
state: RwLock<RegistryState>,
shared: Arc<Shared>,
loops: Mutex<BTreeMap<PeerName, LoopEntry>>,
loops_enabled: bool,
shutdown: CancellationToken,
}
fn initial_status(peer: &PeerConfig) -> PeerStatus {
@@ -52,8 +87,22 @@ fn lock_write<T>(lock: &RwLock<T>) -> std::sync::RwLockWriteGuard<'_, T> {
lock.write().unwrap_or_else(|e| e.into_inner())
}
fn lock_mutex<T>(lock: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
lock.lock().unwrap_or_else(|e| e.into_inner())
}
impl PeerRegistry {
/// Registry without loops, for tests that drive the transport directly.
pub(crate) fn new(config: PeersConfig, transport: Arc<dyn PeerTransport>) -> Self {
Self::build(config, transport, CancellationToken::new(), false)
}
fn build(
config: PeersConfig,
transport: Arc<dyn PeerTransport>,
shutdown: CancellationToken,
loops_enabled: bool,
) -> Self {
let statuses = config
.peer
.iter()
@@ -61,10 +110,15 @@ impl PeerRegistry {
.collect();
Self {
transport,
state: RwLock::new(RegistryState {
config: Arc::new(config),
statuses,
shared: Arc::new(Shared {
state: RwLock::new(RegistryState {
config: Arc::new(config),
statuses,
}),
}),
loops: Mutex::new(BTreeMap::new()),
loops_enabled,
shutdown,
}
}
@@ -73,40 +127,114 @@ impl PeerRegistry {
config: PeersConfig,
transport: Arc<dyn PeerTransport>,
) -> Self {
Self {
transport,
state: RwLock::new(RegistryState {
config: Arc::new(config),
statuses: statuses.into_iter().map(|s| (s.name.clone(), s)).collect(),
}),
let reg = Self::build(config, transport, CancellationToken::new(), false);
lock_write(&reg.shared.state).statuses =
statuses.into_iter().map(|s| (s.name.clone(), s)).collect();
reg
}
/// Real registry for the server: builds the outbound transport and starts
/// one loop per enabled peer. `shutdown` is the process token; cancelling
/// it stops the loops.
pub fn start(
config: PeersConfig,
shutdown: CancellationToken,
) -> Result<Arc<Self>, PeerConfigError> {
let transport = Arc::new(ReqwestTransport::new(&config)?);
Ok(Self::start_with_transport(config, transport, shutdown))
}
/// Loop-enabled registry over a supplied transport, for tests.
pub fn start_with_transport(
config: PeersConfig,
transport: Arc<dyn PeerTransport>,
shutdown: CancellationToken,
) -> Arc<Self> {
let registry = Arc::new(Self::build(config, transport, shutdown.child_token(), true));
registry.spawn_all();
registry
}
fn spawn_all(&self) {
let peers: Vec<PeerConfig> = if self.config().enabled {
self.config()
.peer
.iter()
.filter(|peer| peer.enabled)
.cloned()
.collect()
} else {
Vec::new()
};
for peer in peers {
self.spawn_peer(&peer);
}
}
fn spawn_peer(&self, peer: &PeerConfig) {
let params = LoopConfig::from_config(&self.config());
let initial = self.status_internal(&peer.name);
let token = self.shutdown.child_token();
let sink: Arc<dyn StatusSink> = self.shared.clone();
let transport = self.transport.clone();
let peer_copy = peer.clone();
let name = peer.name.clone();
let task_token = token.clone();
let join = tokio::spawn(async move {
health::run_peer(transport, peer_copy, params, initial, sink, task_token).await;
});
lock_mutex(&self.loops).insert(
name,
LoopEntry {
peer: peer.clone(),
params,
token,
join,
},
);
}
fn stop_loop(&self, entry: LoopEntry) {
// Cancel, then let the task finish its bounded probe on its own.
entry.token.cancel();
tokio::spawn(async move {
let _ = entry.join.await;
});
}
/// Sorted by name; disabled peers included.
pub fn snapshot(&self) -> Vec<PeerStatus> {
lock_read(&self.state).statuses.values().cloned().collect()
lock_read(&self.shared.state)
.statuses
.values()
.cloned()
.collect()
}
pub fn status(&self, name: &str) -> Result<PeerStatus, AppError> {
let Ok(key) = PeerName::parse(name) else {
return Err(AppError::NotFound(format!("peer {name}: unknown peer")));
};
lock_read(&self.state)
lock_read(&self.shared.state)
.statuses
.get(&key)
.cloned()
.ok_or_else(|| AppError::NotFound(format!("peer {name}: unknown peer")))
}
fn status_internal(&self, name: &PeerName) -> Option<PeerStatus> {
lock_read(&self.shared.state).statuses.get(name).cloned()
}
pub fn contains(&self, name: &str) -> bool {
let Ok(key) = PeerName::parse(name) else {
return false;
};
lock_read(&self.state).statuses.contains_key(&key)
lock_read(&self.shared.state).statuses.contains_key(&key)
}
pub fn config(&self) -> Arc<PeersConfig> {
lock_read(&self.state).config.clone()
lock_read(&self.shared.state).config.clone()
}
/// Single proxy entry point for REST and MCP.
@@ -119,7 +247,7 @@ impl PeerRegistry {
) -> Result<PeerResponse, AppError> {
let key = PeerName::parse(name).ok();
let (peer, timeout) = {
let st = lock_read(&self.state);
let st = lock_read(&self.shared.state);
let peer = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key).cloned());
(peer, Duration::from_secs(st.config.request_timeout_secs))
};
@@ -128,30 +256,114 @@ impl PeerRegistry {
}
/// Replace the peer set. Statuses of peers that survive a reload are kept
/// so `since` survives a config change. Validation arrives with the
/// config and secrets tasks; this body only swaps state.
/// so `since` survives a config change. Loops for new or changed peers are
/// started, loops for removed or disabled peers are cancelled.
///
/// A bad `[peers]` block never reaches this method: the handler logs the
/// config error and keeps the old registry, as TLS reload does.
pub async fn reload(&self, new: PeersConfig) -> Result<(), PeerConfigError> {
let mut st = lock_write(&self.state);
let old = std::mem::take(&mut st.statuses);
st.statuses = new
.peer
.iter()
.map(|peer| {
let mut status = old
.get(&peer.name)
.cloned()
.unwrap_or_else(|| initial_status(peer));
status.enabled = peer.enabled;
status.url = peer.display_url();
(peer.name.clone(), status)
})
.collect();
st.config = Arc::new(new);
let params = LoopConfig::from_config(&new);
{
let mut st = lock_write(&self.shared.state);
let old = std::mem::take(&mut st.statuses);
st.statuses = new
.peer
.iter()
.map(|peer| {
let mut status = old
.get(&peer.name)
.cloned()
.unwrap_or_else(|| initial_status(peer));
status.enabled = peer.enabled;
status.url = peer.display_url();
(peer.name.clone(), status)
})
.collect();
st.config = Arc::new(new.clone());
}
if self.loops_enabled {
self.resync_loops(&new, params);
}
Ok(())
}
/// Health loops stop on this; nothing is running yet.
pub async fn shutdown(&self) {}
fn resync_loops(&self, new: &PeersConfig, params: LoopConfig) {
let mut old = std::mem::take(&mut *lock_mutex(&self.loops));
let mut kept = BTreeMap::new();
for (name, entry) in std::mem::take(&mut old) {
let wanted = new
.enabled
.then(|| {
new.peer
.iter()
.find(|peer| peer.name == name && peer.enabled)
})
.flatten();
match wanted {
Some(peer) if entry.peer == *peer && entry.params == params => {
kept.insert(name, entry);
}
_ => self.stop_loop(entry),
}
}
*lock_mutex(&self.loops) = kept;
if new.enabled {
for peer in new.peer.iter().filter(|peer| peer.enabled) {
if !lock_mutex(&self.loops).contains_key(&peer.name) {
self.spawn_peer(peer);
}
}
}
}
/// Stop every loop and wait (bounded) for the tasks to exit.
pub async fn shutdown(&self) {
self.shutdown.cancel();
let joins: Vec<JoinHandle<()>> = std::mem::take(&mut *lock_mutex(&self.loops))
.into_values()
.map(|entry| entry.join)
.collect();
let join_all = async {
for join in joins {
let _ = join.await;
}
};
if tokio::time::timeout(Duration::from_secs(5), join_all)
.await
.is_err()
{
tracing::warn!("peer health loops did not stop in time");
}
}
}
/// Everything the SIGHUP handler needs to re-read the `[peers]` block.
#[derive(Clone)]
pub struct PeersReloader {
registry: Arc<PeerRegistry>,
path: Option<std::path::PathBuf>,
overrides: CliOverrides,
}
impl PeersReloader {
pub fn new(
registry: Arc<PeerRegistry>,
path: Option<std::path::PathBuf>,
overrides: CliOverrides,
) -> Self {
Self {
registry,
path,
overrides,
}
}
/// Re-read the whole config and apply only `[peers]`. A failure returns
/// the config error; the old registry stays in place.
pub async fn reload(&self) -> Result<(), PeerConfigError> {
let cfg = Config::load(self.path.as_deref(), &self.overrides)?;
self.registry.reload(cfg.peers).await
}
}
#[cfg(test)]
@@ -159,6 +371,7 @@ mod tests {
use super::*;
use crate::peers::testing::{fake_peers_config, registry_with, static_registry, FakeTransport};
use crate::peers::transport::PeerBody;
use crate::peers::types::AuthState;
fn status(name: &str, up: bool) -> PeerStatus {
PeerStatus {
@@ -184,7 +397,7 @@ mod tests {
assert_eq!(names, ["beta", "gamma"]);
let b = reg.status("beta").unwrap();
assert!(b.server.up);
assert_eq!(b.api.auth, crate::peers::types::AuthState::Unknown);
assert_eq!(b.api.auth, AuthState::Unknown);
}
#[test]
@@ -247,7 +460,7 @@ mod tests {
async fn reload_keeps_surviving_status_and_prunes_removed() {
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta", "gamma"]));
{
let mut st = lock_write(&reg.state);
let mut st = lock_write(&reg.shared.state);
let beta = st
.statuses
.get_mut(&PeerName::parse("beta").unwrap())
@@ -256,8 +469,7 @@ mod tests {
}
let mut new = fake_peers_config(&["beta", "delta"]);
new.peer[0].enabled = false;
let reloaded: Result<(), PeerConfigError> = reg.reload(new).await;
reloaded.unwrap();
reg.reload(new).await.unwrap();
let snap = reg.snapshot();
assert_eq!(snap.len(), 2);
@@ -267,4 +479,149 @@ mod tests {
assert_eq!(snap[1].name.as_str(), "delta");
assert!(!snap[1].server.up);
}
async fn wait_for<F: Fn(&PeerRegistry) -> bool>(reg: &PeerRegistry, f: F) {
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
loop {
if f(reg) {
return;
}
assert!(
tokio::time::Instant::now() < deadline,
"condition not met: {:?}",
reg.snapshot()
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
}
#[tokio::test]
async fn loops_mark_up_and_honour_the_failure_threshold() {
let fake = FakeTransport::new();
let mut cfg = fake_peers_config(&["beta"]);
cfg.fail_threshold = 2;
let reg = PeerRegistry::start_with_transport(cfg, fake.clone(), CancellationToken::new());
wait_for(&reg, |r| {
let s = r.status("beta").unwrap();
s.server.up && s.api.up && s.api.auth == AuthState::Ok
})
.await;
let up = reg.status("beta").unwrap();
assert!(up.server.since.is_some());
assert_eq!(up.api.peer_version.as_deref(), Some("0.0.0-fake"));
fake.peer("beta").tcp(Err(crate::peers::PeerError::Timeout));
fake.peer("beta")
.respond_err("/api/v1/ping", crate::peers::PeerError::Timeout);
// One failure each: still up with the error visible.
wait_for(&reg, |r| {
let s = r.status("beta").unwrap();
s.server.last_error.as_deref() == Some("timeout")
&& s.api.last_error.as_deref() == Some("timeout")
})
.await;
assert!(reg.status("beta").unwrap().server.up);
reg.shutdown().await;
let after = reg.status("beta").unwrap();
assert!(after.server.up, "one failure below threshold = still up");
}
#[tokio::test]
async fn a_hung_peer_does_not_delay_another_peers_checks() {
let fake = FakeTransport::new();
fake.peer("beta").delay(Duration::from_secs(2));
fake.peer("gamma")
.respond("/api/v1/ping", 200, "application/json", "{}");
let reg = PeerRegistry::start_with_transport(
fake_peers_config(&["beta", "gamma"]),
fake.clone(),
CancellationToken::new(),
);
wait_for(&reg, |r| {
r.status("gamma").unwrap().api.last_checked.is_some()
})
.await;
let gamma_checked = reg.status("gamma").unwrap().api.last_checked;
assert!(reg.status("beta").unwrap().api.last_checked.is_none());
assert!(gamma_checked.is_some());
reg.shutdown().await;
}
#[test]
#[allow(clippy::result_large_err)] // figment::Jail closures return figment::Error
fn reloader_rereads_the_peers_block_and_keeps_the_old_set_on_error() {
use figment::Jail;
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
Jail::expect_with(|jail| {
jail.create_file(
"config.toml",
r#"
[peers]
enabled = true
api_interval_secs = 2
[[peers.peer]]
name = "gamma"
url = "http://gamma:1984"
token_file = "/run/credentials/skyg-linux-tracker.service/peer-gamma"
allow_plain_http = true
"#,
)?;
let reloader = PeersReloader::new(
reg.clone(),
Some(std::path::PathBuf::from("config.toml")),
CliOverrides::default(),
);
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(reloader.reload())
.unwrap();
let snap = reg.snapshot();
assert_eq!(snap.len(), 1);
assert_eq!(snap[0].name.as_str(), "gamma");
assert_eq!(reg.config().api_interval_secs, 2);
let broken = PeersReloader::new(
reg.clone(),
Some(std::path::PathBuf::from("missing.toml")),
CliOverrides::default(),
);
assert!(tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(broken.reload())
.is_err());
assert!(reg.contains("gamma"), "a bad reload keeps the old peers");
Ok(())
});
}
#[tokio::test]
async fn reload_moves_loops_to_the_new_peer_set() {
let fake = FakeTransport::new();
let reg = PeerRegistry::start_with_transport(
fake_peers_config(&["beta"]),
fake.clone(),
CancellationToken::new(),
);
wait_for(&reg, |r| r.status("beta").unwrap().server.up).await;
reg.reload(fake_peers_config(&["beta", "gamma"]))
.await
.unwrap();
wait_for(&reg, |r| r.status("gamma").unwrap().server.up).await;
assert!(
reg.status("beta").unwrap().server.up,
"kept peer keeps state"
);
reg.reload(fake_peers_config(&["gamma"])).await.unwrap();
assert!(reg.status("beta").is_err());
assert!(reg.contains("gamma"));
reg.shutdown().await;
}
}