diff --git a/src/peers/registry.rs b/src/peers/registry.rs index fddf901..85fd887 100644 --- a/src/peers/registry.rs +++ b/src/peers/registry.rs @@ -65,6 +65,9 @@ pub struct PeerRegistry { transport: Arc, shared: Arc, loops: Mutex>, + /// Loops cancelled by a reload. Kept so `shutdown` can wait for them + /// instead of detaching their join handles. + retired: Mutex>>, loops_enabled: bool, shutdown: CancellationToken, } @@ -117,6 +120,7 @@ impl PeerRegistry { }), }), loops: Mutex::new(BTreeMap::new()), + retired: Mutex::new(Vec::new()), loops_enabled, shutdown, } @@ -197,9 +201,11 @@ impl PeerRegistry { fn stop_loop(&self, entry: LoopEntry) { // Cancel, then let the task finish its bounded probe on its own. entry.token.cancel(); - tokio::spawn(async move { - let _ = entry.join.await; - }); + lock_mutex(&self.retired).push(entry.join); + } + + fn prune_retired(&self) { + lock_mutex(&self.retired).retain(|join| !join.is_finished()); } /// Sorted by name; disabled peers included. @@ -246,12 +252,30 @@ impl PeerRegistry { opts: ProxyOpts, ) -> Result { let key = PeerName::parse(name).ok(); - let (peer, timeout) = { + let (peer, timeout, configured, feature_on) = { let st = lock_read(&self.shared.state); - let peer = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key).cloned()); - (peer, Duration::from_secs(st.config.request_timeout_secs)) + let found = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key)); + let peer = found.filter(|p| p.enabled).cloned(); + ( + peer, + Duration::from_secs(st.config.request_timeout_secs), + found.is_some(), + st.config.enabled, + ) }; - let peer = peer.ok_or_else(|| AppError::NotFound(format!("peer {name}: unknown peer")))?; + // A switched-off peer is never called, so its bearer token never + // leaves the process. This mirrors what `status` and `snapshot` + // report: no entry for a disabled feature, `enabled = false` for a + // disabled peer. + if !feature_on || peer.is_none() { + let reason = if feature_on && configured { + "disabled" + } else { + "unknown peer" + }; + return Err(AppError::NotFound(format!("peer {name}: {reason}"))); + } + let peer = peer.expect("checked above"); crate::peers::proxy::get(self.transport.as_ref(), &peer, path, query, opts, timeout).await } @@ -262,8 +286,28 @@ impl PeerRegistry { /// A bad `[peers]` block never reaches this method: the handler logs the /// config error and keeps the old registry, as TLS reload does. pub async fn reload(&self, new: PeersConfig) -> Result<(), PeerConfigError> { + // Limits and the deny list live inside the transport for its whole + // lifetime. Rejecting the change keeps `config()` honest: what it + // reports is what is enforced. An invalid CIDR is rejected here for + // the same reason, instead of being accepted and then ignored. + { + let current = self.config(); + let old = &*current; + if new.deny_cidrs != old.deny_cidrs + || new.max_response_bytes != old.max_response_bytes + || new.max_concurrent_per_peer != old.max_concurrent_per_peer + || new.max_concurrent_total != old.max_concurrent_total + { + return Err(PeerConfigError::Invalid( + "peers.deny_cidrs and the peers concurrency/size limits cannot change on reload; restart the service" + .into(), + )); + } + crate::peers::registry::client::deny_networks(&new.deny_cidrs)?; + } let params = LoopConfig::from_config(&new); { + let old_config = self.config(); let mut st = lock_write(&self.shared.state); let old = std::mem::take(&mut st.statuses); // Turning the feature off renders an empty list rather than a @@ -273,10 +317,20 @@ impl PeerRegistry { new.peer .iter() .map(|peer| { - let mut status = old - .get(&peer.name) - .cloned() - .unwrap_or_else(|| initial_status(peer)); + // A different target has a different state; keeping + // the old one would report the old peer's `up` and + // `since` under the new URL. + let same_target = old_config + .peer + .iter() + .any(|prev| prev.name == peer.name && prev.url == peer.url); + let mut status = if same_target { + old.get(&peer.name) + .cloned() + .unwrap_or_else(|| initial_status(peer)) + } else { + initial_status(peer) + }; status.enabled = peer.enabled; status.url = peer.display_url(); (peer.name.clone(), status) @@ -294,6 +348,7 @@ impl PeerRegistry { } fn resync_loops(&self, new: &PeersConfig, params: LoopConfig) { + self.prune_retired(); let mut old = std::mem::take(&mut *lock_mutex(&self.loops)); let mut kept = BTreeMap::new(); for (name, entry) in std::mem::take(&mut old) { @@ -325,10 +380,11 @@ impl PeerRegistry { /// Stop every loop and wait (bounded) for the tasks to exit. pub async fn shutdown(&self) { self.shutdown.cancel(); - let joins: Vec> = std::mem::take(&mut *lock_mutex(&self.loops)) + let mut joins: Vec> = std::mem::take(&mut *lock_mutex(&self.loops)) .into_values() .map(|entry| entry.join) .collect(); + joins.extend(std::mem::take(&mut *lock_mutex(&self.retired))); let join_all = async { for join in joins { let _ = join.await; @@ -501,7 +557,7 @@ mod tests { } } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn loops_mark_up_and_honour_the_failure_threshold() { let fake = FakeTransport::new(); let mut cfg = fake_peers_config(&["beta"]); @@ -526,11 +582,25 @@ mod tests { && s.api.last_error.as_deref() == Some("timeout") }) .await; - assert!(reg.status("beta").unwrap().server.up); + let below = reg.status("beta").unwrap(); + assert!(below.server.up); + assert_eq!( + below.api.peer_version.as_deref(), + Some("0.0.0-fake"), + "the version stays known while the API is still up" + ); + // The next failure reaches the threshold and flips both checks down. + wait_for(®, |r| { + let s = r.status("beta").unwrap(); + !s.server.up && !s.api.up + }) + .await; + let down = reg.status("beta").unwrap(); + assert_eq!(down.server.last_error.as_deref(), Some("timeout")); + assert_eq!(down.api.auth, AuthState::Unknown); + assert!(down.server.since.is_some()); reg.shutdown().await; - let after = reg.status("beta").unwrap(); - assert!(after.server.up, "one failure below threshold = still up"); } #[tokio::test] @@ -606,6 +676,165 @@ mod tests { }); } + #[tokio::test] + async fn a_disabled_peer_is_never_proxied() { + let fake = FakeTransport::new(); + fake.peer("beta") + .respond("/systemd/units", 200, "application/json", "{}"); + let reg = registry_with(fake.clone(), fake_peers_config(&["beta"])); + reg.get("beta", "/systemd/units", &[], ProxyOpts::default()) + .await + .unwrap(); + let calls = fake.calls().len(); + + let mut peer_off = fake_peers_config(&["beta"]); + peer_off.peer[0].enabled = false; + reg.reload(peer_off).await.unwrap(); + let err = reg + .get("beta", "/systemd/units", &[], ProxyOpts::default()) + .await + .unwrap_err(); + assert_eq!(err.code(), "not_found"); + assert!(err.message().contains("disabled"), "{}", err.message()); + + let mut feature_off = fake_peers_config(&["beta"]); + feature_off.enabled = false; + reg.reload(feature_off).await.unwrap(); + let err = reg + .get("beta", "/systemd/units", &[], ProxyOpts::default()) + .await + .unwrap_err(); + assert_eq!(err.code(), "not_found"); + assert_eq!( + fake.calls().len(), + calls, + "no request (and so no token) leaves while the peer is off" + ); + } + + #[tokio::test] + async fn reload_rejects_changes_the_transport_cannot_apply() { + let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"])); + let old = reg.config(); + + let mut moved = fake_peers_config(&["beta"]); + moved.deny_cidrs = vec!["10.0.0.0/8".into()]; + let err = reg.reload(moved).await.unwrap_err(); + assert!(err.to_string().contains("restart"), "{err}"); + + let mut resized = fake_peers_config(&["beta"]); + resized.max_response_bytes = 4096; + assert!(reg.reload(resized).await.is_err()); + + let mut capped = fake_peers_config(&["beta"]); + capped.max_concurrent_per_peer = 1; + assert!(reg.reload(capped).await.is_err()); + assert_eq!(reg.config().deny_cidrs, old.deny_cidrs); + assert_eq!(reg.config().max_response_bytes, old.max_response_bytes); + + let mut invalid = fake_peers_config(&["beta"]); + invalid.deny_cidrs = vec!["not-a-cidr".into()]; + let broken = registry_with(FakeTransport::new(), invalid.clone()); + let err = broken.reload(invalid).await.unwrap_err(); + assert!(err.to_string().contains("CIDR"), "{err}"); + } + + #[tokio::test] + async fn a_changed_peer_url_starts_with_a_clean_status() { + let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"])); + { + let mut st = lock_write(®.shared.state); + let beta = st + .statuses + .get_mut(&PeerName::parse("beta").unwrap()) + .unwrap(); + beta.server.up = true; + beta.server.since = Some(jiff::Timestamp::now()); + beta.api.up = true; + beta.api.peer_version = Some("1.0.0".into()); + } + let mut moved = fake_peers_config(&["beta"]); + moved.peer[0].url = url::Url::parse("http://beta-2:1984").unwrap(); + reg.reload(moved).await.unwrap(); + let status = reg.status("beta").unwrap(); + assert_eq!(status.url, "http://beta-2:1984"); + assert!(!status.server.up); + assert!(status.server.since.is_none()); + assert!(status.api.since.is_none()); + assert_eq!(status.api.peer_version, None); + } + + /// The SIGHUP handler fans out to peers even with TLS off, and the + /// peer-less default (both off) still consumes the signal. + #[tokio::test] + async fn sighup_reloads_peers_when_tls_is_off() { + use tokio::signal::unix::{signal, SignalKind}; + + let dir = std::env::temp_dir().join(format!("slinuxm-sighup-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("config.toml"); + std::fs::write( + &path, + r#" +[peers] +enabled = true +[[peers.peer]] +name = "gamma" +url = "http://gamma:1984" +token_file = "/run/peers/gamma.token" +allow_plain_http = true +"#, + ) + .unwrap(); + + let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"])); + let hup = signal(SignalKind::hangup()).unwrap(); + let shutdown = CancellationToken::new(); + crate::tls::spawn_sighup_reload( + None, + hup, + shutdown.clone(), + Some(PeersReloader::new( + reg.clone(), + Some(path.clone()), + CliOverrides::default(), + )), + ); + tokio::time::sleep(Duration::from_millis(50)).await; + let sent = unsafe { libc::kill(std::process::id() as libc::pid_t, libc::SIGHUP) }; + assert_eq!(sent, 0, "sending SIGHUP failed"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while !reg.contains("gamma") { + assert!( + tokio::time::Instant::now() < deadline, + "SIGHUP did not reach the peer reloader: {:?}", + reg.snapshot() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + shutdown.cancel(); + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test(start_paused = true)] + async fn shutdown_waits_for_loops_retired_by_a_reload() { + let fake = FakeTransport::new(); + let reg = PeerRegistry::start_with_transport( + fake_peers_config(&["beta", "gamma"]), + fake.clone(), + CancellationToken::new(), + ); + wait_for(®, |r| r.status("beta").unwrap().server.up).await; + reg.reload(fake_peers_config(&["gamma"])).await.unwrap(); + assert!( + !lock_mutex(®.retired).is_empty(), + "the removed peer's loop is retained, not detached" + ); + reg.shutdown().await; + assert!(lock_mutex(®.retired).is_empty()); + } + #[tokio::test] async fn reload_disabling_the_feature_empties_the_snapshot() { let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"])); @@ -617,7 +846,7 @@ mod tests { assert!(!reg.config().enabled); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn reload_moves_loops_to_the_new_peer_set() { let fake = FakeTransport::new(); let reg = PeerRegistry::start_with_transport(