B: refuse to proxy disabled peers and reject reloads the transport cannot apply

Also reset status when a peer URL changes, keep loops retired by a
reload and await them at shutdown, and cover the SIGHUP fan-out.
This commit is contained in:
ari committed 2026-10-19 20:55:25 -04:00
1 parent 26b9f8bdff
commit c1a12abc19
1 file changed
+246 -17
+246 -17
View File
@@ -65,6 +65,9 @@ pub struct PeerRegistry {
transport: Arc<dyn PeerTransport>,
shared: Arc<Shared>,
loops: Mutex<BTreeMap<PeerName, LoopEntry>>,
/// Loops cancelled by a reload. Kept so `shutdown` can wait for them
/// instead of detaching their join handles.
retired: Mutex<Vec<JoinHandle<()>>>,
loops_enabled: bool,
shutdown: CancellationToken,
}
@@ -117,6 +120,7 @@ impl PeerRegistry {
}),
}),
loops: Mutex::new(BTreeMap::new()),
retired: Mutex::new(Vec::new()),
loops_enabled,
shutdown,
}
@@ -197,9 +201,11 @@ impl PeerRegistry {
fn stop_loop(&self, entry: LoopEntry) {
// Cancel, then let the task finish its bounded probe on its own.
entry.token.cancel();
tokio::spawn(async move {
let _ = entry.join.await;
});
lock_mutex(&self.retired).push(entry.join);
}
fn prune_retired(&self) {
lock_mutex(&self.retired).retain(|join| !join.is_finished());
}
/// Sorted by name; disabled peers included.
@@ -246,12 +252,30 @@ impl PeerRegistry {
opts: ProxyOpts,
) -> Result<PeerResponse, AppError> {
let key = PeerName::parse(name).ok();
let (peer, timeout) = {
let (peer, timeout, configured, feature_on) = {
let st = lock_read(&self.shared.state);
let peer = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key).cloned());
(peer, Duration::from_secs(st.config.request_timeout_secs))
let found = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key));
let peer = found.filter(|p| p.enabled).cloned();
(
peer,
Duration::from_secs(st.config.request_timeout_secs),
found.is_some(),
st.config.enabled,
)
};
let peer = peer.ok_or_else(|| AppError::NotFound(format!("peer {name}: unknown peer")))?;
// A switched-off peer is never called, so its bearer token never
// leaves the process. This mirrors what `status` and `snapshot`
// report: no entry for a disabled feature, `enabled = false` for a
// disabled peer.
if !feature_on || peer.is_none() {
let reason = if feature_on && configured {
"disabled"
} else {
"unknown peer"
};
return Err(AppError::NotFound(format!("peer {name}: {reason}")));
}
let peer = peer.expect("checked above");
crate::peers::proxy::get(self.transport.as_ref(), &peer, path, query, opts, timeout).await
}
@@ -262,8 +286,28 @@ impl PeerRegistry {
/// A bad `[peers]` block never reaches this method: the handler logs the
/// config error and keeps the old registry, as TLS reload does.
pub async fn reload(&self, new: PeersConfig) -> Result<(), PeerConfigError> {
// Limits and the deny list live inside the transport for its whole
// lifetime. Rejecting the change keeps `config()` honest: what it
// reports is what is enforced. An invalid CIDR is rejected here for
// the same reason, instead of being accepted and then ignored.
{
let current = self.config();
let old = &*current;
if new.deny_cidrs != old.deny_cidrs
|| new.max_response_bytes != old.max_response_bytes
|| new.max_concurrent_per_peer != old.max_concurrent_per_peer
|| new.max_concurrent_total != old.max_concurrent_total
{
return Err(PeerConfigError::Invalid(
"peers.deny_cidrs and the peers concurrency/size limits cannot change on reload; restart the service"
.into(),
));
}
crate::peers::registry::client::deny_networks(&new.deny_cidrs)?;
}
let params = LoopConfig::from_config(&new);
{
let old_config = self.config();
let mut st = lock_write(&self.shared.state);
let old = std::mem::take(&mut st.statuses);
// Turning the feature off renders an empty list rather than a
@@ -273,10 +317,20 @@ impl PeerRegistry {
new.peer
.iter()
.map(|peer| {
let mut status = old
.get(&peer.name)
.cloned()
.unwrap_or_else(|| initial_status(peer));
// A different target has a different state; keeping
// the old one would report the old peer's `up` and
// `since` under the new URL.
let same_target = old_config
.peer
.iter()
.any(|prev| prev.name == peer.name && prev.url == peer.url);
let mut status = if same_target {
old.get(&peer.name)
.cloned()
.unwrap_or_else(|| initial_status(peer))
} else {
initial_status(peer)
};
status.enabled = peer.enabled;
status.url = peer.display_url();
(peer.name.clone(), status)
@@ -294,6 +348,7 @@ impl PeerRegistry {
}
fn resync_loops(&self, new: &PeersConfig, params: LoopConfig) {
self.prune_retired();
let mut old = std::mem::take(&mut *lock_mutex(&self.loops));
let mut kept = BTreeMap::new();
for (name, entry) in std::mem::take(&mut old) {
@@ -325,10 +380,11 @@ impl PeerRegistry {
/// Stop every loop and wait (bounded) for the tasks to exit.
pub async fn shutdown(&self) {
self.shutdown.cancel();
let joins: Vec<JoinHandle<()>> = std::mem::take(&mut *lock_mutex(&self.loops))
let mut joins: Vec<JoinHandle<()>> = std::mem::take(&mut *lock_mutex(&self.loops))
.into_values()
.map(|entry| entry.join)
.collect();
joins.extend(std::mem::take(&mut *lock_mutex(&self.retired)));
let join_all = async {
for join in joins {
let _ = join.await;
@@ -501,7 +557,7 @@ mod tests {
}
}
#[tokio::test]
#[tokio::test(start_paused = true)]
async fn loops_mark_up_and_honour_the_failure_threshold() {
let fake = FakeTransport::new();
let mut cfg = fake_peers_config(&["beta"]);
@@ -526,11 +582,25 @@ mod tests {
&& s.api.last_error.as_deref() == Some("timeout")
})
.await;
assert!(reg.status("beta").unwrap().server.up);
let below = reg.status("beta").unwrap();
assert!(below.server.up);
assert_eq!(
below.api.peer_version.as_deref(),
Some("0.0.0-fake"),
"the version stays known while the API is still up"
);
// The next failure reaches the threshold and flips both checks down.
wait_for(&reg, |r| {
let s = r.status("beta").unwrap();
!s.server.up && !s.api.up
})
.await;
let down = reg.status("beta").unwrap();
assert_eq!(down.server.last_error.as_deref(), Some("timeout"));
assert_eq!(down.api.auth, AuthState::Unknown);
assert!(down.server.since.is_some());
reg.shutdown().await;
let after = reg.status("beta").unwrap();
assert!(after.server.up, "one failure below threshold = still up");
}
#[tokio::test]
@@ -606,6 +676,165 @@ mod tests {
});
}
#[tokio::test]
async fn a_disabled_peer_is_never_proxied() {
let fake = FakeTransport::new();
fake.peer("beta")
.respond("/systemd/units", 200, "application/json", "{}");
let reg = registry_with(fake.clone(), fake_peers_config(&["beta"]));
reg.get("beta", "/systemd/units", &[], ProxyOpts::default())
.await
.unwrap();
let calls = fake.calls().len();
let mut peer_off = fake_peers_config(&["beta"]);
peer_off.peer[0].enabled = false;
reg.reload(peer_off).await.unwrap();
let err = reg
.get("beta", "/systemd/units", &[], ProxyOpts::default())
.await
.unwrap_err();
assert_eq!(err.code(), "not_found");
assert!(err.message().contains("disabled"), "{}", err.message());
let mut feature_off = fake_peers_config(&["beta"]);
feature_off.enabled = false;
reg.reload(feature_off).await.unwrap();
let err = reg
.get("beta", "/systemd/units", &[], ProxyOpts::default())
.await
.unwrap_err();
assert_eq!(err.code(), "not_found");
assert_eq!(
fake.calls().len(),
calls,
"no request (and so no token) leaves while the peer is off"
);
}
#[tokio::test]
async fn reload_rejects_changes_the_transport_cannot_apply() {
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
let old = reg.config();
let mut moved = fake_peers_config(&["beta"]);
moved.deny_cidrs = vec!["10.0.0.0/8".into()];
let err = reg.reload(moved).await.unwrap_err();
assert!(err.to_string().contains("restart"), "{err}");
let mut resized = fake_peers_config(&["beta"]);
resized.max_response_bytes = 4096;
assert!(reg.reload(resized).await.is_err());
let mut capped = fake_peers_config(&["beta"]);
capped.max_concurrent_per_peer = 1;
assert!(reg.reload(capped).await.is_err());
assert_eq!(reg.config().deny_cidrs, old.deny_cidrs);
assert_eq!(reg.config().max_response_bytes, old.max_response_bytes);
let mut invalid = fake_peers_config(&["beta"]);
invalid.deny_cidrs = vec!["not-a-cidr".into()];
let broken = registry_with(FakeTransport::new(), invalid.clone());
let err = broken.reload(invalid).await.unwrap_err();
assert!(err.to_string().contains("CIDR"), "{err}");
}
#[tokio::test]
async fn a_changed_peer_url_starts_with_a_clean_status() {
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
{
let mut st = lock_write(&reg.shared.state);
let beta = st
.statuses
.get_mut(&PeerName::parse("beta").unwrap())
.unwrap();
beta.server.up = true;
beta.server.since = Some(jiff::Timestamp::now());
beta.api.up = true;
beta.api.peer_version = Some("1.0.0".into());
}
let mut moved = fake_peers_config(&["beta"]);
moved.peer[0].url = url::Url::parse("http://beta-2:1984").unwrap();
reg.reload(moved).await.unwrap();
let status = reg.status("beta").unwrap();
assert_eq!(status.url, "http://beta-2:1984");
assert!(!status.server.up);
assert!(status.server.since.is_none());
assert!(status.api.since.is_none());
assert_eq!(status.api.peer_version, None);
}
/// The SIGHUP handler fans out to peers even with TLS off, and the
/// peer-less default (both off) still consumes the signal.
#[tokio::test]
async fn sighup_reloads_peers_when_tls_is_off() {
use tokio::signal::unix::{signal, SignalKind};
let dir = std::env::temp_dir().join(format!("slinuxm-sighup-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("config.toml");
std::fs::write(
&path,
r#"
[peers]
enabled = true
[[peers.peer]]
name = "gamma"
url = "http://gamma:1984"
token_file = "/run/peers/gamma.token"
allow_plain_http = true
"#,
)
.unwrap();
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
let hup = signal(SignalKind::hangup()).unwrap();
let shutdown = CancellationToken::new();
crate::tls::spawn_sighup_reload(
None,
hup,
shutdown.clone(),
Some(PeersReloader::new(
reg.clone(),
Some(path.clone()),
CliOverrides::default(),
)),
);
tokio::time::sleep(Duration::from_millis(50)).await;
let sent = unsafe { libc::kill(std::process::id() as libc::pid_t, libc::SIGHUP) };
assert_eq!(sent, 0, "sending SIGHUP failed");
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
while !reg.contains("gamma") {
assert!(
tokio::time::Instant::now() < deadline,
"SIGHUP did not reach the peer reloader: {:?}",
reg.snapshot()
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
shutdown.cancel();
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(start_paused = true)]
async fn shutdown_waits_for_loops_retired_by_a_reload() {
let fake = FakeTransport::new();
let reg = PeerRegistry::start_with_transport(
fake_peers_config(&["beta", "gamma"]),
fake.clone(),
CancellationToken::new(),
);
wait_for(&reg, |r| r.status("beta").unwrap().server.up).await;
reg.reload(fake_peers_config(&["gamma"])).await.unwrap();
assert!(
!lock_mutex(&reg.retired).is_empty(),
"the removed peer's loop is retained, not detached"
);
reg.shutdown().await;
assert!(lock_mutex(&reg.retired).is_empty());
}
#[tokio::test]
async fn reload_disabling_the_feature_empties_the_snapshot() {
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
@@ -617,7 +846,7 @@ mod tests {
assert!(!reg.config().enabled);
}
#[tokio::test]
#[tokio::test(start_paused = true)]
async fn reload_moves_loops_to_the_new_peer_set() {
let fake = FakeTransport::new();
let reg = PeerRegistry::start_with_transport(