B: refuse to proxy disabled peers and reject reloads the transport cannot apply
Also reset status when a peer URL changes, keep loops retired by a reload and await them at shutdown, and cover the SIGHUP fan-out.
This commit is contained in:
1 parent
26b9f8bdff
commit
c1a12abc19
1 file changed
+246
-17
+246
-17
@@ -65,6 +65,9 @@ pub struct PeerRegistry {
|
||||
transport: Arc<dyn PeerTransport>,
|
||||
shared: Arc<Shared>,
|
||||
loops: Mutex<BTreeMap<PeerName, LoopEntry>>,
|
||||
/// Loops cancelled by a reload. Kept so `shutdown` can wait for them
|
||||
/// instead of detaching their join handles.
|
||||
retired: Mutex<Vec<JoinHandle<()>>>,
|
||||
loops_enabled: bool,
|
||||
shutdown: CancellationToken,
|
||||
}
|
||||
@@ -117,6 +120,7 @@ impl PeerRegistry {
|
||||
}),
|
||||
}),
|
||||
loops: Mutex::new(BTreeMap::new()),
|
||||
retired: Mutex::new(Vec::new()),
|
||||
loops_enabled,
|
||||
shutdown,
|
||||
}
|
||||
@@ -197,9 +201,11 @@ impl PeerRegistry {
|
||||
fn stop_loop(&self, entry: LoopEntry) {
|
||||
// Cancel, then let the task finish its bounded probe on its own.
|
||||
entry.token.cancel();
|
||||
tokio::spawn(async move {
|
||||
let _ = entry.join.await;
|
||||
});
|
||||
lock_mutex(&self.retired).push(entry.join);
|
||||
}
|
||||
|
||||
fn prune_retired(&self) {
|
||||
lock_mutex(&self.retired).retain(|join| !join.is_finished());
|
||||
}
|
||||
|
||||
/// Sorted by name; disabled peers included.
|
||||
@@ -246,12 +252,30 @@ impl PeerRegistry {
|
||||
opts: ProxyOpts,
|
||||
) -> Result<PeerResponse, AppError> {
|
||||
let key = PeerName::parse(name).ok();
|
||||
let (peer, timeout) = {
|
||||
let (peer, timeout, configured, feature_on) = {
|
||||
let st = lock_read(&self.shared.state);
|
||||
let peer = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key).cloned());
|
||||
(peer, Duration::from_secs(st.config.request_timeout_secs))
|
||||
let found = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key));
|
||||
let peer = found.filter(|p| p.enabled).cloned();
|
||||
(
|
||||
peer,
|
||||
Duration::from_secs(st.config.request_timeout_secs),
|
||||
found.is_some(),
|
||||
st.config.enabled,
|
||||
)
|
||||
};
|
||||
let peer = peer.ok_or_else(|| AppError::NotFound(format!("peer {name}: unknown peer")))?;
|
||||
// A switched-off peer is never called, so its bearer token never
|
||||
// leaves the process. This mirrors what `status` and `snapshot`
|
||||
// report: no entry for a disabled feature, `enabled = false` for a
|
||||
// disabled peer.
|
||||
if !feature_on || peer.is_none() {
|
||||
let reason = if feature_on && configured {
|
||||
"disabled"
|
||||
} else {
|
||||
"unknown peer"
|
||||
};
|
||||
return Err(AppError::NotFound(format!("peer {name}: {reason}")));
|
||||
}
|
||||
let peer = peer.expect("checked above");
|
||||
crate::peers::proxy::get(self.transport.as_ref(), &peer, path, query, opts, timeout).await
|
||||
}
|
||||
|
||||
@@ -262,8 +286,28 @@ impl PeerRegistry {
|
||||
/// A bad `[peers]` block never reaches this method: the handler logs the
|
||||
/// config error and keeps the old registry, as TLS reload does.
|
||||
pub async fn reload(&self, new: PeersConfig) -> Result<(), PeerConfigError> {
|
||||
// Limits and the deny list live inside the transport for its whole
|
||||
// lifetime. Rejecting the change keeps `config()` honest: what it
|
||||
// reports is what is enforced. An invalid CIDR is rejected here for
|
||||
// the same reason, instead of being accepted and then ignored.
|
||||
{
|
||||
let current = self.config();
|
||||
let old = &*current;
|
||||
if new.deny_cidrs != old.deny_cidrs
|
||||
|| new.max_response_bytes != old.max_response_bytes
|
||||
|| new.max_concurrent_per_peer != old.max_concurrent_per_peer
|
||||
|| new.max_concurrent_total != old.max_concurrent_total
|
||||
{
|
||||
return Err(PeerConfigError::Invalid(
|
||||
"peers.deny_cidrs and the peers concurrency/size limits cannot change on reload; restart the service"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
crate::peers::registry::client::deny_networks(&new.deny_cidrs)?;
|
||||
}
|
||||
let params = LoopConfig::from_config(&new);
|
||||
{
|
||||
let old_config = self.config();
|
||||
let mut st = lock_write(&self.shared.state);
|
||||
let old = std::mem::take(&mut st.statuses);
|
||||
// Turning the feature off renders an empty list rather than a
|
||||
@@ -273,10 +317,20 @@ impl PeerRegistry {
|
||||
new.peer
|
||||
.iter()
|
||||
.map(|peer| {
|
||||
let mut status = old
|
||||
.get(&peer.name)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| initial_status(peer));
|
||||
// A different target has a different state; keeping
|
||||
// the old one would report the old peer's `up` and
|
||||
// `since` under the new URL.
|
||||
let same_target = old_config
|
||||
.peer
|
||||
.iter()
|
||||
.any(|prev| prev.name == peer.name && prev.url == peer.url);
|
||||
let mut status = if same_target {
|
||||
old.get(&peer.name)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| initial_status(peer))
|
||||
} else {
|
||||
initial_status(peer)
|
||||
};
|
||||
status.enabled = peer.enabled;
|
||||
status.url = peer.display_url();
|
||||
(peer.name.clone(), status)
|
||||
@@ -294,6 +348,7 @@ impl PeerRegistry {
|
||||
}
|
||||
|
||||
fn resync_loops(&self, new: &PeersConfig, params: LoopConfig) {
|
||||
self.prune_retired();
|
||||
let mut old = std::mem::take(&mut *lock_mutex(&self.loops));
|
||||
let mut kept = BTreeMap::new();
|
||||
for (name, entry) in std::mem::take(&mut old) {
|
||||
@@ -325,10 +380,11 @@ impl PeerRegistry {
|
||||
/// Stop every loop and wait (bounded) for the tasks to exit.
|
||||
pub async fn shutdown(&self) {
|
||||
self.shutdown.cancel();
|
||||
let joins: Vec<JoinHandle<()>> = std::mem::take(&mut *lock_mutex(&self.loops))
|
||||
let mut joins: Vec<JoinHandle<()>> = std::mem::take(&mut *lock_mutex(&self.loops))
|
||||
.into_values()
|
||||
.map(|entry| entry.join)
|
||||
.collect();
|
||||
joins.extend(std::mem::take(&mut *lock_mutex(&self.retired)));
|
||||
let join_all = async {
|
||||
for join in joins {
|
||||
let _ = join.await;
|
||||
@@ -501,7 +557,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn loops_mark_up_and_honour_the_failure_threshold() {
|
||||
let fake = FakeTransport::new();
|
||||
let mut cfg = fake_peers_config(&["beta"]);
|
||||
@@ -526,11 +582,25 @@ mod tests {
|
||||
&& s.api.last_error.as_deref() == Some("timeout")
|
||||
})
|
||||
.await;
|
||||
assert!(reg.status("beta").unwrap().server.up);
|
||||
let below = reg.status("beta").unwrap();
|
||||
assert!(below.server.up);
|
||||
assert_eq!(
|
||||
below.api.peer_version.as_deref(),
|
||||
Some("0.0.0-fake"),
|
||||
"the version stays known while the API is still up"
|
||||
);
|
||||
|
||||
// The next failure reaches the threshold and flips both checks down.
|
||||
wait_for(®, |r| {
|
||||
let s = r.status("beta").unwrap();
|
||||
!s.server.up && !s.api.up
|
||||
})
|
||||
.await;
|
||||
let down = reg.status("beta").unwrap();
|
||||
assert_eq!(down.server.last_error.as_deref(), Some("timeout"));
|
||||
assert_eq!(down.api.auth, AuthState::Unknown);
|
||||
assert!(down.server.since.is_some());
|
||||
reg.shutdown().await;
|
||||
let after = reg.status("beta").unwrap();
|
||||
assert!(after.server.up, "one failure below threshold = still up");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -606,6 +676,165 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_disabled_peer_is_never_proxied() {
|
||||
let fake = FakeTransport::new();
|
||||
fake.peer("beta")
|
||||
.respond("/systemd/units", 200, "application/json", "{}");
|
||||
let reg = registry_with(fake.clone(), fake_peers_config(&["beta"]));
|
||||
reg.get("beta", "/systemd/units", &[], ProxyOpts::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let calls = fake.calls().len();
|
||||
|
||||
let mut peer_off = fake_peers_config(&["beta"]);
|
||||
peer_off.peer[0].enabled = false;
|
||||
reg.reload(peer_off).await.unwrap();
|
||||
let err = reg
|
||||
.get("beta", "/systemd/units", &[], ProxyOpts::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), "not_found");
|
||||
assert!(err.message().contains("disabled"), "{}", err.message());
|
||||
|
||||
let mut feature_off = fake_peers_config(&["beta"]);
|
||||
feature_off.enabled = false;
|
||||
reg.reload(feature_off).await.unwrap();
|
||||
let err = reg
|
||||
.get("beta", "/systemd/units", &[], ProxyOpts::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), "not_found");
|
||||
assert_eq!(
|
||||
fake.calls().len(),
|
||||
calls,
|
||||
"no request (and so no token) leaves while the peer is off"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_rejects_changes_the_transport_cannot_apply() {
|
||||
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
|
||||
let old = reg.config();
|
||||
|
||||
let mut moved = fake_peers_config(&["beta"]);
|
||||
moved.deny_cidrs = vec!["10.0.0.0/8".into()];
|
||||
let err = reg.reload(moved).await.unwrap_err();
|
||||
assert!(err.to_string().contains("restart"), "{err}");
|
||||
|
||||
let mut resized = fake_peers_config(&["beta"]);
|
||||
resized.max_response_bytes = 4096;
|
||||
assert!(reg.reload(resized).await.is_err());
|
||||
|
||||
let mut capped = fake_peers_config(&["beta"]);
|
||||
capped.max_concurrent_per_peer = 1;
|
||||
assert!(reg.reload(capped).await.is_err());
|
||||
assert_eq!(reg.config().deny_cidrs, old.deny_cidrs);
|
||||
assert_eq!(reg.config().max_response_bytes, old.max_response_bytes);
|
||||
|
||||
let mut invalid = fake_peers_config(&["beta"]);
|
||||
invalid.deny_cidrs = vec!["not-a-cidr".into()];
|
||||
let broken = registry_with(FakeTransport::new(), invalid.clone());
|
||||
let err = broken.reload(invalid).await.unwrap_err();
|
||||
assert!(err.to_string().contains("CIDR"), "{err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_changed_peer_url_starts_with_a_clean_status() {
|
||||
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
|
||||
{
|
||||
let mut st = lock_write(®.shared.state);
|
||||
let beta = st
|
||||
.statuses
|
||||
.get_mut(&PeerName::parse("beta").unwrap())
|
||||
.unwrap();
|
||||
beta.server.up = true;
|
||||
beta.server.since = Some(jiff::Timestamp::now());
|
||||
beta.api.up = true;
|
||||
beta.api.peer_version = Some("1.0.0".into());
|
||||
}
|
||||
let mut moved = fake_peers_config(&["beta"]);
|
||||
moved.peer[0].url = url::Url::parse("http://beta-2:1984").unwrap();
|
||||
reg.reload(moved).await.unwrap();
|
||||
let status = reg.status("beta").unwrap();
|
||||
assert_eq!(status.url, "http://beta-2:1984");
|
||||
assert!(!status.server.up);
|
||||
assert!(status.server.since.is_none());
|
||||
assert!(status.api.since.is_none());
|
||||
assert_eq!(status.api.peer_version, None);
|
||||
}
|
||||
|
||||
/// The SIGHUP handler fans out to peers even with TLS off, and the
|
||||
/// peer-less default (both off) still consumes the signal.
|
||||
#[tokio::test]
|
||||
async fn sighup_reloads_peers_when_tls_is_off() {
|
||||
use tokio::signal::unix::{signal, SignalKind};
|
||||
|
||||
let dir = std::env::temp_dir().join(format!("slinuxm-sighup-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("config.toml");
|
||||
std::fs::write(
|
||||
&path,
|
||||
r#"
|
||||
[peers]
|
||||
enabled = true
|
||||
[[peers.peer]]
|
||||
name = "gamma"
|
||||
url = "http://gamma:1984"
|
||||
token_file = "/run/peers/gamma.token"
|
||||
allow_plain_http = true
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
|
||||
let hup = signal(SignalKind::hangup()).unwrap();
|
||||
let shutdown = CancellationToken::new();
|
||||
crate::tls::spawn_sighup_reload(
|
||||
None,
|
||||
hup,
|
||||
shutdown.clone(),
|
||||
Some(PeersReloader::new(
|
||||
reg.clone(),
|
||||
Some(path.clone()),
|
||||
CliOverrides::default(),
|
||||
)),
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let sent = unsafe { libc::kill(std::process::id() as libc::pid_t, libc::SIGHUP) };
|
||||
assert_eq!(sent, 0, "sending SIGHUP failed");
|
||||
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
||||
while !reg.contains("gamma") {
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"SIGHUP did not reach the peer reloader: {:?}",
|
||||
reg.snapshot()
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
shutdown.cancel();
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shutdown_waits_for_loops_retired_by_a_reload() {
|
||||
let fake = FakeTransport::new();
|
||||
let reg = PeerRegistry::start_with_transport(
|
||||
fake_peers_config(&["beta", "gamma"]),
|
||||
fake.clone(),
|
||||
CancellationToken::new(),
|
||||
);
|
||||
wait_for(®, |r| r.status("beta").unwrap().server.up).await;
|
||||
reg.reload(fake_peers_config(&["gamma"])).await.unwrap();
|
||||
assert!(
|
||||
!lock_mutex(®.retired).is_empty(),
|
||||
"the removed peer's loop is retained, not detached"
|
||||
);
|
||||
reg.shutdown().await;
|
||||
assert!(lock_mutex(®.retired).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_disabling_the_feature_empties_the_snapshot() {
|
||||
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
|
||||
@@ -617,7 +846,7 @@ mod tests {
|
||||
assert!(!reg.config().enabled);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn reload_moves_loops_to_the_new_peer_set() {
|
||||
let fake = FakeTransport::new();
|
||||
let reg = PeerRegistry::start_with_transport(
|
||||
|
||||
Reference in new issue
Block a user