From 26b9f8bdff06864e5d5ef90d74cbd7f1ee296b95 Mon Sep 17 00:00:00 2001 From: ari Date: Mon, 19 Oct 2026 20:51:12 -0400 Subject: [PATCH] B: report healthz latency and keep the last peer version below the threshold --- src/peers/health.rs | 141 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 118 insertions(+), 23 deletions(-) diff --git a/src/peers/health.rs b/src/peers/health.rs index 3357dbd..2346c54 100644 --- a/src/peers/health.rs +++ b/src/peers/health.rs @@ -148,6 +148,7 @@ struct ApiTracker { since: Option, failures: u32, auth: AuthState, + version: Option, } impl ApiTracker { @@ -157,6 +158,7 @@ impl ApiTracker { since: status.since, failures: 0, auth: status.auth, + version: status.peer_version.clone(), } } @@ -169,6 +171,9 @@ impl ApiTracker { } self.up = true; self.auth = observation.auth; + if let Some(version) = observation.version { + self.version = Some(version); + } } else { self.failures = self.failures.saturating_add(1); if first { @@ -182,6 +187,14 @@ impl ApiTracker { self.auth = AuthState::Unknown; } } + // A failure below the threshold leaves the peer up, so the version + // it reported before is still the last known one. + let peer_version = if self.up { + self.version.clone() + } else { + self.version = None; + None + }; ApiStatus { up: self.up, since: self.since, @@ -189,7 +202,7 @@ impl ApiTracker { last_error: observation.error, latency_ms: observation.latency.map(millis), auth: self.auth, - peer_version: observation.version, + peer_version, } } } @@ -211,13 +224,7 @@ pub async fn probe_api( timeout: Duration, ) -> ApiObservation { let started = Instant::now(); - let req = PeerRequest { - path: PING_PATH.into(), - query: Vec::new(), - headers: Vec::new(), - accept: None, - }; - let resp = match transport.get(peer, req, timeout).await { + let resp = match transport.get(peer, probe_request(PING_PATH), timeout).await { Ok(resp) => resp, Err(PeerError::Rejected) => { return ApiObservation::running(AuthState::Rejected, started.elapsed(), None) @@ -228,7 +235,7 @@ pub async fn probe_api( match resp.status { 200..=299 => ApiObservation::running(AuthState::Ok, latency, version_of(&resp)), 401 | 403 => ApiObservation::running(AuthState::Rejected, latency, None), - 404 => probe_healthz(transport, peer, timeout, latency).await, + 404 => probe_healthz(transport, peer, timeout).await, status => ApiObservation::down(&PeerError::Http(status)), } } @@ -237,24 +244,32 @@ async fn probe_healthz( transport: &dyn PeerTransport, peer: &PeerConfig, timeout: Duration, - so_far: Duration, ) -> ApiObservation { - let req = PeerRequest { - path: HEALTHZ_PATH.into(), + let started = Instant::now(); + match transport + .get(peer, probe_request(HEALTHZ_PATH), timeout) + .await + { + Ok(resp) if (200..300).contains(&resp.status) => { + ApiObservation::running(AuthState::Unknown, started.elapsed(), version_of(&resp)) + } + Ok(resp) if resp.status == 401 || resp.status == 403 => { + ApiObservation::running(AuthState::Rejected, started.elapsed(), None) + } + Ok(resp) => ApiObservation::down(&PeerError::Http(resp.status)), + Err(PeerError::Rejected) => { + ApiObservation::running(AuthState::Rejected, started.elapsed(), None) + } + Err(err) => ApiObservation::down(&err), + } +} + +fn probe_request(path: &str) -> PeerRequest { + PeerRequest { + path: path.into(), query: Vec::new(), headers: Vec::new(), accept: None, - }; - match transport.get(peer, req, timeout).await { - Ok(resp) if (200..300).contains(&resp.status) => { - ApiObservation::running(AuthState::Unknown, so_far, version_of(&resp)) - } - Ok(resp) if resp.status == 401 || resp.status == 403 => { - ApiObservation::running(AuthState::Rejected, so_far, None) - } - Ok(resp) => ApiObservation::down(&PeerError::Http(resp.status)), - Err(PeerError::Rejected) => ApiObservation::running(AuthState::Rejected, so_far, None), - Err(err) => ApiObservation::down(&err), } } @@ -533,6 +548,86 @@ mod tests { assert_eq!(down.peer_version, None); } + #[test] + fn a_failure_below_the_threshold_keeps_the_known_peer_version() { + let mut api = ApiTracker::seeded(&ApiStatus::default()); + let t0 = Timestamp::now(); + let up = api.record( + ApiObservation::running( + AuthState::Ok, + Duration::from_millis(1), + Some("1.2.3".into()), + ), + t0, + 2, + ); + assert_eq!(up.peer_version.as_deref(), Some("1.2.3")); + + let t1 = t0 + Duration::from_secs(1); + let still_up = api.record(ApiObservation::down(&PeerError::Timeout), t1, 2); + assert!(still_up.up); + assert_eq!(still_up.peer_version.as_deref(), Some("1.2.3")); + + let t2 = t0 + Duration::from_secs(2); + let down = api.record(ApiObservation::down(&PeerError::Timeout), t2, 2); + assert!(!down.up); + assert_eq!(down.peer_version, None); + } + + struct SlowHealthz; + + #[async_trait::async_trait] + impl PeerTransport for SlowHealthz { + async fn tcp_check( + &self, + _peer: &PeerConfig, + _timeout: Duration, + ) -> Result { + Ok(Duration::ZERO) + } + + async fn get( + &self, + _peer: &PeerConfig, + req: PeerRequest, + _timeout: Duration, + ) -> Result { + match req.path.as_str() { + "/api/v1/ping" => Ok(PeerResponse { + status: 404, + content_type: None, + body: PeerBody::Bytes(bytes::Bytes::new()), + }), + _ => { + tokio::time::sleep(Duration::from_millis(30)).await; + Ok(PeerResponse { + status: 200, + content_type: None, + body: PeerBody::Bytes(bytes::Bytes::from_static(br#"{"status":"ok"}"#)), + }) + } + } + } + } + + #[tokio::test] + async fn healthz_fallback_reports_its_own_latency() { + let cfg = cfg(); + let (_, api) = check_once( + &SlowHealthz, + &cfg.peer[0], + Duration::from_secs(1), + Duration::from_secs(1), + ) + .await; + assert!(api.up); + assert!( + api.latency_ms.unwrap() >= 25, + "healthz call latency, not the ping's: {:?}", + api.latency_ms + ); + } + #[test] fn jitter_is_bounded_and_stable() { let interval = Duration::from_secs(10);