B: report healthz latency and keep the last peer version below the threshold

This commit is contained in:
ari committed 2026-10-19 20:55:25 -04:00
1 parent 5ad94f249e
commit 26b9f8bdff
1 file changed
+118 -23
+118 -23
View File
@@ -148,6 +148,7 @@ struct ApiTracker {
since: Option<Timestamp>,
failures: u32,
auth: AuthState,
version: Option<String>,
}
impl ApiTracker {
@@ -157,6 +158,7 @@ impl ApiTracker {
since: status.since,
failures: 0,
auth: status.auth,
version: status.peer_version.clone(),
}
}
@@ -169,6 +171,9 @@ impl ApiTracker {
}
self.up = true;
self.auth = observation.auth;
if let Some(version) = observation.version {
self.version = Some(version);
}
} else {
self.failures = self.failures.saturating_add(1);
if first {
@@ -182,6 +187,14 @@ impl ApiTracker {
self.auth = AuthState::Unknown;
}
}
// A failure below the threshold leaves the peer up, so the version
// it reported before is still the last known one.
let peer_version = if self.up {
self.version.clone()
} else {
self.version = None;
None
};
ApiStatus {
up: self.up,
since: self.since,
@@ -189,7 +202,7 @@ impl ApiTracker {
last_error: observation.error,
latency_ms: observation.latency.map(millis),
auth: self.auth,
peer_version: observation.version,
peer_version,
}
}
}
@@ -211,13 +224,7 @@ pub async fn probe_api(
timeout: Duration,
) -> ApiObservation {
let started = Instant::now();
let req = PeerRequest {
path: PING_PATH.into(),
query: Vec::new(),
headers: Vec::new(),
accept: None,
};
let resp = match transport.get(peer, req, timeout).await {
let resp = match transport.get(peer, probe_request(PING_PATH), timeout).await {
Ok(resp) => resp,
Err(PeerError::Rejected) => {
return ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
@@ -228,7 +235,7 @@ pub async fn probe_api(
match resp.status {
200..=299 => ApiObservation::running(AuthState::Ok, latency, version_of(&resp)),
401 | 403 => ApiObservation::running(AuthState::Rejected, latency, None),
404 => probe_healthz(transport, peer, timeout, latency).await,
404 => probe_healthz(transport, peer, timeout).await,
status => ApiObservation::down(&PeerError::Http(status)),
}
}
@@ -237,24 +244,32 @@ async fn probe_healthz(
transport: &dyn PeerTransport,
peer: &PeerConfig,
timeout: Duration,
so_far: Duration,
) -> ApiObservation {
let req = PeerRequest {
path: HEALTHZ_PATH.into(),
let started = Instant::now();
match transport
.get(peer, probe_request(HEALTHZ_PATH), timeout)
.await
{
Ok(resp) if (200..300).contains(&resp.status) => {
ApiObservation::running(AuthState::Unknown, started.elapsed(), version_of(&resp))
}
Ok(resp) if resp.status == 401 || resp.status == 403 => {
ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
}
Ok(resp) => ApiObservation::down(&PeerError::Http(resp.status)),
Err(PeerError::Rejected) => {
ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
}
Err(err) => ApiObservation::down(&err),
}
}
fn probe_request(path: &str) -> PeerRequest {
PeerRequest {
path: path.into(),
query: Vec::new(),
headers: Vec::new(),
accept: None,
};
match transport.get(peer, req, timeout).await {
Ok(resp) if (200..300).contains(&resp.status) => {
ApiObservation::running(AuthState::Unknown, so_far, version_of(&resp))
}
Ok(resp) if resp.status == 401 || resp.status == 403 => {
ApiObservation::running(AuthState::Rejected, so_far, None)
}
Ok(resp) => ApiObservation::down(&PeerError::Http(resp.status)),
Err(PeerError::Rejected) => ApiObservation::running(AuthState::Rejected, so_far, None),
Err(err) => ApiObservation::down(&err),
}
}
@@ -533,6 +548,86 @@ mod tests {
assert_eq!(down.peer_version, None);
}
#[test]
fn a_failure_below_the_threshold_keeps_the_known_peer_version() {
let mut api = ApiTracker::seeded(&ApiStatus::default());
let t0 = Timestamp::now();
let up = api.record(
ApiObservation::running(
AuthState::Ok,
Duration::from_millis(1),
Some("1.2.3".into()),
),
t0,
2,
);
assert_eq!(up.peer_version.as_deref(), Some("1.2.3"));
let t1 = t0 + Duration::from_secs(1);
let still_up = api.record(ApiObservation::down(&PeerError::Timeout), t1, 2);
assert!(still_up.up);
assert_eq!(still_up.peer_version.as_deref(), Some("1.2.3"));
let t2 = t0 + Duration::from_secs(2);
let down = api.record(ApiObservation::down(&PeerError::Timeout), t2, 2);
assert!(!down.up);
assert_eq!(down.peer_version, None);
}
struct SlowHealthz;
#[async_trait::async_trait]
impl PeerTransport for SlowHealthz {
async fn tcp_check(
&self,
_peer: &PeerConfig,
_timeout: Duration,
) -> Result<Duration, PeerError> {
Ok(Duration::ZERO)
}
async fn get(
&self,
_peer: &PeerConfig,
req: PeerRequest,
_timeout: Duration,
) -> Result<PeerResponse, PeerError> {
match req.path.as_str() {
"/api/v1/ping" => Ok(PeerResponse {
status: 404,
content_type: None,
body: PeerBody::Bytes(bytes::Bytes::new()),
}),
_ => {
tokio::time::sleep(Duration::from_millis(30)).await;
Ok(PeerResponse {
status: 200,
content_type: None,
body: PeerBody::Bytes(bytes::Bytes::from_static(br#"{"status":"ok"}"#)),
})
}
}
}
}
#[tokio::test]
async fn healthz_fallback_reports_its_own_latency() {
let cfg = cfg();
let (_, api) = check_once(
&SlowHealthz,
&cfg.peer[0],
Duration::from_secs(1),
Duration::from_secs(1),
)
.await;
assert!(api.up);
assert!(
api.latency_ms.unwrap() >= 25,
"healthz call latency, not the ping's: {:?}",
api.latency_ms
);
}
#[test]
fn jitter_is_bounded_and_stable() {
let interval = Duration::from_secs(10);