B: report healthz latency and keep the last peer version below the threshold
This commit is contained in:
1 parent
5ad94f249e
commit
26b9f8bdff
1 file changed
+118
-23
+118
-23
@@ -148,6 +148,7 @@ struct ApiTracker {
|
||||
since: Option<Timestamp>,
|
||||
failures: u32,
|
||||
auth: AuthState,
|
||||
version: Option<String>,
|
||||
}
|
||||
|
||||
impl ApiTracker {
|
||||
@@ -157,6 +158,7 @@ impl ApiTracker {
|
||||
since: status.since,
|
||||
failures: 0,
|
||||
auth: status.auth,
|
||||
version: status.peer_version.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,6 +171,9 @@ impl ApiTracker {
|
||||
}
|
||||
self.up = true;
|
||||
self.auth = observation.auth;
|
||||
if let Some(version) = observation.version {
|
||||
self.version = Some(version);
|
||||
}
|
||||
} else {
|
||||
self.failures = self.failures.saturating_add(1);
|
||||
if first {
|
||||
@@ -182,6 +187,14 @@ impl ApiTracker {
|
||||
self.auth = AuthState::Unknown;
|
||||
}
|
||||
}
|
||||
// A failure below the threshold leaves the peer up, so the version
|
||||
// it reported before is still the last known one.
|
||||
let peer_version = if self.up {
|
||||
self.version.clone()
|
||||
} else {
|
||||
self.version = None;
|
||||
None
|
||||
};
|
||||
ApiStatus {
|
||||
up: self.up,
|
||||
since: self.since,
|
||||
@@ -189,7 +202,7 @@ impl ApiTracker {
|
||||
last_error: observation.error,
|
||||
latency_ms: observation.latency.map(millis),
|
||||
auth: self.auth,
|
||||
peer_version: observation.version,
|
||||
peer_version,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,13 +224,7 @@ pub async fn probe_api(
|
||||
timeout: Duration,
|
||||
) -> ApiObservation {
|
||||
let started = Instant::now();
|
||||
let req = PeerRequest {
|
||||
path: PING_PATH.into(),
|
||||
query: Vec::new(),
|
||||
headers: Vec::new(),
|
||||
accept: None,
|
||||
};
|
||||
let resp = match transport.get(peer, req, timeout).await {
|
||||
let resp = match transport.get(peer, probe_request(PING_PATH), timeout).await {
|
||||
Ok(resp) => resp,
|
||||
Err(PeerError::Rejected) => {
|
||||
return ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
|
||||
@@ -228,7 +235,7 @@ pub async fn probe_api(
|
||||
match resp.status {
|
||||
200..=299 => ApiObservation::running(AuthState::Ok, latency, version_of(&resp)),
|
||||
401 | 403 => ApiObservation::running(AuthState::Rejected, latency, None),
|
||||
404 => probe_healthz(transport, peer, timeout, latency).await,
|
||||
404 => probe_healthz(transport, peer, timeout).await,
|
||||
status => ApiObservation::down(&PeerError::Http(status)),
|
||||
}
|
||||
}
|
||||
@@ -237,24 +244,32 @@ async fn probe_healthz(
|
||||
transport: &dyn PeerTransport,
|
||||
peer: &PeerConfig,
|
||||
timeout: Duration,
|
||||
so_far: Duration,
|
||||
) -> ApiObservation {
|
||||
let req = PeerRequest {
|
||||
path: HEALTHZ_PATH.into(),
|
||||
let started = Instant::now();
|
||||
match transport
|
||||
.get(peer, probe_request(HEALTHZ_PATH), timeout)
|
||||
.await
|
||||
{
|
||||
Ok(resp) if (200..300).contains(&resp.status) => {
|
||||
ApiObservation::running(AuthState::Unknown, started.elapsed(), version_of(&resp))
|
||||
}
|
||||
Ok(resp) if resp.status == 401 || resp.status == 403 => {
|
||||
ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
|
||||
}
|
||||
Ok(resp) => ApiObservation::down(&PeerError::Http(resp.status)),
|
||||
Err(PeerError::Rejected) => {
|
||||
ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
|
||||
}
|
||||
Err(err) => ApiObservation::down(&err),
|
||||
}
|
||||
}
|
||||
|
||||
fn probe_request(path: &str) -> PeerRequest {
|
||||
PeerRequest {
|
||||
path: path.into(),
|
||||
query: Vec::new(),
|
||||
headers: Vec::new(),
|
||||
accept: None,
|
||||
};
|
||||
match transport.get(peer, req, timeout).await {
|
||||
Ok(resp) if (200..300).contains(&resp.status) => {
|
||||
ApiObservation::running(AuthState::Unknown, so_far, version_of(&resp))
|
||||
}
|
||||
Ok(resp) if resp.status == 401 || resp.status == 403 => {
|
||||
ApiObservation::running(AuthState::Rejected, so_far, None)
|
||||
}
|
||||
Ok(resp) => ApiObservation::down(&PeerError::Http(resp.status)),
|
||||
Err(PeerError::Rejected) => ApiObservation::running(AuthState::Rejected, so_far, None),
|
||||
Err(err) => ApiObservation::down(&err),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -533,6 +548,86 @@ mod tests {
|
||||
assert_eq!(down.peer_version, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failure_below_the_threshold_keeps_the_known_peer_version() {
|
||||
let mut api = ApiTracker::seeded(&ApiStatus::default());
|
||||
let t0 = Timestamp::now();
|
||||
let up = api.record(
|
||||
ApiObservation::running(
|
||||
AuthState::Ok,
|
||||
Duration::from_millis(1),
|
||||
Some("1.2.3".into()),
|
||||
),
|
||||
t0,
|
||||
2,
|
||||
);
|
||||
assert_eq!(up.peer_version.as_deref(), Some("1.2.3"));
|
||||
|
||||
let t1 = t0 + Duration::from_secs(1);
|
||||
let still_up = api.record(ApiObservation::down(&PeerError::Timeout), t1, 2);
|
||||
assert!(still_up.up);
|
||||
assert_eq!(still_up.peer_version.as_deref(), Some("1.2.3"));
|
||||
|
||||
let t2 = t0 + Duration::from_secs(2);
|
||||
let down = api.record(ApiObservation::down(&PeerError::Timeout), t2, 2);
|
||||
assert!(!down.up);
|
||||
assert_eq!(down.peer_version, None);
|
||||
}
|
||||
|
||||
struct SlowHealthz;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl PeerTransport for SlowHealthz {
|
||||
async fn tcp_check(
|
||||
&self,
|
||||
_peer: &PeerConfig,
|
||||
_timeout: Duration,
|
||||
) -> Result<Duration, PeerError> {
|
||||
Ok(Duration::ZERO)
|
||||
}
|
||||
|
||||
async fn get(
|
||||
&self,
|
||||
_peer: &PeerConfig,
|
||||
req: PeerRequest,
|
||||
_timeout: Duration,
|
||||
) -> Result<PeerResponse, PeerError> {
|
||||
match req.path.as_str() {
|
||||
"/api/v1/ping" => Ok(PeerResponse {
|
||||
status: 404,
|
||||
content_type: None,
|
||||
body: PeerBody::Bytes(bytes::Bytes::new()),
|
||||
}),
|
||||
_ => {
|
||||
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||
Ok(PeerResponse {
|
||||
status: 200,
|
||||
content_type: None,
|
||||
body: PeerBody::Bytes(bytes::Bytes::from_static(br#"{"status":"ok"}"#)),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn healthz_fallback_reports_its_own_latency() {
|
||||
let cfg = cfg();
|
||||
let (_, api) = check_once(
|
||||
&SlowHealthz,
|
||||
&cfg.peer[0],
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await;
|
||||
assert!(api.up);
|
||||
assert!(
|
||||
api.latency_ms.unwrap() >= 25,
|
||||
"healthz call latency, not the ping's: {:?}",
|
||||
api.latency_ms
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jitter_is_bounded_and_stable() {
|
||||
let interval = Duration::from_secs(10);
|
||||
|
||||
Reference in new issue
Block a user