B: report healthz latency and keep the last peer version below the threshold
This commit is contained in:
1 parent
5ad94f249e
commit
26b9f8bdff
1 file changed
+118
-23
+118
-23
@@ -148,6 +148,7 @@ struct ApiTracker {
|
|||||||
since: Option<Timestamp>,
|
since: Option<Timestamp>,
|
||||||
failures: u32,
|
failures: u32,
|
||||||
auth: AuthState,
|
auth: AuthState,
|
||||||
|
version: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ApiTracker {
|
impl ApiTracker {
|
||||||
@@ -157,6 +158,7 @@ impl ApiTracker {
|
|||||||
since: status.since,
|
since: status.since,
|
||||||
failures: 0,
|
failures: 0,
|
||||||
auth: status.auth,
|
auth: status.auth,
|
||||||
|
version: status.peer_version.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,6 +171,9 @@ impl ApiTracker {
|
|||||||
}
|
}
|
||||||
self.up = true;
|
self.up = true;
|
||||||
self.auth = observation.auth;
|
self.auth = observation.auth;
|
||||||
|
if let Some(version) = observation.version {
|
||||||
|
self.version = Some(version);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
self.failures = self.failures.saturating_add(1);
|
self.failures = self.failures.saturating_add(1);
|
||||||
if first {
|
if first {
|
||||||
@@ -182,6 +187,14 @@ impl ApiTracker {
|
|||||||
self.auth = AuthState::Unknown;
|
self.auth = AuthState::Unknown;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// A failure below the threshold leaves the peer up, so the version
|
||||||
|
// it reported before is still the last known one.
|
||||||
|
let peer_version = if self.up {
|
||||||
|
self.version.clone()
|
||||||
|
} else {
|
||||||
|
self.version = None;
|
||||||
|
None
|
||||||
|
};
|
||||||
ApiStatus {
|
ApiStatus {
|
||||||
up: self.up,
|
up: self.up,
|
||||||
since: self.since,
|
since: self.since,
|
||||||
@@ -189,7 +202,7 @@ impl ApiTracker {
|
|||||||
last_error: observation.error,
|
last_error: observation.error,
|
||||||
latency_ms: observation.latency.map(millis),
|
latency_ms: observation.latency.map(millis),
|
||||||
auth: self.auth,
|
auth: self.auth,
|
||||||
peer_version: observation.version,
|
peer_version,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -211,13 +224,7 @@ pub async fn probe_api(
|
|||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
) -> ApiObservation {
|
) -> ApiObservation {
|
||||||
let started = Instant::now();
|
let started = Instant::now();
|
||||||
let req = PeerRequest {
|
let resp = match transport.get(peer, probe_request(PING_PATH), timeout).await {
|
||||||
path: PING_PATH.into(),
|
|
||||||
query: Vec::new(),
|
|
||||||
headers: Vec::new(),
|
|
||||||
accept: None,
|
|
||||||
};
|
|
||||||
let resp = match transport.get(peer, req, timeout).await {
|
|
||||||
Ok(resp) => resp,
|
Ok(resp) => resp,
|
||||||
Err(PeerError::Rejected) => {
|
Err(PeerError::Rejected) => {
|
||||||
return ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
|
return ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
|
||||||
@@ -228,7 +235,7 @@ pub async fn probe_api(
|
|||||||
match resp.status {
|
match resp.status {
|
||||||
200..=299 => ApiObservation::running(AuthState::Ok, latency, version_of(&resp)),
|
200..=299 => ApiObservation::running(AuthState::Ok, latency, version_of(&resp)),
|
||||||
401 | 403 => ApiObservation::running(AuthState::Rejected, latency, None),
|
401 | 403 => ApiObservation::running(AuthState::Rejected, latency, None),
|
||||||
404 => probe_healthz(transport, peer, timeout, latency).await,
|
404 => probe_healthz(transport, peer, timeout).await,
|
||||||
status => ApiObservation::down(&PeerError::Http(status)),
|
status => ApiObservation::down(&PeerError::Http(status)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -237,24 +244,32 @@ async fn probe_healthz(
|
|||||||
transport: &dyn PeerTransport,
|
transport: &dyn PeerTransport,
|
||||||
peer: &PeerConfig,
|
peer: &PeerConfig,
|
||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
so_far: Duration,
|
|
||||||
) -> ApiObservation {
|
) -> ApiObservation {
|
||||||
let req = PeerRequest {
|
let started = Instant::now();
|
||||||
path: HEALTHZ_PATH.into(),
|
match transport
|
||||||
|
.get(peer, probe_request(HEALTHZ_PATH), timeout)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(resp) if (200..300).contains(&resp.status) => {
|
||||||
|
ApiObservation::running(AuthState::Unknown, started.elapsed(), version_of(&resp))
|
||||||
|
}
|
||||||
|
Ok(resp) if resp.status == 401 || resp.status == 403 => {
|
||||||
|
ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
|
||||||
|
}
|
||||||
|
Ok(resp) => ApiObservation::down(&PeerError::Http(resp.status)),
|
||||||
|
Err(PeerError::Rejected) => {
|
||||||
|
ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
|
||||||
|
}
|
||||||
|
Err(err) => ApiObservation::down(&err),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn probe_request(path: &str) -> PeerRequest {
|
||||||
|
PeerRequest {
|
||||||
|
path: path.into(),
|
||||||
query: Vec::new(),
|
query: Vec::new(),
|
||||||
headers: Vec::new(),
|
headers: Vec::new(),
|
||||||
accept: None,
|
accept: None,
|
||||||
};
|
|
||||||
match transport.get(peer, req, timeout).await {
|
|
||||||
Ok(resp) if (200..300).contains(&resp.status) => {
|
|
||||||
ApiObservation::running(AuthState::Unknown, so_far, version_of(&resp))
|
|
||||||
}
|
|
||||||
Ok(resp) if resp.status == 401 || resp.status == 403 => {
|
|
||||||
ApiObservation::running(AuthState::Rejected, so_far, None)
|
|
||||||
}
|
|
||||||
Ok(resp) => ApiObservation::down(&PeerError::Http(resp.status)),
|
|
||||||
Err(PeerError::Rejected) => ApiObservation::running(AuthState::Rejected, so_far, None),
|
|
||||||
Err(err) => ApiObservation::down(&err),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -533,6 +548,86 @@ mod tests {
|
|||||||
assert_eq!(down.peer_version, None);
|
assert_eq!(down.peer_version, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_failure_below_the_threshold_keeps_the_known_peer_version() {
|
||||||
|
let mut api = ApiTracker::seeded(&ApiStatus::default());
|
||||||
|
let t0 = Timestamp::now();
|
||||||
|
let up = api.record(
|
||||||
|
ApiObservation::running(
|
||||||
|
AuthState::Ok,
|
||||||
|
Duration::from_millis(1),
|
||||||
|
Some("1.2.3".into()),
|
||||||
|
),
|
||||||
|
t0,
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
assert_eq!(up.peer_version.as_deref(), Some("1.2.3"));
|
||||||
|
|
||||||
|
let t1 = t0 + Duration::from_secs(1);
|
||||||
|
let still_up = api.record(ApiObservation::down(&PeerError::Timeout), t1, 2);
|
||||||
|
assert!(still_up.up);
|
||||||
|
assert_eq!(still_up.peer_version.as_deref(), Some("1.2.3"));
|
||||||
|
|
||||||
|
let t2 = t0 + Duration::from_secs(2);
|
||||||
|
let down = api.record(ApiObservation::down(&PeerError::Timeout), t2, 2);
|
||||||
|
assert!(!down.up);
|
||||||
|
assert_eq!(down.peer_version, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct SlowHealthz;
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl PeerTransport for SlowHealthz {
|
||||||
|
async fn tcp_check(
|
||||||
|
&self,
|
||||||
|
_peer: &PeerConfig,
|
||||||
|
_timeout: Duration,
|
||||||
|
) -> Result<Duration, PeerError> {
|
||||||
|
Ok(Duration::ZERO)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get(
|
||||||
|
&self,
|
||||||
|
_peer: &PeerConfig,
|
||||||
|
req: PeerRequest,
|
||||||
|
_timeout: Duration,
|
||||||
|
) -> Result<PeerResponse, PeerError> {
|
||||||
|
match req.path.as_str() {
|
||||||
|
"/api/v1/ping" => Ok(PeerResponse {
|
||||||
|
status: 404,
|
||||||
|
content_type: None,
|
||||||
|
body: PeerBody::Bytes(bytes::Bytes::new()),
|
||||||
|
}),
|
||||||
|
_ => {
|
||||||
|
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||||
|
Ok(PeerResponse {
|
||||||
|
status: 200,
|
||||||
|
content_type: None,
|
||||||
|
body: PeerBody::Bytes(bytes::Bytes::from_static(br#"{"status":"ok"}"#)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn healthz_fallback_reports_its_own_latency() {
|
||||||
|
let cfg = cfg();
|
||||||
|
let (_, api) = check_once(
|
||||||
|
&SlowHealthz,
|
||||||
|
&cfg.peer[0],
|
||||||
|
Duration::from_secs(1),
|
||||||
|
Duration::from_secs(1),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(api.up);
|
||||||
|
assert!(
|
||||||
|
api.latency_ms.unwrap() >= 25,
|
||||||
|
"healthz call latency, not the ping's: {:?}",
|
||||||
|
api.latency_ms
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn jitter_is_bounded_and_stable() {
|
fn jitter_is_bounded_and_stable() {
|
||||||
let interval = Duration::from_secs(10);
|
let interval = Duration::from_secs(10);
|
||||||
|
|||||||
Reference in new issue
Block a user