B: B2/B3 health probes, loops and the in-memory status model
This commit is contained in:
1 parent
bd5b21bcc4
commit
d94c2b3a1d
3 files changed
+994
-126
No files matched your search
+439
-81
@@ -1,47 +1,278 @@
|
||||
//! One-shot peer probes used by the diagnostic CLI.
|
||||
//! Peer probes and health state.
|
||||
//!
|
||||
//! The registry's periodic loops reuse the transport; this module only
|
||||
//! classifies a single round so `slinuxm peer check` can report it.
|
||||
//! One probe is cheap and bounded: a TCP connect for "server up" and an
|
||||
//! authenticated `GET /api/v1/ping` for "API running". The registry schedules
|
||||
//! them and stores the resulting status; `slinuxm peer check` calls the same
|
||||
//! probes once.
|
||||
|
||||
use std::time::Duration;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use jiff::Timestamp;
|
||||
use serde_json::Value;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::error::PeerError;
|
||||
use super::transport::{PeerRequest, PeerTransport};
|
||||
use super::types::{ApiStatus, AuthState, PeerConfig, ServerStatus};
|
||||
use super::transport::{PeerBody, PeerRequest, PeerResponse, PeerTransport};
|
||||
use super::types::{
|
||||
ApiStatus, AuthState, PeerConfig, PeerName, PeerStatus, PeersConfig, ServerStatus,
|
||||
};
|
||||
|
||||
const PING_PATH: &str = "/api/v1/ping";
|
||||
const HEALTHZ_PATH: &str = "/healthz";
|
||||
|
||||
fn millis(d: Duration) -> u64 {
|
||||
u64::try_from(d.as_millis()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
fn failed_server(err: &PeerError, at: Timestamp) -> ServerStatus {
|
||||
ServerStatus {
|
||||
up: false,
|
||||
since: None,
|
||||
last_checked: Some(at),
|
||||
last_error: Some(err.class_str()),
|
||||
latency_ms: None,
|
||||
/// Timing for one peer's loops, copied out of [`PeersConfig`].
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct LoopConfig {
|
||||
pub tcp_interval: Duration,
|
||||
pub tcp_timeout: Duration,
|
||||
pub api_interval: Duration,
|
||||
pub api_timeout: Duration,
|
||||
pub fail_threshold: u32,
|
||||
}
|
||||
|
||||
impl LoopConfig {
|
||||
pub fn from_config(cfg: &PeersConfig) -> Self {
|
||||
Self {
|
||||
tcp_interval: Duration::from_secs(cfg.tcp_interval_secs.max(1)),
|
||||
tcp_timeout: Duration::from_secs(cfg.tcp_timeout_secs.max(1)),
|
||||
api_interval: Duration::from_secs(cfg.api_interval_secs.max(1)),
|
||||
api_timeout: Duration::from_secs(cfg.api_timeout_secs.max(1)),
|
||||
fail_threshold: cfg.fail_threshold.max(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn failed_api(err: &PeerError, at: Timestamp) -> ApiStatus {
|
||||
ApiStatus {
|
||||
up: false,
|
||||
since: None,
|
||||
last_checked: Some(at),
|
||||
last_error: Some(err.class_str()),
|
||||
latency_ms: None,
|
||||
auth: AuthState::Unknown,
|
||||
peer_version: None,
|
||||
/// Outcome of the authenticated API probe.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ApiObservation {
|
||||
/// The API answered as running, even if it rejected our credential.
|
||||
pub ok: bool,
|
||||
pub auth: AuthState,
|
||||
pub version: Option<String>,
|
||||
pub latency: Option<Duration>,
|
||||
/// Class token, never peer text or a URL.
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
impl ApiObservation {
|
||||
fn down(err: &PeerError) -> Self {
|
||||
Self {
|
||||
ok: false,
|
||||
auth: AuthState::Unknown,
|
||||
version: None,
|
||||
latency: None,
|
||||
error: Some(err.class_str()),
|
||||
}
|
||||
}
|
||||
|
||||
fn running(auth: AuthState, latency: Duration, version: Option<String>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
auth,
|
||||
version,
|
||||
latency: Some(latency),
|
||||
error: (auth == AuthState::Rejected).then(|| "unauthorized".to_owned()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One TCP connect and one authenticated `GET /api/v1/ping`.
|
||||
/// Server-side state accumulated between checks. The `since` timestamp lives
|
||||
/// here rather than in the serialized status so it survives a reload of an
|
||||
/// unchanged peer.
|
||||
#[derive(Debug, Clone)]
|
||||
struct ServerTracker {
|
||||
up: bool,
|
||||
since: Option<Timestamp>,
|
||||
failures: u32,
|
||||
}
|
||||
|
||||
impl ServerTracker {
|
||||
fn seeded(status: &ServerStatus) -> Self {
|
||||
Self {
|
||||
up: status.up,
|
||||
since: status.since,
|
||||
failures: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn record(
|
||||
&mut self,
|
||||
result: Result<Duration, PeerError>,
|
||||
now: Timestamp,
|
||||
threshold: u32,
|
||||
) -> ServerStatus {
|
||||
let first = self.since.is_none();
|
||||
let (last_error, latency) = match &result {
|
||||
Ok(latency) => {
|
||||
self.failures = 0;
|
||||
if !self.up || first {
|
||||
self.since = Some(now);
|
||||
}
|
||||
self.up = true;
|
||||
(None, Some(millis(*latency)))
|
||||
}
|
||||
Err(err) => {
|
||||
self.failures = self.failures.saturating_add(1);
|
||||
if first && self.up {
|
||||
self.since = Some(now);
|
||||
}
|
||||
if self.failures >= threshold && self.up {
|
||||
self.up = false;
|
||||
self.since = Some(now);
|
||||
}
|
||||
if first && self.since.is_none() {
|
||||
self.since = Some(now);
|
||||
}
|
||||
(Some(err.class_str()), None)
|
||||
}
|
||||
};
|
||||
ServerStatus {
|
||||
up: self.up,
|
||||
since: self.since,
|
||||
last_checked: Some(now),
|
||||
last_error,
|
||||
latency_ms: latency,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct ApiTracker {
|
||||
up: bool,
|
||||
since: Option<Timestamp>,
|
||||
failures: u32,
|
||||
auth: AuthState,
|
||||
}
|
||||
|
||||
impl ApiTracker {
|
||||
fn seeded(status: &ApiStatus) -> Self {
|
||||
Self {
|
||||
up: status.up,
|
||||
since: status.since,
|
||||
failures: 0,
|
||||
auth: status.auth,
|
||||
}
|
||||
}
|
||||
|
||||
fn record(&mut self, observation: ApiObservation, now: Timestamp, threshold: u32) -> ApiStatus {
|
||||
let first = self.since.is_none();
|
||||
if observation.ok {
|
||||
self.failures = 0;
|
||||
if !self.up || first {
|
||||
self.since = Some(now);
|
||||
}
|
||||
self.up = true;
|
||||
self.auth = observation.auth;
|
||||
} else {
|
||||
self.failures = self.failures.saturating_add(1);
|
||||
if first {
|
||||
self.since = Some(now);
|
||||
}
|
||||
if self.failures >= threshold && self.up {
|
||||
self.up = false;
|
||||
self.since = Some(now);
|
||||
}
|
||||
if !self.up {
|
||||
self.auth = AuthState::Unknown;
|
||||
}
|
||||
}
|
||||
ApiStatus {
|
||||
up: self.up,
|
||||
since: self.since,
|
||||
last_checked: Some(now),
|
||||
last_error: observation.error,
|
||||
latency_ms: observation.latency.map(millis),
|
||||
auth: self.auth,
|
||||
peer_version: observation.version,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// TCP connect, returning the connect latency.
|
||||
pub async fn probe_server(
|
||||
transport: &dyn PeerTransport,
|
||||
peer: &PeerConfig,
|
||||
timeout: Duration,
|
||||
) -> Result<Duration, PeerError> {
|
||||
transport.tcp_check(peer, timeout).await
|
||||
}
|
||||
|
||||
/// Authenticated ping. A 401/403 means the API is up and our credential is
|
||||
/// not; older peers without the route fall back to public `/healthz`.
|
||||
pub async fn probe_api(
|
||||
transport: &dyn PeerTransport,
|
||||
peer: &PeerConfig,
|
||||
timeout: Duration,
|
||||
) -> ApiObservation {
|
||||
let started = Instant::now();
|
||||
let req = PeerRequest {
|
||||
path: PING_PATH.into(),
|
||||
query: Vec::new(),
|
||||
headers: Vec::new(),
|
||||
accept: None,
|
||||
};
|
||||
let resp = match transport.get(peer, req, timeout).await {
|
||||
Ok(resp) => resp,
|
||||
Err(PeerError::Rejected) => {
|
||||
return ApiObservation::running(AuthState::Rejected, started.elapsed(), None)
|
||||
}
|
||||
Err(err) => return ApiObservation::down(&err),
|
||||
};
|
||||
let latency = started.elapsed();
|
||||
match resp.status {
|
||||
200..=299 => ApiObservation::running(AuthState::Ok, latency, version_of(&resp)),
|
||||
401 | 403 => ApiObservation::running(AuthState::Rejected, latency, None),
|
||||
404 => probe_healthz(transport, peer, timeout, latency).await,
|
||||
status => ApiObservation::down(&PeerError::Http(status)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_healthz(
|
||||
transport: &dyn PeerTransport,
|
||||
peer: &PeerConfig,
|
||||
timeout: Duration,
|
||||
so_far: Duration,
|
||||
) -> ApiObservation {
|
||||
let req = PeerRequest {
|
||||
path: HEALTHZ_PATH.into(),
|
||||
query: Vec::new(),
|
||||
headers: Vec::new(),
|
||||
accept: None,
|
||||
};
|
||||
match transport.get(peer, req, timeout).await {
|
||||
Ok(resp) if (200..300).contains(&resp.status) => {
|
||||
ApiObservation::running(AuthState::Unknown, so_far, version_of(&resp))
|
||||
}
|
||||
Ok(resp) if resp.status == 401 || resp.status == 403 => {
|
||||
ApiObservation::running(AuthState::Rejected, so_far, None)
|
||||
}
|
||||
Ok(resp) => ApiObservation::down(&PeerError::Http(resp.status)),
|
||||
Err(PeerError::Rejected) => ApiObservation::running(AuthState::Rejected, so_far, None),
|
||||
Err(err) => ApiObservation::down(&err),
|
||||
}
|
||||
}
|
||||
|
||||
fn version_of(resp: &PeerResponse) -> Option<String> {
|
||||
let PeerBody::Bytes(body) = &resp.body else {
|
||||
return None;
|
||||
};
|
||||
let value: Value = serde_json::from_slice(body).ok()?;
|
||||
value
|
||||
.get("version")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned)
|
||||
}
|
||||
|
||||
/// One TCP connect and one API probe, for the diagnostic CLI.
|
||||
///
|
||||
/// A peer that answers 401 is up with `auth = rejected`: the API is running
|
||||
/// and our credential is the fault. Version and error classification beyond
|
||||
/// that belong to the registry loops.
|
||||
/// Both probes run concurrently so the caller waits for the slower one, not
|
||||
/// the sum. The threshold is one: a single failure is reported as down.
|
||||
pub async fn check_once(
|
||||
transport: &dyn PeerTransport,
|
||||
peer: &PeerConfig,
|
||||
@@ -49,69 +280,93 @@ pub async fn check_once(
|
||||
api_timeout: Duration,
|
||||
) -> (ServerStatus, ApiStatus) {
|
||||
let now = Timestamp::now();
|
||||
let server = match transport.tcp_check(peer, tcp_timeout).await {
|
||||
Ok(latency) => ServerStatus {
|
||||
up: true,
|
||||
since: Some(now),
|
||||
last_checked: Some(now),
|
||||
last_error: None,
|
||||
latency_ms: Some(millis(latency)),
|
||||
},
|
||||
Err(err) => failed_server(&err, now),
|
||||
};
|
||||
|
||||
let req = PeerRequest {
|
||||
path: "/api/v1/ping".into(),
|
||||
query: Vec::new(),
|
||||
headers: Vec::new(),
|
||||
accept: None,
|
||||
};
|
||||
// The GET is not timed, so `latency_ms` stays empty. The probe that
|
||||
// replaces this stub measures it; `peer check` prints that value.
|
||||
let api = match transport.get(peer, req, api_timeout).await {
|
||||
Ok(resp) if (200..300).contains(&resp.status) => ApiStatus {
|
||||
up: true,
|
||||
since: Some(now),
|
||||
last_checked: Some(now),
|
||||
last_error: None,
|
||||
latency_ms: None,
|
||||
auth: AuthState::Ok,
|
||||
peer_version: None,
|
||||
},
|
||||
Ok(resp) if resp.status == 401 || resp.status == 403 => ApiStatus {
|
||||
up: true,
|
||||
since: Some(now),
|
||||
last_checked: Some(now),
|
||||
last_error: Some("unauthorized".into()),
|
||||
latency_ms: None,
|
||||
auth: AuthState::Rejected,
|
||||
peer_version: None,
|
||||
},
|
||||
Ok(resp) => failed_api(&PeerError::Http(resp.status), now),
|
||||
Err(PeerError::Rejected) => ApiStatus {
|
||||
up: true,
|
||||
since: Some(now),
|
||||
last_checked: Some(now),
|
||||
last_error: Some("unauthorized".into()),
|
||||
latency_ms: None,
|
||||
auth: AuthState::Rejected,
|
||||
peer_version: None,
|
||||
},
|
||||
Err(err) => failed_api(&err, now),
|
||||
};
|
||||
let (server, api) = tokio::join!(
|
||||
probe_server(transport, peer, tcp_timeout),
|
||||
probe_api(transport, peer, api_timeout)
|
||||
);
|
||||
let server = ServerTracker::seeded(&ServerStatus::default()).record(server, now, 1);
|
||||
let api = ApiTracker::seeded(&ApiStatus::default()).record(api, now, 1);
|
||||
(server, api)
|
||||
}
|
||||
|
||||
/// Written by the loop after every probe.
|
||||
pub trait StatusSink: Send + Sync {
|
||||
fn apply_server(&self, name: &PeerName, status: ServerStatus);
|
||||
fn apply_api(&self, name: &PeerName, status: ApiStatus);
|
||||
}
|
||||
|
||||
/// How long to wait before a peer's first check, so a restart of the
|
||||
/// consolidator does not hit every peer at the same instant.
|
||||
pub fn start_jitter(name: &PeerName, interval: Duration) -> Duration {
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
name.hash(&mut hasher);
|
||||
let window_ms = (u64::try_from(interval.as_millis()).unwrap_or(u64::MAX) / 4).max(1);
|
||||
Duration::from_millis(hasher.finish() % window_ms)
|
||||
}
|
||||
|
||||
/// One loop per peer: both checks share a task so a peer's own slow check
|
||||
/// cannot fan out into more tasks. A hung peer only delays its own next
|
||||
/// round; every await is bounded by its timeout and cancellation.
|
||||
pub async fn run_peer(
|
||||
transport: Arc<dyn PeerTransport>,
|
||||
peer: PeerConfig,
|
||||
cfg: LoopConfig,
|
||||
initial: Option<PeerStatus>,
|
||||
sink: Arc<dyn StatusSink>,
|
||||
token: CancellationToken,
|
||||
) {
|
||||
let mut server = ServerTracker::seeded(
|
||||
initial
|
||||
.as_ref()
|
||||
.map(|s| &s.server)
|
||||
.unwrap_or(&ServerStatus::default()),
|
||||
);
|
||||
let mut api = ApiTracker::seeded(
|
||||
initial
|
||||
.as_ref()
|
||||
.map(|s| &s.api)
|
||||
.unwrap_or(&ApiStatus::default()),
|
||||
);
|
||||
let jitter = start_jitter(&peer.name, cfg.tcp_interval.min(cfg.api_interval));
|
||||
let first = tokio::time::Instant::now() + jitter;
|
||||
let mut tcp_deadline = first;
|
||||
let mut api_deadline = first;
|
||||
|
||||
tokio::select! {
|
||||
_ = token.cancelled() => return,
|
||||
_ = tokio::time::sleep_until(first) => {}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = token.cancelled() => break,
|
||||
_ = tokio::time::sleep_until(tcp_deadline) => {
|
||||
let result = probe_server(transport.as_ref(), &peer, cfg.tcp_timeout).await;
|
||||
sink.apply_server(&peer.name, server.record(result, Timestamp::now(), cfg.fail_threshold));
|
||||
tcp_deadline = tokio::time::Instant::now() + cfg.tcp_interval;
|
||||
}
|
||||
_ = tokio::time::sleep_until(api_deadline) => {
|
||||
let observation = probe_api(transport.as_ref(), &peer, cfg.api_timeout).await;
|
||||
sink.apply_api(&peer.name, api.record(observation, Timestamp::now(), cfg.fail_threshold));
|
||||
api_deadline = tokio::time::Instant::now() + cfg.api_interval;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::peers::error::PeerError;
|
||||
use crate::peers::testing::{fake_peers_config, FakeTransport};
|
||||
|
||||
fn cfg() -> PeersConfig {
|
||||
fake_peers_config(&["beta"])
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn healthy_peer() {
|
||||
let fake = FakeTransport::new();
|
||||
let cfg = fake_peers_config(&["beta"]);
|
||||
let cfg = cfg();
|
||||
let (server, api) = check_once(
|
||||
fake.as_ref(),
|
||||
&cfg.peer[0],
|
||||
@@ -123,6 +378,7 @@ mod tests {
|
||||
assert_eq!(server.latency_ms, Some(1));
|
||||
assert!(api.up);
|
||||
assert_eq!(api.auth, AuthState::Ok);
|
||||
assert_eq!(api.peer_version.as_deref(), Some("0.0.0-fake"));
|
||||
assert_eq!(fake.calls().len(), 1);
|
||||
}
|
||||
|
||||
@@ -131,7 +387,7 @@ mod tests {
|
||||
let fake = FakeTransport::new();
|
||||
fake.peer("beta")
|
||||
.respond_err("/api/v1/ping", PeerError::Rejected);
|
||||
let cfg = fake_peers_config(&["beta"]);
|
||||
let cfg = cfg();
|
||||
let (_, api) = check_once(
|
||||
fake.as_ref(),
|
||||
&cfg.peer[0],
|
||||
@@ -144,13 +400,50 @@ mod tests {
|
||||
assert_eq!(api.last_error.as_deref(), Some("unauthorized"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_401_status_is_rejected_not_down() {
|
||||
let fake = FakeTransport::new();
|
||||
fake.peer("beta")
|
||||
.respond("/api/v1/ping", 401, "application/json", "{}");
|
||||
let cfg = cfg();
|
||||
let (_, api) = check_once(
|
||||
fake.as_ref(),
|
||||
&cfg.peer[0],
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await;
|
||||
assert!(api.up);
|
||||
assert_eq!(api.auth, AuthState::Rejected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn older_peer_falls_back_to_healthz_with_unknown_auth() {
|
||||
let fake = FakeTransport::new();
|
||||
fake.peer("beta")
|
||||
.respond("/api/v1/ping", 404, "application/json", "{}")
|
||||
.respond("/healthz", 200, "application/json", r#"{"status":"ok"}"#);
|
||||
let cfg = cfg();
|
||||
let (_, api) = check_once(
|
||||
fake.as_ref(),
|
||||
&cfg.peer[0],
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await;
|
||||
assert!(api.up);
|
||||
assert_eq!(api.auth, AuthState::Unknown);
|
||||
let paths: Vec<_> = fake.calls().into_iter().map(|(_, r)| r.path).collect();
|
||||
assert_eq!(paths, ["/api/v1/ping", "/healthz"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn down_peer_reports_class() {
|
||||
let fake = FakeTransport::new();
|
||||
fake.peer("beta").tcp(Err(PeerError::Timeout));
|
||||
fake.peer("beta")
|
||||
.respond_err("/api/v1/ping", PeerError::Timeout);
|
||||
let cfg = fake_peers_config(&["beta"]);
|
||||
let cfg = cfg();
|
||||
let (server, api) = check_once(
|
||||
fake.as_ref(),
|
||||
&cfg.peer[0],
|
||||
@@ -170,7 +463,7 @@ mod tests {
|
||||
let fake = FakeTransport::new();
|
||||
fake.peer("beta")
|
||||
.respond_err("/api/v1/ping", PeerError::BadRequest(leaked.into()));
|
||||
let cfg = fake_peers_config(&["beta"]);
|
||||
let cfg = cfg();
|
||||
let (_, api) = check_once(
|
||||
fake.as_ref(),
|
||||
&cfg.peer[0],
|
||||
@@ -183,4 +476,69 @@ mod tests {
|
||||
assert!(!stored.contains("token"));
|
||||
assert!(!stored.contains("http://"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_needs_consecutive_failures_and_recovers_immediately() {
|
||||
let mut server = ServerTracker::seeded(&ServerStatus::default());
|
||||
let t0 = Timestamp::now();
|
||||
let ok = Duration::from_millis(2);
|
||||
let first = server.record(Ok(ok), t0, 2);
|
||||
assert!(first.up);
|
||||
assert_eq!(first.since, Some(t0));
|
||||
let since_up = first.since;
|
||||
|
||||
let t1 = t0 + Duration::from_secs(1);
|
||||
let still_up = server.record(Err(PeerError::Timeout), t1, 2);
|
||||
assert!(still_up.up, "one failure below the threshold");
|
||||
assert_eq!(still_up.since, since_up);
|
||||
assert_eq!(still_up.last_error.as_deref(), Some("timeout"));
|
||||
|
||||
let t2 = t0 + Duration::from_secs(2);
|
||||
let down = server.record(Err(PeerError::Timeout), t2, 2);
|
||||
assert!(!down.up);
|
||||
assert_eq!(down.since, Some(t2));
|
||||
|
||||
let t3 = t0 + Duration::from_secs(3);
|
||||
let back = server.record(Ok(ok), t3, 2);
|
||||
assert!(back.up, "recovery does not wait for a threshold");
|
||||
assert_eq!(back.since, Some(t3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_failed_check_still_records_a_since() {
|
||||
let mut server = ServerTracker::seeded(&ServerStatus::default());
|
||||
let t0 = Timestamp::now();
|
||||
let st = server.record(Err(PeerError::Unreachable), t0, 2);
|
||||
assert!(!st.up);
|
||||
assert_eq!(st.since, Some(t0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_tracker_keeps_auth_when_up_and_clears_it_when_down() {
|
||||
let mut api = ApiTracker::seeded(&ApiStatus::default());
|
||||
let t0 = Timestamp::now();
|
||||
let up = api.record(
|
||||
ApiObservation::running(AuthState::Rejected, Duration::from_millis(3), None),
|
||||
t0,
|
||||
2,
|
||||
);
|
||||
assert!(up.up);
|
||||
assert_eq!(up.auth, AuthState::Rejected);
|
||||
assert_eq!(up.last_error.as_deref(), Some("unauthorized"));
|
||||
|
||||
let t1 = t0 + Duration::from_secs(1);
|
||||
let down = api.record(ApiObservation::down(&PeerError::Unreachable), t1, 1);
|
||||
assert!(!down.up);
|
||||
assert_eq!(down.auth, AuthState::Unknown);
|
||||
assert_eq!(down.peer_version, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jitter_is_bounded_and_stable() {
|
||||
let interval = Duration::from_secs(10);
|
||||
let name = PeerName::parse("beta").unwrap();
|
||||
let j = start_jitter(&name, interval);
|
||||
assert!(j < interval / 3, "{j:?}");
|
||||
assert_eq!(j, start_jitter(&name, interval));
|
||||
}
|
||||
}
|
||||
+397
-40
@@ -2,11 +2,16 @@
|
||||
//! side never awaits on a peer and is cheap enough for a metrics scrape.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::config::{CliOverrides, Config};
|
||||
use crate::error::AppError;
|
||||
use crate::peers::error::PeerConfigError;
|
||||
use crate::peers::health::{self, LoopConfig, StatusSink};
|
||||
use crate::peers::transport::{PeerResponse, PeerTransport};
|
||||
use crate::peers::types::{ApiStatus, PeerConfig, PeerName, PeerStatus, PeersConfig, ServerStatus};
|
||||
|
||||
@@ -29,9 +34,39 @@ struct RegistryState {
|
||||
statuses: BTreeMap<PeerName, PeerStatus>,
|
||||
}
|
||||
|
||||
/// Status storage shared with the loops; this is the only mutable state a
|
||||
/// loop needs, so the registry can be dropped without leaking a task.
|
||||
struct Shared {
|
||||
state: RwLock<RegistryState>,
|
||||
}
|
||||
|
||||
impl StatusSink for Shared {
|
||||
fn apply_server(&self, name: &PeerName, status: ServerStatus) {
|
||||
if let Some(entry) = lock_write(&self.state).statuses.get_mut(name) {
|
||||
entry.server = status;
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_api(&self, name: &PeerName, status: ApiStatus) {
|
||||
if let Some(entry) = lock_write(&self.state).statuses.get_mut(name) {
|
||||
entry.api = status;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct LoopEntry {
|
||||
peer: PeerConfig,
|
||||
params: LoopConfig,
|
||||
token: CancellationToken,
|
||||
join: JoinHandle<()>,
|
||||
}
|
||||
|
||||
pub struct PeerRegistry {
|
||||
transport: Arc<dyn PeerTransport>,
|
||||
state: RwLock<RegistryState>,
|
||||
shared: Arc<Shared>,
|
||||
loops: Mutex<BTreeMap<PeerName, LoopEntry>>,
|
||||
loops_enabled: bool,
|
||||
shutdown: CancellationToken,
|
||||
}
|
||||
|
||||
fn initial_status(peer: &PeerConfig) -> PeerStatus {
|
||||
@@ -52,8 +87,22 @@ fn lock_write<T>(lock: &RwLock<T>) -> std::sync::RwLockWriteGuard<'_, T> {
|
||||
lock.write().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
fn lock_mutex<T>(lock: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
|
||||
lock.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
impl PeerRegistry {
|
||||
/// Registry without loops, for tests that drive the transport directly.
|
||||
pub(crate) fn new(config: PeersConfig, transport: Arc<dyn PeerTransport>) -> Self {
|
||||
Self::build(config, transport, CancellationToken::new(), false)
|
||||
}
|
||||
|
||||
fn build(
|
||||
config: PeersConfig,
|
||||
transport: Arc<dyn PeerTransport>,
|
||||
shutdown: CancellationToken,
|
||||
loops_enabled: bool,
|
||||
) -> Self {
|
||||
let statuses = config
|
||||
.peer
|
||||
.iter()
|
||||
@@ -61,10 +110,15 @@ impl PeerRegistry {
|
||||
.collect();
|
||||
Self {
|
||||
transport,
|
||||
state: RwLock::new(RegistryState {
|
||||
config: Arc::new(config),
|
||||
statuses,
|
||||
shared: Arc::new(Shared {
|
||||
state: RwLock::new(RegistryState {
|
||||
config: Arc::new(config),
|
||||
statuses,
|
||||
}),
|
||||
}),
|
||||
loops: Mutex::new(BTreeMap::new()),
|
||||
loops_enabled,
|
||||
shutdown,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,40 +127,114 @@ impl PeerRegistry {
|
||||
config: PeersConfig,
|
||||
transport: Arc<dyn PeerTransport>,
|
||||
) -> Self {
|
||||
Self {
|
||||
transport,
|
||||
state: RwLock::new(RegistryState {
|
||||
config: Arc::new(config),
|
||||
statuses: statuses.into_iter().map(|s| (s.name.clone(), s)).collect(),
|
||||
}),
|
||||
let reg = Self::build(config, transport, CancellationToken::new(), false);
|
||||
lock_write(®.shared.state).statuses =
|
||||
statuses.into_iter().map(|s| (s.name.clone(), s)).collect();
|
||||
reg
|
||||
}
|
||||
|
||||
/// Real registry for the server: builds the outbound transport and starts
|
||||
/// one loop per enabled peer. `shutdown` is the process token; cancelling
|
||||
/// it stops the loops.
|
||||
pub fn start(
|
||||
config: PeersConfig,
|
||||
shutdown: CancellationToken,
|
||||
) -> Result<Arc<Self>, PeerConfigError> {
|
||||
let transport = Arc::new(ReqwestTransport::new(&config)?);
|
||||
Ok(Self::start_with_transport(config, transport, shutdown))
|
||||
}
|
||||
|
||||
/// Loop-enabled registry over a supplied transport, for tests.
|
||||
pub fn start_with_transport(
|
||||
config: PeersConfig,
|
||||
transport: Arc<dyn PeerTransport>,
|
||||
shutdown: CancellationToken,
|
||||
) -> Arc<Self> {
|
||||
let registry = Arc::new(Self::build(config, transport, shutdown.child_token(), true));
|
||||
registry.spawn_all();
|
||||
registry
|
||||
}
|
||||
|
||||
fn spawn_all(&self) {
|
||||
let peers: Vec<PeerConfig> = if self.config().enabled {
|
||||
self.config()
|
||||
.peer
|
||||
.iter()
|
||||
.filter(|peer| peer.enabled)
|
||||
.cloned()
|
||||
.collect()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
for peer in peers {
|
||||
self.spawn_peer(&peer);
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_peer(&self, peer: &PeerConfig) {
|
||||
let params = LoopConfig::from_config(&self.config());
|
||||
let initial = self.status_internal(&peer.name);
|
||||
let token = self.shutdown.child_token();
|
||||
let sink: Arc<dyn StatusSink> = self.shared.clone();
|
||||
let transport = self.transport.clone();
|
||||
let peer_copy = peer.clone();
|
||||
let name = peer.name.clone();
|
||||
let task_token = token.clone();
|
||||
let join = tokio::spawn(async move {
|
||||
health::run_peer(transport, peer_copy, params, initial, sink, task_token).await;
|
||||
});
|
||||
lock_mutex(&self.loops).insert(
|
||||
name,
|
||||
LoopEntry {
|
||||
peer: peer.clone(),
|
||||
params,
|
||||
token,
|
||||
join,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn stop_loop(&self, entry: LoopEntry) {
|
||||
// Cancel, then let the task finish its bounded probe on its own.
|
||||
entry.token.cancel();
|
||||
tokio::spawn(async move {
|
||||
let _ = entry.join.await;
|
||||
});
|
||||
}
|
||||
|
||||
/// Sorted by name; disabled peers included.
|
||||
pub fn snapshot(&self) -> Vec<PeerStatus> {
|
||||
lock_read(&self.state).statuses.values().cloned().collect()
|
||||
lock_read(&self.shared.state)
|
||||
.statuses
|
||||
.values()
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn status(&self, name: &str) -> Result<PeerStatus, AppError> {
|
||||
let Ok(key) = PeerName::parse(name) else {
|
||||
return Err(AppError::NotFound(format!("peer {name}: unknown peer")));
|
||||
};
|
||||
lock_read(&self.state)
|
||||
lock_read(&self.shared.state)
|
||||
.statuses
|
||||
.get(&key)
|
||||
.cloned()
|
||||
.ok_or_else(|| AppError::NotFound(format!("peer {name}: unknown peer")))
|
||||
}
|
||||
|
||||
fn status_internal(&self, name: &PeerName) -> Option<PeerStatus> {
|
||||
lock_read(&self.shared.state).statuses.get(name).cloned()
|
||||
}
|
||||
|
||||
pub fn contains(&self, name: &str) -> bool {
|
||||
let Ok(key) = PeerName::parse(name) else {
|
||||
return false;
|
||||
};
|
||||
lock_read(&self.state).statuses.contains_key(&key)
|
||||
lock_read(&self.shared.state).statuses.contains_key(&key)
|
||||
}
|
||||
|
||||
pub fn config(&self) -> Arc<PeersConfig> {
|
||||
lock_read(&self.state).config.clone()
|
||||
lock_read(&self.shared.state).config.clone()
|
||||
}
|
||||
|
||||
/// Single proxy entry point for REST and MCP.
|
||||
@@ -119,7 +247,7 @@ impl PeerRegistry {
|
||||
) -> Result<PeerResponse, AppError> {
|
||||
let key = PeerName::parse(name).ok();
|
||||
let (peer, timeout) = {
|
||||
let st = lock_read(&self.state);
|
||||
let st = lock_read(&self.shared.state);
|
||||
let peer = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key).cloned());
|
||||
(peer, Duration::from_secs(st.config.request_timeout_secs))
|
||||
};
|
||||
@@ -128,30 +256,114 @@ impl PeerRegistry {
|
||||
}
|
||||
|
||||
/// Replace the peer set. Statuses of peers that survive a reload are kept
|
||||
/// so `since` survives a config change. Validation arrives with the
|
||||
/// config and secrets tasks; this body only swaps state.
|
||||
/// so `since` survives a config change. Loops for new or changed peers are
|
||||
/// started, loops for removed or disabled peers are cancelled.
|
||||
///
|
||||
/// A bad `[peers]` block never reaches this method: the handler logs the
|
||||
/// config error and keeps the old registry, as TLS reload does.
|
||||
pub async fn reload(&self, new: PeersConfig) -> Result<(), PeerConfigError> {
|
||||
let mut st = lock_write(&self.state);
|
||||
let old = std::mem::take(&mut st.statuses);
|
||||
st.statuses = new
|
||||
.peer
|
||||
.iter()
|
||||
.map(|peer| {
|
||||
let mut status = old
|
||||
.get(&peer.name)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| initial_status(peer));
|
||||
status.enabled = peer.enabled;
|
||||
status.url = peer.display_url();
|
||||
(peer.name.clone(), status)
|
||||
})
|
||||
.collect();
|
||||
st.config = Arc::new(new);
|
||||
let params = LoopConfig::from_config(&new);
|
||||
{
|
||||
let mut st = lock_write(&self.shared.state);
|
||||
let old = std::mem::take(&mut st.statuses);
|
||||
st.statuses = new
|
||||
.peer
|
||||
.iter()
|
||||
.map(|peer| {
|
||||
let mut status = old
|
||||
.get(&peer.name)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| initial_status(peer));
|
||||
status.enabled = peer.enabled;
|
||||
status.url = peer.display_url();
|
||||
(peer.name.clone(), status)
|
||||
})
|
||||
.collect();
|
||||
st.config = Arc::new(new.clone());
|
||||
}
|
||||
if self.loops_enabled {
|
||||
self.resync_loops(&new, params);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Health loops stop on this; nothing is running yet.
|
||||
pub async fn shutdown(&self) {}
|
||||
fn resync_loops(&self, new: &PeersConfig, params: LoopConfig) {
|
||||
let mut old = std::mem::take(&mut *lock_mutex(&self.loops));
|
||||
let mut kept = BTreeMap::new();
|
||||
for (name, entry) in std::mem::take(&mut old) {
|
||||
let wanted = new
|
||||
.enabled
|
||||
.then(|| {
|
||||
new.peer
|
||||
.iter()
|
||||
.find(|peer| peer.name == name && peer.enabled)
|
||||
})
|
||||
.flatten();
|
||||
match wanted {
|
||||
Some(peer) if entry.peer == *peer && entry.params == params => {
|
||||
kept.insert(name, entry);
|
||||
}
|
||||
_ => self.stop_loop(entry),
|
||||
}
|
||||
}
|
||||
*lock_mutex(&self.loops) = kept;
|
||||
if new.enabled {
|
||||
for peer in new.peer.iter().filter(|peer| peer.enabled) {
|
||||
if !lock_mutex(&self.loops).contains_key(&peer.name) {
|
||||
self.spawn_peer(peer);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop every loop and wait (bounded) for the tasks to exit.
|
||||
pub async fn shutdown(&self) {
|
||||
self.shutdown.cancel();
|
||||
let joins: Vec<JoinHandle<()>> = std::mem::take(&mut *lock_mutex(&self.loops))
|
||||
.into_values()
|
||||
.map(|entry| entry.join)
|
||||
.collect();
|
||||
let join_all = async {
|
||||
for join in joins {
|
||||
let _ = join.await;
|
||||
}
|
||||
};
|
||||
if tokio::time::timeout(Duration::from_secs(5), join_all)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!("peer health loops did not stop in time");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything the SIGHUP handler needs to re-read the `[peers]` block.
|
||||
#[derive(Clone)]
|
||||
pub struct PeersReloader {
|
||||
registry: Arc<PeerRegistry>,
|
||||
path: Option<std::path::PathBuf>,
|
||||
overrides: CliOverrides,
|
||||
}
|
||||
|
||||
impl PeersReloader {
|
||||
pub fn new(
|
||||
registry: Arc<PeerRegistry>,
|
||||
path: Option<std::path::PathBuf>,
|
||||
overrides: CliOverrides,
|
||||
) -> Self {
|
||||
Self {
|
||||
registry,
|
||||
path,
|
||||
overrides,
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-read the whole config and apply only `[peers]`. A failure returns
|
||||
/// the config error; the old registry stays in place.
|
||||
pub async fn reload(&self) -> Result<(), PeerConfigError> {
|
||||
let cfg = Config::load(self.path.as_deref(), &self.overrides)?;
|
||||
self.registry.reload(cfg.peers).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -159,6 +371,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::peers::testing::{fake_peers_config, registry_with, static_registry, FakeTransport};
|
||||
use crate::peers::transport::PeerBody;
|
||||
use crate::peers::types::AuthState;
|
||||
|
||||
fn status(name: &str, up: bool) -> PeerStatus {
|
||||
PeerStatus {
|
||||
@@ -184,7 +397,7 @@ mod tests {
|
||||
assert_eq!(names, ["beta", "gamma"]);
|
||||
let b = reg.status("beta").unwrap();
|
||||
assert!(b.server.up);
|
||||
assert_eq!(b.api.auth, crate::peers::types::AuthState::Unknown);
|
||||
assert_eq!(b.api.auth, AuthState::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -247,7 +460,7 @@ mod tests {
|
||||
async fn reload_keeps_surviving_status_and_prunes_removed() {
|
||||
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta", "gamma"]));
|
||||
{
|
||||
let mut st = lock_write(®.state);
|
||||
let mut st = lock_write(®.shared.state);
|
||||
let beta = st
|
||||
.statuses
|
||||
.get_mut(&PeerName::parse("beta").unwrap())
|
||||
@@ -256,8 +469,7 @@ mod tests {
|
||||
}
|
||||
let mut new = fake_peers_config(&["beta", "delta"]);
|
||||
new.peer[0].enabled = false;
|
||||
let reloaded: Result<(), PeerConfigError> = reg.reload(new).await;
|
||||
reloaded.unwrap();
|
||||
reg.reload(new).await.unwrap();
|
||||
|
||||
let snap = reg.snapshot();
|
||||
assert_eq!(snap.len(), 2);
|
||||
@@ -267,4 +479,149 @@ mod tests {
|
||||
assert_eq!(snap[1].name.as_str(), "delta");
|
||||
assert!(!snap[1].server.up);
|
||||
}
|
||||
|
||||
async fn wait_for<F: Fn(&PeerRegistry) -> bool>(reg: &PeerRegistry, f: F) {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
if f(reg) {
|
||||
return;
|
||||
}
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"condition not met: {:?}",
|
||||
reg.snapshot()
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn loops_mark_up_and_honour_the_failure_threshold() {
|
||||
let fake = FakeTransport::new();
|
||||
let mut cfg = fake_peers_config(&["beta"]);
|
||||
cfg.fail_threshold = 2;
|
||||
let reg = PeerRegistry::start_with_transport(cfg, fake.clone(), CancellationToken::new());
|
||||
wait_for(®, |r| {
|
||||
let s = r.status("beta").unwrap();
|
||||
s.server.up && s.api.up && s.api.auth == AuthState::Ok
|
||||
})
|
||||
.await;
|
||||
let up = reg.status("beta").unwrap();
|
||||
assert!(up.server.since.is_some());
|
||||
assert_eq!(up.api.peer_version.as_deref(), Some("0.0.0-fake"));
|
||||
|
||||
fake.peer("beta").tcp(Err(crate::peers::PeerError::Timeout));
|
||||
fake.peer("beta")
|
||||
.respond_err("/api/v1/ping", crate::peers::PeerError::Timeout);
|
||||
// One failure each: still up with the error visible.
|
||||
wait_for(®, |r| {
|
||||
let s = r.status("beta").unwrap();
|
||||
s.server.last_error.as_deref() == Some("timeout")
|
||||
&& s.api.last_error.as_deref() == Some("timeout")
|
||||
})
|
||||
.await;
|
||||
assert!(reg.status("beta").unwrap().server.up);
|
||||
|
||||
reg.shutdown().await;
|
||||
let after = reg.status("beta").unwrap();
|
||||
assert!(after.server.up, "one failure below threshold = still up");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_hung_peer_does_not_delay_another_peers_checks() {
|
||||
let fake = FakeTransport::new();
|
||||
fake.peer("beta").delay(Duration::from_secs(2));
|
||||
fake.peer("gamma")
|
||||
.respond("/api/v1/ping", 200, "application/json", "{}");
|
||||
let reg = PeerRegistry::start_with_transport(
|
||||
fake_peers_config(&["beta", "gamma"]),
|
||||
fake.clone(),
|
||||
CancellationToken::new(),
|
||||
);
|
||||
wait_for(®, |r| {
|
||||
r.status("gamma").unwrap().api.last_checked.is_some()
|
||||
})
|
||||
.await;
|
||||
let gamma_checked = reg.status("gamma").unwrap().api.last_checked;
|
||||
assert!(reg.status("beta").unwrap().api.last_checked.is_none());
|
||||
assert!(gamma_checked.is_some());
|
||||
reg.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::result_large_err)] // figment::Jail closures return figment::Error
|
||||
fn reloader_rereads_the_peers_block_and_keeps_the_old_set_on_error() {
|
||||
use figment::Jail;
|
||||
|
||||
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
|
||||
Jail::expect_with(|jail| {
|
||||
jail.create_file(
|
||||
"config.toml",
|
||||
r#"
|
||||
[peers]
|
||||
enabled = true
|
||||
api_interval_secs = 2
|
||||
[[peers.peer]]
|
||||
name = "gamma"
|
||||
url = "http://gamma:1984"
|
||||
token_file = "/run/credentials/skyg-linux-tracker.service/peer-gamma"
|
||||
allow_plain_http = true
|
||||
"#,
|
||||
)?;
|
||||
let reloader = PeersReloader::new(
|
||||
reg.clone(),
|
||||
Some(std::path::PathBuf::from("config.toml")),
|
||||
CliOverrides::default(),
|
||||
);
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(reloader.reload())
|
||||
.unwrap();
|
||||
let snap = reg.snapshot();
|
||||
assert_eq!(snap.len(), 1);
|
||||
assert_eq!(snap[0].name.as_str(), "gamma");
|
||||
assert_eq!(reg.config().api_interval_secs, 2);
|
||||
|
||||
let broken = PeersReloader::new(
|
||||
reg.clone(),
|
||||
Some(std::path::PathBuf::from("missing.toml")),
|
||||
CliOverrides::default(),
|
||||
);
|
||||
assert!(tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(broken.reload())
|
||||
.is_err());
|
||||
assert!(reg.contains("gamma"), "a bad reload keeps the old peers");
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_moves_loops_to_the_new_peer_set() {
|
||||
let fake = FakeTransport::new();
|
||||
let reg = PeerRegistry::start_with_transport(
|
||||
fake_peers_config(&["beta"]),
|
||||
fake.clone(),
|
||||
CancellationToken::new(),
|
||||
);
|
||||
wait_for(®, |r| r.status("beta").unwrap().server.up).await;
|
||||
|
||||
reg.reload(fake_peers_config(&["beta", "gamma"]))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for(®, |r| r.status("gamma").unwrap().server.up).await;
|
||||
assert!(
|
||||
reg.status("beta").unwrap().server.up,
|
||||
"kept peer keeps state"
|
||||
);
|
||||
|
||||
reg.reload(fake_peers_config(&["gamma"])).await.unwrap();
|
||||
assert!(reg.status("beta").is_err());
|
||||
assert!(reg.contains("gamma"));
|
||||
reg.shutdown().await;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user