645 lines
22 KiB
Rust
645 lines
22 KiB
Rust
//! In-memory peer views. Health loops refresh the status entries; the read
|
|
//! side never awaits on a peer and is cheap enough for a metrics scrape.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::sync::{Arc, Mutex, RwLock};
|
|
use std::time::Duration;
|
|
|
|
use tokio::task::JoinHandle;
|
|
use tokio_util::sync::CancellationToken;
|
|
|
|
use crate::config::{CliOverrides, Config};
|
|
use crate::error::AppError;
|
|
use crate::peers::error::PeerConfigError;
|
|
use crate::peers::health::{self, LoopConfig, StatusSink};
|
|
use crate::peers::transport::{PeerResponse, PeerTransport};
|
|
use crate::peers::types::{ApiStatus, PeerConfig, PeerName, PeerStatus, PeersConfig, ServerStatus};
|
|
|
|
// `client.rs` lives next to this file. The peers module list does not declare it.
|
|
#[path = "client.rs"]
|
|
pub mod client;
|
|
|
|
pub use client::ReqwestTransport;
|
|
|
|
/// Options a surface passes down to the proxy layer. Trace context is a
|
|
/// pre-built `traceparent` header value; the peer hop is always marked.
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
pub struct ProxyOpts {
|
|
pub traceparent: Option<String>,
|
|
pub accept_sse: bool,
|
|
}
|
|
|
|
struct RegistryState {
|
|
config: Arc<PeersConfig>,
|
|
statuses: BTreeMap<PeerName, PeerStatus>,
|
|
}
|
|
|
|
/// Status storage shared with the loops; this is the only mutable state a
|
|
/// loop needs, so the registry can be dropped without leaking a task.
|
|
struct Shared {
|
|
state: RwLock<RegistryState>,
|
|
}
|
|
|
|
impl StatusSink for Shared {
|
|
fn apply_server(&self, name: &PeerName, status: ServerStatus) {
|
|
if let Some(entry) = lock_write(&self.state).statuses.get_mut(name) {
|
|
entry.server = status;
|
|
}
|
|
}
|
|
|
|
fn apply_api(&self, name: &PeerName, status: ApiStatus) {
|
|
if let Some(entry) = lock_write(&self.state).statuses.get_mut(name) {
|
|
entry.api = status;
|
|
}
|
|
}
|
|
}
|
|
|
|
struct LoopEntry {
|
|
peer: PeerConfig,
|
|
params: LoopConfig,
|
|
token: CancellationToken,
|
|
join: JoinHandle<()>,
|
|
}
|
|
|
|
pub struct PeerRegistry {
|
|
transport: Arc<dyn PeerTransport>,
|
|
shared: Arc<Shared>,
|
|
loops: Mutex<BTreeMap<PeerName, LoopEntry>>,
|
|
loops_enabled: bool,
|
|
shutdown: CancellationToken,
|
|
}
|
|
|
|
fn initial_status(peer: &PeerConfig) -> PeerStatus {
|
|
PeerStatus {
|
|
name: peer.name.clone(),
|
|
enabled: peer.enabled,
|
|
url: peer.display_url(),
|
|
server: ServerStatus::default(),
|
|
api: ApiStatus::default(),
|
|
}
|
|
}
|
|
|
|
fn lock_read<T>(lock: &RwLock<T>) -> std::sync::RwLockReadGuard<'_, T> {
|
|
lock.read().unwrap_or_else(|e| e.into_inner())
|
|
}
|
|
|
|
fn lock_write<T>(lock: &RwLock<T>) -> std::sync::RwLockWriteGuard<'_, T> {
|
|
lock.write().unwrap_or_else(|e| e.into_inner())
|
|
}
|
|
|
|
fn lock_mutex<T>(lock: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
|
|
lock.lock().unwrap_or_else(|e| e.into_inner())
|
|
}
|
|
|
|
impl PeerRegistry {
|
|
/// Registry without loops, for tests that drive the transport directly.
|
|
pub(crate) fn new(config: PeersConfig, transport: Arc<dyn PeerTransport>) -> Self {
|
|
Self::build(config, transport, CancellationToken::new(), false)
|
|
}
|
|
|
|
fn build(
|
|
config: PeersConfig,
|
|
transport: Arc<dyn PeerTransport>,
|
|
shutdown: CancellationToken,
|
|
loops_enabled: bool,
|
|
) -> Self {
|
|
let statuses = config
|
|
.peer
|
|
.iter()
|
|
.map(|peer| (peer.name.clone(), initial_status(peer)))
|
|
.collect();
|
|
Self {
|
|
transport,
|
|
shared: Arc::new(Shared {
|
|
state: RwLock::new(RegistryState {
|
|
config: Arc::new(config),
|
|
statuses,
|
|
}),
|
|
}),
|
|
loops: Mutex::new(BTreeMap::new()),
|
|
loops_enabled,
|
|
shutdown,
|
|
}
|
|
}
|
|
|
|
pub(crate) fn from_statuses(
|
|
statuses: Vec<PeerStatus>,
|
|
config: PeersConfig,
|
|
transport: Arc<dyn PeerTransport>,
|
|
) -> Self {
|
|
let reg = Self::build(config, transport, CancellationToken::new(), false);
|
|
lock_write(®.shared.state).statuses =
|
|
statuses.into_iter().map(|s| (s.name.clone(), s)).collect();
|
|
reg
|
|
}
|
|
|
|
/// Real registry for the server: builds the outbound transport and starts
|
|
/// one loop per enabled peer. `shutdown` is the process token; cancelling
|
|
/// it stops the loops.
|
|
pub fn start(
|
|
config: PeersConfig,
|
|
shutdown: CancellationToken,
|
|
) -> Result<Arc<Self>, PeerConfigError> {
|
|
let transport = Arc::new(ReqwestTransport::new(&config)?);
|
|
Ok(Self::start_with_transport(config, transport, shutdown))
|
|
}
|
|
|
|
/// Loop-enabled registry over a supplied transport, for tests.
|
|
pub fn start_with_transport(
|
|
config: PeersConfig,
|
|
transport: Arc<dyn PeerTransport>,
|
|
shutdown: CancellationToken,
|
|
) -> Arc<Self> {
|
|
let registry = Arc::new(Self::build(config, transport, shutdown.child_token(), true));
|
|
registry.spawn_all();
|
|
registry
|
|
}
|
|
|
|
fn spawn_all(&self) {
|
|
let peers: Vec<PeerConfig> = if self.config().enabled {
|
|
self.config()
|
|
.peer
|
|
.iter()
|
|
.filter(|peer| peer.enabled)
|
|
.cloned()
|
|
.collect()
|
|
} else {
|
|
Vec::new()
|
|
};
|
|
for peer in peers {
|
|
self.spawn_peer(&peer);
|
|
}
|
|
}
|
|
|
|
fn spawn_peer(&self, peer: &PeerConfig) {
|
|
let params = LoopConfig::from_config(&self.config());
|
|
let initial = self.status_internal(&peer.name);
|
|
let token = self.shutdown.child_token();
|
|
let sink: Arc<dyn StatusSink> = self.shared.clone();
|
|
let transport = self.transport.clone();
|
|
let peer_copy = peer.clone();
|
|
let name = peer.name.clone();
|
|
let task_token = token.clone();
|
|
let join = tokio::spawn(async move {
|
|
health::run_peer(transport, peer_copy, params, initial, sink, task_token).await;
|
|
});
|
|
lock_mutex(&self.loops).insert(
|
|
name,
|
|
LoopEntry {
|
|
peer: peer.clone(),
|
|
params,
|
|
token,
|
|
join,
|
|
},
|
|
);
|
|
}
|
|
|
|
fn stop_loop(&self, entry: LoopEntry) {
|
|
// Cancel, then let the task finish its bounded probe on its own.
|
|
entry.token.cancel();
|
|
tokio::spawn(async move {
|
|
let _ = entry.join.await;
|
|
});
|
|
}
|
|
|
|
/// Sorted by name; disabled peers included.
|
|
pub fn snapshot(&self) -> Vec<PeerStatus> {
|
|
lock_read(&self.shared.state)
|
|
.statuses
|
|
.values()
|
|
.cloned()
|
|
.collect()
|
|
}
|
|
|
|
pub fn status(&self, name: &str) -> Result<PeerStatus, AppError> {
|
|
let Ok(key) = PeerName::parse(name) else {
|
|
return Err(AppError::NotFound(format!("peer {name}: unknown peer")));
|
|
};
|
|
lock_read(&self.shared.state)
|
|
.statuses
|
|
.get(&key)
|
|
.cloned()
|
|
.ok_or_else(|| AppError::NotFound(format!("peer {name}: unknown peer")))
|
|
}
|
|
|
|
fn status_internal(&self, name: &PeerName) -> Option<PeerStatus> {
|
|
lock_read(&self.shared.state).statuses.get(name).cloned()
|
|
}
|
|
|
|
pub fn contains(&self, name: &str) -> bool {
|
|
let Ok(key) = PeerName::parse(name) else {
|
|
return false;
|
|
};
|
|
lock_read(&self.shared.state).statuses.contains_key(&key)
|
|
}
|
|
|
|
pub fn config(&self) -> Arc<PeersConfig> {
|
|
lock_read(&self.shared.state).config.clone()
|
|
}
|
|
|
|
/// Single proxy entry point for REST and MCP.
|
|
pub async fn get(
|
|
&self,
|
|
name: &str,
|
|
path: &str,
|
|
query: &[(String, String)],
|
|
opts: ProxyOpts,
|
|
) -> Result<PeerResponse, AppError> {
|
|
let key = PeerName::parse(name).ok();
|
|
let (peer, timeout) = {
|
|
let st = lock_read(&self.shared.state);
|
|
let peer = key.and_then(|key| st.config.peer.iter().find(|p| p.name == key).cloned());
|
|
(peer, Duration::from_secs(st.config.request_timeout_secs))
|
|
};
|
|
let peer = peer.ok_or_else(|| AppError::NotFound(format!("peer {name}: unknown peer")))?;
|
|
crate::peers::proxy::get(self.transport.as_ref(), &peer, path, query, opts, timeout).await
|
|
}
|
|
|
|
/// Replace the peer set. Statuses of peers that survive a reload are kept
|
|
/// so `since` survives a config change. Loops for new or changed peers are
|
|
/// started, loops for removed or disabled peers are cancelled.
|
|
///
|
|
/// A bad `[peers]` block never reaches this method: the handler logs the
|
|
/// config error and keeps the old registry, as TLS reload does.
|
|
pub async fn reload(&self, new: PeersConfig) -> Result<(), PeerConfigError> {
|
|
let params = LoopConfig::from_config(&new);
|
|
{
|
|
let mut st = lock_write(&self.shared.state);
|
|
let old = std::mem::take(&mut st.statuses);
|
|
// Turning the feature off renders an empty list rather than a
|
|
// stale snapshot; the routes themselves stay mounted until
|
|
// restart because the router is built once.
|
|
st.statuses = if new.enabled {
|
|
new.peer
|
|
.iter()
|
|
.map(|peer| {
|
|
let mut status = old
|
|
.get(&peer.name)
|
|
.cloned()
|
|
.unwrap_or_else(|| initial_status(peer));
|
|
status.enabled = peer.enabled;
|
|
status.url = peer.display_url();
|
|
(peer.name.clone(), status)
|
|
})
|
|
.collect()
|
|
} else {
|
|
BTreeMap::new()
|
|
};
|
|
st.config = Arc::new(new.clone());
|
|
}
|
|
if self.loops_enabled {
|
|
self.resync_loops(&new, params);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn resync_loops(&self, new: &PeersConfig, params: LoopConfig) {
|
|
let mut old = std::mem::take(&mut *lock_mutex(&self.loops));
|
|
let mut kept = BTreeMap::new();
|
|
for (name, entry) in std::mem::take(&mut old) {
|
|
let wanted = new
|
|
.enabled
|
|
.then(|| {
|
|
new.peer
|
|
.iter()
|
|
.find(|peer| peer.name == name && peer.enabled)
|
|
})
|
|
.flatten();
|
|
match wanted {
|
|
Some(peer) if entry.peer == *peer && entry.params == params => {
|
|
kept.insert(name, entry);
|
|
}
|
|
_ => self.stop_loop(entry),
|
|
}
|
|
}
|
|
*lock_mutex(&self.loops) = kept;
|
|
if new.enabled {
|
|
for peer in new.peer.iter().filter(|peer| peer.enabled) {
|
|
if !lock_mutex(&self.loops).contains_key(&peer.name) {
|
|
self.spawn_peer(peer);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Stop every loop and wait (bounded) for the tasks to exit.
|
|
pub async fn shutdown(&self) {
|
|
self.shutdown.cancel();
|
|
let joins: Vec<JoinHandle<()>> = std::mem::take(&mut *lock_mutex(&self.loops))
|
|
.into_values()
|
|
.map(|entry| entry.join)
|
|
.collect();
|
|
let join_all = async {
|
|
for join in joins {
|
|
let _ = join.await;
|
|
}
|
|
};
|
|
if tokio::time::timeout(Duration::from_secs(5), join_all)
|
|
.await
|
|
.is_err()
|
|
{
|
|
tracing::warn!("peer health loops did not stop in time");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Everything the SIGHUP handler needs to re-read the `[peers]` block.
|
|
#[derive(Clone)]
|
|
pub struct PeersReloader {
|
|
registry: Arc<PeerRegistry>,
|
|
path: Option<std::path::PathBuf>,
|
|
overrides: CliOverrides,
|
|
}
|
|
|
|
impl PeersReloader {
|
|
pub fn new(
|
|
registry: Arc<PeerRegistry>,
|
|
path: Option<std::path::PathBuf>,
|
|
overrides: CliOverrides,
|
|
) -> Self {
|
|
Self {
|
|
registry,
|
|
path,
|
|
overrides,
|
|
}
|
|
}
|
|
|
|
/// Re-read the whole config and apply only `[peers]`. A failure returns
|
|
/// the config error; the old registry stays in place.
|
|
pub async fn reload(&self) -> Result<(), PeerConfigError> {
|
|
let cfg = Config::load(self.path.as_deref(), &self.overrides)?;
|
|
self.registry.reload(cfg.peers).await
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::peers::testing::{fake_peers_config, registry_with, static_registry, FakeTransport};
|
|
use crate::peers::transport::PeerBody;
|
|
use crate::peers::types::AuthState;
|
|
|
|
fn status(name: &str, up: bool) -> PeerStatus {
|
|
PeerStatus {
|
|
name: PeerName::parse(name).unwrap(),
|
|
enabled: true,
|
|
url: format!("http://{name}:1984"),
|
|
server: ServerStatus {
|
|
up,
|
|
..ServerStatus::default()
|
|
},
|
|
api: ApiStatus::default(),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn snapshot_is_sorted_and_includes_initial_state() {
|
|
let reg = static_registry(vec![status("gamma", false), status("beta", true)]);
|
|
let names: Vec<_> = reg
|
|
.snapshot()
|
|
.into_iter()
|
|
.map(|s| s.name.as_str().to_owned())
|
|
.collect();
|
|
assert_eq!(names, ["beta", "gamma"]);
|
|
let b = reg.status("beta").unwrap();
|
|
assert!(b.server.up);
|
|
assert_eq!(b.api.auth, AuthState::Unknown);
|
|
}
|
|
|
|
#[test]
|
|
fn unknown_peer_status_is_not_found() {
|
|
let reg = static_registry(vec![status("beta", true)]);
|
|
let err = reg.status("nope").unwrap_err();
|
|
assert_eq!(err.code(), "not_found");
|
|
assert_eq!(err.message(), "peer nope: unknown peer");
|
|
assert!(reg.contains("beta"));
|
|
assert!(!reg.contains("nope"));
|
|
assert!(!reg.contains("Beta"));
|
|
assert!(!reg.contains("-beta"));
|
|
let err = reg.status("Beta").unwrap_err();
|
|
assert_eq!(err.code(), "not_found");
|
|
assert_eq!(err.message(), "peer Beta: unknown peer");
|
|
}
|
|
|
|
#[test]
|
|
fn config_is_shared() {
|
|
let cfg = fake_peers_config(&["beta"]);
|
|
let reg = registry_with(FakeTransport::new(), cfg);
|
|
let got = reg.config();
|
|
assert!(got.enabled);
|
|
assert_eq!(got.peer[0].name.as_str(), "beta");
|
|
assert_eq!(got.request_timeout_secs, 2);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn get_forwards_and_maps_errors() {
|
|
let fake = FakeTransport::new();
|
|
fake.peer("beta")
|
|
.respond("/systemd/units", 200, "application/json", "{\"ok\":true}");
|
|
let reg = registry_with(fake, fake_peers_config(&["beta"]));
|
|
|
|
let resp = reg
|
|
.get("beta", "/systemd/units", &[], ProxyOpts::default())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status, 200);
|
|
match resp.body {
|
|
PeerBody::Bytes(b) => assert_eq!(&b[..], b"{\"ok\":true}"),
|
|
PeerBody::Stream(_) => panic!("expected buffered body"),
|
|
}
|
|
|
|
let err = reg
|
|
.get("nope", "/x", &[], ProxyOpts::default())
|
|
.await
|
|
.unwrap_err();
|
|
assert_eq!(err.code(), "not_found");
|
|
assert_eq!(err.message(), "peer nope: unknown peer");
|
|
|
|
let err = reg
|
|
.get("Beta", "/x", &[], ProxyOpts::default())
|
|
.await
|
|
.unwrap_err();
|
|
assert_eq!(err.message(), "peer Beta: unknown peer");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn reload_keeps_surviving_status_and_prunes_removed() {
|
|
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta", "gamma"]));
|
|
{
|
|
let mut st = lock_write(®.shared.state);
|
|
let beta = st
|
|
.statuses
|
|
.get_mut(&PeerName::parse("beta").unwrap())
|
|
.unwrap();
|
|
beta.server.up = true;
|
|
}
|
|
let mut new = fake_peers_config(&["beta", "delta"]);
|
|
new.peer[0].enabled = false;
|
|
reg.reload(new).await.unwrap();
|
|
|
|
let snap = reg.snapshot();
|
|
assert_eq!(snap.len(), 2);
|
|
assert_eq!(snap[0].name.as_str(), "beta");
|
|
assert!(snap[0].server.up, "health survives a reload");
|
|
assert!(!snap[0].enabled);
|
|
assert_eq!(snap[1].name.as_str(), "delta");
|
|
assert!(!snap[1].server.up);
|
|
}
|
|
|
|
async fn wait_for<F: Fn(&PeerRegistry) -> bool>(reg: &PeerRegistry, f: F) {
|
|
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
|
loop {
|
|
if f(reg) {
|
|
return;
|
|
}
|
|
assert!(
|
|
tokio::time::Instant::now() < deadline,
|
|
"condition not met: {:?}",
|
|
reg.snapshot()
|
|
);
|
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn loops_mark_up_and_honour_the_failure_threshold() {
|
|
let fake = FakeTransport::new();
|
|
let mut cfg = fake_peers_config(&["beta"]);
|
|
cfg.fail_threshold = 2;
|
|
let reg = PeerRegistry::start_with_transport(cfg, fake.clone(), CancellationToken::new());
|
|
wait_for(®, |r| {
|
|
let s = r.status("beta").unwrap();
|
|
s.server.up && s.api.up && s.api.auth == AuthState::Ok
|
|
})
|
|
.await;
|
|
let up = reg.status("beta").unwrap();
|
|
assert!(up.server.since.is_some());
|
|
assert_eq!(up.api.peer_version.as_deref(), Some("0.0.0-fake"));
|
|
|
|
fake.peer("beta").tcp(Err(crate::peers::PeerError::Timeout));
|
|
fake.peer("beta")
|
|
.respond_err("/api/v1/ping", crate::peers::PeerError::Timeout);
|
|
// One failure each: still up with the error visible.
|
|
wait_for(®, |r| {
|
|
let s = r.status("beta").unwrap();
|
|
s.server.last_error.as_deref() == Some("timeout")
|
|
&& s.api.last_error.as_deref() == Some("timeout")
|
|
})
|
|
.await;
|
|
assert!(reg.status("beta").unwrap().server.up);
|
|
|
|
reg.shutdown().await;
|
|
let after = reg.status("beta").unwrap();
|
|
assert!(after.server.up, "one failure below threshold = still up");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_hung_peer_does_not_delay_another_peers_checks() {
|
|
let fake = FakeTransport::new();
|
|
fake.peer("beta").delay(Duration::from_secs(2));
|
|
fake.peer("gamma")
|
|
.respond("/api/v1/ping", 200, "application/json", "{}");
|
|
let reg = PeerRegistry::start_with_transport(
|
|
fake_peers_config(&["beta", "gamma"]),
|
|
fake.clone(),
|
|
CancellationToken::new(),
|
|
);
|
|
wait_for(®, |r| {
|
|
r.status("gamma").unwrap().api.last_checked.is_some()
|
|
})
|
|
.await;
|
|
let gamma_checked = reg.status("gamma").unwrap().api.last_checked;
|
|
assert!(reg.status("beta").unwrap().api.last_checked.is_none());
|
|
assert!(gamma_checked.is_some());
|
|
reg.shutdown().await;
|
|
}
|
|
|
|
#[test]
|
|
#[allow(clippy::result_large_err)] // figment::Jail closures return figment::Error
|
|
fn reloader_rereads_the_peers_block_and_keeps_the_old_set_on_error() {
|
|
use figment::Jail;
|
|
|
|
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
|
|
Jail::expect_with(|jail| {
|
|
jail.create_file(
|
|
"config.toml",
|
|
r#"
|
|
[peers]
|
|
enabled = true
|
|
api_interval_secs = 2
|
|
[[peers.peer]]
|
|
name = "gamma"
|
|
url = "http://gamma:1984"
|
|
token_file = "/run/credentials/skyg-linux-tracker.service/peer-gamma"
|
|
allow_plain_http = true
|
|
"#,
|
|
)?;
|
|
let reloader = PeersReloader::new(
|
|
reg.clone(),
|
|
Some(std::path::PathBuf::from("config.toml")),
|
|
CliOverrides::default(),
|
|
);
|
|
tokio::runtime::Builder::new_current_thread()
|
|
.enable_all()
|
|
.build()
|
|
.unwrap()
|
|
.block_on(reloader.reload())
|
|
.unwrap();
|
|
let snap = reg.snapshot();
|
|
assert_eq!(snap.len(), 1);
|
|
assert_eq!(snap[0].name.as_str(), "gamma");
|
|
assert_eq!(reg.config().api_interval_secs, 2);
|
|
|
|
let broken = PeersReloader::new(
|
|
reg.clone(),
|
|
Some(std::path::PathBuf::from("missing.toml")),
|
|
CliOverrides::default(),
|
|
);
|
|
assert!(tokio::runtime::Builder::new_current_thread()
|
|
.enable_all()
|
|
.build()
|
|
.unwrap()
|
|
.block_on(broken.reload())
|
|
.is_err());
|
|
assert!(reg.contains("gamma"), "a bad reload keeps the old peers");
|
|
Ok(())
|
|
});
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn reload_disabling_the_feature_empties_the_snapshot() {
|
|
let reg = registry_with(FakeTransport::new(), fake_peers_config(&["beta"]));
|
|
let mut new = fake_peers_config(&["beta"]);
|
|
new.enabled = false;
|
|
reg.reload(new).await.unwrap();
|
|
assert!(reg.snapshot().is_empty());
|
|
assert!(reg.status("beta").is_err());
|
|
assert!(!reg.config().enabled);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn reload_moves_loops_to_the_new_peer_set() {
|
|
let fake = FakeTransport::new();
|
|
let reg = PeerRegistry::start_with_transport(
|
|
fake_peers_config(&["beta"]),
|
|
fake.clone(),
|
|
CancellationToken::new(),
|
|
);
|
|
wait_for(®, |r| r.status("beta").unwrap().server.up).await;
|
|
|
|
reg.reload(fake_peers_config(&["beta", "gamma"]))
|
|
.await
|
|
.unwrap();
|
|
wait_for(®, |r| r.status("gamma").unwrap().server.up).await;
|
|
assert!(
|
|
reg.status("beta").unwrap().server.up,
|
|
"kept peer keeps state"
|
|
);
|
|
|
|
reg.reload(fake_peers_config(&["gamma"])).await.unwrap();
|
|
assert!(reg.status("beta").is_err());
|
|
assert!(reg.contains("gamma"));
|
|
reg.shutdown().await;
|
|
}
|
|
}
|