Files

765 lines
26 KiB
Rust

//! End-to-end peer behaviour: a consolidator served by the real router and
//! registry, with each peer served in-process by another `slinuxm` router
//! (fake sources) or by a hand-written listener when the peer has to misbehave.
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use skyg_linux_tracker::auth::tokens;
use skyg_linux_tracker::config::Config;
use skyg_linux_tracker::peers::registry::PeerRegistry;
use skyg_linux_tracker::peers::testing::TempDir;
use skyg_linux_tracker::peers::types::{AuthState, PeerConfig, PeerName, PeerStatus, PeersConfig};
use skyg_linux_tracker::server;
use skyg_linux_tracker::sources::fake::fake_state;
use skyg_linux_tracker::tls;
use sqlx::sqlite::SqlitePoolOptions;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
mod common;
/// Deadline for one proxied request; well above the configured timeouts.
const EXCHANGE: Duration = Duration::from_secs(20);
/// Deadline for a peer to flip its status.
const FLIP: Duration = Duration::from_secs(10);
/// One request over a fresh connection, read to EOF.
async fn send(port: u16, raw: &str) -> (u16, String, String) {
let mut sock = tokio::net::TcpStream::connect(("127.0.0.1", port))
.await
.unwrap();
sock.write_all(raw.as_bytes()).await.unwrap();
let mut buf = Vec::new();
tokio::time::timeout(EXCHANGE, sock.read_to_end(&mut buf))
.await
.expect("request did not finish")
.unwrap();
common::parse_response(&buf)
}
fn get_raw(path: &str, token: &str, accept: Option<&str>) -> String {
let auth = format!("Bearer {token}");
let mut headers = vec![("Authorization", auth.as_str())];
if let Some(accept) = accept {
headers.push(("Accept", accept));
}
common::request("GET", path, "localhost", &headers, None)
}
/// One in-process `slinuxm`: real router and listener over fake sources, auth
/// on, and one minted token. `prepare` runs before the router is built, so a
/// caller can install its peer registry, and its value is returned.
struct Served {
port: u16,
token: String,
shutdown: CancellationToken,
join: JoinHandle<anyhow::Result<()>>,
}
impl Served {
async fn stop(self) {
self.shutdown.cancel();
let _ = self.join.await;
}
}
async fn serve<T, F>(
mut config: Config,
tls_pair: Option<(&Path, &Path)>,
prepare: F,
) -> (Served, T)
where
F: FnOnce(&mut skyg_linux_tracker::state::AppState) -> T,
{
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
config.server.port = port;
config.auth.enabled = true;
if let Some((cert, key)) = tls_pair {
config.server.tls.enabled = true;
config.server.tls.cert_file = cert.to_path_buf();
config.server.tls.key_file = key.to_path_buf();
}
let mut state = fake_state(config);
let pool = SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.unwrap();
skyg_linux_tracker::db::migrate(&pool).await.unwrap();
let token = tokens::create(&pool, "test").await.unwrap();
state.db = Some(pool);
let prepared = prepare(&mut state);
let acceptor = tls_pair.map(|_| {
let cert = tls::ReloadableCert::load(&state.config.server.tls).unwrap();
tls::acceptor(cert).unwrap()
});
let shutdown = state.shutdown.clone();
let join = tokio::spawn(server::serve_listener(
listener,
acceptor,
server::build_router(state),
shutdown.clone(),
server::Limits::default(),
));
(
Served {
port,
token,
shutdown,
join,
},
prepared,
)
}
/// A second `slinuxm` acting as the peer: real router, fake sources.
struct Peer {
served: Served,
}
impl Peer {
async fn start() -> Self {
Self::spawn(None).await
}
async fn start_tls(cert: &Path, key: &Path) -> Self {
Self::spawn(Some((cert, key))).await
}
async fn spawn(tls_pair: Option<(&Path, &Path)>) -> Self {
let (served, ()) = serve(Config::default(), tls_pair, |_| ()).await;
Self { served }
}
fn url(&self, scheme: &str) -> String {
format!("{scheme}://127.0.0.1:{}", self.served.port)
}
fn port(&self) -> u16 {
self.served.port
}
fn token(&self) -> &str {
&self.served.token
}
fn token_file(&self, dir: &TempDir) -> PathBuf {
dir.write(
"peer-token",
format!("{}\n", self.served.token).as_bytes(),
0o600,
)
}
async fn stop(self) {
self.served.stop().await;
}
}
/// The consolidator under test: real registry, real router, own caller token.
struct Consolidator {
served: Served,
registry: Arc<PeerRegistry>,
}
impl Consolidator {
async fn start(peers: PeersConfig) -> Self {
let config = Config {
peers: peers.clone(),
..Config::default()
};
let (served, registry) = serve(config, None, |state| {
let registry = PeerRegistry::start(peers, state.shutdown.clone()).unwrap();
state.peers = Some(registry.clone());
registry
})
.await;
Self { served, registry }
}
fn port(&self) -> u16 {
self.served.port
}
fn token(&self) -> &str {
&self.served.token
}
/// One GET with the caller's own token unless `token` is given.
async fn get(&self, path: &str, token: Option<&str>) -> (u16, String, String) {
let raw = get_raw(path, token.unwrap_or(&self.served.token), None);
send(self.served.port, &raw).await
}
async fn wait_status(&self, peer: &str, ok: impl Fn(&PeerStatus) -> bool) -> PeerStatus {
let deadline = tokio::time::Instant::now() + FLIP;
loop {
let status = self.registry.status(peer).unwrap();
if ok(&status) {
return status;
}
assert!(
tokio::time::Instant::now() < deadline,
"{peer} stayed {status:?}"
);
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
async fn stop(self) {
self.served.stop().await;
self.registry.shutdown().await;
}
}
fn peers_config(entries: &[(&str, String, Option<PathBuf>)]) -> PeersConfig {
PeersConfig {
enabled: true,
tcp_interval_secs: 2,
tcp_timeout_secs: 1,
api_interval_secs: 2,
api_timeout_secs: 1,
fail_threshold: 1,
request_timeout_secs: 2,
peer: entries
.iter()
.map(|(name, url, ca)| PeerConfig {
name: PeerName::parse(name).unwrap(),
url: url::Url::parse(url).unwrap(),
token_file: PathBuf::from(format!("/run/credentials/peer-{name}")),
ca_file: ca.clone(),
allow_plain_http: true,
enabled: true,
})
.collect(),
..PeersConfig::default()
}
}
/// One peer entry, with its token file replaced by `path`.
fn one_peer(name: &str, url: String, ca: Option<PathBuf>, path: &Path) -> PeersConfig {
let mut cfg = peers_config(&[(name, url, ca)]);
cfg.peer[0].token_file = path.to_path_buf();
cfg
}
fn json(body: &str) -> serde_json::Value {
serde_json::from_str(body).unwrap_or_else(|e| panic!("not JSON ({e}): {body}"))
}
/// A CA plus a leaf the consolidator can be told to trust.
struct Ca {
params: rcgen::CertificateParams,
key: rcgen::KeyPair,
cert_pem: String,
}
impl Ca {
fn new(name: &str) -> Self {
let mut params = rcgen::CertificateParams::new(Vec::new()).unwrap();
params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained);
params
.distinguished_name
.push(rcgen::DnType::CommonName, name);
let key = rcgen::KeyPair::generate().unwrap();
let cert_pem = params.self_signed(&key).unwrap().pem();
Self {
params,
key,
cert_pem,
}
}
/// Writes the CA and a leaf for 127.0.0.1 into `dir`; returns
/// (ca, cert, key) paths. Consumes the CA: it holds the signing key.
fn leaf_for_loopback(self, dir: &TempDir, tag: &str) -> (PathBuf, PathBuf, PathBuf) {
let params = rcgen::CertificateParams::new(vec!["127.0.0.1".to_owned()]).unwrap();
let key = rcgen::KeyPair::generate().unwrap();
let issuer = rcgen::Issuer::new(self.params, self.key);
let cert = params.signed_by(&key, &issuer).unwrap();
(
dir.write(&format!("{tag}-ca.pem"), self.cert_pem.as_bytes(), 0o600),
dir.write(&format!("{tag}-cert.pem"), cert.pem().as_bytes(), 0o600),
dir.write(
&format!("{tag}-key.pem"),
key.serialize_pem().as_bytes(),
0o600,
),
)
}
}
/// What a hand-written listener does with one accepted connection.
enum Answer {
/// The whole response; the socket closes afterwards.
Reply(Vec<u8>),
/// Status line and headers, then the socket stays open and silent.
Hold(Vec<u8>),
/// Status line and headers, then one body byte every `every`.
Trickle { head: Vec<u8>, every: Duration },
}
/// A hand-written listener for the cases a real `slinuxm` cannot produce.
///
/// `respond` returns the answer for each accepted connection, or `None` to
/// accept and answer nothing.
struct Listener {
port: u16,
hits: Arc<AtomicUsize>,
stop: CancellationToken,
}
impl Listener {
async fn start<F>(respond: F) -> Self
where
F: Fn() -> Option<Answer> + Send + Sync + 'static,
{
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let stop = CancellationToken::new();
let hits = Arc::new(AtomicUsize::new(0));
let respond = Arc::new(respond);
let counter = hits.clone();
let cancelled = stop.clone();
tokio::spawn(async move {
loop {
let accepted = tokio::select! {
_ = cancelled.cancelled() => return,
accepted = listener.accept() => accepted,
};
let Ok((mut sock, _)) = accepted else {
return;
};
counter.fetch_add(1, Ordering::SeqCst);
// `None` means "accept and say nothing": headers with no bytes,
// which leaves the client waiting.
let answer = respond().unwrap_or(Answer::Hold(Vec::new()));
let cancelled = cancelled.clone();
tokio::spawn(async move {
// Read the head first so the client is not reset mid-send.
let mut buf = [0u8; 1024];
let _ =
tokio::time::timeout(Duration::from_millis(500), sock.read(&mut buf)).await;
match answer {
Answer::Reply(body) => {
let _ = sock.write_all(&body).await;
}
Answer::Hold(head) => {
let _ = sock.write_all(&head).await;
tokio::select! {
_ = cancelled.cancelled() => {}
_ = tokio::time::sleep(Duration::from_secs(60)) => {}
}
}
Answer::Trickle { head, every } => {
let _ = sock.write_all(&head).await;
loop {
tokio::select! {
_ = cancelled.cancelled() => return,
_ = tokio::time::sleep(every) => {}
}
if sock.write_all(b" ").await.is_err() {
return;
}
}
}
}
});
}
});
Self { port, hits, stop }
}
fn hits(&self) -> usize {
self.hits.load(Ordering::SeqCst)
}
fn url(&self, scheme: &str) -> String {
format!("{scheme}://127.0.0.1:{}", self.port)
}
}
impl Drop for Listener {
fn drop(&mut self) {
self.stop.cancel();
}
}
fn response(status: &str, extra: &str, body: &[u8]) -> Vec<u8> {
let mut out = format!(
"HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n{extra}\r\n",
body.len()
)
.into_bytes();
out.extend_from_slice(body);
out
}
/// Asserts that a reply over `max_response_bytes` is refused before its body
/// is read.
async fn assert_oversize_refused(source: &Listener, token: &Path) {
let mut cfg = one_peer("beta", source.url("http"), None, token);
cfg.max_response_bytes = 64;
let consolidator = Consolidator::start(cfg).await;
let (code, _, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 503, "{body}");
assert!(body.contains("oversize"), "{body}");
consolidator.stop().await;
}
/// Asserts that the 1 s request deadline ends the call as a timeout. The bound
/// is 4 s rather than 10 s so a deadline that never fired cannot pass.
async fn assert_peer_times_out(source: &Listener, token: &Path) {
let mut cfg = one_peer("beta", source.url("http"), None, token);
cfg.request_timeout_secs = 1;
let consolidator = Consolidator::start(cfg).await;
let started = std::time::Instant::now();
let (code, _, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
let elapsed = started.elapsed();
assert_eq!(code, 503, "{body}");
assert!(body.contains("timeout"), "{body}");
assert!(elapsed < Duration::from_secs(4), "took {elapsed:?}");
consolidator.stop().await;
}
#[tokio::test]
async fn a_plain_peer_reports_up_and_auth_ok() {
let dir = TempDir::new("plain");
let peer = Peer::start().await;
let token = peer.token_file(&dir);
let consolidator = Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await;
let mut status = consolidator
.wait_status("beta", |s| s.server.up && s.api.up)
.await;
assert_eq!(status.api.auth, AuthState::Ok);
assert!(status.api.peer_version.is_some(), "{status:?}");
let (code, _, body) = consolidator.get("/api/v1/peers", None).await;
assert_eq!(code, 200);
let listed = json(&body);
assert_eq!(listed["peers"][0]["name"], "beta");
assert_eq!(listed["peers"][0]["api"]["auth"], "ok");
assert_eq!(listed["peers"][0]["url"], peer.url("http"));
// The caller still needs a token of this host, and unknown names are
// local 404s.
let (code, _, _) = consolidator
.get("/api/v1/peers/beta/systemd/units", Some("wrong"))
.await;
assert_eq!(code, 401);
let (code, _, body) = consolidator.get("/api/v1/peers/nope", None).await;
assert_eq!(code, 404, "{body}");
assert!(body.contains("unknown peer"), "{body}");
let auth = format!("Bearer {}", consolidator.token());
let raw = common::request(
"POST",
"/api/v1/peers/beta/systemd/units",
"localhost",
&[("Authorization", &auth)],
Some("{}"),
);
assert_eq!(send(consolidator.port(), &raw).await.0, 405);
// A dead peer flips both checks; the proxy fails with the peer's name and
// the caller's own token keeps working.
peer.stop().await;
status = consolidator
.wait_status("beta", |s| !s.server.up && !s.api.up)
.await;
assert_eq!(status.api.auth, AuthState::Unknown);
let (code, _, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 503, "{body}");
assert!(body.contains("beta"), "{body}");
assert_eq!(consolidator.get("/api/v1/systemd/units", None).await.0, 200);
consolidator.stop().await;
}
#[tokio::test]
async fn a_plain_peer_serves_its_own_data_through_the_proxy() {
let dir = TempDir::new("proxy");
let peer = Peer::start().await;
let token = peer.token_file(&dir);
let consolidator = Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await;
consolidator
.wait_status("beta", |s| s.server.up && s.api.up)
.await;
// The proxied answer is the peer's own: compare it with a direct call.
let raw = get_raw("/api/v1/systemd/units", peer.token(), None);
let (direct_code, _, direct) = send(peer.port(), &raw).await;
assert_eq!(direct_code, 200);
let (code, head, proxied) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 200, "{proxied}");
assert!(head.contains("x-slinuxm-peer: beta"), "{head}");
assert_eq!(json(&proxied), json(&direct));
// An allowlisted query reaches the peer too.
let (code, _, failed) = consolidator
.get("/api/v1/peers/beta/systemd/units?state=failed", None)
.await;
assert_eq!(code, 200, "{failed}");
consolidator.stop().await;
peer.stop().await;
}
/// Declared by name, not by IP literal: `localhost` resolves to both loopback
/// families while the peer listens on IPv4 only, so a transport that pins just
/// the first resolved address reports this peer down.
#[tokio::test]
async fn a_peer_declared_by_name_is_reached_on_the_family_it_listens_on() {
let dir = TempDir::new("localhost");
let peer = Peer::start().await;
let token = peer.token_file(&dir);
let url = format!("http://localhost:{}", peer.port());
let consolidator = Consolidator::start(one_peer("beta", url, None, &token)).await;
let status = consolidator
.wait_status("beta", |s| s.server.up && s.api.up)
.await;
assert_eq!(status.api.auth, AuthState::Ok);
let (code, _, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 200, "{body}");
consolidator.stop().await;
peer.stop().await;
}
#[tokio::test]
async fn a_tls_peer_is_trusted_through_its_ca_file() {
let dir = TempDir::new("tls");
let ca = Ca::new("slinuxm-e2e-ca");
let (ca_path, cert, key) = ca.leaf_for_loopback(&dir, "good");
let peer = Peer::start_tls(&cert, &key).await;
let token = peer.token_file(&dir);
let consolidator =
Consolidator::start(one_peer("beta", peer.url("https"), Some(ca_path), &token)).await;
// `api.up` needs a verified handshake and our token: that is the CA file
// working end to end.
let status = consolidator
.wait_status("beta", |s| s.server.up && s.api.up)
.await;
assert_eq!(status.api.auth, AuthState::Ok);
assert!(status.url.starts_with("https://"), "{status:?}");
peer.stop().await;
consolidator.stop().await;
}
#[tokio::test]
async fn a_peer_signed_by_another_ca_is_a_tls_error() {
let dir = TempDir::new("wrong-ca");
let trusted = Ca::new("trusted-ca");
let other = Ca::new("other-ca");
let (trusted_ca, _, _) = trusted.leaf_for_loopback(&dir, "trusted");
let (_, cert, key) = other.leaf_for_loopback(&dir, "other");
let peer = Peer::start_tls(&cert, &key).await;
let token = peer.token_file(&dir);
let consolidator = Consolidator::start(one_peer(
"beta",
peer.url("https"),
Some(trusted_ca),
&token,
))
.await;
// The port answers, the handshake does not: a certificate fault must not
// read as an outage.
let status = consolidator
.wait_status("beta", |s| s.api.last_checked.is_some() && !s.api.up)
.await;
assert!(status.server.up, "{status:?}");
assert_eq!(status.api.last_error.as_deref(), Some("tls_error"));
let (code, _, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 503);
assert!(body.contains("tls_error"), "{body}");
assert!(body.contains("beta"), "{body}");
peer.stop().await;
consolidator.stop().await;
}
#[tokio::test]
async fn a_redirect_is_reported_and_never_followed() {
let dir = TempDir::new("redirect");
let target = Listener::start(|| Some(Answer::Reply(response("200 OK", "", b"{}")))).await;
let location = format!("{}/elsewhere", target.url("http"));
let source = Listener::start(move || {
Some(Answer::Reply(response(
"302 Found",
&format!("Location: {location}\r\n"),
b"moved",
)))
})
.await;
let token = dir.write("peer-token", "redirect-token\n".as_bytes(), 0o600);
let consolidator =
Consolidator::start(one_peer("beta", source.url("http"), None, &token)).await;
let (code, head, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 503, "{body}");
assert!(body.contains("http_302"), "{body}");
assert!(head.contains("x-slinuxm-peer: beta"), "{head}");
// The health loop probes on its own interval; waiting for its verdict is
// what proves the loop refused the redirect too.
consolidator
.wait_status("beta", |s| {
!s.api.up && s.api.last_error.as_deref() == Some("http_302")
})
.await;
assert!(source.hits() > 0);
assert_eq!(target.hits(), 0, "the redirect was followed");
consolidator.stop().await;
}
#[tokio::test]
async fn an_oversize_body_is_refused() {
let dir = TempDir::new("oversize");
let source =
Listener::start(|| Some(Answer::Reply(response("200 OK", "", &[b'x'; 4096])))).await;
let token = dir.write("peer-token", "big-token\n".as_bytes(), 0o600);
assert_oversize_refused(&source, &token).await;
}
#[tokio::test]
async fn a_silent_peer_times_out_within_the_request_timeout() {
let dir = TempDir::new("slow");
let source = Listener::start(|| None).await;
let token = dir.write("peer-token", "slow-token\n".as_bytes(), 0o600);
assert_peer_times_out(&source, &token).await;
}
/// No Content-Length to precheck: only the counting of chunk bytes can refuse
/// this body.
#[tokio::test]
async fn a_chunked_body_over_the_cap_is_refused() {
let dir = TempDir::new("chunked");
let chunk = vec![b'x'; 64];
let mut raw =
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nTransfer-Encoding: chunked\r\n\r\n"
.to_vec();
for _ in 0..2 {
raw.extend(format!("{:x}\r\n", chunk.len()).into_bytes());
raw.extend_from_slice(&chunk);
raw.extend(b"\r\n");
}
raw.extend(b"0\r\n\r\n");
let source = Listener::start(move || Some(Answer::Reply(raw.clone()))).await;
let token = dir.write("peer-token", "chunked-token\n".as_bytes(), 0o600);
assert_oversize_refused(&source, &token).await;
}
/// Slow loris: the peer answers the headers at once and then drips the body, so
/// only the request deadline can end the call.
#[tokio::test]
async fn a_trickling_peer_times_out_within_the_request_timeout() {
let dir = TempDir::new("trickle");
let head =
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 1000000\r\n\r\n"
.to_vec();
let every = Duration::from_millis(300);
let source = Listener::start(move || {
Some(Answer::Trickle {
head: head.clone(),
every,
})
})
.await;
let token = dir.write("peer-token", "trickle-token\n".as_bytes(), 0o600);
assert_peer_times_out(&source, &token).await;
assert!(source.hits() > 0);
}
#[test]
fn at_most_64_peers_may_be_configured() {
let peer = |i: usize| PeerConfig {
name: PeerName::parse(&format!("peer{i}")).unwrap(),
url: url::Url::parse(&format!("http://127.0.0.1:{}", 2000 + i)).unwrap(),
token_file: PathBuf::from(format!("/run/credentials/peer{i}")),
ca_file: None,
allow_plain_http: true,
enabled: true,
};
let mut config = Config::default();
config.server.bind = "127.0.0.1".parse().unwrap();
config.peers = PeersConfig {
enabled: true,
peer: (0..skyg_linux_tracker::peers::types::MAX_PEERS)
.map(peer)
.collect(),
..PeersConfig::default()
};
config.validate().expect("64 peers are accepted");
config.peers.peer.push(peer(64));
let err = config.validate().unwrap_err().to_string();
assert!(err.contains("at most 64 peers"), "{err}");
}
/// The peer answers any path with SSE headers and then holds the socket, so
/// nothing but shutdown can end the proxied stream.
#[tokio::test]
async fn shutdown_ends_a_proxied_follow() {
let dir = TempDir::new("follow");
let source = Listener::start(|| {
Some(Answer::Hold(
b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\r\n".to_vec(),
))
})
.await;
let token = dir.write("peer-token", "follow-token\n".as_bytes(), 0o600);
let consolidator =
Consolidator::start(one_peer("beta", source.url("http"), None, &token)).await;
let raw = get_raw(
"/api/v1/peers/beta/journal/follow",
consolidator.token(),
Some("text/event-stream"),
);
let mut sock = tokio::net::TcpStream::connect(("127.0.0.1", consolidator.port()))
.await
.unwrap();
sock.write_all(raw.as_bytes()).await.unwrap();
let mut buf = [0u8; 1024];
let n = tokio::time::timeout(EXCHANGE, sock.read(&mut buf))
.await
.expect("the follow stream never started")
.unwrap();
assert!(n > 0);
assert!(String::from_utf8_lossy(&buf[..n]).contains("200 OK"));
// The stream is open and idle on the peer; shutdown must end it.
consolidator.served.shutdown.cancel();
let mut rest = Vec::new();
tokio::time::timeout(Duration::from_secs(10), sock.read_to_end(&mut rest))
.await
.expect("the proxied follow outlived shutdown")
.unwrap();
tokio::time::timeout(Duration::from_secs(10), consolidator.served.join)
.await
.expect("the server did not drain after shutdown")
.unwrap()
.unwrap();
assert!(source.hits() > 0, "the peer was never called");
}