Files
skyg-linux-tracker/tests/peers.rs
T
ari dde6a12ce4 F: try every resolved peer address instead of the first
A peer whose name resolves to both families but which listens on only one
of them was reported unreachable, which the documented 0.0.0.0 binding
produces on a dual-stack LAN and in the NixOS test network.

resolve_target now returns the whole address set (still refusing the name
if any address is denied), the HTTP client pins it with
resolve_to_addrs so the connector falls through to the next address, and
tcp_check tries each address in order with a per-attempt share of the
remaining deadline. ClientKey keys on the set.

Regression cover: a client test that resolves localhost against a
single-family listener, a tests/peers.rs case declaring the peer as
http://localhost:<port>, and the VM test's TLS peer gamma now binds
0.0.0.0 (IPv4 only) while alpha and beta stay dual stack. Docs and the
contracts section 10 address note describe the new behaviour.
2026-10-20 15:04:09 -04:00

688 lines
23 KiB
Rust

//! End-to-end peer behaviour: a consolidator served by the real router and
//! registry, with each peer served in-process by another `slinuxm` router
//! (fake sources) or by a hand-written listener when the peer has to misbehave.
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use skyg_linux_tracker::auth::tokens;
use skyg_linux_tracker::config::Config;
use skyg_linux_tracker::peers::registry::PeerRegistry;
use skyg_linux_tracker::peers::types::{AuthState, PeerConfig, PeerName, PeerStatus, PeersConfig};
use skyg_linux_tracker::server;
use skyg_linux_tracker::sources::fake::fake_state;
use skyg_linux_tracker::tls;
use sqlx::sqlite::SqlitePoolOptions;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
mod common;
/// Deadline for one proxied request; well above the configured timeouts.
const EXCHANGE: Duration = Duration::from_secs(20);
/// Deadline for a peer to flip its status.
const FLIP: Duration = Duration::from_secs(10);
struct TempDir(PathBuf);
impl TempDir {
fn new(tag: &str) -> Self {
static N: AtomicUsize = AtomicUsize::new(0);
let p = std::env::temp_dir().join(format!(
"slinuxm-peers-e2e-{tag}-{}-{}",
std::process::id(),
N.fetch_add(1, Ordering::SeqCst)
));
std::fs::create_dir_all(&p).unwrap();
Self(p)
}
/// A token file as a credential manager leaves it: private to this user.
fn token(&self, name: &str, contents: &str) -> PathBuf {
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let path = self.0.join(name);
std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(0o600)
.open(&path)
.unwrap()
.write_all(contents.as_bytes())
.unwrap();
path
}
fn file(&self, name: &str, contents: &str) -> PathBuf {
let path = self.0.join(name);
std::fs::write(&path, contents).unwrap();
path
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
/// One request over a fresh connection, read to EOF.
async fn send(port: u16, raw: &str) -> (u16, String, String) {
let mut sock = tokio::net::TcpStream::connect(("127.0.0.1", port))
.await
.unwrap();
sock.write_all(raw.as_bytes()).await.unwrap();
let mut buf = Vec::new();
tokio::time::timeout(EXCHANGE, sock.read_to_end(&mut buf))
.await
.expect("request did not finish")
.unwrap();
common::parse_response(&buf)
}
fn get_raw(path: &str, token: &str, accept: Option<&str>) -> String {
let auth = format!("Bearer {token}");
let mut headers = vec![("Authorization", auth.as_str())];
if let Some(accept) = accept {
headers.push(("Accept", accept));
}
common::request("GET", path, "localhost", &headers, None)
}
/// A second `slinuxm` acting as the peer: real router, fake sources.
struct Peer {
port: u16,
/// The token this peer accepts.
token: String,
shutdown: CancellationToken,
join: JoinHandle<anyhow::Result<()>>,
}
impl Peer {
async fn start() -> Self {
Self::spawn(None).await
}
async fn start_tls(cert: &Path, key: &Path) -> Self {
Self::spawn(Some((cert, key))).await
}
async fn spawn(pair: Option<(&Path, &Path)>) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let mut config = Config::default();
config.server.port = port;
config.auth.enabled = true;
if let Some((cert, key)) = pair {
config.server.tls.enabled = true;
config.server.tls.cert_file = cert.to_path_buf();
config.server.tls.key_file = key.to_path_buf();
}
let mut state = fake_state(config);
let pool = SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.unwrap();
skyg_linux_tracker::db::migrate(&pool).await.unwrap();
let token = tokens::create(&pool, "consolidator").await.unwrap();
state.db = Some(pool);
let acceptor = pair.map(|_| {
let cert = tls::ReloadableCert::load(&state.config.server.tls).unwrap();
tls::acceptor(cert).unwrap()
});
let shutdown = state.shutdown.clone();
let join = tokio::spawn(server::serve_listener(
listener,
acceptor,
server::build_router(state),
shutdown.clone(),
server::Limits::default(),
));
Self {
port,
token,
shutdown,
join,
}
}
fn url(&self, scheme: &str) -> String {
format!("{scheme}://127.0.0.1:{}", self.port)
}
fn token_file(&self, dir: &TempDir) -> PathBuf {
dir.token("peer-token", &format!("{}\n", self.token))
}
async fn stop(self) {
self.shutdown.cancel();
let _ = self.join.await;
}
}
/// The consolidator under test: real registry, real router, own caller token.
struct Consolidator {
port: u16,
token: String,
registry: Arc<PeerRegistry>,
shutdown: CancellationToken,
join: JoinHandle<anyhow::Result<()>>,
}
impl Consolidator {
async fn start(peers: PeersConfig) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let mut config = Config::default();
config.server.port = port;
config.auth.enabled = true;
config.peers = peers.clone();
let mut state = fake_state(config);
let pool = SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.unwrap();
skyg_linux_tracker::db::migrate(&pool).await.unwrap();
let token = tokens::create(&pool, "caller").await.unwrap();
state.db = Some(pool);
let shutdown = state.shutdown.clone();
let registry = PeerRegistry::start(peers, shutdown.clone()).unwrap();
state.peers = Some(registry.clone());
let join = tokio::spawn(server::serve_listener(
listener,
None,
server::build_router(state),
shutdown.clone(),
server::Limits::default(),
));
Self {
port,
token,
registry,
shutdown,
join,
}
}
/// One GET with the caller's own token unless `token` is given.
async fn get(&self, path: &str, token: Option<&str>) -> (u16, String, String) {
let raw = get_raw(path, token.unwrap_or(&self.token), None);
send(self.port, &raw).await
}
async fn wait_status(&self, peer: &str, ok: impl Fn(&PeerStatus) -> bool) -> PeerStatus {
let deadline = tokio::time::Instant::now() + FLIP;
loop {
let status = self.registry.status(peer).unwrap();
if ok(&status) {
return status;
}
assert!(
tokio::time::Instant::now() < deadline,
"{peer} stayed {status:?}"
);
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
async fn stop(self) {
self.shutdown.cancel();
let _ = self.join.await;
self.registry.shutdown().await;
}
}
fn peers_config(entries: &[(&str, String, Option<PathBuf>)]) -> PeersConfig {
PeersConfig {
enabled: true,
tcp_interval_secs: 2,
tcp_timeout_secs: 1,
api_interval_secs: 2,
api_timeout_secs: 1,
fail_threshold: 1,
request_timeout_secs: 2,
peer: entries
.iter()
.map(|(name, url, ca)| PeerConfig {
name: PeerName::parse(name).unwrap(),
url: url::Url::parse(url).unwrap(),
token_file: PathBuf::from(format!("/run/credentials/peer-{name}")),
ca_file: ca.clone(),
allow_plain_http: true,
enabled: true,
})
.collect(),
..PeersConfig::default()
}
}
/// One peer entry, with its token file replaced by `path`.
fn one_peer(name: &str, url: String, ca: Option<PathBuf>, path: &Path) -> PeersConfig {
let mut cfg = peers_config(&[(name, url, ca)]);
cfg.peer[0].token_file = path.to_path_buf();
cfg
}
fn json(body: &str) -> serde_json::Value {
serde_json::from_str(body).unwrap_or_else(|e| panic!("not JSON ({e}): {body}"))
}
/// A CA plus a leaf the consolidator can be told to trust.
struct Ca {
params: rcgen::CertificateParams,
key: rcgen::KeyPair,
cert_pem: String,
}
impl Ca {
fn new(name: &str) -> Self {
let mut params = rcgen::CertificateParams::new(Vec::new()).unwrap();
params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained);
params
.distinguished_name
.push(rcgen::DnType::CommonName, name);
let key = rcgen::KeyPair::generate().unwrap();
let cert_pem = params.self_signed(&key).unwrap().pem();
Self {
params,
key,
cert_pem,
}
}
/// Writes the CA and a leaf for 127.0.0.1 into `dir`; returns
/// (ca, cert, key) paths. Consumes the CA: it holds the signing key.
fn leaf_for_loopback(self, dir: &TempDir, tag: &str) -> (PathBuf, PathBuf, PathBuf) {
let params = rcgen::CertificateParams::new(vec!["127.0.0.1".to_owned()]).unwrap();
let key = rcgen::KeyPair::generate().unwrap();
let issuer = rcgen::Issuer::new(self.params, self.key);
let cert = params.signed_by(&key, &issuer).unwrap();
(
dir.file(&format!("{tag}-ca.pem"), &self.cert_pem),
dir.file(&format!("{tag}-cert.pem"), &cert.pem()),
dir.file(&format!("{tag}-key.pem"), &key.serialize_pem()),
)
}
}
/// A hand-written listener for the cases a real `slinuxm` cannot produce.
///
/// `respond` returns the raw response for each accepted connection, or
/// `None` to accept and answer nothing.
struct Listener {
port: u16,
hits: Arc<AtomicUsize>,
stop: CancellationToken,
}
impl Listener {
async fn start<F>(respond: F) -> Self
where
F: Fn() -> Option<Vec<u8>> + Send + Sync + 'static,
{
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let stop = CancellationToken::new();
let hits = Arc::new(AtomicUsize::new(0));
let respond = Arc::new(respond);
let counter = hits.clone();
let cancelled = stop.clone();
tokio::spawn(async move {
let mut held: Vec<tokio::net::TcpStream> = Vec::new();
loop {
let accepted = tokio::select! {
_ = cancelled.cancelled() => return,
accepted = listener.accept() => accepted,
};
let Ok((mut sock, _)) = accepted else {
return;
};
counter.fetch_add(1, Ordering::SeqCst);
let Some(body) = respond() else {
held.push(sock);
continue;
};
tokio::spawn(async move {
// Read the head first so the client is not reset mid-send.
let mut buf = [0u8; 1024];
let _ =
tokio::time::timeout(Duration::from_millis(500), sock.read(&mut buf)).await;
let _ = sock.write_all(&body).await;
});
}
});
Self { port, hits, stop }
}
fn hits(&self) -> usize {
self.hits.load(Ordering::SeqCst)
}
fn url(&self, scheme: &str) -> String {
format!("{scheme}://127.0.0.1:{}", self.port)
}
}
impl Drop for Listener {
fn drop(&mut self) {
self.stop.cancel();
}
}
fn response(status: &str, extra: &str, body: &[u8]) -> Vec<u8> {
let mut out = format!(
"HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n{extra}\r\n",
body.len()
)
.into_bytes();
out.extend_from_slice(body);
out
}
#[tokio::test]
async fn a_plain_peer_reports_up_and_auth_ok() {
let dir = TempDir::new("plain");
let peer = Peer::start().await;
let token = peer.token_file(&dir);
let consolidator = Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await;
let mut status = consolidator
.wait_status("beta", |s| s.server.up && s.api.up)
.await;
assert_eq!(status.api.auth, AuthState::Ok);
assert!(status.api.peer_version.is_some(), "{status:?}");
let (code, _, body) = consolidator.get("/api/v1/peers", None).await;
assert_eq!(code, 200);
let listed = json(&body);
assert_eq!(listed["peers"][0]["name"], "beta");
assert_eq!(listed["peers"][0]["api"]["auth"], "ok");
assert_eq!(listed["peers"][0]["url"], peer.url("http"));
// The caller still needs a token of this host, and unknown names are
// local 404s.
let (code, _, _) = consolidator
.get("/api/v1/peers/beta/systemd/units", Some("wrong"))
.await;
assert_eq!(code, 401);
let (code, _, body) = consolidator.get("/api/v1/peers/nope", None).await;
assert_eq!(code, 404, "{body}");
assert!(body.contains("unknown peer"), "{body}");
let auth = format!("Bearer {}", consolidator.token);
let raw = common::request(
"POST",
"/api/v1/peers/beta/systemd/units",
"localhost",
&[("Authorization", &auth)],
Some("{}"),
);
assert_eq!(send(consolidator.port, &raw).await.0, 405);
// A dead peer flips both checks; the proxy fails with the peer's name and
// the caller's own token keeps working.
peer.stop().await;
status = consolidator
.wait_status("beta", |s| !s.server.up && !s.api.up)
.await;
assert_eq!(status.api.auth, AuthState::Unknown);
let (code, _, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 503, "{body}");
assert!(body.contains("beta"), "{body}");
assert_eq!(consolidator.get("/api/v1/systemd/units", None).await.0, 200);
consolidator.stop().await;
}
#[tokio::test]
async fn a_plain_peer_serves_its_own_data_through_the_proxy() {
let dir = TempDir::new("proxy");
let peer = Peer::start().await;
let token = peer.token_file(&dir);
let consolidator = Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await;
consolidator
.wait_status("beta", |s| s.server.up && s.api.up)
.await;
// The proxied answer is the peer's own: compare it with a direct call.
let raw = get_raw("/api/v1/systemd/units", &peer.token, None);
let (direct_code, _, direct) = send(peer.port, &raw).await;
assert_eq!(direct_code, 200);
let (code, head, proxied) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 200, "{proxied}");
assert!(head.contains("x-slinuxm-peer: beta"), "{head}");
assert_eq!(json(&proxied), json(&direct));
// An allowlisted query reaches the peer too.
let (code, _, failed) = consolidator
.get("/api/v1/peers/beta/systemd/units?state=failed", None)
.await;
assert_eq!(code, 200, "{failed}");
consolidator.stop().await;
peer.stop().await;
}
/// Declared by name, not by IP literal: `localhost` resolves to both loopback
/// families while the peer listens on IPv4 only, so a transport that pins just
/// the first resolved address reports this peer down.
#[tokio::test]
async fn a_peer_declared_by_name_is_reached_on_the_family_it_listens_on() {
let dir = TempDir::new("localhost");
let peer = Peer::start().await;
let token = peer.token_file(&dir);
let url = format!("http://localhost:{}", peer.port);
let consolidator = Consolidator::start(one_peer("beta", url, None, &token)).await;
let status = consolidator
.wait_status("beta", |s| s.server.up && s.api.up)
.await;
assert_eq!(status.api.auth, AuthState::Ok);
let (code, _, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 200, "{body}");
consolidator.stop().await;
peer.stop().await;
}
#[tokio::test]
async fn a_tls_peer_is_trusted_through_its_ca_file() {
let dir = TempDir::new("tls");
let ca = Ca::new("slinuxm-e2e-ca");
let (ca_path, cert, key) = ca.leaf_for_loopback(&dir, "good");
let peer = Peer::start_tls(&cert, &key).await;
let token = peer.token_file(&dir);
let consolidator =
Consolidator::start(one_peer("beta", peer.url("https"), Some(ca_path), &token)).await;
// `api.up` needs a verified handshake and our token: that is the CA file
// working end to end.
let status = consolidator
.wait_status("beta", |s| s.server.up && s.api.up)
.await;
assert_eq!(status.api.auth, AuthState::Ok);
assert!(status.url.starts_with("https://"), "{status:?}");
peer.stop().await;
consolidator.stop().await;
}
#[tokio::test]
async fn a_peer_signed_by_another_ca_is_a_tls_error() {
let dir = TempDir::new("wrong-ca");
let trusted = Ca::new("trusted-ca");
let other = Ca::new("other-ca");
let (trusted_ca, _, _) = trusted.leaf_for_loopback(&dir, "trusted");
let (_, cert, key) = other.leaf_for_loopback(&dir, "other");
let peer = Peer::start_tls(&cert, &key).await;
let token = peer.token_file(&dir);
let consolidator = Consolidator::start(one_peer(
"beta",
peer.url("https"),
Some(trusted_ca),
&token,
))
.await;
// The port answers, the handshake does not: a certificate fault must not
// read as an outage.
let status = consolidator
.wait_status("beta", |s| s.api.last_checked.is_some() && !s.api.up)
.await;
assert!(status.server.up, "{status:?}");
assert_eq!(status.api.last_error.as_deref(), Some("tls_error"));
let (code, _, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 503);
assert!(body.contains("tls_error"), "{body}");
assert!(body.contains("beta"), "{body}");
peer.stop().await;
consolidator.stop().await;
}
#[tokio::test]
async fn a_redirect_is_reported_and_never_followed() {
let dir = TempDir::new("redirect");
let target = Listener::start(|| Some(response("200 OK", "", b"{}"))).await;
let location = format!("{}/elsewhere", target.url("http"));
let source = Listener::start(move || {
Some(response(
"302 Found",
&format!("Location: {location}\r\n"),
b"moved",
))
})
.await;
let token = dir.token("peer-token", "redirect-token\n");
let consolidator =
Consolidator::start(one_peer("beta", source.url("http"), None, &token)).await;
let (code, head, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 503, "{body}");
assert!(body.contains("http_302"), "{body}");
assert!(head.contains("x-slinuxm-peer: beta"), "{head}");
// Let the health loop probe as well, then check the Location was never
// acted on.
tokio::time::sleep(Duration::from_millis(500)).await;
consolidator.wait_status("beta", |s| !s.api.up).await;
assert!(source.hits() > 0);
assert_eq!(target.hits(), 0, "the redirect was followed");
consolidator.stop().await;
}
#[tokio::test]
async fn an_oversize_body_is_refused() {
let dir = TempDir::new("oversize");
let source = Listener::start(|| Some(response("200 OK", "", &[b'x'; 4096]))).await;
let token = dir.token("peer-token", "big-token\n");
let mut cfg = one_peer("beta", source.url("http"), None, &token);
cfg.max_response_bytes = 64;
let consolidator = Consolidator::start(cfg).await;
let (code, _, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
assert_eq!(code, 503, "{body}");
assert!(body.contains("oversize"), "{body}");
consolidator.stop().await;
}
#[tokio::test]
async fn a_silent_peer_times_out_within_the_request_timeout() {
let dir = TempDir::new("slow");
let source = Listener::start(|| None).await;
let token = dir.token("peer-token", "slow-token\n");
let mut cfg = one_peer("beta", source.url("http"), None, &token);
cfg.request_timeout_secs = 1;
let consolidator = Consolidator::start(cfg).await;
let started = std::time::Instant::now();
let (code, _, body) = consolidator
.get("/api/v1/peers/beta/systemd/units", None)
.await;
let elapsed = started.elapsed();
assert_eq!(code, 503, "{body}");
assert!(body.contains("timeout"), "{body}");
assert!(elapsed < Duration::from_secs(10), "took {elapsed:?}");
consolidator.stop().await;
}
#[test]
fn at_most_64_peers_may_be_configured() {
let peer = |i: usize| PeerConfig {
name: PeerName::parse(&format!("peer{i}")).unwrap(),
url: url::Url::parse(&format!("http://127.0.0.1:{}", 2000 + i)).unwrap(),
token_file: PathBuf::from(format!("/run/credentials/peer{i}")),
ca_file: None,
allow_plain_http: true,
enabled: true,
};
let mut config = Config::default();
config.server.bind = "127.0.0.1".parse().unwrap();
config.peers = PeersConfig {
enabled: true,
peer: (0..skyg_linux_tracker::peers::types::MAX_PEERS)
.map(peer)
.collect(),
..PeersConfig::default()
};
config.validate().expect("64 peers are accepted");
config.peers.peer.push(peer(64));
let err = config.validate().unwrap_err().to_string();
assert!(err.contains("64"), "{err}");
}
#[tokio::test]
async fn shutdown_ends_a_proxied_follow() {
let dir = TempDir::new("follow");
let peer = Peer::start().await;
let token = peer.token_file(&dir);
let consolidator = Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await;
consolidator
.wait_status("beta", |s| s.server.up && s.api.up)
.await;
let raw = get_raw(
"/api/v1/peers/beta/journal/follow",
&consolidator.token,
Some("text/event-stream"),
);
let mut sock = tokio::net::TcpStream::connect(("127.0.0.1", consolidator.port))
.await
.unwrap();
sock.write_all(raw.as_bytes()).await.unwrap();
let mut buf = [0u8; 1024];
let n = tokio::time::timeout(EXCHANGE, sock.read(&mut buf))
.await
.expect("the follow stream never started")
.unwrap();
assert!(n > 0);
assert!(String::from_utf8_lossy(&buf[..n]).contains("200 OK"));
// The stream is open and idle on the peer; shutdown must end it.
consolidator.shutdown.cancel();
let mut rest = Vec::new();
tokio::time::timeout(Duration::from_secs(10), sock.read_to_end(&mut rest))
.await
.expect("the proxied follow outlived shutdown")
.unwrap();
tokio::time::timeout(Duration::from_secs(10), consolidator.join)
.await
.expect("the server did not drain after shutdown")
.unwrap()
.unwrap();
peer.stop().await;
}