//! End-to-end peer behaviour: a consolidator served by the real router and //! registry, with each peer served in-process by another `slinuxm` router //! (fake sources) or by a hand-written listener when the peer has to misbehave. use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; use skyg_linux_tracker::auth::tokens; use skyg_linux_tracker::config::Config; use skyg_linux_tracker::peers::registry::PeerRegistry; use skyg_linux_tracker::peers::testing::TempDir; use skyg_linux_tracker::peers::types::{AuthState, PeerConfig, PeerName, PeerStatus, PeersConfig}; use skyg_linux_tracker::server; use skyg_linux_tracker::sources::fake::fake_state; use skyg_linux_tracker::tls; use sqlx::sqlite::SqlitePoolOptions; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; mod common; /// Deadline for one proxied request; well above the configured timeouts. const EXCHANGE: Duration = Duration::from_secs(20); /// Deadline for a peer to flip its status. const FLIP: Duration = Duration::from_secs(10); /// One request over a fresh connection, read to EOF. async fn send(port: u16, raw: &str) -> (u16, String, String) { let mut sock = tokio::net::TcpStream::connect(("127.0.0.1", port)) .await .unwrap(); sock.write_all(raw.as_bytes()).await.unwrap(); let mut buf = Vec::new(); tokio::time::timeout(EXCHANGE, sock.read_to_end(&mut buf)) .await .expect("request did not finish") .unwrap(); common::parse_response(&buf) } fn get_raw(path: &str, token: &str, accept: Option<&str>) -> String { let auth = format!("Bearer {token}"); let mut headers = vec![("Authorization", auth.as_str())]; if let Some(accept) = accept { headers.push(("Accept", accept)); } common::request("GET", path, "localhost", &headers, None) } /// One in-process `slinuxm`: real router and listener over fake sources, auth /// on, and one minted token. `prepare` runs before the router is built, so a /// caller can install its peer registry, and its value is returned. struct Served { port: u16, token: String, shutdown: CancellationToken, join: JoinHandle>, } impl Served { async fn stop(self) { self.shutdown.cancel(); let _ = self.join.await; } } async fn serve( mut config: Config, tls_pair: Option<(&Path, &Path)>, prepare: F, ) -> (Served, T) where F: FnOnce(&mut skyg_linux_tracker::state::AppState) -> T, { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); config.server.port = port; config.auth.enabled = true; if let Some((cert, key)) = tls_pair { config.server.tls.enabled = true; config.server.tls.cert_file = cert.to_path_buf(); config.server.tls.key_file = key.to_path_buf(); } let mut state = fake_state(config); let pool = SqlitePoolOptions::new() .max_connections(1) .connect("sqlite::memory:") .await .unwrap(); skyg_linux_tracker::db::migrate(&pool).await.unwrap(); let token = tokens::create(&pool, "test").await.unwrap(); state.db = Some(pool); let prepared = prepare(&mut state); let acceptor = tls_pair.map(|_| { let cert = tls::ReloadableCert::load(&state.config.server.tls).unwrap(); tls::acceptor(cert).unwrap() }); let shutdown = state.shutdown.clone(); let join = tokio::spawn(server::serve_listener( listener, acceptor, server::build_router(state), shutdown.clone(), server::Limits::default(), )); ( Served { port, token, shutdown, join, }, prepared, ) } /// A second `slinuxm` acting as the peer: real router, fake sources. struct Peer { served: Served, } impl Peer { async fn start() -> Self { Self::spawn(None).await } async fn start_tls(cert: &Path, key: &Path) -> Self { Self::spawn(Some((cert, key))).await } async fn spawn(tls_pair: Option<(&Path, &Path)>) -> Self { let (served, ()) = serve(Config::default(), tls_pair, |_| ()).await; Self { served } } fn url(&self, scheme: &str) -> String { format!("{scheme}://127.0.0.1:{}", self.served.port) } fn port(&self) -> u16 { self.served.port } fn token(&self) -> &str { &self.served.token } fn token_file(&self, dir: &TempDir) -> PathBuf { dir.write( "peer-token", format!("{}\n", self.served.token).as_bytes(), 0o600, ) } async fn stop(self) { self.served.stop().await; } } /// The consolidator under test: real registry, real router, own caller token. struct Consolidator { served: Served, registry: Arc, } impl Consolidator { async fn start(peers: PeersConfig) -> Self { let config = Config { peers: peers.clone(), ..Config::default() }; let (served, registry) = serve(config, None, |state| { let registry = PeerRegistry::start(peers, state.shutdown.clone()).unwrap(); state.peers = Some(registry.clone()); registry }) .await; Self { served, registry } } fn port(&self) -> u16 { self.served.port } fn token(&self) -> &str { &self.served.token } /// One GET with the caller's own token unless `token` is given. async fn get(&self, path: &str, token: Option<&str>) -> (u16, String, String) { let raw = get_raw(path, token.unwrap_or(&self.served.token), None); send(self.served.port, &raw).await } async fn wait_status(&self, peer: &str, ok: impl Fn(&PeerStatus) -> bool) -> PeerStatus { let deadline = tokio::time::Instant::now() + FLIP; loop { let status = self.registry.status(peer).unwrap(); if ok(&status) { return status; } assert!( tokio::time::Instant::now() < deadline, "{peer} stayed {status:?}" ); tokio::time::sleep(Duration::from_millis(50)).await; } } async fn stop(self) { self.served.stop().await; self.registry.shutdown().await; } } fn peers_config(entries: &[(&str, String, Option)]) -> PeersConfig { PeersConfig { enabled: true, tcp_interval_secs: 2, tcp_timeout_secs: 1, api_interval_secs: 2, api_timeout_secs: 1, fail_threshold: 1, request_timeout_secs: 2, peer: entries .iter() .map(|(name, url, ca)| PeerConfig { name: PeerName::parse(name).unwrap(), url: url::Url::parse(url).unwrap(), token_file: PathBuf::from(format!("/run/credentials/peer-{name}")), ca_file: ca.clone(), allow_plain_http: true, enabled: true, }) .collect(), ..PeersConfig::default() } } /// One peer entry, with its token file replaced by `path`. fn one_peer(name: &str, url: String, ca: Option, path: &Path) -> PeersConfig { let mut cfg = peers_config(&[(name, url, ca)]); cfg.peer[0].token_file = path.to_path_buf(); cfg } fn json(body: &str) -> serde_json::Value { serde_json::from_str(body).unwrap_or_else(|e| panic!("not JSON ({e}): {body}")) } /// A CA plus a leaf the consolidator can be told to trust. struct Ca { params: rcgen::CertificateParams, key: rcgen::KeyPair, cert_pem: String, } impl Ca { fn new(name: &str) -> Self { let mut params = rcgen::CertificateParams::new(Vec::new()).unwrap(); params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); params .distinguished_name .push(rcgen::DnType::CommonName, name); let key = rcgen::KeyPair::generate().unwrap(); let cert_pem = params.self_signed(&key).unwrap().pem(); Self { params, key, cert_pem, } } /// Writes the CA and a leaf for 127.0.0.1 into `dir`; returns /// (ca, cert, key) paths. Consumes the CA: it holds the signing key. fn leaf_for_loopback(self, dir: &TempDir, tag: &str) -> (PathBuf, PathBuf, PathBuf) { let params = rcgen::CertificateParams::new(vec!["127.0.0.1".to_owned()]).unwrap(); let key = rcgen::KeyPair::generate().unwrap(); let issuer = rcgen::Issuer::new(self.params, self.key); let cert = params.signed_by(&key, &issuer).unwrap(); ( dir.write(&format!("{tag}-ca.pem"), self.cert_pem.as_bytes(), 0o600), dir.write(&format!("{tag}-cert.pem"), cert.pem().as_bytes(), 0o600), dir.write( &format!("{tag}-key.pem"), key.serialize_pem().as_bytes(), 0o600, ), ) } } /// What a hand-written listener does with one accepted connection. enum Answer { /// The whole response; the socket closes afterwards. Reply(Vec), /// Status line and headers, then the socket stays open and silent. Hold(Vec), /// Status line and headers, then one body byte every `every`. Trickle { head: Vec, every: Duration }, } /// A hand-written listener for the cases a real `slinuxm` cannot produce. /// /// `respond` returns the answer for each accepted connection, or `None` to /// accept and answer nothing. struct Listener { port: u16, hits: Arc, stop: CancellationToken, } impl Listener { async fn start(respond: F) -> Self where F: Fn() -> Option + Send + Sync + 'static, { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let stop = CancellationToken::new(); let hits = Arc::new(AtomicUsize::new(0)); let respond = Arc::new(respond); let counter = hits.clone(); let cancelled = stop.clone(); tokio::spawn(async move { loop { let accepted = tokio::select! { _ = cancelled.cancelled() => return, accepted = listener.accept() => accepted, }; let Ok((mut sock, _)) = accepted else { return; }; counter.fetch_add(1, Ordering::SeqCst); // `None` means "accept and say nothing": headers with no bytes, // which leaves the client waiting. let answer = respond().unwrap_or(Answer::Hold(Vec::new())); let cancelled = cancelled.clone(); tokio::spawn(async move { // Read the head first so the client is not reset mid-send. let mut buf = [0u8; 1024]; let _ = tokio::time::timeout(Duration::from_millis(500), sock.read(&mut buf)).await; match answer { Answer::Reply(body) => { let _ = sock.write_all(&body).await; } Answer::Hold(head) => { let _ = sock.write_all(&head).await; tokio::select! { _ = cancelled.cancelled() => {} _ = tokio::time::sleep(Duration::from_secs(60)) => {} } } Answer::Trickle { head, every } => { let _ = sock.write_all(&head).await; loop { tokio::select! { _ = cancelled.cancelled() => return, _ = tokio::time::sleep(every) => {} } if sock.write_all(b" ").await.is_err() { return; } } } } }); } }); Self { port, hits, stop } } fn hits(&self) -> usize { self.hits.load(Ordering::SeqCst) } fn url(&self, scheme: &str) -> String { format!("{scheme}://127.0.0.1:{}", self.port) } } impl Drop for Listener { fn drop(&mut self) { self.stop.cancel(); } } fn response(status: &str, extra: &str, body: &[u8]) -> Vec { let mut out = format!( "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n{extra}\r\n", body.len() ) .into_bytes(); out.extend_from_slice(body); out } /// Asserts that a reply over `max_response_bytes` is refused before its body /// is read. async fn assert_oversize_refused(source: &Listener, token: &Path) { let mut cfg = one_peer("beta", source.url("http"), None, token); cfg.max_response_bytes = 64; let consolidator = Consolidator::start(cfg).await; let (code, _, body) = consolidator .get("/api/v1/peers/beta/systemd/units", None) .await; assert_eq!(code, 503, "{body}"); assert!(body.contains("oversize"), "{body}"); consolidator.stop().await; } /// Asserts that the 1 s request deadline ends the call as a timeout. The bound /// is 4 s rather than 10 s so a deadline that never fired cannot pass. async fn assert_peer_times_out(source: &Listener, token: &Path) { let mut cfg = one_peer("beta", source.url("http"), None, token); cfg.request_timeout_secs = 1; let consolidator = Consolidator::start(cfg).await; let started = std::time::Instant::now(); let (code, _, body) = consolidator .get("/api/v1/peers/beta/systemd/units", None) .await; let elapsed = started.elapsed(); assert_eq!(code, 503, "{body}"); assert!(body.contains("timeout"), "{body}"); assert!(elapsed < Duration::from_secs(4), "took {elapsed:?}"); consolidator.stop().await; } #[tokio::test] async fn a_plain_peer_reports_up_and_auth_ok() { let dir = TempDir::new("plain"); let peer = Peer::start().await; let token = peer.token_file(&dir); let consolidator = Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await; let mut status = consolidator .wait_status("beta", |s| s.server.up && s.api.up) .await; assert_eq!(status.api.auth, AuthState::Ok); assert!(status.api.peer_version.is_some(), "{status:?}"); let (code, _, body) = consolidator.get("/api/v1/peers", None).await; assert_eq!(code, 200); let listed = json(&body); assert_eq!(listed["peers"][0]["name"], "beta"); assert_eq!(listed["peers"][0]["api"]["auth"], "ok"); assert_eq!(listed["peers"][0]["url"], peer.url("http")); // The caller still needs a token of this host, and unknown names are // local 404s. let (code, _, _) = consolidator .get("/api/v1/peers/beta/systemd/units", Some("wrong")) .await; assert_eq!(code, 401); let (code, _, body) = consolidator.get("/api/v1/peers/nope", None).await; assert_eq!(code, 404, "{body}"); assert!(body.contains("unknown peer"), "{body}"); let auth = format!("Bearer {}", consolidator.token()); let raw = common::request( "POST", "/api/v1/peers/beta/systemd/units", "localhost", &[("Authorization", &auth)], Some("{}"), ); assert_eq!(send(consolidator.port(), &raw).await.0, 405); // A dead peer flips both checks; the proxy fails with the peer's name and // the caller's own token keeps working. peer.stop().await; status = consolidator .wait_status("beta", |s| !s.server.up && !s.api.up) .await; assert_eq!(status.api.auth, AuthState::Unknown); let (code, _, body) = consolidator .get("/api/v1/peers/beta/systemd/units", None) .await; assert_eq!(code, 503, "{body}"); assert!(body.contains("beta"), "{body}"); assert_eq!(consolidator.get("/api/v1/systemd/units", None).await.0, 200); consolidator.stop().await; } #[tokio::test] async fn a_plain_peer_serves_its_own_data_through_the_proxy() { let dir = TempDir::new("proxy"); let peer = Peer::start().await; let token = peer.token_file(&dir); let consolidator = Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await; consolidator .wait_status("beta", |s| s.server.up && s.api.up) .await; // The proxied answer is the peer's own: compare it with a direct call. let raw = get_raw("/api/v1/systemd/units", peer.token(), None); let (direct_code, _, direct) = send(peer.port(), &raw).await; assert_eq!(direct_code, 200); let (code, head, proxied) = consolidator .get("/api/v1/peers/beta/systemd/units", None) .await; assert_eq!(code, 200, "{proxied}"); assert!(head.contains("x-slinuxm-peer: beta"), "{head}"); assert_eq!(json(&proxied), json(&direct)); // An allowlisted query reaches the peer too. let (code, _, failed) = consolidator .get("/api/v1/peers/beta/systemd/units?state=failed", None) .await; assert_eq!(code, 200, "{failed}"); consolidator.stop().await; peer.stop().await; } /// Declared by name, not by IP literal: `localhost` resolves to both loopback /// families while the peer listens on IPv4 only, so a transport that pins just /// the first resolved address reports this peer down. #[tokio::test] async fn a_peer_declared_by_name_is_reached_on_the_family_it_listens_on() { let dir = TempDir::new("localhost"); let peer = Peer::start().await; let token = peer.token_file(&dir); let url = format!("http://localhost:{}", peer.port()); let consolidator = Consolidator::start(one_peer("beta", url, None, &token)).await; let status = consolidator .wait_status("beta", |s| s.server.up && s.api.up) .await; assert_eq!(status.api.auth, AuthState::Ok); let (code, _, body) = consolidator .get("/api/v1/peers/beta/systemd/units", None) .await; assert_eq!(code, 200, "{body}"); consolidator.stop().await; peer.stop().await; } #[tokio::test] async fn a_tls_peer_is_trusted_through_its_ca_file() { let dir = TempDir::new("tls"); let ca = Ca::new("slinuxm-e2e-ca"); let (ca_path, cert, key) = ca.leaf_for_loopback(&dir, "good"); let peer = Peer::start_tls(&cert, &key).await; let token = peer.token_file(&dir); let consolidator = Consolidator::start(one_peer("beta", peer.url("https"), Some(ca_path), &token)).await; // `api.up` needs a verified handshake and our token: that is the CA file // working end to end. let status = consolidator .wait_status("beta", |s| s.server.up && s.api.up) .await; assert_eq!(status.api.auth, AuthState::Ok); assert!(status.url.starts_with("https://"), "{status:?}"); peer.stop().await; consolidator.stop().await; } #[tokio::test] async fn a_peer_signed_by_another_ca_is_a_tls_error() { let dir = TempDir::new("wrong-ca"); let trusted = Ca::new("trusted-ca"); let other = Ca::new("other-ca"); let (trusted_ca, _, _) = trusted.leaf_for_loopback(&dir, "trusted"); let (_, cert, key) = other.leaf_for_loopback(&dir, "other"); let peer = Peer::start_tls(&cert, &key).await; let token = peer.token_file(&dir); let consolidator = Consolidator::start(one_peer( "beta", peer.url("https"), Some(trusted_ca), &token, )) .await; // The port answers, the handshake does not: a certificate fault must not // read as an outage. let status = consolidator .wait_status("beta", |s| s.api.last_checked.is_some() && !s.api.up) .await; assert!(status.server.up, "{status:?}"); assert_eq!(status.api.last_error.as_deref(), Some("tls_error")); let (code, _, body) = consolidator .get("/api/v1/peers/beta/systemd/units", None) .await; assert_eq!(code, 503); assert!(body.contains("tls_error"), "{body}"); assert!(body.contains("beta"), "{body}"); peer.stop().await; consolidator.stop().await; } #[tokio::test] async fn a_redirect_is_reported_and_never_followed() { let dir = TempDir::new("redirect"); let target = Listener::start(|| Some(Answer::Reply(response("200 OK", "", b"{}")))).await; let location = format!("{}/elsewhere", target.url("http")); let source = Listener::start(move || { Some(Answer::Reply(response( "302 Found", &format!("Location: {location}\r\n"), b"moved", ))) }) .await; let token = dir.write("peer-token", "redirect-token\n".as_bytes(), 0o600); let consolidator = Consolidator::start(one_peer("beta", source.url("http"), None, &token)).await; let (code, head, body) = consolidator .get("/api/v1/peers/beta/systemd/units", None) .await; assert_eq!(code, 503, "{body}"); assert!(body.contains("http_302"), "{body}"); assert!(head.contains("x-slinuxm-peer: beta"), "{head}"); // The health loop probes on its own interval; waiting for its verdict is // what proves the loop refused the redirect too. consolidator .wait_status("beta", |s| { !s.api.up && s.api.last_error.as_deref() == Some("http_302") }) .await; assert!(source.hits() > 0); assert_eq!(target.hits(), 0, "the redirect was followed"); consolidator.stop().await; } #[tokio::test] async fn an_oversize_body_is_refused() { let dir = TempDir::new("oversize"); let source = Listener::start(|| Some(Answer::Reply(response("200 OK", "", &[b'x'; 4096])))).await; let token = dir.write("peer-token", "big-token\n".as_bytes(), 0o600); assert_oversize_refused(&source, &token).await; } #[tokio::test] async fn a_silent_peer_times_out_within_the_request_timeout() { let dir = TempDir::new("slow"); let source = Listener::start(|| None).await; let token = dir.write("peer-token", "slow-token\n".as_bytes(), 0o600); assert_peer_times_out(&source, &token).await; } /// No Content-Length to precheck: only the counting of chunk bytes can refuse /// this body. #[tokio::test] async fn a_chunked_body_over_the_cap_is_refused() { let dir = TempDir::new("chunked"); let chunk = vec![b'x'; 64]; let mut raw = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nTransfer-Encoding: chunked\r\n\r\n" .to_vec(); for _ in 0..2 { raw.extend(format!("{:x}\r\n", chunk.len()).into_bytes()); raw.extend_from_slice(&chunk); raw.extend(b"\r\n"); } raw.extend(b"0\r\n\r\n"); let source = Listener::start(move || Some(Answer::Reply(raw.clone()))).await; let token = dir.write("peer-token", "chunked-token\n".as_bytes(), 0o600); assert_oversize_refused(&source, &token).await; } /// Slow loris: the peer answers the headers at once and then drips the body, so /// only the request deadline can end the call. #[tokio::test] async fn a_trickling_peer_times_out_within_the_request_timeout() { let dir = TempDir::new("trickle"); let head = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 1000000\r\n\r\n" .to_vec(); let every = Duration::from_millis(300); let source = Listener::start(move || { Some(Answer::Trickle { head: head.clone(), every, }) }) .await; let token = dir.write("peer-token", "trickle-token\n".as_bytes(), 0o600); assert_peer_times_out(&source, &token).await; assert!(source.hits() > 0); } #[test] fn at_most_64_peers_may_be_configured() { let peer = |i: usize| PeerConfig { name: PeerName::parse(&format!("peer{i}")).unwrap(), url: url::Url::parse(&format!("http://127.0.0.1:{}", 2000 + i)).unwrap(), token_file: PathBuf::from(format!("/run/credentials/peer{i}")), ca_file: None, allow_plain_http: true, enabled: true, }; let mut config = Config::default(); config.server.bind = "127.0.0.1".parse().unwrap(); config.peers = PeersConfig { enabled: true, peer: (0..skyg_linux_tracker::peers::types::MAX_PEERS) .map(peer) .collect(), ..PeersConfig::default() }; config.validate().expect("64 peers are accepted"); config.peers.peer.push(peer(64)); let err = config.validate().unwrap_err().to_string(); assert!(err.contains("at most 64 peers"), "{err}"); } /// The peer answers any path with SSE headers and then holds the socket, so /// nothing but shutdown can end the proxied stream. #[tokio::test] async fn shutdown_ends_a_proxied_follow() { let dir = TempDir::new("follow"); let source = Listener::start(|| { Some(Answer::Hold( b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\r\n".to_vec(), )) }) .await; let token = dir.write("peer-token", "follow-token\n".as_bytes(), 0o600); let consolidator = Consolidator::start(one_peer("beta", source.url("http"), None, &token)).await; let raw = get_raw( "/api/v1/peers/beta/journal/follow", consolidator.token(), Some("text/event-stream"), ); let mut sock = tokio::net::TcpStream::connect(("127.0.0.1", consolidator.port())) .await .unwrap(); sock.write_all(raw.as_bytes()).await.unwrap(); let mut buf = [0u8; 1024]; let n = tokio::time::timeout(EXCHANGE, sock.read(&mut buf)) .await .expect("the follow stream never started") .unwrap(); assert!(n > 0); assert!(String::from_utf8_lossy(&buf[..n]).contains("200 OK")); // The stream is open and idle on the peer; shutdown must end it. consolidator.served.shutdown.cancel(); let mut rest = Vec::new(); tokio::time::timeout(Duration::from_secs(10), sock.read_to_end(&mut rest)) .await .expect("the proxied follow outlived shutdown") .unwrap(); tokio::time::timeout(Duration::from_secs(10), consolidator.served.join) .await .expect("the server did not drain after shutdown") .unwrap() .unwrap(); assert!(source.hits() > 0, "the peer was never called"); }