//! Outbound GET client used by health checks and the proxy. //! //! `reqwest` is built on a rustls `ClientConfig` whose crypto provider is //! ring, passed in here. Nothing is installed process-wide, so the OTLP //! client's provider choice is left alone. Redirects stay off: a peer URL is //! the only address we will talk to. use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; use std::time::Duration; use async_trait::async_trait; use axum::http::{HeaderName, HeaderValue}; use bytes::Bytes; use futures::Stream; use ipnet::IpNet; use rustls::pki_types::pem::PemObject; use rustls::pki_types::CertificateDer; use rustls::{ClientConfig, RootCertStore}; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use crate::config::ConfigError; use crate::peers::error::{PeerConfigError, PeerError}; use crate::peers::transport::{PeerBody, PeerRequest, PeerResponse, PeerTransport}; use crate::peers::types::{PeerConfig, PeerName, PeersConfig, Secret}; const MAX_TOKEN_BYTES: u64 = 4096; struct Limits { global: Arc, per_peer_n: usize, per_peer: HashMap>, max_response_bytes: u64, deny: Vec, } #[derive(Clone, PartialEq, Eq, Hash)] enum CaKey { Webpki, Extra(Vec), } /// One client per CA plus pinned address. Pinning the resolved address both /// keeps DNS rebinding from slipping past the deny list and makes the check /// and the connection use the same address. #[derive(Clone, PartialEq, Eq, Hash)] struct ClientKey { ca: CaKey, host: String, addr: SocketAddr, } struct Permits { _global: OwnedSemaphorePermit, _peer: OwnedSemaphorePermit, } /// Real [`PeerTransport`]. One shared client for peers that use the webpki /// roots, plus one extra client per distinct custom CA. pub struct ReqwestTransport { limits: Mutex, clients: Mutex>, plain_warned: Mutex>, } impl ReqwestTransport { pub fn new(cfg: &PeersConfig) -> Result { if cfg.max_concurrent_per_peer == 0 || cfg.max_concurrent_total == 0 { return Err(ConfigError::Invalid( "peers concurrency limits must be greater than zero".into(), )); } if cfg.max_response_bytes == 0 { return Err(ConfigError::Invalid( "peers.max_response_bytes must be greater than zero".into(), )); } let deny = cfg .deny_cidrs .iter() .map(|s| { s.parse::().map_err(|_| { ConfigError::Invalid(format!("peers.deny_cidrs entry {s:?} is not a CIDR")) }) }) .collect::, _>>()?; Ok(Self { limits: Mutex::new(Limits { global: Arc::new(Semaphore::new(cfg.max_concurrent_total)), per_peer_n: cfg.max_concurrent_per_peer, per_peer: HashMap::new(), max_response_bytes: cfg.max_response_bytes, deny, }), clients: Mutex::new(HashMap::new()), plain_warned: Mutex::new(HashSet::new()), }) } fn lock<'a, T>(m: &'a Mutex) -> std::sync::MutexGuard<'a, T> { m.lock().unwrap_or_else(|e| e.into_inner()) } fn max_bytes(&self) -> u64 { Self::lock(&self.limits).max_response_bytes } fn deny_list(&self) -> Vec { Self::lock(&self.limits).deny.clone() } fn acquire(&self, name: &PeerName) -> Result { let mut limits = Self::lock(&self.limits); let global = limits.global.clone(); let permits = limits.per_peer_n; let per = limits .per_peer .entry(name.clone()) .or_insert_with(|| Arc::new(Semaphore::new(permits))) .clone(); drop(limits); let global = global .try_acquire_owned() .map_err(|_| PeerError::Saturated)?; let peer = per.try_acquire_owned().map_err(|_| PeerError::Saturated)?; Ok(Permits { _global: global, _peer: peer, }) } fn client_for( &self, ca_pem: Option>, host: &str, addr: SocketAddr, ) -> Result { let key = ClientKey { ca: match &ca_pem { None => CaKey::Webpki, Some(pem) => CaKey::Extra(pem.clone()), }, host: host.to_owned(), addr, }; let mut cache = Self::lock(&self.clients); if let Some(client) = cache.get(&key) { return Ok(client.clone()); } if cache.len() >= crate::peers::types::MAX_PEERS { cache.clear(); } let client = build_client(ca_pem.as_deref(), host, addr)?; cache.insert(key, client.clone()); Ok(client) } fn warn_plain_http(&self, peer: &PeerConfig) { if peer.url.scheme() != "http" || host_is_loopback(peer.url.host_str()) { return; } let mut seen = Self::lock(&self.plain_warned); if seen.insert(peer.name.clone()) { tracing::warn!( peer = %peer.name, "plain HTTP to a non-loopback peer; the bearer token travels in clear" ); } } } fn host_is_loopback(host: Option<&str>) -> bool { match host { Some("localhost") => true, Some(h) => h .parse::() .is_ok_and(|ip| ip.is_loopback()), None => false, } } /// Rules that must hold before resolving anything. The same rules are /// enforced when the config is validated; repeating them here keeps the /// transport safe when it is built from an unvalidated `PeersConfig`. fn validate_target(peer: &PeerConfig) -> Result<(), PeerError> { if !matches!(peer.url.scheme(), "http" | "https") { return Err(PeerError::Unreachable); } if !peer.url.username().is_empty() || peer.url.password().is_some() { return Err(PeerError::Unreachable); } if peer.url.query().is_some() || peer.url.fragment().is_some() { return Err(PeerError::Unreachable); } if peer.url.scheme() == "http" && !host_is_loopback(peer.url.host_str()) && !peer.allow_plain_http { return Err(PeerError::Unreachable); } Ok(()) } fn denied(ip: IpAddr, deny: &[IpNet]) -> bool { deny.iter().any(|net| net.contains(&ip)) } /// Resolve the peer host and reject denied addresses before any connection is /// made. The returned address is pinned into the HTTP client. async fn resolve_target( peer: &PeerConfig, deny: &[IpNet], ) -> Result<(String, SocketAddr), PeerError> { validate_target(peer)?; let host = peer .url .host_str() .ok_or(PeerError::Unreachable)? .to_owned(); let port = peer .url .port_or_known_default() .ok_or(PeerError::Unreachable)?; let addrs: Vec = match host.parse::() { Ok(ip) => vec![SocketAddr::new(ip, port)], Err(_) => tokio::net::lookup_host((host.as_str(), port)) .await .map_err(|_| PeerError::Dns)? .collect(), }; let Some(addr) = addrs.first().copied() else { return Err(PeerError::Dns); }; if addrs.iter().any(|addr| denied(addr.ip(), deny)) { return Err(PeerError::Unreachable); } Ok((host, addr)) } /// Ring is passed in so this process never calls `install_default`. fn build_client( extra_ca: Option<&[u8]>, host: &str, addr: SocketAddr, ) -> Result { let tls = rustls_config(extra_ca)?; reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .no_proxy() .http1_only() .tls_backend_preconfigured(tls) .resolve(host, addr) .build() .map_err(|_| PeerError::Tls) } fn rustls_config(extra_ca: Option<&[u8]>) -> Result { let provider = Arc::new(rustls::crypto::ring::default_provider()); let mut roots = RootCertStore::empty(); roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); if let Some(pem) = extra_ca { let mut any = false; for cert in CertificateDer::pem_slice_iter(pem) { let cert = cert.map_err(|_| PeerError::Tls)?; roots.add(cert).map_err(|_| PeerError::Tls)?; any = true; } if !any { return Err(PeerError::Tls); } } let mut cfg = ClientConfig::builder_with_provider(provider) .with_safe_default_protocol_versions() .map_err(|_| PeerError::Tls)? .with_root_certificates(roots) .with_no_client_auth(); cfg.alpn_protocols = vec![b"http/1.1".to_vec()]; Ok(cfg) } fn request_url(peer: &PeerConfig, req: &PeerRequest) -> Result { if peer.url.scheme() != "http" && peer.url.scheme() != "https" { return Err(PeerError::Unreachable); } if !peer.url.username().is_empty() || peer.url.password().is_some() { return Err(PeerError::Unreachable); } if peer.url.query().is_some() || peer.url.fragment().is_some() { return Err(PeerError::Unreachable); } let mut url = peer.url.clone(); let base = url.path().trim_end_matches('/'); let rel = req.path.trim_start_matches('/'); let path = if rel.is_empty() { if base.is_empty() { "/".to_owned() } else { base.to_owned() } } else { format!("{base}/{rel}") }; url.set_path(&path); url.set_query(None); url.set_fragment(None); if !req.query.is_empty() { { let mut pairs = url.query_pairs_mut(); for (k, v) in &req.query { pairs.append_pair(k, v); } } } Ok(url) } fn load_token(path: &Path) -> Result { let creds = std::env::var_os("CREDENTIALS_DIRECTORY").map(PathBuf::from); load_token_in(path, creds.as_deref()) } fn load_token_in(path: &Path, credentials_dir: Option<&Path>) -> Result { let meta = std::fs::metadata(path).map_err(|_| PeerError::TokenFile("missing".into()))?; if !meta.is_file() { return Err(PeerError::TokenFile("not a file".into())); } if meta.len() > MAX_TOKEN_BYTES { return Err(PeerError::TokenFile("too large".into())); } if !under_credentials(path, credentials_dir) { let mode = { use std::os::unix::fs::PermissionsExt; meta.permissions().mode() }; if mode & 0o077 != 0 { return Err(PeerError::TokenFile("group or world readable".into())); } } let mut bytes = std::fs::read(path).map_err(|_| PeerError::TokenFile("unreadable".into()))?; if bytes.len() as u64 > MAX_TOKEN_BYTES { return Err(PeerError::TokenFile("too large".into())); } if bytes.last() == Some(&b'\n') { bytes.pop(); if bytes.last() == Some(&b'\r') { bytes.pop(); } } if bytes.is_empty() { return Err(PeerError::TokenFile("empty".into())); } if bytes.iter().any(|b| *b == b'\n' || *b == b'\r' || *b == 0) { return Err(PeerError::TokenFile("invalid".into())); } let text = String::from_utf8(bytes).map_err(|_| PeerError::TokenFile("invalid".into()))?; if HeaderValue::from_str(&format!("Bearer {text}")).is_err() { return Err(PeerError::TokenFile("invalid".into())); } Ok(Secret::new(text)) } fn under_credentials(path: &Path, credentials_dir: Option<&Path>) -> bool { match credentials_dir { Some(dir) if !dir.as_os_str().is_empty() => path.starts_with(dir), _ => false, } } fn map_io(err: std::io::Error) -> PeerError { if err.kind() == std::io::ErrorKind::TimedOut { return PeerError::Timeout; } let msg = err.to_string(); if msg.contains("Name or service not known") || msg.contains("nodename nor servname") || msg.contains("Temporary failure in name resolution") || msg.contains("No address associated") || msg.contains("Name does not resolve") { return PeerError::Dns; } PeerError::Unreachable } fn map_reqwest(err: reqwest::Error) -> PeerError { if err.is_timeout() { return PeerError::Timeout; } if err.is_dns() { return PeerError::Dns; } if error_is_tls(&err) { return PeerError::Tls; } if err.is_connect() { return PeerError::Unreachable; } PeerError::Unreachable } fn error_is_tls(err: &(dyn std::error::Error + 'static)) -> bool { if err.downcast_ref::().is_some() { return true; } let msg = err.to_string(); if msg.contains("certificate") || msg.contains("TLS") || msg.contains("tls handshake") { return true; } err.source().is_some_and(error_is_tls) } fn content_type(resp: &reqwest::Response) -> Option { resp.headers() .get(axum::http::header::CONTENT_TYPE) .and_then(|v| v.to_str().ok()) .map(str::to_owned) } fn wants_stream(req: &PeerRequest, resp: &reqwest::Response) -> bool { req.accept.as_deref() == Some("text/event-stream") || content_type(resp).is_some_and(|ct| ct.starts_with("text/event-stream")) } async fn read_capped(mut resp: reqwest::Response, max: u64) -> Result { if resp.content_length().is_some_and(|n| n > max) { return Err(PeerError::Oversize); } let mut buf = Vec::new(); while let Some(chunk) = resp.chunk().await.map_err(map_reqwest)? { let next = (buf.len() as u64).saturating_add(chunk.len() as u64); if next > max { return Err(PeerError::Oversize); } buf.extend_from_slice(&chunk); } Ok(Bytes::from(buf)) } struct CappedStream { inner: Pin> + Send>>, seen: u64, max: u64, done: bool, _permits: Permits, } impl Stream for CappedStream { type Item = Result; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); if this.done { return Poll::Ready(None); } match this.inner.as_mut().poll_next(cx) { Poll::Pending => Poll::Pending, Poll::Ready(None) => { this.done = true; Poll::Ready(None) } Poll::Ready(Some(Err(e))) => { this.done = true; Poll::Ready(Some(Err(map_reqwest(e)))) } Poll::Ready(Some(Ok(chunk))) => { this.seen = this.seen.saturating_add(chunk.len() as u64); if this.seen > this.max { this.done = true; Poll::Ready(Some(Err(PeerError::Oversize))) } else { Poll::Ready(Some(Ok(chunk))) } } } } } #[async_trait] impl PeerTransport for ReqwestTransport { async fn tcp_check(&self, peer: &PeerConfig, timeout: Duration) -> Result { let deny = self.deny_list(); let (_, addr) = resolve_target(peer, &deny).await?; let started = tokio::time::Instant::now(); let connect = tokio::net::TcpStream::connect(addr); match tokio::time::timeout(timeout, connect).await { Err(_) => Err(PeerError::Timeout), Ok(Err(e)) => Err(map_io(e)), Ok(Ok(_stream)) => Ok(started.elapsed()), } } async fn get( &self, peer: &PeerConfig, req: PeerRequest, timeout: Duration, ) -> Result { let url = request_url(peer, &req)?; let deny = self.deny_list(); let (host, addr) = resolve_target(peer, &deny).await?; self.warn_plain_http(peer); let token_path = peer.token_file.clone(); let secret = tokio::task::spawn_blocking(move || load_token(&token_path)) .await .map_err(|_| PeerError::TokenFile("unreadable".into()))??; let ca_path = peer.ca_file.clone(); let ca = tokio::task::spawn_blocking(move || match ca_path { None => Ok(None), Some(path) => load_ca_path(&path), }) .await .map_err(|_| PeerError::Tls)??; let auth = HeaderValue::from_str(&format!("Bearer {}", secret.expose())) .map_err(|_| PeerError::TokenFile("invalid".into()))?; drop(secret); let permits = self.acquire(&peer.name)?; let client = self.client_for(ca, &host, addr)?; let max = self.max_bytes(); let mut builder = client.get(url).timeout(timeout); for (name, value) in &req.headers { if name.as_str().eq_ignore_ascii_case("authorization") { continue; } builder = builder.header(name.clone(), value.clone()); } builder = builder.header(HeaderName::from_static("authorization"), auth); if let Some(accept) = &req.accept { builder = builder.header(HeaderName::from_static("accept"), accept); } let pending = async { let resp = builder.send().await.map_err(map_reqwest)?; let status = resp.status().as_u16(); let ct = content_type(&resp); if wants_stream(&req, &resp) { if resp.content_length().is_some_and(|n| n > max) { return Err(PeerError::Oversize); } // `chunk()` rather than reqwest's `stream` feature: that // feature pulls a wasm-streams pin this crate cannot select. let inner = Box::pin(futures::stream::unfold(resp, |mut resp| async move { match resp.chunk().await { Ok(Some(chunk)) => Some((Ok(chunk), resp)), Ok(None) => None, Err(err) => Some((Err(err), resp)), } })); return Ok(PeerResponse { status, content_type: ct, body: PeerBody::Stream(Box::pin(CappedStream { inner, seen: 0, max, done: false, _permits: permits, })), }); } let body = read_capped(resp, max).await?; drop(permits); Ok(PeerResponse { status, content_type: ct, body: PeerBody::Bytes(body), }) }; match tokio::time::timeout(timeout, pending).await { Err(_) => Err(PeerError::Timeout), Ok(result) => result, } } } fn load_ca_path(path: &Path) -> Result>, PeerError> { let bytes = std::fs::read(path).map_err(|_| PeerError::Tls)?; if bytes.is_empty() || bytes.len() > 1024 * 1024 { return Err(PeerError::Tls); } Ok(Some(bytes)) } #[cfg(test)] mod tests { use super::*; use std::io::Write; use std::os::unix::fs::OpenOptionsExt; use axum::http::HeaderValue as Hv; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; fn limits(per: usize, total: usize, max_bytes: u64) -> PeersConfig { PeersConfig { enabled: true, max_concurrent_per_peer: per, max_concurrent_total: total, max_response_bytes: max_bytes, ..PeersConfig::default() } } fn write_token(contents: &str, mode: u32) -> PathBuf { static N: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let path = std::env::temp_dir().join(format!( "slinuxm-peer-token-{}-{}", std::process::id(), N.fetch_add(1, std::sync::atomic::Ordering::Relaxed) )); let mut opts = std::fs::OpenOptions::new(); opts.write(true).create(true).truncate(true).mode(mode); opts.open(&path) .unwrap() .write_all(contents.as_bytes()) .unwrap(); path } fn peer(port: u16, token: &Path, ca: Option) -> PeerConfig { PeerConfig { name: PeerName::parse("beta").unwrap(), url: url::Url::parse(&format!("http://127.0.0.1:{port}")).unwrap(), token_file: token.to_path_buf(), ca_file: ca, allow_plain_http: false, enabled: true, } } fn req(path: &str) -> PeerRequest { PeerRequest { path: path.into(), query: vec![("n".into(), "1".into())], headers: vec![( HeaderName::from_static("x-slinuxm-hop"), Hv::from_static("1"), )], accept: None, } } async fn read_req(sock: &mut tokio::net::TcpStream) -> String { let mut buf = Vec::new(); let mut tmp = [0u8; 1024]; while !buf.windows(4).any(|w| w == b"\r\n\r\n") { let n = tokio::time::timeout(Duration::from_secs(2), sock.read(&mut tmp)) .await .unwrap() .unwrap(); if n == 0 { break; } buf.extend_from_slice(&tmp[..n]); } String::from_utf8_lossy(&buf).into_owned() } async fn serve(response: Vec) -> u16 { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); tokio::spawn(async move { let (mut sock, _) = listener.accept().await.unwrap(); let _ = read_req(&mut sock).await; let _ = sock.write_all(&response).await; }); port } #[test] fn token_file_rules() { let ok = write_token("abc\n", 0o600); let secret = load_token_in(&ok, None).unwrap(); assert_eq!(secret.expose(), "abc"); assert!(!format!("{secret:?}").contains("abc")); let _ = std::fs::remove_file(&ok); let crlf = write_token("abc\r\n", 0o600); assert_eq!(load_token_in(&crlf, None).unwrap().expose(), "abc"); let _ = std::fs::remove_file(&crlf); // One trailing newline is stripped. A second one would land in the // header, so the file is rejected rather than forwarded. let doubled = write_token("abc\n\n", 0o600); assert!(load_token_in(&doubled, None).is_err()); let _ = std::fs::remove_file(&doubled); for (body, mode) in [("\n", 0o600), ("", 0o600), ("x", 0o644), ("x", 0o640)] { let path = write_token(body, mode); assert!(load_token_in(&path, None).is_err(), "{body:?} {mode:o}"); let _ = std::fs::remove_file(&path); } let big = write_token(&"a".repeat(4097), 0o600); assert!(matches!( load_token_in(&big, None), Err(PeerError::TokenFile(_)) )); let _ = std::fs::remove_file(&big); let loose = write_token("secret-value\n", 0o644); let dir = loose.parent().unwrap(); assert!(load_token_in(&loose, Some(dir)).is_ok()); let err = load_token_in(&loose, None).unwrap_err(); assert!(matches!(err, PeerError::TokenFile(_))); assert!(!format!("{err:?}").contains("secret-value")); let _ = std::fs::remove_file(&loose); assert!(load_token_in(Path::new("/no/such/token"), None).is_err()); } #[test] fn rejects_zero_limits_and_bad_urls() { let mut cfg = limits(0, 1, 10); assert!(ReqwestTransport::new(&cfg).is_err()); cfg.max_concurrent_per_peer = 1; cfg.max_concurrent_total = 0; assert!(ReqwestTransport::new(&cfg).is_err()); cfg.max_concurrent_total = 1; cfg.max_response_bytes = 0; assert!(ReqwestTransport::new(&cfg).is_err()); let token = write_token("t", 0o600); let mut p = peer(9, &token, None); p.url = url::Url::parse("http://user:pw@127.0.0.1:9/").unwrap(); assert!(request_url(&p, &req("/api/v1/ping")).is_err()); p.url = url::Url::parse("http://127.0.0.1:9/?q=1").unwrap(); assert!(request_url(&p, &req("/x")).is_err()); p.url = url::Url::parse("ftp://127.0.0.1/").unwrap(); assert!(request_url(&p, &req("/x")).is_err()); let _ = std::fs::remove_file(&token); } #[tokio::test] async fn get_is_get_and_sends_the_token_not_a_redirect() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let token = write_token("peer-token-a\n", 0o600); tokio::spawn(async move { let (mut sock, _) = listener.accept().await.unwrap(); let head = read_req(&mut sock).await; assert!(head.starts_with("GET /api/v1/ping?n=1 HTTP/1.1")); assert!(head .to_ascii_lowercase() .contains("authorization: bearer peer-token-a")); assert!(head.contains("x-slinuxm-hop: 1")); let body = "{\"status\":\"ok\",\"version\":\"9\"}"; let resp = format!( "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:1/gone\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len() ); let _ = sock.write_all(resp.as_bytes()).await; }); let transport = ReqwestTransport::new(&limits(4, 8, 1024)).unwrap(); let resp = transport .get( &peer(port, &token, None), req("/api/v1/ping"), Duration::from_secs(2), ) .await .unwrap(); assert_eq!(resp.status, 302, "redirects are not followed"); match resp.body { PeerBody::Bytes(b) => assert!(b.windows(3).any(|w| w == b"9\"}")), PeerBody::Stream(_) => panic!("buffered"), } let _ = std::fs::remove_file(&token); } #[tokio::test] async fn rereads_the_token_file() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let token = write_token("first-token\n", 0o600); tokio::spawn(async move { for _ in 0..2 { let (mut sock, _) = listener.accept().await.unwrap(); let head = read_req(&mut sock).await; let line = head .lines() .find(|l| l.to_ascii_lowercase().starts_with("authorization:")) .unwrap() .to_owned(); let body = line; let resp = format!( "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len() ); let _ = sock.write_all(resp.as_bytes()).await; } }); let transport = ReqwestTransport::new(&limits(4, 8, 4096)).unwrap(); let p = peer(port, &token, None); let first = transport .get(&p, req("/api/v1/ping"), Duration::from_secs(2)) .await .unwrap(); std::fs::OpenOptions::new() .write(true) .truncate(true) .mode(0o600) .open(&token) .unwrap() .write_all(b"second-token\n") .unwrap(); let second = transport .get(&p, req("/api/v1/ping"), Duration::from_secs(2)) .await .unwrap(); let text = |r: PeerResponse| match r.body { PeerBody::Bytes(b) => String::from_utf8(b.to_vec()).unwrap(), PeerBody::Stream(_) => panic!("buffered"), }; assert!(text(first).contains("first-token")); assert!(text(second).contains("second-token")); let _ = std::fs::remove_file(&token); } #[tokio::test] async fn timeout_and_closed_port() { let open = TcpListener::bind("127.0.0.1:0").await.unwrap(); let open_port = open.local_addr().unwrap().port(); tokio::spawn(async move { let (_sock, _) = open.accept().await.unwrap(); tokio::time::sleep(Duration::from_secs(2)).await; }); let hang = TcpListener::bind("127.0.0.1:0").await.unwrap(); let hang_port = hang.local_addr().unwrap().port(); tokio::spawn(async move { let (mut sock, _) = hang.accept().await.unwrap(); let _ = read_req(&mut sock).await; tokio::time::sleep(Duration::from_secs(5)).await; }); let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap(); let token = write_token("t", 0o600); assert!(transport .tcp_check(&peer(open_port, &token, None), Duration::from_secs(1)) .await .is_ok()); let started = tokio::time::Instant::now(); let err = transport .get( &peer(hang_port, &token, None), req("/api/v1/ping"), Duration::from_millis(200), ) .await .unwrap_err(); assert!(matches!(err, PeerError::Timeout), "{err:?}"); assert!(started.elapsed() < Duration::from_secs(2)); let closed = TcpListener::bind("127.0.0.1:0").await.unwrap(); let closed_port = closed.local_addr().unwrap().port(); drop(closed); let err = transport .tcp_check(&peer(closed_port, &token, None), Duration::from_secs(1)) .await .unwrap_err(); assert!(matches!(err, PeerError::Unreachable), "{err:?}"); let _ = std::fs::remove_file(&token); } #[tokio::test] async fn byte_cap_ignores_a_lying_content_length_and_counts_chunks() { let transport = ReqwestTransport::new(&limits(2, 2, 32)).unwrap(); let token = write_token("t", 0o600); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); tokio::spawn(async move { let (mut sock, _) = listener.accept().await.unwrap(); let _ = read_req(&mut sock).await; let _ = sock .write_all( b"HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\nConnection: close\r\n\r\n", ) .await; tokio::time::sleep(Duration::from_secs(5)).await; }); let started = tokio::time::Instant::now(); let err = transport .get( &peer(port, &token, None), req("/big"), Duration::from_secs(2), ) .await .unwrap_err(); assert!(matches!(err, PeerError::Oversize), "{err:?}"); assert!(started.elapsed() < Duration::from_secs(1)); let body = vec![b'x'; 200]; let mut bytes = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n".to_vec(); bytes.extend(format!("{:x}\r\n", body.len()).into_bytes()); bytes.extend_from_slice(&body); bytes.extend(b"\r\n0\r\n\r\n"); let port = serve(bytes).await; let err = transport .get( &peer(port, &token, None), req("/chunked"), Duration::from_secs(2), ) .await .unwrap_err(); assert!(matches!(err, PeerError::Oversize), "{err:?}"); let _ = std::fs::remove_file(&token); } #[tokio::test] async fn semaphores_are_per_peer_and_global() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); tokio::spawn(async move { let (mut sock, _) = listener.accept().await.unwrap(); let _ = read_req(&mut sock).await; let _ = ready_tx.send(()); tokio::time::sleep(Duration::from_secs(5)).await; }); let transport = Arc::new(ReqwestTransport::new(&limits(1, 4, 1024)).unwrap()); let token = write_token("t", 0o600); let p = peer(port, &token, None); let t2 = transport.clone(); let p2 = p.clone(); let hung = tokio::spawn(async move { t2.get(&p2, req("/hang"), Duration::from_secs(4)).await }); ready_rx.await.unwrap(); let started = tokio::time::Instant::now(); let err = transport .get(&p, req("/other"), Duration::from_millis(500)) .await .unwrap_err(); assert!(matches!(err, PeerError::Saturated), "{err:?}"); assert!(started.elapsed() < Duration::from_millis(400)); let mut gamma = p.clone(); gamma.name = PeerName::parse("gamma").unwrap(); // gamma is not blocked by beta's permit; it fails because nothing is listening twice. // Use a fresh accepting server so the call is not Saturated. let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let gport = listener.local_addr().unwrap().port(); tokio::spawn(async move { let (mut sock, _) = listener.accept().await.unwrap(); let _ = read_req(&mut sock).await; let _ = sock .write_all( b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", ) .await; }); gamma.url = url::Url::parse(&format!("http://127.0.0.1:{gport}")).unwrap(); let resp = transport .get(&gamma, req("/ok"), Duration::from_secs(2)) .await .unwrap(); assert_eq!(resp.status, 204); hung.abort(); let _ = std::fs::remove_file(&token); } #[tokio::test] async fn global_semaphore() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); tokio::spawn(async move { let (mut sock, _) = listener.accept().await.unwrap(); let _ = read_req(&mut sock).await; let _ = ready_tx.send(()); tokio::time::sleep(Duration::from_secs(5)).await; }); let transport = Arc::new(ReqwestTransport::new(&limits(4, 1, 1024)).unwrap()); let token = write_token("t", 0o600); let beta = peer(port, &token, None); let mut gamma = beta.clone(); gamma.name = PeerName::parse("gamma").unwrap(); let t2 = transport.clone(); let b2 = beta.clone(); let hung = tokio::spawn(async move { t2.get(&b2, req("/hang"), Duration::from_secs(4)).await }); ready_rx.await.unwrap(); let err = transport .get(&gamma, req("/x"), Duration::from_millis(500)) .await .unwrap_err(); assert!(matches!(err, PeerError::Saturated), "{err:?}"); hung.abort(); let _ = std::fs::remove_file(&token); } #[tokio::test] async fn custom_ca_is_required_for_a_private_cert() { let ck = rcgen::generate_simple_self_signed(vec!["localhost".into(), "127.0.0.1".into()]) .unwrap(); let cert_pem = ck.cert.pem(); let key_pem = ck.signing_key.serialize_pem(); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let certs: Vec<_> = CertificateDer::pem_slice_iter(cert_pem.as_bytes()) .map(|c| c.unwrap()) .collect(); let key = rustls::pki_types::PrivateKeyDer::from_pem_slice(key_pem.as_bytes()).unwrap(); let provider = Arc::new(rustls::crypto::ring::default_provider()); let mut scfg = rustls::ServerConfig::builder_with_provider(provider) .with_safe_default_protocol_versions() .unwrap() .with_no_client_auth() .with_single_cert(certs, key) .unwrap(); scfg.alpn_protocols = vec![b"http/1.1".to_vec()]; let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(scfg)); tokio::spawn(async move { for _ in 0..2 { let (sock, _) = listener.accept().await.unwrap(); let Ok(mut tls) = acceptor.accept(sock).await else { continue; }; let mut buf = [0u8; 2048]; let _ = tokio::time::timeout(Duration::from_secs(2), tls.read(&mut buf)).await; let _ = tls .write_all( b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", ) .await; } }); let token = write_token("t", 0o600); let ca_path = std::env::temp_dir().join(format!("slinuxm-ca-{}-{}", std::process::id(), port)); std::fs::write(&ca_path, &cert_pem).unwrap(); let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap(); let mut https = peer(port, &token, None); https.url = url::Url::parse(&format!("https://127.0.0.1:{port}")).unwrap(); let err = transport .get(&https, req("/api/v1/ping"), Duration::from_secs(2)) .await .unwrap_err(); assert!(matches!(err, PeerError::Tls), "{err:?}"); https.ca_file = Some(ca_path.clone()); let resp = transport .get(&https, req("/api/v1/ping"), Duration::from_secs(2)) .await .unwrap(); assert_eq!(resp.status, 200); let _ = std::fs::remove_file(&token); let _ = std::fs::remove_file(&ca_path); } #[tokio::test] async fn deny_cidrs_are_enforced_at_connect_time() { let mut cfg = limits(2, 2, 1024); cfg.deny_cidrs = vec!["127.0.0.0/8".into()]; let transport = ReqwestTransport::new(&cfg).unwrap(); let token = write_token("t", 0o600); let p = peer(9, &token, None); assert!(matches!( transport.tcp_check(&p, Duration::from_secs(1)).await, Err(PeerError::Unreachable) )); assert!(matches!( transport .get(&p, req("/api/v1/ping"), Duration::from_secs(1)) .await, Err(PeerError::Unreachable) )); let _ = std::fs::remove_file(&token); } #[test] fn invalid_deny_cidr_and_plain_http_opt_in() { let mut cfg = limits(2, 2, 1024); cfg.deny_cidrs = vec!["not-a-cidr".into()]; assert!(ReqwestTransport::new(&cfg).is_err()); let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap(); let token = write_token("t", 0o600); let mut p = peer(9, &token, None); p.url = url::Url::parse("http://10.0.0.5:1984").unwrap(); assert!(!p.allow_plain_http); assert!(validate_target(&p).is_err()); p.allow_plain_http = true; assert!(validate_target(&p).is_ok()); assert!(transport .deny_list() .iter() .any(|n| n.to_string() == "169.254.0.0/16")); let _ = std::fs::remove_file(&token); } #[test] fn loopback_names() { assert!(host_is_loopback(Some("127.0.0.1"))); assert!(host_is_loopback(Some("::1"))); assert!(host_is_loopback(Some("localhost"))); assert!(!host_is_loopback(Some("10.0.0.1"))); assert!(!host_is_loopback(Some("beta"))); } }