diff --git a/src/peers/client.rs b/src/peers/client.rs index 762086d..c1c4d35 100644 --- a/src/peers/client.rs +++ b/src/peers/client.rs @@ -16,6 +16,7 @@ use std::time::Duration; use async_trait::async_trait; use axum::http::{HeaderName, HeaderValue}; use bytes::Bytes; +use futures::stream::BoxStream; use futures::Stream; use ipnet::IpNet; use rustls::pki_types::pem::PemObject; @@ -30,6 +31,27 @@ use crate::peers::types::{PeerConfig, PeerName, PeersConfig, Secret}; const MAX_TOKEN_BYTES: u64 = 4096; +/// Paths the health loops probe. They skip the concurrency limits: a held +/// follow stream must not make a healthy peer look saturated and down. +const PING_PATH: &str = "/api/v1/ping"; +const HEALTHZ_PATH: &str = "/healthz"; + +/// A stream that is silent for longer than this is dead. Peers send SSE +/// keep-alives and end their own follow streams at `limits.max_follow_secs`, +/// so this only catches a connection that stopped without closing. +const STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(120); + +/// Parse the deny list once, so startup and reload reject the same values. +pub(crate) fn deny_networks(list: &[String]) -> Result, PeerConfigError> { + list.iter() + .map(|s| { + s.parse::().map_err(|_| { + ConfigError::Invalid(format!("peers.deny_cidrs entry {s:?} is not a CIDR")) + }) + }) + .collect() +} + struct Limits { global: Arc, per_peer_n: usize, @@ -50,7 +72,7 @@ enum CaKey { #[derive(Clone, PartialEq, Eq, Hash)] struct ClientKey { ca: CaKey, - host: String, + host: Option, addr: SocketAddr, } @@ -79,15 +101,7 @@ impl ReqwestTransport { "peers.max_response_bytes must be greater than zero".into(), )); } - let deny = cfg - .deny_cidrs - .iter() - .map(|s| { - s.parse::().map_err(|_| { - ConfigError::Invalid(format!("peers.deny_cidrs entry {s:?} is not a CIDR")) - }) - }) - .collect::, _>>()?; + let deny = deny_networks(&cfg.deny_cidrs)?; Ok(Self { limits: Mutex::new(Limits { global: Arc::new(Semaphore::new(cfg.max_concurrent_total)), @@ -136,7 +150,7 @@ impl ReqwestTransport { fn client_for( &self, ca_pem: Option>, - host: &str, + host: Option<&str>, addr: SocketAddr, ) -> Result { let key = ClientKey { @@ -144,7 +158,7 @@ impl ReqwestTransport { None => CaKey::Webpki, Some(pem) => CaKey::Extra(pem.clone()), }, - host: host.to_owned(), + host: host.map(str::to_owned), addr, }; let mut cache = Self::lock(&self.clients); @@ -160,7 +174,7 @@ impl ReqwestTransport { } fn warn_plain_http(&self, peer: &PeerConfig) { - if peer.url.scheme() != "http" || host_is_loopback(peer.url.host_str()) { + if peer.url.scheme() != "http" || url_is_loopback(&peer.url) { return; } let mut seen = Self::lock(&self.plain_warned); @@ -173,12 +187,13 @@ impl ReqwestTransport { } } -fn host_is_loopback(host: Option<&str>) -> bool { - match host { - Some("localhost") => true, - Some(h) => h - .parse::() - .is_ok_and(|ip| ip.is_loopback()), +/// The URL parser already knows the host kind: an IPv6 literal is +/// `Host::Ipv6`, never the bracketed string `host_str()` renders. +fn url_is_loopback(url: &url::Url) -> bool { + match url.host() { + Some(url::Host::Ipv4(ip)) => ip.is_loopback(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback(), + Some(url::Host::Domain(name)) => name.eq_ignore_ascii_case("localhost"), None => false, } } @@ -196,10 +211,7 @@ fn validate_target(peer: &PeerConfig) -> Result<(), PeerError> { if peer.url.query().is_some() || peer.url.fragment().is_some() { return Err(PeerError::Unreachable); } - if peer.url.scheme() == "http" - && !host_is_loopback(peer.url.host_str()) - && !peer.allow_plain_http - { + if peer.url.scheme() == "http" && !url_is_loopback(&peer.url) && !peer.allow_plain_http { return Err(PeerError::Unreachable); } Ok(()) @@ -210,27 +222,25 @@ fn denied(ip: IpAddr, deny: &[IpNet]) -> bool { } /// Resolve the peer host and reject denied addresses before any connection is -/// made. The returned address is pinned into the HTTP client. +/// made. The returned address is pinned into the HTTP client; an IP literal +/// needs no pin because there is no name to rebind. async fn resolve_target( peer: &PeerConfig, deny: &[IpNet], -) -> Result<(String, SocketAddr), PeerError> { +) -> Result<(Option, SocketAddr), PeerError> { validate_target(peer)?; - let host = peer - .url - .host_str() - .ok_or(PeerError::Unreachable)? - .to_owned(); let port = peer .url .port_or_known_default() .ok_or(PeerError::Unreachable)?; - let addrs: Vec = match host.parse::() { - Ok(ip) => vec![SocketAddr::new(ip, port)], - Err(_) => tokio::net::lookup_host((host.as_str(), port)) + let addrs: Vec = match peer.url.host() { + Some(url::Host::Ipv4(ip)) => vec![SocketAddr::new(IpAddr::V4(ip), port)], + Some(url::Host::Ipv6(ip)) => vec![SocketAddr::new(IpAddr::V6(ip), port)], + Some(url::Host::Domain(name)) => tokio::net::lookup_host((name, port)) .await .map_err(|_| PeerError::Dns)? .collect(), + None => return Err(PeerError::Unreachable), }; let Some(addr) = addrs.first().copied() else { return Err(PeerError::Dns); @@ -238,24 +248,29 @@ async fn resolve_target( if addrs.iter().any(|addr| denied(addr.ip(), deny)) { return Err(PeerError::Unreachable); } - Ok((host, addr)) + let pin = match peer.url.host() { + Some(url::Host::Domain(name)) => Some(name.to_owned()), + _ => None, + }; + Ok((pin, addr)) } /// Ring is passed in so this process never calls `install_default`. fn build_client( extra_ca: Option<&[u8]>, - host: &str, + host: Option<&str>, addr: SocketAddr, ) -> Result { let tls = rustls_config(extra_ca)?; - reqwest::Client::builder() + let mut builder = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .no_proxy() .http1_only() - .tls_backend_preconfigured(tls) - .resolve(host, addr) - .build() - .map_err(|_| PeerError::Tls) + .tls_backend_preconfigured(tls); + if let Some(host) = host { + builder = builder.resolve(host, addr); + } + builder.build().map_err(|_| PeerError::Tls) } fn rustls_config(extra_ca: Option<&[u8]>) -> Result { @@ -283,15 +298,7 @@ fn rustls_config(extra_ca: Option<&[u8]>) -> Result { } fn request_url(peer: &PeerConfig, req: &PeerRequest) -> Result { - if peer.url.scheme() != "http" && peer.url.scheme() != "https" { - return Err(PeerError::Unreachable); - } - if !peer.url.username().is_empty() || peer.url.password().is_some() { - return Err(PeerError::Unreachable); - } - if peer.url.query().is_some() || peer.url.fragment().is_some() { - return Err(PeerError::Unreachable); - } + validate_target(peer)?; let mut url = peer.url.clone(); let base = url.path().trim_end_matches('/'); let rel = req.path.trim_start_matches('/'); @@ -374,15 +381,6 @@ fn map_io(err: std::io::Error) -> PeerError { if err.kind() == std::io::ErrorKind::TimedOut { return PeerError::Timeout; } - let msg = err.to_string(); - if msg.contains("Name or service not known") - || msg.contains("nodename nor servname") - || msg.contains("Temporary failure in name resolution") - || msg.contains("No address associated") - || msg.contains("Name does not resolve") - { - return PeerError::Dns; - } PeerError::Unreachable } @@ -402,15 +400,31 @@ fn map_reqwest(err: reqwest::Error) -> PeerError { PeerError::Unreachable } +/// hyper-rustls reports a handshake failure as an `io::Error` with a +/// non-socket kind, while a refused or reset connection has a socket kind. +/// The kind separates them without matching message text. fn error_is_tls(err: &(dyn std::error::Error + 'static)) -> bool { if err.downcast_ref::().is_some() { return true; } - let msg = err.to_string(); - if msg.contains("certificate") || msg.contains("TLS") || msg.contains("tls handshake") { - return true; + match socket_failure(err) { + Some(is_socket) => !is_socket, + None => false, } - err.source().is_some_and(error_is_tls) +} + +fn socket_failure(err: &(dyn std::error::Error + 'static)) -> Option { + if let Some(io) = err.downcast_ref::() { + return Some(matches!( + io.kind(), + std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::BrokenPipe + )); + } + err.source().and_then(socket_failure) } fn content_type(resp: &reqwest::Response) -> Option { @@ -420,9 +434,34 @@ fn content_type(resp: &reqwest::Response) -> Option { .map(str::to_owned) } -fn wants_stream(req: &PeerRequest, resp: &reqwest::Response) -> bool { - req.accept.as_deref() == Some("text/event-stream") - || content_type(resp).is_some_and(|ct| ct.starts_with("text/event-stream")) +fn is_probe(path: &str) -> bool { + path == PING_PATH || path == HEALTHZ_PATH +} + +/// One deadline for the whole call. DNS, the token and CA reads, the connect, +/// the headers and a buffered body all count; a slow phase must not outlive +/// the timeout the caller configured. +async fn within(deadline: tokio::time::Instant, fut: F) -> Result +where + F: std::future::Future>, +{ + match tokio::time::timeout_at(deadline, fut).await { + Err(_) => Err(PeerError::Timeout), + Ok(result) => result, + } +} + +/// SSE body: no total deadline (a follow stream is long-lived by intent), +/// only an idle bound so a silently dead connection ends. +fn stream_body(resp: reqwest::Response) -> BoxStream<'static, Result> { + Box::pin(futures::stream::unfold(resp, |mut resp| async move { + match tokio::time::timeout(STREAM_IDLE_TIMEOUT, resp.chunk()).await { + Err(_) => Some((Err(PeerError::Timeout), resp)), + Ok(Ok(Some(chunk))) => Some((Ok(chunk), resp)), + Ok(Ok(None)) => None, + Ok(Err(err)) => Some((Err(map_reqwest(err)), resp)), + } + })) } async fn read_capped(mut resp: reqwest::Response, max: u64) -> Result { @@ -441,11 +480,11 @@ async fn read_capped(mut resp: reqwest::Response, max: u64) -> Result> + Send>>, + inner: BoxStream<'static, Result>, seen: u64, max: u64, done: bool, - _permits: Permits, + _permits: Option, } impl Stream for CappedStream { @@ -464,7 +503,7 @@ impl Stream for CappedStream { } Poll::Ready(Some(Err(e))) => { this.done = true; - Poll::Ready(Some(Err(map_reqwest(e)))) + Poll::Ready(Some(Err(e))) } Poll::Ready(Some(Ok(chunk))) => { this.seen = this.seen.saturating_add(chunk.len() as u64); @@ -482,15 +521,16 @@ impl Stream for CappedStream { #[async_trait] impl PeerTransport for ReqwestTransport { async fn tcp_check(&self, peer: &PeerConfig, timeout: Duration) -> Result { - let deny = self.deny_list(); - let (_, addr) = resolve_target(peer, &deny).await?; - let started = tokio::time::Instant::now(); - let connect = tokio::net::TcpStream::connect(addr); - match tokio::time::timeout(timeout, connect).await { - Err(_) => Err(PeerError::Timeout), - Ok(Err(e)) => Err(map_io(e)), - Ok(Ok(_stream)) => Ok(started.elapsed()), - } + let deadline = tokio::time::Instant::now() + timeout; + within(deadline, async { + let deny = self.deny_list(); + let (_, addr) = resolve_target(peer, &deny).await?; + let started = tokio::time::Instant::now(); + let stream = tokio::net::TcpStream::connect(addr).await.map_err(map_io)?; + drop(stream); + Ok(started.elapsed()) + }) + .await } async fn get( @@ -499,79 +539,83 @@ impl PeerTransport for ReqwestTransport { req: PeerRequest, timeout: Duration, ) -> Result { - let url = request_url(peer, &req)?; - let deny = self.deny_list(); - let (host, addr) = resolve_target(peer, &deny).await?; - self.warn_plain_http(peer); - let token_path = peer.token_file.clone(); - let secret = tokio::task::spawn_blocking(move || load_token(&token_path)) + // A stream may outlive the deadline: only its preparation does. + let deadline = tokio::time::Instant::now() + timeout; + let probe = is_probe(&req.path); + let wants_sse = req.accept.as_deref() == Some("text/event-stream"); + let (resp, permits, max) = within(deadline, async { + let url = request_url(peer, &req)?; + let deny = self.deny_list(); + let (pin, addr) = resolve_target(peer, &deny).await?; + self.warn_plain_http(peer); + let token_path = peer.token_file.clone(); + let secret = tokio::task::spawn_blocking(move || load_token(&token_path)) + .await + .map_err(|_| PeerError::TokenFile("unreadable".into()))??; + let ca_path = peer.ca_file.clone(); + let ca = tokio::task::spawn_blocking(move || match ca_path { + None => Ok(None), + Some(path) => load_ca_path(&path), + }) .await - .map_err(|_| PeerError::TokenFile("unreadable".into()))??; - let ca_path = peer.ca_file.clone(); - let ca = tokio::task::spawn_blocking(move || match ca_path { - None => Ok(None), - Some(path) => load_ca_path(&path), - }) - .await - .map_err(|_| PeerError::Tls)??; - let auth = HeaderValue::from_str(&format!("Bearer {}", secret.expose())) - .map_err(|_| PeerError::TokenFile("invalid".into()))?; - drop(secret); - let permits = self.acquire(&peer.name)?; - let client = self.client_for(ca, &host, addr)?; - let max = self.max_bytes(); - let mut builder = client.get(url).timeout(timeout); - for (name, value) in &req.headers { - if name.as_str().eq_ignore_ascii_case("authorization") { - continue; - } - builder = builder.header(name.clone(), value.clone()); - } - builder = builder.header(HeaderName::from_static("authorization"), auth); - if let Some(accept) = &req.accept { - builder = builder.header(HeaderName::from_static("accept"), accept); - } - let pending = async { - let resp = builder.send().await.map_err(map_reqwest)?; - let status = resp.status().as_u16(); - let ct = content_type(&resp); - if wants_stream(&req, &resp) { - if resp.content_length().is_some_and(|n| n > max) { - return Err(PeerError::Oversize); + .map_err(|_| PeerError::Tls)??; + let mut auth = HeaderValue::from_str(&format!("Bearer {}", secret.expose())) + .map_err(|_| PeerError::TokenFile("invalid".into()))?; + auth.set_sensitive(true); + drop(secret); + // Probes skip the limits so a held follow stream cannot make a + // healthy peer look saturated and then down. + let permits = if probe { + None + } else { + Some(self.acquire(&peer.name)?) + }; + let client = self.client_for(ca, pin.as_deref(), addr)?; + let max = self.max_bytes(); + let mut builder = client.get(url); + for (name, value) in &req.headers { + if name.as_str().eq_ignore_ascii_case("authorization") { + continue; } - // `chunk()` rather than reqwest's `stream` feature: that - // feature pulls a wasm-streams pin this crate cannot select. - let inner = Box::pin(futures::stream::unfold(resp, |mut resp| async move { - match resp.chunk().await { - Ok(Some(chunk)) => Some((Ok(chunk), resp)), - Ok(None) => None, - Err(err) => Some((Err(err), resp)), - } - })); - return Ok(PeerResponse { - status, - content_type: ct, - body: PeerBody::Stream(Box::pin(CappedStream { - inner, - seen: 0, - max, - done: false, - _permits: permits, - })), - }); + builder = builder.header(name.clone(), value.clone()); } - let body = read_capped(resp, max).await?; - drop(permits); - Ok(PeerResponse { + builder = builder.header(HeaderName::from_static("authorization"), auth); + if let Some(accept) = &req.accept { + builder = builder.header(HeaderName::from_static("accept"), accept); + } + let resp = builder.send().await.map_err(map_reqwest)?; + Ok((resp, permits, max)) + }) + .await?; + let status = resp.status().as_u16(); + let ct = content_type(&resp); + let stream = wants_sse + || ct + .as_deref() + .is_some_and(|ct| ct.starts_with("text/event-stream")); + if stream { + if resp.content_length().is_some_and(|n| n > max) { + return Err(PeerError::Oversize); + } + return Ok(PeerResponse { status, content_type: ct, - body: PeerBody::Bytes(body), - }) - }; - match tokio::time::timeout(timeout, pending).await { - Err(_) => Err(PeerError::Timeout), - Ok(result) => result, + body: PeerBody::Stream(Box::pin(CappedStream { + inner: stream_body(resp), + seen: 0, + max, + done: false, + _permits: permits, + })), + }); } + let body = within(deadline, read_capped(resp, max)).await?; + drop(permits); + Ok(PeerResponse { + status, + content_type: ct, + body: PeerBody::Bytes(body), + }) } } @@ -1095,12 +1139,180 @@ mod tests { let _ = std::fs::remove_file(&token); } + fn url(s: &str) -> url::Url { + url::Url::parse(s).unwrap() + } + #[test] - fn loopback_names() { - assert!(host_is_loopback(Some("127.0.0.1"))); - assert!(host_is_loopback(Some("::1"))); - assert!(host_is_loopback(Some("localhost"))); - assert!(!host_is_loopback(Some("10.0.0.1"))); - assert!(!host_is_loopback(Some("beta"))); + fn loopback_hosts_as_the_url_parser_reports_them() { + assert!(url_is_loopback(&url("http://127.0.0.1:9"))); + assert!(url_is_loopback(&url("http://[::1]:9"))); + assert!(url_is_loopback(&url("http://localhost:9"))); + assert!(url_is_loopback(&url("http://LOCALHOST:9"))); + assert!(!url_is_loopback(&url("http://10.0.0.1:9"))); + assert!(!url_is_loopback(&url("http://beta:9"))); + } + + #[test] + fn ipv6_plain_http_to_loopback_needs_no_opt_in() { + let mut p = peer(9, Path::new("/tmp/does-not-matter"), None); + p.url = url("http://[::1]:9"); + assert!(!p.allow_plain_http); + assert!(validate_target(&p).is_ok()); + } + + #[tokio::test] + async fn ipv6_literal_peers_resolve_and_connect() { + let Ok(listener) = TcpListener::bind("[::1]:0").await else { + return; // no IPv6 loopback in this environment + }; + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { + return; + }; + let _ = read_req(&mut sock).await; + let _ = sock + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", + ) + .await; + } + }); + let token = write_token("t", 0o600); + let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap(); + let mut p = peer(0, &token, None); + p.url = url(&format!("http://[::1]:{port}")); + assert!(transport + .tcp_check(&p, Duration::from_secs(1)) + .await + .is_ok()); + let resp = transport + .get(&p, req("/api/v1/ping"), Duration::from_secs(2)) + .await; + let resp = resp.unwrap(); + assert_eq!(resp.status, 200); + let _ = std::fs::remove_file(&token); + } + + /// The request timeout bounds preparation, not a follow stream: chunks + /// keep arriving long after it would have expired. + #[tokio::test] + async fn a_stream_outlives_the_request_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let _ = read_req(&mut sock).await; + let _ = sock + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n", + ) + .await; + for i in 0..3u8 { + let chunk = format!("data: {i}\n\n"); + let frame = format!("{:x}\r\n{}\r\n", chunk.len(), chunk); + if sock.write_all(frame.as_bytes()).await.is_err() { + return; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + let _ = sock.write_all(b"0\r\n\r\n").await; + }); + let token = write_token("t", 0o600); + let transport = ReqwestTransport::new(&limits(2, 2, 4096)).unwrap(); + let mut r = req("/journal/follow"); + r.query.clear(); + r.accept = Some("text/event-stream".into()); + let resp = transport + .get(&peer(port, &token, None), r, Duration::from_millis(150)) + .await + .unwrap(); + assert_eq!(resp.status, 200); + let PeerBody::Stream(stream) = resp.body else { + panic!("expected a stream"); + }; + use futures::StreamExt; + let chunks: Vec<_> = stream.collect().await; + assert_eq!(chunks.len(), 3, "{chunks:?}"); + assert!(chunks.into_iter().all(|c| c.is_ok())); + let _ = std::fs::remove_file(&token); + } + + /// A held stream must not consume the slot health probes need. + #[tokio::test] + async fn probes_bypass_the_concurrency_limits() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { + return; + }; + tokio::spawn(async move { + let head = read_req(&mut sock).await; + if head.starts_with("GET /journal/follow") { + let _ = sock + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n5\r\ndata:\r\n", + ) + .await; + tokio::time::sleep(Duration::from_secs(3)).await; + } else { + let _ = sock + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", + ) + .await; + } + }); + } + }); + let token = write_token("t", 0o600); + let transport = ReqwestTransport::new(&limits(1, 4, 4096)).unwrap(); + let p = peer(port, &token, None); + let mut follow = req("/journal/follow"); + follow.query.clear(); + follow.accept = Some("text/event-stream".into()); + let held = transport + .get(&p, follow, Duration::from_secs(2)) + .await + .unwrap(); + assert!(matches!(held.body, PeerBody::Stream(_))); + + let mut ping = req("/api/v1/ping"); + ping.query.clear(); + let resp = transport.get(&p, ping, Duration::from_secs(1)).await; + assert!(resp.is_ok(), "a probe is not blocked by the held stream"); + + let err = transport + .get(&p, req("/systemd/units"), Duration::from_secs(1)) + .await + .unwrap_err(); + assert!(matches!(err, PeerError::Saturated), "{err:?}"); + drop(held); + let _ = std::fs::remove_file(&token); + } + + #[tokio::test(start_paused = true)] + async fn one_deadline_covers_every_phase() { + let deadline = tokio::time::Instant::now() + Duration::from_millis(50); + let err = within(deadline, std::future::pending::>()) + .await + .unwrap_err(); + assert!(matches!(err, PeerError::Timeout)); + + let deadline = tokio::time::Instant::now() + Duration::from_millis(50); + let ok = within(deadline, async { Ok::<_, PeerError>(7) }) + .await + .unwrap(); + assert_eq!(ok, 7); + + let deadline = tokio::time::Instant::now() + Duration::from_millis(50); + let err = within(deadline, async { Err::<(), _>(PeerError::Dns) }) + .await + .unwrap_err(); + assert!(matches!(err, PeerError::Dns)); } }