B: bound peer calls with one deadline and let streams outlive it

Count DNS, token/CA reads and headers inside the timeout, treat SSE as
long-lived with an idle bound, take targets from url::Host so IPv6
literals work, exempt health probes from the concurrency limits and
tighten error classification. Reuse validate_target in request_url and
mark the Authorization value sensitive.
This commit is contained in:
ari committed 2026-10-19 20:55:25 -04:00
1 parent 9d6fbd3288
commit 5ad94f249e
1 file changed
+362 -150
+362 -150
View File
@@ -16,6 +16,7 @@ use std::time::Duration;
use async_trait::async_trait; use async_trait::async_trait;
use axum::http::{HeaderName, HeaderValue}; use axum::http::{HeaderName, HeaderValue};
use bytes::Bytes; use bytes::Bytes;
use futures::stream::BoxStream;
use futures::Stream; use futures::Stream;
use ipnet::IpNet; use ipnet::IpNet;
use rustls::pki_types::pem::PemObject; use rustls::pki_types::pem::PemObject;
@@ -30,6 +31,27 @@ use crate::peers::types::{PeerConfig, PeerName, PeersConfig, Secret};
const MAX_TOKEN_BYTES: u64 = 4096; const MAX_TOKEN_BYTES: u64 = 4096;
/// Paths the health loops probe. They skip the concurrency limits: a held
/// follow stream must not make a healthy peer look saturated and down.
const PING_PATH: &str = "/api/v1/ping";
const HEALTHZ_PATH: &str = "/healthz";
/// A stream that is silent for longer than this is dead. Peers send SSE
/// keep-alives and end their own follow streams at `limits.max_follow_secs`,
/// so this only catches a connection that stopped without closing.
const STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(120);
/// Parse the deny list once, so startup and reload reject the same values.
pub(crate) fn deny_networks(list: &[String]) -> Result<Vec<IpNet>, PeerConfigError> {
list.iter()
.map(|s| {
s.parse::<IpNet>().map_err(|_| {
ConfigError::Invalid(format!("peers.deny_cidrs entry {s:?} is not a CIDR"))
})
})
.collect()
}
struct Limits { struct Limits {
global: Arc<Semaphore>, global: Arc<Semaphore>,
per_peer_n: usize, per_peer_n: usize,
@@ -50,7 +72,7 @@ enum CaKey {
#[derive(Clone, PartialEq, Eq, Hash)] #[derive(Clone, PartialEq, Eq, Hash)]
struct ClientKey { struct ClientKey {
ca: CaKey, ca: CaKey,
host: String, host: Option<String>,
addr: SocketAddr, addr: SocketAddr,
} }
@@ -79,15 +101,7 @@ impl ReqwestTransport {
"peers.max_response_bytes must be greater than zero".into(), "peers.max_response_bytes must be greater than zero".into(),
)); ));
} }
let deny = cfg let deny = deny_networks(&cfg.deny_cidrs)?;
.deny_cidrs
.iter()
.map(|s| {
s.parse::<IpNet>().map_err(|_| {
ConfigError::Invalid(format!("peers.deny_cidrs entry {s:?} is not a CIDR"))
})
})
.collect::<Result<Vec<_>, _>>()?;
Ok(Self { Ok(Self {
limits: Mutex::new(Limits { limits: Mutex::new(Limits {
global: Arc::new(Semaphore::new(cfg.max_concurrent_total)), global: Arc::new(Semaphore::new(cfg.max_concurrent_total)),
@@ -136,7 +150,7 @@ impl ReqwestTransport {
fn client_for( fn client_for(
&self, &self,
ca_pem: Option<Vec<u8>>, ca_pem: Option<Vec<u8>>,
host: &str, host: Option<&str>,
addr: SocketAddr, addr: SocketAddr,
) -> Result<reqwest::Client, PeerError> { ) -> Result<reqwest::Client, PeerError> {
let key = ClientKey { let key = ClientKey {
@@ -144,7 +158,7 @@ impl ReqwestTransport {
None => CaKey::Webpki, None => CaKey::Webpki,
Some(pem) => CaKey::Extra(pem.clone()), Some(pem) => CaKey::Extra(pem.clone()),
}, },
host: host.to_owned(), host: host.map(str::to_owned),
addr, addr,
}; };
let mut cache = Self::lock(&self.clients); let mut cache = Self::lock(&self.clients);
@@ -160,7 +174,7 @@ impl ReqwestTransport {
} }
fn warn_plain_http(&self, peer: &PeerConfig) { fn warn_plain_http(&self, peer: &PeerConfig) {
if peer.url.scheme() != "http" || host_is_loopback(peer.url.host_str()) { if peer.url.scheme() != "http" || url_is_loopback(&peer.url) {
return; return;
} }
let mut seen = Self::lock(&self.plain_warned); let mut seen = Self::lock(&self.plain_warned);
@@ -173,12 +187,13 @@ impl ReqwestTransport {
} }
} }
fn host_is_loopback(host: Option<&str>) -> bool { /// The URL parser already knows the host kind: an IPv6 literal is
match host { /// `Host::Ipv6`, never the bracketed string `host_str()` renders.
Some("localhost") => true, fn url_is_loopback(url: &url::Url) -> bool {
Some(h) => h match url.host() {
.parse::<std::net::IpAddr>() Some(url::Host::Ipv4(ip)) => ip.is_loopback(),
.is_ok_and(|ip| ip.is_loopback()), Some(url::Host::Ipv6(ip)) => ip.is_loopback(),
Some(url::Host::Domain(name)) => name.eq_ignore_ascii_case("localhost"),
None => false, None => false,
} }
} }
@@ -196,10 +211,7 @@ fn validate_target(peer: &PeerConfig) -> Result<(), PeerError> {
if peer.url.query().is_some() || peer.url.fragment().is_some() { if peer.url.query().is_some() || peer.url.fragment().is_some() {
return Err(PeerError::Unreachable); return Err(PeerError::Unreachable);
} }
if peer.url.scheme() == "http" if peer.url.scheme() == "http" && !url_is_loopback(&peer.url) && !peer.allow_plain_http {
&& !host_is_loopback(peer.url.host_str())
&& !peer.allow_plain_http
{
return Err(PeerError::Unreachable); return Err(PeerError::Unreachable);
} }
Ok(()) Ok(())
@@ -210,27 +222,25 @@ fn denied(ip: IpAddr, deny: &[IpNet]) -> bool {
} }
/// Resolve the peer host and reject denied addresses before any connection is /// Resolve the peer host and reject denied addresses before any connection is
/// made. The returned address is pinned into the HTTP client. /// made. The returned address is pinned into the HTTP client; an IP literal
/// needs no pin because there is no name to rebind.
async fn resolve_target( async fn resolve_target(
peer: &PeerConfig, peer: &PeerConfig,
deny: &[IpNet], deny: &[IpNet],
) -> Result<(String, SocketAddr), PeerError> { ) -> Result<(Option<String>, SocketAddr), PeerError> {
validate_target(peer)?; validate_target(peer)?;
let host = peer
.url
.host_str()
.ok_or(PeerError::Unreachable)?
.to_owned();
let port = peer let port = peer
.url .url
.port_or_known_default() .port_or_known_default()
.ok_or(PeerError::Unreachable)?; .ok_or(PeerError::Unreachable)?;
let addrs: Vec<SocketAddr> = match host.parse::<IpAddr>() { let addrs: Vec<SocketAddr> = match peer.url.host() {
Ok(ip) => vec![SocketAddr::new(ip, port)], Some(url::Host::Ipv4(ip)) => vec![SocketAddr::new(IpAddr::V4(ip), port)],
Err(_) => tokio::net::lookup_host((host.as_str(), port)) Some(url::Host::Ipv6(ip)) => vec![SocketAddr::new(IpAddr::V6(ip), port)],
Some(url::Host::Domain(name)) => tokio::net::lookup_host((name, port))
.await .await
.map_err(|_| PeerError::Dns)? .map_err(|_| PeerError::Dns)?
.collect(), .collect(),
None => return Err(PeerError::Unreachable),
}; };
let Some(addr) = addrs.first().copied() else { let Some(addr) = addrs.first().copied() else {
return Err(PeerError::Dns); return Err(PeerError::Dns);
@@ -238,24 +248,29 @@ async fn resolve_target(
if addrs.iter().any(|addr| denied(addr.ip(), deny)) { if addrs.iter().any(|addr| denied(addr.ip(), deny)) {
return Err(PeerError::Unreachable); return Err(PeerError::Unreachable);
} }
Ok((host, addr)) let pin = match peer.url.host() {
Some(url::Host::Domain(name)) => Some(name.to_owned()),
_ => None,
};
Ok((pin, addr))
} }
/// Ring is passed in so this process never calls `install_default`. /// Ring is passed in so this process never calls `install_default`.
fn build_client( fn build_client(
extra_ca: Option<&[u8]>, extra_ca: Option<&[u8]>,
host: &str, host: Option<&str>,
addr: SocketAddr, addr: SocketAddr,
) -> Result<reqwest::Client, PeerError> { ) -> Result<reqwest::Client, PeerError> {
let tls = rustls_config(extra_ca)?; let tls = rustls_config(extra_ca)?;
reqwest::Client::builder() let mut builder = reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none()) .redirect(reqwest::redirect::Policy::none())
.no_proxy() .no_proxy()
.http1_only() .http1_only()
.tls_backend_preconfigured(tls) .tls_backend_preconfigured(tls);
.resolve(host, addr) if let Some(host) = host {
.build() builder = builder.resolve(host, addr);
.map_err(|_| PeerError::Tls) }
builder.build().map_err(|_| PeerError::Tls)
} }
fn rustls_config(extra_ca: Option<&[u8]>) -> Result<ClientConfig, PeerError> { fn rustls_config(extra_ca: Option<&[u8]>) -> Result<ClientConfig, PeerError> {
@@ -283,15 +298,7 @@ fn rustls_config(extra_ca: Option<&[u8]>) -> Result<ClientConfig, PeerError> {
} }
fn request_url(peer: &PeerConfig, req: &PeerRequest) -> Result<url::Url, PeerError> { fn request_url(peer: &PeerConfig, req: &PeerRequest) -> Result<url::Url, PeerError> {
if peer.url.scheme() != "http" && peer.url.scheme() != "https" { validate_target(peer)?;
return Err(PeerError::Unreachable);
}
if !peer.url.username().is_empty() || peer.url.password().is_some() {
return Err(PeerError::Unreachable);
}
if peer.url.query().is_some() || peer.url.fragment().is_some() {
return Err(PeerError::Unreachable);
}
let mut url = peer.url.clone(); let mut url = peer.url.clone();
let base = url.path().trim_end_matches('/'); let base = url.path().trim_end_matches('/');
let rel = req.path.trim_start_matches('/'); let rel = req.path.trim_start_matches('/');
@@ -374,15 +381,6 @@ fn map_io(err: std::io::Error) -> PeerError {
if err.kind() == std::io::ErrorKind::TimedOut { if err.kind() == std::io::ErrorKind::TimedOut {
return PeerError::Timeout; return PeerError::Timeout;
} }
let msg = err.to_string();
if msg.contains("Name or service not known")
|| msg.contains("nodename nor servname")
|| msg.contains("Temporary failure in name resolution")
|| msg.contains("No address associated")
|| msg.contains("Name does not resolve")
{
return PeerError::Dns;
}
PeerError::Unreachable PeerError::Unreachable
} }
@@ -402,15 +400,31 @@ fn map_reqwest(err: reqwest::Error) -> PeerError {
PeerError::Unreachable PeerError::Unreachable
} }
/// hyper-rustls reports a handshake failure as an `io::Error` with a
/// non-socket kind, while a refused or reset connection has a socket kind.
/// The kind separates them without matching message text.
fn error_is_tls(err: &(dyn std::error::Error + 'static)) -> bool { fn error_is_tls(err: &(dyn std::error::Error + 'static)) -> bool {
if err.downcast_ref::<rustls::Error>().is_some() { if err.downcast_ref::<rustls::Error>().is_some() {
return true; return true;
} }
let msg = err.to_string(); match socket_failure(err) {
if msg.contains("certificate") || msg.contains("TLS") || msg.contains("tls handshake") { Some(is_socket) => !is_socket,
return true; None => false,
} }
err.source().is_some_and(error_is_tls) }
fn socket_failure(err: &(dyn std::error::Error + 'static)) -> Option<bool> {
if let Some(io) = err.downcast_ref::<std::io::Error>() {
return Some(matches!(
io.kind(),
std::io::ErrorKind::ConnectionRefused
| std::io::ErrorKind::ConnectionReset
| std::io::ErrorKind::ConnectionAborted
| std::io::ErrorKind::TimedOut
| std::io::ErrorKind::BrokenPipe
));
}
err.source().and_then(socket_failure)
} }
fn content_type(resp: &reqwest::Response) -> Option<String> { fn content_type(resp: &reqwest::Response) -> Option<String> {
@@ -420,9 +434,34 @@ fn content_type(resp: &reqwest::Response) -> Option<String> {
.map(str::to_owned) .map(str::to_owned)
} }
fn wants_stream(req: &PeerRequest, resp: &reqwest::Response) -> bool { fn is_probe(path: &str) -> bool {
req.accept.as_deref() == Some("text/event-stream") path == PING_PATH || path == HEALTHZ_PATH
|| content_type(resp).is_some_and(|ct| ct.starts_with("text/event-stream")) }
/// One deadline for the whole call. DNS, the token and CA reads, the connect,
/// the headers and a buffered body all count; a slow phase must not outlive
/// the timeout the caller configured.
async fn within<F, T>(deadline: tokio::time::Instant, fut: F) -> Result<T, PeerError>
where
F: std::future::Future<Output = Result<T, PeerError>>,
{
match tokio::time::timeout_at(deadline, fut).await {
Err(_) => Err(PeerError::Timeout),
Ok(result) => result,
}
}
/// SSE body: no total deadline (a follow stream is long-lived by intent),
/// only an idle bound so a silently dead connection ends.
fn stream_body(resp: reqwest::Response) -> BoxStream<'static, Result<Bytes, PeerError>> {
Box::pin(futures::stream::unfold(resp, |mut resp| async move {
match tokio::time::timeout(STREAM_IDLE_TIMEOUT, resp.chunk()).await {
Err(_) => Some((Err(PeerError::Timeout), resp)),
Ok(Ok(Some(chunk))) => Some((Ok(chunk), resp)),
Ok(Ok(None)) => None,
Ok(Err(err)) => Some((Err(map_reqwest(err)), resp)),
}
}))
} }
async fn read_capped(mut resp: reqwest::Response, max: u64) -> Result<Bytes, PeerError> { async fn read_capped(mut resp: reqwest::Response, max: u64) -> Result<Bytes, PeerError> {
@@ -441,11 +480,11 @@ async fn read_capped(mut resp: reqwest::Response, max: u64) -> Result<Bytes, Pee
} }
struct CappedStream { struct CappedStream {
inner: Pin<Box<dyn Stream<Item = Result<Bytes, reqwest::Error>> + Send>>, inner: BoxStream<'static, Result<Bytes, PeerError>>,
seen: u64, seen: u64,
max: u64, max: u64,
done: bool, done: bool,
_permits: Permits, _permits: Option<Permits>,
} }
impl Stream for CappedStream { impl Stream for CappedStream {
@@ -464,7 +503,7 @@ impl Stream for CappedStream {
} }
Poll::Ready(Some(Err(e))) => { Poll::Ready(Some(Err(e))) => {
this.done = true; this.done = true;
Poll::Ready(Some(Err(map_reqwest(e)))) Poll::Ready(Some(Err(e)))
} }
Poll::Ready(Some(Ok(chunk))) => { Poll::Ready(Some(Ok(chunk))) => {
this.seen = this.seen.saturating_add(chunk.len() as u64); this.seen = this.seen.saturating_add(chunk.len() as u64);
@@ -482,15 +521,16 @@ impl Stream for CappedStream {
#[async_trait] #[async_trait]
impl PeerTransport for ReqwestTransport { impl PeerTransport for ReqwestTransport {
async fn tcp_check(&self, peer: &PeerConfig, timeout: Duration) -> Result<Duration, PeerError> { async fn tcp_check(&self, peer: &PeerConfig, timeout: Duration) -> Result<Duration, PeerError> {
let deny = self.deny_list(); let deadline = tokio::time::Instant::now() + timeout;
let (_, addr) = resolve_target(peer, &deny).await?; within(deadline, async {
let started = tokio::time::Instant::now(); let deny = self.deny_list();
let connect = tokio::net::TcpStream::connect(addr); let (_, addr) = resolve_target(peer, &deny).await?;
match tokio::time::timeout(timeout, connect).await { let started = tokio::time::Instant::now();
Err(_) => Err(PeerError::Timeout), let stream = tokio::net::TcpStream::connect(addr).await.map_err(map_io)?;
Ok(Err(e)) => Err(map_io(e)), drop(stream);
Ok(Ok(_stream)) => Ok(started.elapsed()), Ok(started.elapsed())
} })
.await
} }
async fn get( async fn get(
@@ -499,79 +539,83 @@ impl PeerTransport for ReqwestTransport {
req: PeerRequest, req: PeerRequest,
timeout: Duration, timeout: Duration,
) -> Result<PeerResponse, PeerError> { ) -> Result<PeerResponse, PeerError> {
let url = request_url(peer, &req)?; // A stream may outlive the deadline: only its preparation does.
let deny = self.deny_list(); let deadline = tokio::time::Instant::now() + timeout;
let (host, addr) = resolve_target(peer, &deny).await?; let probe = is_probe(&req.path);
self.warn_plain_http(peer); let wants_sse = req.accept.as_deref() == Some("text/event-stream");
let token_path = peer.token_file.clone(); let (resp, permits, max) = within(deadline, async {
let secret = tokio::task::spawn_blocking(move || load_token(&token_path)) let url = request_url(peer, &req)?;
let deny = self.deny_list();
let (pin, addr) = resolve_target(peer, &deny).await?;
self.warn_plain_http(peer);
let token_path = peer.token_file.clone();
let secret = tokio::task::spawn_blocking(move || load_token(&token_path))
.await
.map_err(|_| PeerError::TokenFile("unreadable".into()))??;
let ca_path = peer.ca_file.clone();
let ca = tokio::task::spawn_blocking(move || match ca_path {
None => Ok(None),
Some(path) => load_ca_path(&path),
})
.await .await
.map_err(|_| PeerError::TokenFile("unreadable".into()))??; .map_err(|_| PeerError::Tls)??;
let ca_path = peer.ca_file.clone(); let mut auth = HeaderValue::from_str(&format!("Bearer {}", secret.expose()))
let ca = tokio::task::spawn_blocking(move || match ca_path { .map_err(|_| PeerError::TokenFile("invalid".into()))?;
None => Ok(None), auth.set_sensitive(true);
Some(path) => load_ca_path(&path), drop(secret);
}) // Probes skip the limits so a held follow stream cannot make a
.await // healthy peer look saturated and then down.
.map_err(|_| PeerError::Tls)??; let permits = if probe {
let auth = HeaderValue::from_str(&format!("Bearer {}", secret.expose())) None
.map_err(|_| PeerError::TokenFile("invalid".into()))?; } else {
drop(secret); Some(self.acquire(&peer.name)?)
let permits = self.acquire(&peer.name)?; };
let client = self.client_for(ca, &host, addr)?; let client = self.client_for(ca, pin.as_deref(), addr)?;
let max = self.max_bytes(); let max = self.max_bytes();
let mut builder = client.get(url).timeout(timeout); let mut builder = client.get(url);
for (name, value) in &req.headers { for (name, value) in &req.headers {
if name.as_str().eq_ignore_ascii_case("authorization") { if name.as_str().eq_ignore_ascii_case("authorization") {
continue; continue;
}
builder = builder.header(name.clone(), value.clone());
}
builder = builder.header(HeaderName::from_static("authorization"), auth);
if let Some(accept) = &req.accept {
builder = builder.header(HeaderName::from_static("accept"), accept);
}
let pending = async {
let resp = builder.send().await.map_err(map_reqwest)?;
let status = resp.status().as_u16();
let ct = content_type(&resp);
if wants_stream(&req, &resp) {
if resp.content_length().is_some_and(|n| n > max) {
return Err(PeerError::Oversize);
} }
// `chunk()` rather than reqwest's `stream` feature: that builder = builder.header(name.clone(), value.clone());
// feature pulls a wasm-streams pin this crate cannot select.
let inner = Box::pin(futures::stream::unfold(resp, |mut resp| async move {
match resp.chunk().await {
Ok(Some(chunk)) => Some((Ok(chunk), resp)),
Ok(None) => None,
Err(err) => Some((Err(err), resp)),
}
}));
return Ok(PeerResponse {
status,
content_type: ct,
body: PeerBody::Stream(Box::pin(CappedStream {
inner,
seen: 0,
max,
done: false,
_permits: permits,
})),
});
} }
let body = read_capped(resp, max).await?; builder = builder.header(HeaderName::from_static("authorization"), auth);
drop(permits); if let Some(accept) = &req.accept {
Ok(PeerResponse { builder = builder.header(HeaderName::from_static("accept"), accept);
}
let resp = builder.send().await.map_err(map_reqwest)?;
Ok((resp, permits, max))
})
.await?;
let status = resp.status().as_u16();
let ct = content_type(&resp);
let stream = wants_sse
|| ct
.as_deref()
.is_some_and(|ct| ct.starts_with("text/event-stream"));
if stream {
if resp.content_length().is_some_and(|n| n > max) {
return Err(PeerError::Oversize);
}
return Ok(PeerResponse {
status, status,
content_type: ct, content_type: ct,
body: PeerBody::Bytes(body), body: PeerBody::Stream(Box::pin(CappedStream {
}) inner: stream_body(resp),
}; seen: 0,
match tokio::time::timeout(timeout, pending).await { max,
Err(_) => Err(PeerError::Timeout), done: false,
Ok(result) => result, _permits: permits,
})),
});
} }
let body = within(deadline, read_capped(resp, max)).await?;
drop(permits);
Ok(PeerResponse {
status,
content_type: ct,
body: PeerBody::Bytes(body),
})
} }
} }
@@ -1095,12 +1139,180 @@ mod tests {
let _ = std::fs::remove_file(&token); let _ = std::fs::remove_file(&token);
} }
fn url(s: &str) -> url::Url {
url::Url::parse(s).unwrap()
}
#[test] #[test]
fn loopback_names() { fn loopback_hosts_as_the_url_parser_reports_them() {
assert!(host_is_loopback(Some("127.0.0.1"))); assert!(url_is_loopback(&url("http://127.0.0.1:9")));
assert!(host_is_loopback(Some("::1"))); assert!(url_is_loopback(&url("http://[::1]:9")));
assert!(host_is_loopback(Some("localhost"))); assert!(url_is_loopback(&url("http://localhost:9")));
assert!(!host_is_loopback(Some("10.0.0.1"))); assert!(url_is_loopback(&url("http://LOCALHOST:9")));
assert!(!host_is_loopback(Some("beta"))); assert!(!url_is_loopback(&url("http://10.0.0.1:9")));
assert!(!url_is_loopback(&url("http://beta:9")));
}
#[test]
fn ipv6_plain_http_to_loopback_needs_no_opt_in() {
let mut p = peer(9, Path::new("/tmp/does-not-matter"), None);
p.url = url("http://[::1]:9");
assert!(!p.allow_plain_http);
assert!(validate_target(&p).is_ok());
}
#[tokio::test]
async fn ipv6_literal_peers_resolve_and_connect() {
let Ok(listener) = TcpListener::bind("[::1]:0").await else {
return; // no IPv6 loopback in this environment
};
let port = listener.local_addr().unwrap().port();
tokio::spawn(async move {
loop {
let Ok((mut sock, _)) = listener.accept().await else {
return;
};
let _ = read_req(&mut sock).await;
let _ = sock
.write_all(
b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok",
)
.await;
}
});
let token = write_token("t", 0o600);
let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap();
let mut p = peer(0, &token, None);
p.url = url(&format!("http://[::1]:{port}"));
assert!(transport
.tcp_check(&p, Duration::from_secs(1))
.await
.is_ok());
let resp = transport
.get(&p, req("/api/v1/ping"), Duration::from_secs(2))
.await;
let resp = resp.unwrap();
assert_eq!(resp.status, 200);
let _ = std::fs::remove_file(&token);
}
/// The request timeout bounds preparation, not a follow stream: chunks
/// keep arriving long after it would have expired.
#[tokio::test]
async fn a_stream_outlives_the_request_timeout() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
tokio::spawn(async move {
let (mut sock, _) = listener.accept().await.unwrap();
let _ = read_req(&mut sock).await;
let _ = sock
.write_all(
b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n",
)
.await;
for i in 0..3u8 {
let chunk = format!("data: {i}\n\n");
let frame = format!("{:x}\r\n{}\r\n", chunk.len(), chunk);
if sock.write_all(frame.as_bytes()).await.is_err() {
return;
}
tokio::time::sleep(Duration::from_millis(250)).await;
}
let _ = sock.write_all(b"0\r\n\r\n").await;
});
let token = write_token("t", 0o600);
let transport = ReqwestTransport::new(&limits(2, 2, 4096)).unwrap();
let mut r = req("/journal/follow");
r.query.clear();
r.accept = Some("text/event-stream".into());
let resp = transport
.get(&peer(port, &token, None), r, Duration::from_millis(150))
.await
.unwrap();
assert_eq!(resp.status, 200);
let PeerBody::Stream(stream) = resp.body else {
panic!("expected a stream");
};
use futures::StreamExt;
let chunks: Vec<_> = stream.collect().await;
assert_eq!(chunks.len(), 3, "{chunks:?}");
assert!(chunks.into_iter().all(|c| c.is_ok()));
let _ = std::fs::remove_file(&token);
}
/// A held stream must not consume the slot health probes need.
#[tokio::test]
async fn probes_bypass_the_concurrency_limits() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
tokio::spawn(async move {
loop {
let Ok((mut sock, _)) = listener.accept().await else {
return;
};
tokio::spawn(async move {
let head = read_req(&mut sock).await;
if head.starts_with("GET /journal/follow") {
let _ = sock
.write_all(
b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n5\r\ndata:\r\n",
)
.await;
tokio::time::sleep(Duration::from_secs(3)).await;
} else {
let _ = sock
.write_all(
b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok",
)
.await;
}
});
}
});
let token = write_token("t", 0o600);
let transport = ReqwestTransport::new(&limits(1, 4, 4096)).unwrap();
let p = peer(port, &token, None);
let mut follow = req("/journal/follow");
follow.query.clear();
follow.accept = Some("text/event-stream".into());
let held = transport
.get(&p, follow, Duration::from_secs(2))
.await
.unwrap();
assert!(matches!(held.body, PeerBody::Stream(_)));
let mut ping = req("/api/v1/ping");
ping.query.clear();
let resp = transport.get(&p, ping, Duration::from_secs(1)).await;
assert!(resp.is_ok(), "a probe is not blocked by the held stream");
let err = transport
.get(&p, req("/systemd/units"), Duration::from_secs(1))
.await
.unwrap_err();
assert!(matches!(err, PeerError::Saturated), "{err:?}");
drop(held);
let _ = std::fs::remove_file(&token);
}
#[tokio::test(start_paused = true)]
async fn one_deadline_covers_every_phase() {
let deadline = tokio::time::Instant::now() + Duration::from_millis(50);
let err = within(deadline, std::future::pending::<Result<(), PeerError>>())
.await
.unwrap_err();
assert!(matches!(err, PeerError::Timeout));
let deadline = tokio::time::Instant::now() + Duration::from_millis(50);
let ok = within(deadline, async { Ok::<_, PeerError>(7) })
.await
.unwrap();
assert_eq!(ok, 7);
let deadline = tokio::time::Instant::now() + Duration::from_millis(50);
let err = within(deadline, async { Err::<(), _>(PeerError::Dns) })
.await
.unwrap_err();
assert!(matches!(err, PeerError::Dns));
} }
} }