B: B1 real reqwest peer client with caps, timeouts and ring TLS

This commit is contained in:
ari committed 2026-10-19 20:55:25 -04:00
1 parent 58b41fa001
commit bd5b21bcc4
6 files changed
+1117

No files matched your search

+971
View File
@@ -0,0 +1,971 @@
//! Outbound GET client used by health checks and the proxy.
//!
//! `reqwest` is built on a rustls `ClientConfig` whose crypto provider is
//! ring, passed in here. Nothing is installed process-wide, so the OTLP
//! client's provider choice is left alone. Redirects stay off: a peer URL is
//! the only address we will talk to.
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use std::time::Duration;
use async_trait::async_trait;
use axum::http::{HeaderName, HeaderValue};
use bytes::Bytes;
use futures::Stream;
use rustls::pki_types::pem::PemObject;
use rustls::pki_types::CertificateDer;
use rustls::{ClientConfig, RootCertStore};
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use crate::config::ConfigError;
use crate::peers::error::{PeerConfigError, PeerError};
use crate::peers::transport::{PeerBody, PeerRequest, PeerResponse, PeerTransport};
use crate::peers::types::{PeerConfig, PeerName, PeersConfig, Secret};
const MAX_TOKEN_BYTES: u64 = 4096;
struct Limits {
global: Arc<Semaphore>,
per_peer_n: usize,
per_peer: HashMap<PeerName, Arc<Semaphore>>,
max_response_bytes: u64,
}
#[derive(Clone, PartialEq, Eq, Hash)]
enum CaKey {
Webpki,
Extra(Vec<u8>),
}
struct Permits {
_global: OwnedSemaphorePermit,
_peer: OwnedSemaphorePermit,
}
/// Real [`PeerTransport`]. One shared client for peers that use the webpki
/// roots, plus one extra client per distinct custom CA.
pub struct ReqwestTransport {
limits: Mutex<Limits>,
clients: Mutex<HashMap<CaKey, reqwest::Client>>,
plain_warned: Mutex<HashSet<PeerName>>,
}
impl ReqwestTransport {
pub fn new(cfg: &PeersConfig) -> Result<Self, PeerConfigError> {
if cfg.max_concurrent_per_peer == 0 || cfg.max_concurrent_total == 0 {
return Err(ConfigError::Invalid(
"peers concurrency limits must be greater than zero".into(),
));
}
if cfg.max_response_bytes == 0 {
return Err(ConfigError::Invalid(
"peers.max_response_bytes must be greater than zero".into(),
));
}
Ok(Self {
limits: Mutex::new(Limits {
global: Arc::new(Semaphore::new(cfg.max_concurrent_total)),
per_peer_n: cfg.max_concurrent_per_peer,
per_peer: HashMap::new(),
max_response_bytes: cfg.max_response_bytes,
}),
clients: Mutex::new(HashMap::new()),
plain_warned: Mutex::new(HashSet::new()),
})
}
fn lock<'a, T>(m: &'a Mutex<T>) -> std::sync::MutexGuard<'a, T> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
fn max_bytes(&self) -> u64 {
Self::lock(&self.limits).max_response_bytes
}
fn acquire(&self, name: &PeerName) -> Result<Permits, PeerError> {
let mut limits = Self::lock(&self.limits);
let global = limits.global.clone();
let permits = limits.per_peer_n;
let per = limits
.per_peer
.entry(name.clone())
.or_insert_with(|| Arc::new(Semaphore::new(permits)))
.clone();
drop(limits);
let global = global
.try_acquire_owned()
.map_err(|_| PeerError::Saturated)?;
let peer = per.try_acquire_owned().map_err(|_| PeerError::Saturated)?;
Ok(Permits {
_global: global,
_peer: peer,
})
}
fn client_for(&self, ca_pem: Option<Vec<u8>>) -> Result<reqwest::Client, PeerError> {
let key = match &ca_pem {
None => CaKey::Webpki,
Some(pem) => CaKey::Extra(pem.clone()),
};
let mut cache = Self::lock(&self.clients);
if let Some(client) = cache.get(&key) {
return Ok(client.clone());
}
if cache.len() >= crate::peers::types::MAX_PEERS {
cache.clear();
}
let client = build_client(ca_pem.as_deref())?;
cache.insert(key, client.clone());
Ok(client)
}
fn warn_plain_http(&self, peer: &PeerConfig) {
if peer.url.scheme() != "http" || host_is_loopback(peer.url.host_str()) {
return;
}
let mut seen = Self::lock(&self.plain_warned);
if seen.insert(peer.name.clone()) {
tracing::warn!(
peer = %peer.name,
"plain HTTP to a non-loopback peer; the bearer token travels in clear"
);
}
}
}
fn host_is_loopback(host: Option<&str>) -> bool {
match host {
Some("localhost") => true,
Some(h) => h
.parse::<std::net::IpAddr>()
.is_ok_and(|ip| ip.is_loopback()),
None => false,
}
}
/// Ring is passed in so this process never calls `install_default`.
fn build_client(extra_ca: Option<&[u8]>) -> Result<reqwest::Client, PeerError> {
let tls = rustls_config(extra_ca)?;
reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.no_proxy()
.http1_only()
.tls_backend_preconfigured(tls)
.build()
.map_err(|_| PeerError::Tls)
}
fn rustls_config(extra_ca: Option<&[u8]>) -> Result<ClientConfig, PeerError> {
let provider = Arc::new(rustls::crypto::ring::default_provider());
let mut roots = RootCertStore::empty();
roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
if let Some(pem) = extra_ca {
let mut any = false;
for cert in CertificateDer::pem_slice_iter(pem) {
let cert = cert.map_err(|_| PeerError::Tls)?;
roots.add(cert).map_err(|_| PeerError::Tls)?;
any = true;
}
if !any {
return Err(PeerError::Tls);
}
}
let mut cfg = ClientConfig::builder_with_provider(provider)
.with_safe_default_protocol_versions()
.map_err(|_| PeerError::Tls)?
.with_root_certificates(roots)
.with_no_client_auth();
cfg.alpn_protocols = vec![b"http/1.1".to_vec()];
Ok(cfg)
}
fn request_url(peer: &PeerConfig, req: &PeerRequest) -> Result<url::Url, PeerError> {
if peer.url.scheme() != "http" && peer.url.scheme() != "https" {
return Err(PeerError::Unreachable);
}
if !peer.url.username().is_empty() || peer.url.password().is_some() {
return Err(PeerError::Unreachable);
}
if peer.url.query().is_some() || peer.url.fragment().is_some() {
return Err(PeerError::Unreachable);
}
let mut url = peer.url.clone();
let base = url.path().trim_end_matches('/');
let rel = req.path.trim_start_matches('/');
let path = if rel.is_empty() {
if base.is_empty() {
"/".to_owned()
} else {
base.to_owned()
}
} else {
format!("{base}/{rel}")
};
url.set_path(&path);
url.set_query(None);
url.set_fragment(None);
if !req.query.is_empty() {
{
let mut pairs = url.query_pairs_mut();
for (k, v) in &req.query {
pairs.append_pair(k, v);
}
}
}
Ok(url)
}
fn load_token(path: &Path) -> Result<Secret, PeerError> {
let creds = std::env::var_os("CREDENTIALS_DIRECTORY").map(PathBuf::from);
load_token_in(path, creds.as_deref())
}
fn load_token_in(path: &Path, credentials_dir: Option<&Path>) -> Result<Secret, PeerError> {
let meta = std::fs::metadata(path).map_err(|_| PeerError::TokenFile("missing".into()))?;
if !meta.is_file() {
return Err(PeerError::TokenFile("not a file".into()));
}
if meta.len() > MAX_TOKEN_BYTES {
return Err(PeerError::TokenFile("too large".into()));
}
if !under_credentials(path, credentials_dir) {
let mode = {
use std::os::unix::fs::PermissionsExt;
meta.permissions().mode()
};
if mode & 0o077 != 0 {
return Err(PeerError::TokenFile("group or world readable".into()));
}
}
let mut bytes = std::fs::read(path).map_err(|_| PeerError::TokenFile("unreadable".into()))?;
if bytes.len() as u64 > MAX_TOKEN_BYTES {
return Err(PeerError::TokenFile("too large".into()));
}
if bytes.last() == Some(&b'\n') {
bytes.pop();
if bytes.last() == Some(&b'\r') {
bytes.pop();
}
}
if bytes.is_empty() {
return Err(PeerError::TokenFile("empty".into()));
}
if bytes.iter().any(|b| *b == b'\n' || *b == b'\r' || *b == 0) {
return Err(PeerError::TokenFile("invalid".into()));
}
let text = String::from_utf8(bytes).map_err(|_| PeerError::TokenFile("invalid".into()))?;
if HeaderValue::from_str(&format!("Bearer {text}")).is_err() {
return Err(PeerError::TokenFile("invalid".into()));
}
Ok(Secret::new(text))
}
fn under_credentials(path: &Path, credentials_dir: Option<&Path>) -> bool {
match credentials_dir {
Some(dir) if !dir.as_os_str().is_empty() => path.starts_with(dir),
_ => false,
}
}
fn map_io(err: std::io::Error) -> PeerError {
if err.kind() == std::io::ErrorKind::TimedOut {
return PeerError::Timeout;
}
let msg = err.to_string();
if msg.contains("Name or service not known")
|| msg.contains("nodename nor servname")
|| msg.contains("Temporary failure in name resolution")
|| msg.contains("No address associated")
|| msg.contains("Name does not resolve")
{
return PeerError::Dns;
}
PeerError::Unreachable
}
fn map_reqwest(err: reqwest::Error) -> PeerError {
if err.is_timeout() {
return PeerError::Timeout;
}
if err.is_dns() {
return PeerError::Dns;
}
if error_is_tls(&err) {
return PeerError::Tls;
}
if err.is_connect() {
return PeerError::Unreachable;
}
PeerError::Unreachable
}
fn error_is_tls(err: &(dyn std::error::Error + 'static)) -> bool {
if err.downcast_ref::<rustls::Error>().is_some() {
return true;
}
let msg = err.to_string();
if msg.contains("certificate") || msg.contains("TLS") || msg.contains("tls handshake") {
return true;
}
err.source().is_some_and(error_is_tls)
}
fn content_type(resp: &reqwest::Response) -> Option<String> {
resp.headers()
.get(axum::http::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.map(str::to_owned)
}
fn wants_stream(req: &PeerRequest, resp: &reqwest::Response) -> bool {
req.accept.as_deref() == Some("text/event-stream")
|| content_type(resp).is_some_and(|ct| ct.starts_with("text/event-stream"))
}
async fn read_capped(mut resp: reqwest::Response, max: u64) -> Result<Bytes, PeerError> {
if resp.content_length().is_some_and(|n| n > max) {
return Err(PeerError::Oversize);
}
let mut buf = Vec::new();
while let Some(chunk) = resp.chunk().await.map_err(map_reqwest)? {
let next = (buf.len() as u64).saturating_add(chunk.len() as u64);
if next > max {
return Err(PeerError::Oversize);
}
buf.extend_from_slice(&chunk);
}
Ok(Bytes::from(buf))
}
struct CappedStream {
inner: Pin<Box<dyn Stream<Item = Result<Bytes, reqwest::Error>> + Send>>,
seen: u64,
max: u64,
done: bool,
_permits: Permits,
}
impl Stream for CappedStream {
type Item = Result<Bytes, PeerError>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
if this.done {
return Poll::Ready(None);
}
match this.inner.as_mut().poll_next(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(None) => {
this.done = true;
Poll::Ready(None)
}
Poll::Ready(Some(Err(e))) => {
this.done = true;
Poll::Ready(Some(Err(map_reqwest(e))))
}
Poll::Ready(Some(Ok(chunk))) => {
this.seen = this.seen.saturating_add(chunk.len() as u64);
if this.seen > this.max {
this.done = true;
Poll::Ready(Some(Err(PeerError::Oversize)))
} else {
Poll::Ready(Some(Ok(chunk)))
}
}
}
}
}
#[async_trait]
impl PeerTransport for ReqwestTransport {
async fn tcp_check(&self, peer: &PeerConfig, timeout: Duration) -> Result<Duration, PeerError> {
let host = peer
.url
.host_str()
.ok_or(PeerError::Unreachable)?
.to_owned();
let port = peer
.url
.port_or_known_default()
.ok_or(PeerError::Unreachable)?;
let started = tokio::time::Instant::now();
let connect = tokio::net::TcpStream::connect((host.as_str(), port));
match tokio::time::timeout(timeout, connect).await {
Err(_) => Err(PeerError::Timeout),
Ok(Err(e)) => Err(map_io(e)),
Ok(Ok(_stream)) => Ok(started.elapsed()),
}
}
async fn get(
&self,
peer: &PeerConfig,
req: PeerRequest,
timeout: Duration,
) -> Result<PeerResponse, PeerError> {
let url = request_url(peer, &req)?;
self.warn_plain_http(peer);
let token_path = peer.token_file.clone();
let secret = tokio::task::spawn_blocking(move || load_token(&token_path))
.await
.map_err(|_| PeerError::TokenFile("unreadable".into()))??;
let ca_path = peer.ca_file.clone();
let ca = tokio::task::spawn_blocking(move || match ca_path {
None => Ok(None),
Some(path) => load_ca_path(&path),
})
.await
.map_err(|_| PeerError::Tls)??;
let auth = HeaderValue::from_str(&format!("Bearer {}", secret.expose()))
.map_err(|_| PeerError::TokenFile("invalid".into()))?;
drop(secret);
let permits = self.acquire(&peer.name)?;
let client = self.client_for(ca)?;
let max = self.max_bytes();
let mut builder = client.get(url).timeout(timeout);
for (name, value) in &req.headers {
if name.as_str().eq_ignore_ascii_case("authorization") {
continue;
}
builder = builder.header(name.clone(), value.clone());
}
builder = builder.header(HeaderName::from_static("authorization"), auth);
if let Some(accept) = &req.accept {
builder = builder.header(HeaderName::from_static("accept"), accept);
}
let pending = async {
let resp = builder.send().await.map_err(map_reqwest)?;
let status = resp.status().as_u16();
let ct = content_type(&resp);
if wants_stream(&req, &resp) {
if resp.content_length().is_some_and(|n| n > max) {
return Err(PeerError::Oversize);
}
// `chunk()` rather than reqwest's `stream` feature: that
// feature pulls a wasm-streams pin this crate cannot select.
let inner = Box::pin(futures::stream::unfold(resp, |mut resp| async move {
match resp.chunk().await {
Ok(Some(chunk)) => Some((Ok(chunk), resp)),
Ok(None) => None,
Err(err) => Some((Err(err), resp)),
}
}));
return Ok(PeerResponse {
status,
content_type: ct,
body: PeerBody::Stream(Box::pin(CappedStream {
inner,
seen: 0,
max,
done: false,
_permits: permits,
})),
});
}
let body = read_capped(resp, max).await?;
drop(permits);
Ok(PeerResponse {
status,
content_type: ct,
body: PeerBody::Bytes(body),
})
};
match tokio::time::timeout(timeout, pending).await {
Err(_) => Err(PeerError::Timeout),
Ok(result) => result,
}
}
}
fn load_ca_path(path: &Path) -> Result<Option<Vec<u8>>, PeerError> {
let bytes = std::fs::read(path).map_err(|_| PeerError::Tls)?;
if bytes.is_empty() || bytes.len() > 1024 * 1024 {
return Err(PeerError::Tls);
}
Ok(Some(bytes))
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
use axum::http::HeaderValue as Hv;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
fn limits(per: usize, total: usize, max_bytes: u64) -> PeersConfig {
PeersConfig {
enabled: true,
max_concurrent_per_peer: per,
max_concurrent_total: total,
max_response_bytes: max_bytes,
..PeersConfig::default()
}
}
fn write_token(contents: &str, mode: u32) -> PathBuf {
static N: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let path = std::env::temp_dir().join(format!(
"slinuxm-peer-token-{}-{}",
std::process::id(),
N.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
));
let mut opts = std::fs::OpenOptions::new();
opts.write(true).create(true).truncate(true).mode(mode);
opts.open(&path)
.unwrap()
.write_all(contents.as_bytes())
.unwrap();
path
}
fn peer(port: u16, token: &Path, ca: Option<PathBuf>) -> PeerConfig {
PeerConfig {
name: PeerName::parse("beta").unwrap(),
url: url::Url::parse(&format!("http://127.0.0.1:{port}")).unwrap(),
token_file: token.to_path_buf(),
ca_file: ca,
allow_plain_http: false,
enabled: true,
}
}
fn req(path: &str) -> PeerRequest {
PeerRequest {
path: path.into(),
query: vec![("n".into(), "1".into())],
headers: vec![(
HeaderName::from_static("x-slinuxm-hop"),
Hv::from_static("1"),
)],
accept: None,
}
}
async fn read_req(sock: &mut tokio::net::TcpStream) -> String {
let mut buf = Vec::new();
let mut tmp = [0u8; 1024];
while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
let n = tokio::time::timeout(Duration::from_secs(2), sock.read(&mut tmp))
.await
.unwrap()
.unwrap();
if n == 0 {
break;
}
buf.extend_from_slice(&tmp[..n]);
}
String::from_utf8_lossy(&buf).into_owned()
}
async fn serve(response: Vec<u8>) -> u16 {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
tokio::spawn(async move {
let (mut sock, _) = listener.accept().await.unwrap();
let _ = read_req(&mut sock).await;
let _ = sock.write_all(&response).await;
});
port
}
#[test]
fn token_file_rules() {
let ok = write_token("abc\n", 0o600);
let secret = load_token_in(&ok, None).unwrap();
assert_eq!(secret.expose(), "abc");
assert!(!format!("{secret:?}").contains("abc"));
let _ = std::fs::remove_file(&ok);
let crlf = write_token("abc\r\n", 0o600);
assert_eq!(load_token_in(&crlf, None).unwrap().expose(), "abc");
let _ = std::fs::remove_file(&crlf);
// One trailing newline is stripped. A second one would land in the
// header, so the file is rejected rather than forwarded.
let doubled = write_token("abc\n\n", 0o600);
assert!(load_token_in(&doubled, None).is_err());
let _ = std::fs::remove_file(&doubled);
for (body, mode) in [("\n", 0o600), ("", 0o600), ("x", 0o644), ("x", 0o640)] {
let path = write_token(body, mode);
assert!(load_token_in(&path, None).is_err(), "{body:?} {mode:o}");
let _ = std::fs::remove_file(&path);
}
let big = write_token(&"a".repeat(4097), 0o600);
assert!(matches!(
load_token_in(&big, None),
Err(PeerError::TokenFile(_))
));
let _ = std::fs::remove_file(&big);
let loose = write_token("secret-value\n", 0o644);
let dir = loose.parent().unwrap();
assert!(load_token_in(&loose, Some(dir)).is_ok());
let err = load_token_in(&loose, None).unwrap_err();
assert!(matches!(err, PeerError::TokenFile(_)));
assert!(!format!("{err:?}").contains("secret-value"));
let _ = std::fs::remove_file(&loose);
assert!(load_token_in(Path::new("/no/such/token"), None).is_err());
}
#[test]
fn rejects_zero_limits_and_bad_urls() {
let mut cfg = limits(0, 1, 10);
assert!(ReqwestTransport::new(&cfg).is_err());
cfg.max_concurrent_per_peer = 1;
cfg.max_concurrent_total = 0;
assert!(ReqwestTransport::new(&cfg).is_err());
cfg.max_concurrent_total = 1;
cfg.max_response_bytes = 0;
assert!(ReqwestTransport::new(&cfg).is_err());
let token = write_token("t", 0o600);
let mut p = peer(9, &token, None);
p.url = url::Url::parse("http://user:pw@127.0.0.1:9/").unwrap();
assert!(request_url(&p, &req("/api/v1/ping")).is_err());
p.url = url::Url::parse("http://127.0.0.1:9/?q=1").unwrap();
assert!(request_url(&p, &req("/x")).is_err());
p.url = url::Url::parse("ftp://127.0.0.1/").unwrap();
assert!(request_url(&p, &req("/x")).is_err());
let _ = std::fs::remove_file(&token);
}
#[tokio::test]
async fn get_is_get_and_sends_the_token_not_a_redirect() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let token = write_token("peer-token-a\n", 0o600);
tokio::spawn(async move {
let (mut sock, _) = listener.accept().await.unwrap();
let head = read_req(&mut sock).await;
assert!(head.starts_with("GET /api/v1/ping?n=1 HTTP/1.1"));
assert!(head
.to_ascii_lowercase()
.contains("authorization: bearer peer-token-a"));
assert!(head.contains("x-slinuxm-hop: 1"));
let body = "{\"status\":\"ok\",\"version\":\"9\"}";
let resp = format!(
"HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:1/gone\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
let _ = sock.write_all(resp.as_bytes()).await;
});
let transport = ReqwestTransport::new(&limits(4, 8, 1024)).unwrap();
let resp = transport
.get(
&peer(port, &token, None),
req("/api/v1/ping"),
Duration::from_secs(2),
)
.await
.unwrap();
assert_eq!(resp.status, 302, "redirects are not followed");
match resp.body {
PeerBody::Bytes(b) => assert!(b.windows(3).any(|w| w == b"9\"}")),
PeerBody::Stream(_) => panic!("buffered"),
}
let _ = std::fs::remove_file(&token);
}
#[tokio::test]
async fn rereads_the_token_file() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let token = write_token("first-token\n", 0o600);
tokio::spawn(async move {
for _ in 0..2 {
let (mut sock, _) = listener.accept().await.unwrap();
let head = read_req(&mut sock).await;
let line = head
.lines()
.find(|l| l.to_ascii_lowercase().starts_with("authorization:"))
.unwrap()
.to_owned();
let body = line;
let resp = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
let _ = sock.write_all(resp.as_bytes()).await;
}
});
let transport = ReqwestTransport::new(&limits(4, 8, 4096)).unwrap();
let p = peer(port, &token, None);
let first = transport
.get(&p, req("/api/v1/ping"), Duration::from_secs(2))
.await
.unwrap();
std::fs::OpenOptions::new()
.write(true)
.truncate(true)
.mode(0o600)
.open(&token)
.unwrap()
.write_all(b"second-token\n")
.unwrap();
let second = transport
.get(&p, req("/api/v1/ping"), Duration::from_secs(2))
.await
.unwrap();
let text = |r: PeerResponse| match r.body {
PeerBody::Bytes(b) => String::from_utf8(b.to_vec()).unwrap(),
PeerBody::Stream(_) => panic!("buffered"),
};
assert!(text(first).contains("first-token"));
assert!(text(second).contains("second-token"));
let _ = std::fs::remove_file(&token);
}
#[tokio::test]
async fn timeout_and_closed_port() {
let open = TcpListener::bind("127.0.0.1:0").await.unwrap();
let open_port = open.local_addr().unwrap().port();
tokio::spawn(async move {
let (_sock, _) = open.accept().await.unwrap();
tokio::time::sleep(Duration::from_secs(2)).await;
});
let hang = TcpListener::bind("127.0.0.1:0").await.unwrap();
let hang_port = hang.local_addr().unwrap().port();
tokio::spawn(async move {
let (mut sock, _) = hang.accept().await.unwrap();
let _ = read_req(&mut sock).await;
tokio::time::sleep(Duration::from_secs(5)).await;
});
let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap();
let token = write_token("t", 0o600);
assert!(transport
.tcp_check(&peer(open_port, &token, None), Duration::from_secs(1))
.await
.is_ok());
let started = tokio::time::Instant::now();
let err = transport
.get(
&peer(hang_port, &token, None),
req("/api/v1/ping"),
Duration::from_millis(200),
)
.await
.unwrap_err();
assert!(matches!(err, PeerError::Timeout), "{err:?}");
assert!(started.elapsed() < Duration::from_secs(2));
let closed = TcpListener::bind("127.0.0.1:0").await.unwrap();
let closed_port = closed.local_addr().unwrap().port();
drop(closed);
let err = transport
.tcp_check(&peer(closed_port, &token, None), Duration::from_secs(1))
.await
.unwrap_err();
assert!(matches!(err, PeerError::Unreachable), "{err:?}");
let _ = std::fs::remove_file(&token);
}
#[tokio::test]
async fn byte_cap_ignores_a_lying_content_length_and_counts_chunks() {
let transport = ReqwestTransport::new(&limits(2, 2, 32)).unwrap();
let token = write_token("t", 0o600);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
tokio::spawn(async move {
let (mut sock, _) = listener.accept().await.unwrap();
let _ = read_req(&mut sock).await;
let _ = sock
.write_all(
b"HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\nConnection: close\r\n\r\n",
)
.await;
tokio::time::sleep(Duration::from_secs(5)).await;
});
let started = tokio::time::Instant::now();
let err = transport
.get(
&peer(port, &token, None),
req("/big"),
Duration::from_secs(2),
)
.await
.unwrap_err();
assert!(matches!(err, PeerError::Oversize), "{err:?}");
assert!(started.elapsed() < Duration::from_secs(1));
let body = vec![b'x'; 200];
let mut bytes =
b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n".to_vec();
bytes.extend(format!("{:x}\r\n", body.len()).into_bytes());
bytes.extend_from_slice(&body);
bytes.extend(b"\r\n0\r\n\r\n");
let port = serve(bytes).await;
let err = transport
.get(
&peer(port, &token, None),
req("/chunked"),
Duration::from_secs(2),
)
.await
.unwrap_err();
assert!(matches!(err, PeerError::Oversize), "{err:?}");
let _ = std::fs::remove_file(&token);
}
#[tokio::test]
async fn semaphores_are_per_peer_and_global() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
let (mut sock, _) = listener.accept().await.unwrap();
let _ = read_req(&mut sock).await;
let _ = ready_tx.send(());
tokio::time::sleep(Duration::from_secs(5)).await;
});
let transport = Arc::new(ReqwestTransport::new(&limits(1, 4, 1024)).unwrap());
let token = write_token("t", 0o600);
let p = peer(port, &token, None);
let t2 = transport.clone();
let p2 = p.clone();
let hung =
tokio::spawn(async move { t2.get(&p2, req("/hang"), Duration::from_secs(4)).await });
ready_rx.await.unwrap();
let started = tokio::time::Instant::now();
let err = transport
.get(&p, req("/other"), Duration::from_millis(500))
.await
.unwrap_err();
assert!(matches!(err, PeerError::Saturated), "{err:?}");
assert!(started.elapsed() < Duration::from_millis(400));
let mut gamma = p.clone();
gamma.name = PeerName::parse("gamma").unwrap();
// gamma is not blocked by beta's permit; it fails because nothing is listening twice.
// Use a fresh accepting server so the call is not Saturated.
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let gport = listener.local_addr().unwrap().port();
tokio::spawn(async move {
let (mut sock, _) = listener.accept().await.unwrap();
let _ = read_req(&mut sock).await;
let _ = sock
.write_all(
b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
)
.await;
});
gamma.url = url::Url::parse(&format!("http://127.0.0.1:{gport}")).unwrap();
let resp = transport
.get(&gamma, req("/ok"), Duration::from_secs(2))
.await
.unwrap();
assert_eq!(resp.status, 204);
hung.abort();
let _ = std::fs::remove_file(&token);
}
#[tokio::test]
async fn global_semaphore() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
let (mut sock, _) = listener.accept().await.unwrap();
let _ = read_req(&mut sock).await;
let _ = ready_tx.send(());
tokio::time::sleep(Duration::from_secs(5)).await;
});
let transport = Arc::new(ReqwestTransport::new(&limits(4, 1, 1024)).unwrap());
let token = write_token("t", 0o600);
let beta = peer(port, &token, None);
let mut gamma = beta.clone();
gamma.name = PeerName::parse("gamma").unwrap();
let t2 = transport.clone();
let b2 = beta.clone();
let hung =
tokio::spawn(async move { t2.get(&b2, req("/hang"), Duration::from_secs(4)).await });
ready_rx.await.unwrap();
let err = transport
.get(&gamma, req("/x"), Duration::from_millis(500))
.await
.unwrap_err();
assert!(matches!(err, PeerError::Saturated), "{err:?}");
hung.abort();
let _ = std::fs::remove_file(&token);
}
#[tokio::test]
async fn custom_ca_is_required_for_a_private_cert() {
let ck = rcgen::generate_simple_self_signed(vec!["localhost".into(), "127.0.0.1".into()])
.unwrap();
let cert_pem = ck.cert.pem();
let key_pem = ck.signing_key.serialize_pem();
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let certs: Vec<_> = CertificateDer::pem_slice_iter(cert_pem.as_bytes())
.map(|c| c.unwrap())
.collect();
let key = rustls::pki_types::PrivateKeyDer::from_pem_slice(key_pem.as_bytes()).unwrap();
let provider = Arc::new(rustls::crypto::ring::default_provider());
let mut scfg = rustls::ServerConfig::builder_with_provider(provider)
.with_safe_default_protocol_versions()
.unwrap()
.with_no_client_auth()
.with_single_cert(certs, key)
.unwrap();
scfg.alpn_protocols = vec![b"http/1.1".to_vec()];
let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(scfg));
tokio::spawn(async move {
for _ in 0..2 {
let (sock, _) = listener.accept().await.unwrap();
let Ok(mut tls) = acceptor.accept(sock).await else {
continue;
};
let mut buf = [0u8; 2048];
let _ = tokio::time::timeout(Duration::from_secs(2), tls.read(&mut buf)).await;
let _ = tls
.write_all(
b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok",
)
.await;
}
});
let token = write_token("t", 0o600);
let ca_path =
std::env::temp_dir().join(format!("slinuxm-ca-{}-{}", std::process::id(), port));
std::fs::write(&ca_path, &cert_pem).unwrap();
let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap();
let mut https = peer(port, &token, None);
https.url = url::Url::parse(&format!("https://127.0.0.1:{port}")).unwrap();
let err = transport
.get(&https, req("/api/v1/ping"), Duration::from_secs(2))
.await
.unwrap_err();
assert!(matches!(err, PeerError::Tls), "{err:?}");
https.ca_file = Some(ca_path.clone());
let resp = transport
.get(&https, req("/api/v1/ping"), Duration::from_secs(2))
.await
.unwrap();
assert_eq!(resp.status, 200);
let _ = std::fs::remove_file(&token);
let _ = std::fs::remove_file(&ca_path);
}
#[test]
fn loopback_names() {
assert!(host_is_loopback(Some("127.0.0.1")));
assert!(host_is_loopback(Some("::1")));
assert!(host_is_loopback(Some("localhost")));
assert!(!host_is_loopback(Some("10.0.0.1")));
assert!(!host_is_loopback(Some("beta")));
}
}