B: B5 enforce deny CIDRs and plain-HTTP opt-in before connecting

This commit is contained in:
ari committed 2026-10-19 20:55:25 -04:00
1 parent ed1c3a608e
commit 000659a552
3 files changed
+155 -18

No files matched your search

Generated
+1
View File
@@ -2820,6 +2820,7 @@ dependencies = [
"figment",
"futures",
"hyper-util",
"ipnet",
"jiff",
"libc",
"opentelemetry",
+1
View File
@@ -32,6 +32,7 @@ tower-http = { version = "0.7.1", features = ["trace"] }
# default-features is off so native-tls (openssl) stays out.
reqwest = { version = "0.13.5", default-features = false, features = ["rustls-no-provider"] }
webpki-roots = "1.0.9"
ipnet = "2.11.0"
# Native TLS (DEC-12): rustls with the ring provider passed explicitly, no openssl
rustls = { version = "0.23.45", default-features = false, features = ["ring", "std", "tls12"] }
+153 -18
View File
@@ -6,6 +6,7 @@
//! the only address we will talk to.
use std::collections::{HashMap, HashSet};
use std::net::{IpAddr, SocketAddr};
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::{Arc, Mutex};
@@ -16,6 +17,7 @@ use async_trait::async_trait;
use axum::http::{HeaderName, HeaderValue};
use bytes::Bytes;
use futures::Stream;
use ipnet::IpNet;
use rustls::pki_types::pem::PemObject;
use rustls::pki_types::CertificateDer;
use rustls::{ClientConfig, RootCertStore};
@@ -33,6 +35,7 @@ struct Limits {
per_peer_n: usize,
per_peer: HashMap<PeerName, Arc<Semaphore>>,
max_response_bytes: u64,
deny: Vec<IpNet>,
}
#[derive(Clone, PartialEq, Eq, Hash)]
@@ -41,6 +44,16 @@ enum CaKey {
Extra(Vec<u8>),
}
/// One client per CA plus pinned address. Pinning the resolved address both
/// keeps DNS rebinding from slipping past the deny list and makes the check
/// and the connection use the same address.
#[derive(Clone, PartialEq, Eq, Hash)]
struct ClientKey {
ca: CaKey,
host: String,
addr: SocketAddr,
}
struct Permits {
_global: OwnedSemaphorePermit,
_peer: OwnedSemaphorePermit,
@@ -50,7 +63,7 @@ struct Permits {
/// roots, plus one extra client per distinct custom CA.
pub struct ReqwestTransport {
limits: Mutex<Limits>,
clients: Mutex<HashMap<CaKey, reqwest::Client>>,
clients: Mutex<HashMap<ClientKey, reqwest::Client>>,
plain_warned: Mutex<HashSet<PeerName>>,
}
@@ -66,12 +79,22 @@ impl ReqwestTransport {
"peers.max_response_bytes must be greater than zero".into(),
));
}
let deny = cfg
.deny_cidrs
.iter()
.map(|s| {
s.parse::<IpNet>().map_err(|_| {
ConfigError::Invalid(format!("peers.deny_cidrs entry {s:?} is not a CIDR"))
})
})
.collect::<Result<Vec<_>, _>>()?;
Ok(Self {
limits: Mutex::new(Limits {
global: Arc::new(Semaphore::new(cfg.max_concurrent_total)),
per_peer_n: cfg.max_concurrent_per_peer,
per_peer: HashMap::new(),
max_response_bytes: cfg.max_response_bytes,
deny,
}),
clients: Mutex::new(HashMap::new()),
plain_warned: Mutex::new(HashSet::new()),
@@ -86,6 +109,10 @@ impl ReqwestTransport {
Self::lock(&self.limits).max_response_bytes
}
fn deny_list(&self) -> Vec<IpNet> {
Self::lock(&self.limits).deny.clone()
}
fn acquire(&self, name: &PeerName) -> Result<Permits, PeerError> {
let mut limits = Self::lock(&self.limits);
let global = limits.global.clone();
@@ -106,10 +133,19 @@ impl ReqwestTransport {
})
}
fn client_for(&self, ca_pem: Option<Vec<u8>>) -> Result<reqwest::Client, PeerError> {
let key = match &ca_pem {
None => CaKey::Webpki,
Some(pem) => CaKey::Extra(pem.clone()),
fn client_for(
&self,
ca_pem: Option<Vec<u8>>,
host: &str,
addr: SocketAddr,
) -> Result<reqwest::Client, PeerError> {
let key = ClientKey {
ca: match &ca_pem {
None => CaKey::Webpki,
Some(pem) => CaKey::Extra(pem.clone()),
},
host: host.to_owned(),
addr,
};
let mut cache = Self::lock(&self.clients);
if let Some(client) = cache.get(&key) {
@@ -118,7 +154,7 @@ impl ReqwestTransport {
if cache.len() >= crate::peers::types::MAX_PEERS {
cache.clear();
}
let client = build_client(ca_pem.as_deref())?;
let client = build_client(ca_pem.as_deref(), host, addr)?;
cache.insert(key, client.clone());
Ok(client)
}
@@ -147,14 +183,77 @@ fn host_is_loopback(host: Option<&str>) -> bool {
}
}
/// Rules that must hold before resolving anything. The same rules are
/// enforced when the config is validated; repeating them here keeps the
/// transport safe when it is built from an unvalidated `PeersConfig`.
fn validate_target(peer: &PeerConfig) -> Result<(), PeerError> {
if !matches!(peer.url.scheme(), "http" | "https") {
return Err(PeerError::Unreachable);
}
if !peer.url.username().is_empty() || peer.url.password().is_some() {
return Err(PeerError::Unreachable);
}
if peer.url.query().is_some() || peer.url.fragment().is_some() {
return Err(PeerError::Unreachable);
}
if peer.url.scheme() == "http"
&& !host_is_loopback(peer.url.host_str())
&& !peer.allow_plain_http
{
return Err(PeerError::Unreachable);
}
Ok(())
}
fn denied(ip: IpAddr, deny: &[IpNet]) -> bool {
deny.iter().any(|net| net.contains(&ip))
}
/// Resolve the peer host and reject denied addresses before any connection is
/// made. The returned address is pinned into the HTTP client.
async fn resolve_target(
peer: &PeerConfig,
deny: &[IpNet],
) -> Result<(String, SocketAddr), PeerError> {
validate_target(peer)?;
let host = peer
.url
.host_str()
.ok_or(PeerError::Unreachable)?
.to_owned();
let port = peer
.url
.port_or_known_default()
.ok_or(PeerError::Unreachable)?;
let addrs: Vec<SocketAddr> = match host.parse::<IpAddr>() {
Ok(ip) => vec![SocketAddr::new(ip, port)],
Err(_) => tokio::net::lookup_host((host.as_str(), port))
.await
.map_err(|_| PeerError::Dns)?
.collect(),
};
let Some(addr) = addrs.first().copied() else {
return Err(PeerError::Dns);
};
if addrs.iter().any(|addr| denied(addr.ip(), deny)) {
return Err(PeerError::Unreachable);
}
Ok((host, addr))
}
/// Ring is passed in so this process never calls `install_default`.
fn build_client(extra_ca: Option<&[u8]>) -> Result<reqwest::Client, PeerError> {
fn build_client(
extra_ca: Option<&[u8]>,
host: &str,
addr: SocketAddr,
) -> Result<reqwest::Client, PeerError> {
let tls = rustls_config(extra_ca)?;
reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.no_proxy()
.http1_only()
.tls_backend_preconfigured(tls)
.resolve(host, addr)
.build()
.map_err(|_| PeerError::Tls)
}
@@ -383,17 +482,10 @@ impl Stream for CappedStream {
#[async_trait]
impl PeerTransport for ReqwestTransport {
async fn tcp_check(&self, peer: &PeerConfig, timeout: Duration) -> Result<Duration, PeerError> {
let host = peer
.url
.host_str()
.ok_or(PeerError::Unreachable)?
.to_owned();
let port = peer
.url
.port_or_known_default()
.ok_or(PeerError::Unreachable)?;
let deny = self.deny_list();
let (_, addr) = resolve_target(peer, &deny).await?;
let started = tokio::time::Instant::now();
let connect = tokio::net::TcpStream::connect((host.as_str(), port));
let connect = tokio::net::TcpStream::connect(addr);
match tokio::time::timeout(timeout, connect).await {
Err(_) => Err(PeerError::Timeout),
Ok(Err(e)) => Err(map_io(e)),
@@ -408,6 +500,8 @@ impl PeerTransport for ReqwestTransport {
timeout: Duration,
) -> Result<PeerResponse, PeerError> {
let url = request_url(peer, &req)?;
let deny = self.deny_list();
let (host, addr) = resolve_target(peer, &deny).await?;
self.warn_plain_http(peer);
let token_path = peer.token_file.clone();
let secret = tokio::task::spawn_blocking(move || load_token(&token_path))
@@ -424,7 +518,7 @@ impl PeerTransport for ReqwestTransport {
.map_err(|_| PeerError::TokenFile("invalid".into()))?;
drop(secret);
let permits = self.acquire(&peer.name)?;
let client = self.client_for(ca)?;
let client = self.client_for(ca, &host, addr)?;
let max = self.max_bytes();
let mut builder = client.get(url).timeout(timeout);
for (name, value) in &req.headers {
@@ -960,6 +1054,47 @@ mod tests {
let _ = std::fs::remove_file(&ca_path);
}
#[tokio::test]
async fn deny_cidrs_are_enforced_at_connect_time() {
let mut cfg = limits(2, 2, 1024);
cfg.deny_cidrs = vec!["127.0.0.0/8".into()];
let transport = ReqwestTransport::new(&cfg).unwrap();
let token = write_token("t", 0o600);
let p = peer(9, &token, None);
assert!(matches!(
transport.tcp_check(&p, Duration::from_secs(1)).await,
Err(PeerError::Unreachable)
));
assert!(matches!(
transport
.get(&p, req("/api/v1/ping"), Duration::from_secs(1))
.await,
Err(PeerError::Unreachable)
));
let _ = std::fs::remove_file(&token);
}
#[test]
fn invalid_deny_cidr_and_plain_http_opt_in() {
let mut cfg = limits(2, 2, 1024);
cfg.deny_cidrs = vec!["not-a-cidr".into()];
assert!(ReqwestTransport::new(&cfg).is_err());
let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap();
let token = write_token("t", 0o600);
let mut p = peer(9, &token, None);
p.url = url::Url::parse("http://10.0.0.5:1984").unwrap();
assert!(!p.allow_plain_http);
assert!(validate_target(&p).is_err());
p.allow_plain_http = true;
assert!(validate_target(&p).is_ok());
assert!(transport
.deny_list()
.iter()
.any(|n| n.to_string() == "169.254.0.0/16"));
let _ = std::fs::remove_file(&token);
}
#[test]
fn loopback_names() {
assert!(host_is_loopback(Some("127.0.0.1")));