1107 lines
39 KiB
Rust
1107 lines
39 KiB
Rust
//! Outbound GET client used by health checks and the proxy.
|
|
//!
|
|
//! `reqwest` is built on a rustls `ClientConfig` whose crypto provider is
|
|
//! ring, passed in here. Nothing is installed process-wide, so the OTLP
|
|
//! client's provider choice is left alone. Redirects stay off: a peer URL is
|
|
//! the only address we will talk to.
|
|
|
|
use std::collections::{HashMap, HashSet};
|
|
use std::net::{IpAddr, SocketAddr};
|
|
use std::path::{Path, PathBuf};
|
|
use std::pin::Pin;
|
|
use std::sync::{Arc, Mutex};
|
|
use std::task::{Context, Poll};
|
|
use std::time::Duration;
|
|
|
|
use async_trait::async_trait;
|
|
use axum::http::{HeaderName, HeaderValue};
|
|
use bytes::Bytes;
|
|
use futures::Stream;
|
|
use ipnet::IpNet;
|
|
use rustls::pki_types::pem::PemObject;
|
|
use rustls::pki_types::CertificateDer;
|
|
use rustls::{ClientConfig, RootCertStore};
|
|
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
|
|
|
use crate::config::ConfigError;
|
|
use crate::peers::error::{PeerConfigError, PeerError};
|
|
use crate::peers::transport::{PeerBody, PeerRequest, PeerResponse, PeerTransport};
|
|
use crate::peers::types::{PeerConfig, PeerName, PeersConfig, Secret};
|
|
|
|
const MAX_TOKEN_BYTES: u64 = 4096;
|
|
|
|
struct Limits {
|
|
global: Arc<Semaphore>,
|
|
per_peer_n: usize,
|
|
per_peer: HashMap<PeerName, Arc<Semaphore>>,
|
|
max_response_bytes: u64,
|
|
deny: Vec<IpNet>,
|
|
}
|
|
|
|
#[derive(Clone, PartialEq, Eq, Hash)]
|
|
enum CaKey {
|
|
Webpki,
|
|
Extra(Vec<u8>),
|
|
}
|
|
|
|
/// One client per CA plus pinned address. Pinning the resolved address both
|
|
/// keeps DNS rebinding from slipping past the deny list and makes the check
|
|
/// and the connection use the same address.
|
|
#[derive(Clone, PartialEq, Eq, Hash)]
|
|
struct ClientKey {
|
|
ca: CaKey,
|
|
host: String,
|
|
addr: SocketAddr,
|
|
}
|
|
|
|
struct Permits {
|
|
_global: OwnedSemaphorePermit,
|
|
_peer: OwnedSemaphorePermit,
|
|
}
|
|
|
|
/// Real [`PeerTransport`]. One shared client for peers that use the webpki
|
|
/// roots, plus one extra client per distinct custom CA.
|
|
pub struct ReqwestTransport {
|
|
limits: Mutex<Limits>,
|
|
clients: Mutex<HashMap<ClientKey, reqwest::Client>>,
|
|
plain_warned: Mutex<HashSet<PeerName>>,
|
|
}
|
|
|
|
impl ReqwestTransport {
|
|
pub fn new(cfg: &PeersConfig) -> Result<Self, PeerConfigError> {
|
|
if cfg.max_concurrent_per_peer == 0 || cfg.max_concurrent_total == 0 {
|
|
return Err(ConfigError::Invalid(
|
|
"peers concurrency limits must be greater than zero".into(),
|
|
));
|
|
}
|
|
if cfg.max_response_bytes == 0 {
|
|
return Err(ConfigError::Invalid(
|
|
"peers.max_response_bytes must be greater than zero".into(),
|
|
));
|
|
}
|
|
let deny = cfg
|
|
.deny_cidrs
|
|
.iter()
|
|
.map(|s| {
|
|
s.parse::<IpNet>().map_err(|_| {
|
|
ConfigError::Invalid(format!("peers.deny_cidrs entry {s:?} is not a CIDR"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
Ok(Self {
|
|
limits: Mutex::new(Limits {
|
|
global: Arc::new(Semaphore::new(cfg.max_concurrent_total)),
|
|
per_peer_n: cfg.max_concurrent_per_peer,
|
|
per_peer: HashMap::new(),
|
|
max_response_bytes: cfg.max_response_bytes,
|
|
deny,
|
|
}),
|
|
clients: Mutex::new(HashMap::new()),
|
|
plain_warned: Mutex::new(HashSet::new()),
|
|
})
|
|
}
|
|
|
|
fn lock<'a, T>(m: &'a Mutex<T>) -> std::sync::MutexGuard<'a, T> {
|
|
m.lock().unwrap_or_else(|e| e.into_inner())
|
|
}
|
|
|
|
fn max_bytes(&self) -> u64 {
|
|
Self::lock(&self.limits).max_response_bytes
|
|
}
|
|
|
|
fn deny_list(&self) -> Vec<IpNet> {
|
|
Self::lock(&self.limits).deny.clone()
|
|
}
|
|
|
|
fn acquire(&self, name: &PeerName) -> Result<Permits, PeerError> {
|
|
let mut limits = Self::lock(&self.limits);
|
|
let global = limits.global.clone();
|
|
let permits = limits.per_peer_n;
|
|
let per = limits
|
|
.per_peer
|
|
.entry(name.clone())
|
|
.or_insert_with(|| Arc::new(Semaphore::new(permits)))
|
|
.clone();
|
|
drop(limits);
|
|
let global = global
|
|
.try_acquire_owned()
|
|
.map_err(|_| PeerError::Saturated)?;
|
|
let peer = per.try_acquire_owned().map_err(|_| PeerError::Saturated)?;
|
|
Ok(Permits {
|
|
_global: global,
|
|
_peer: peer,
|
|
})
|
|
}
|
|
|
|
fn client_for(
|
|
&self,
|
|
ca_pem: Option<Vec<u8>>,
|
|
host: &str,
|
|
addr: SocketAddr,
|
|
) -> Result<reqwest::Client, PeerError> {
|
|
let key = ClientKey {
|
|
ca: match &ca_pem {
|
|
None => CaKey::Webpki,
|
|
Some(pem) => CaKey::Extra(pem.clone()),
|
|
},
|
|
host: host.to_owned(),
|
|
addr,
|
|
};
|
|
let mut cache = Self::lock(&self.clients);
|
|
if let Some(client) = cache.get(&key) {
|
|
return Ok(client.clone());
|
|
}
|
|
if cache.len() >= crate::peers::types::MAX_PEERS {
|
|
cache.clear();
|
|
}
|
|
let client = build_client(ca_pem.as_deref(), host, addr)?;
|
|
cache.insert(key, client.clone());
|
|
Ok(client)
|
|
}
|
|
|
|
fn warn_plain_http(&self, peer: &PeerConfig) {
|
|
if peer.url.scheme() != "http" || host_is_loopback(peer.url.host_str()) {
|
|
return;
|
|
}
|
|
let mut seen = Self::lock(&self.plain_warned);
|
|
if seen.insert(peer.name.clone()) {
|
|
tracing::warn!(
|
|
peer = %peer.name,
|
|
"plain HTTP to a non-loopback peer; the bearer token travels in clear"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn host_is_loopback(host: Option<&str>) -> bool {
|
|
match host {
|
|
Some("localhost") => true,
|
|
Some(h) => h
|
|
.parse::<std::net::IpAddr>()
|
|
.is_ok_and(|ip| ip.is_loopback()),
|
|
None => false,
|
|
}
|
|
}
|
|
|
|
/// Rules that must hold before resolving anything. The same rules are
|
|
/// enforced when the config is validated; repeating them here keeps the
|
|
/// transport safe when it is built from an unvalidated `PeersConfig`.
|
|
fn validate_target(peer: &PeerConfig) -> Result<(), PeerError> {
|
|
if !matches!(peer.url.scheme(), "http" | "https") {
|
|
return Err(PeerError::Unreachable);
|
|
}
|
|
if !peer.url.username().is_empty() || peer.url.password().is_some() {
|
|
return Err(PeerError::Unreachable);
|
|
}
|
|
if peer.url.query().is_some() || peer.url.fragment().is_some() {
|
|
return Err(PeerError::Unreachable);
|
|
}
|
|
if peer.url.scheme() == "http"
|
|
&& !host_is_loopback(peer.url.host_str())
|
|
&& !peer.allow_plain_http
|
|
{
|
|
return Err(PeerError::Unreachable);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn denied(ip: IpAddr, deny: &[IpNet]) -> bool {
|
|
deny.iter().any(|net| net.contains(&ip))
|
|
}
|
|
|
|
/// Resolve the peer host and reject denied addresses before any connection is
|
|
/// made. The returned address is pinned into the HTTP client.
|
|
async fn resolve_target(
|
|
peer: &PeerConfig,
|
|
deny: &[IpNet],
|
|
) -> Result<(String, SocketAddr), PeerError> {
|
|
validate_target(peer)?;
|
|
let host = peer
|
|
.url
|
|
.host_str()
|
|
.ok_or(PeerError::Unreachable)?
|
|
.to_owned();
|
|
let port = peer
|
|
.url
|
|
.port_or_known_default()
|
|
.ok_or(PeerError::Unreachable)?;
|
|
let addrs: Vec<SocketAddr> = match host.parse::<IpAddr>() {
|
|
Ok(ip) => vec![SocketAddr::new(ip, port)],
|
|
Err(_) => tokio::net::lookup_host((host.as_str(), port))
|
|
.await
|
|
.map_err(|_| PeerError::Dns)?
|
|
.collect(),
|
|
};
|
|
let Some(addr) = addrs.first().copied() else {
|
|
return Err(PeerError::Dns);
|
|
};
|
|
if addrs.iter().any(|addr| denied(addr.ip(), deny)) {
|
|
return Err(PeerError::Unreachable);
|
|
}
|
|
Ok((host, addr))
|
|
}
|
|
|
|
/// Ring is passed in so this process never calls `install_default`.
|
|
fn build_client(
|
|
extra_ca: Option<&[u8]>,
|
|
host: &str,
|
|
addr: SocketAddr,
|
|
) -> Result<reqwest::Client, PeerError> {
|
|
let tls = rustls_config(extra_ca)?;
|
|
reqwest::Client::builder()
|
|
.redirect(reqwest::redirect::Policy::none())
|
|
.no_proxy()
|
|
.http1_only()
|
|
.tls_backend_preconfigured(tls)
|
|
.resolve(host, addr)
|
|
.build()
|
|
.map_err(|_| PeerError::Tls)
|
|
}
|
|
|
|
fn rustls_config(extra_ca: Option<&[u8]>) -> Result<ClientConfig, PeerError> {
|
|
let provider = Arc::new(rustls::crypto::ring::default_provider());
|
|
let mut roots = RootCertStore::empty();
|
|
roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
|
|
if let Some(pem) = extra_ca {
|
|
let mut any = false;
|
|
for cert in CertificateDer::pem_slice_iter(pem) {
|
|
let cert = cert.map_err(|_| PeerError::Tls)?;
|
|
roots.add(cert).map_err(|_| PeerError::Tls)?;
|
|
any = true;
|
|
}
|
|
if !any {
|
|
return Err(PeerError::Tls);
|
|
}
|
|
}
|
|
let mut cfg = ClientConfig::builder_with_provider(provider)
|
|
.with_safe_default_protocol_versions()
|
|
.map_err(|_| PeerError::Tls)?
|
|
.with_root_certificates(roots)
|
|
.with_no_client_auth();
|
|
cfg.alpn_protocols = vec![b"http/1.1".to_vec()];
|
|
Ok(cfg)
|
|
}
|
|
|
|
fn request_url(peer: &PeerConfig, req: &PeerRequest) -> Result<url::Url, PeerError> {
|
|
if peer.url.scheme() != "http" && peer.url.scheme() != "https" {
|
|
return Err(PeerError::Unreachable);
|
|
}
|
|
if !peer.url.username().is_empty() || peer.url.password().is_some() {
|
|
return Err(PeerError::Unreachable);
|
|
}
|
|
if peer.url.query().is_some() || peer.url.fragment().is_some() {
|
|
return Err(PeerError::Unreachable);
|
|
}
|
|
let mut url = peer.url.clone();
|
|
let base = url.path().trim_end_matches('/');
|
|
let rel = req.path.trim_start_matches('/');
|
|
let path = if rel.is_empty() {
|
|
if base.is_empty() {
|
|
"/".to_owned()
|
|
} else {
|
|
base.to_owned()
|
|
}
|
|
} else {
|
|
format!("{base}/{rel}")
|
|
};
|
|
url.set_path(&path);
|
|
url.set_query(None);
|
|
url.set_fragment(None);
|
|
if !req.query.is_empty() {
|
|
{
|
|
let mut pairs = url.query_pairs_mut();
|
|
for (k, v) in &req.query {
|
|
pairs.append_pair(k, v);
|
|
}
|
|
}
|
|
}
|
|
Ok(url)
|
|
}
|
|
|
|
fn load_token(path: &Path) -> Result<Secret, PeerError> {
|
|
let creds = std::env::var_os("CREDENTIALS_DIRECTORY").map(PathBuf::from);
|
|
load_token_in(path, creds.as_deref())
|
|
}
|
|
|
|
fn load_token_in(path: &Path, credentials_dir: Option<&Path>) -> Result<Secret, PeerError> {
|
|
let meta = std::fs::metadata(path).map_err(|_| PeerError::TokenFile("missing".into()))?;
|
|
if !meta.is_file() {
|
|
return Err(PeerError::TokenFile("not a file".into()));
|
|
}
|
|
if meta.len() > MAX_TOKEN_BYTES {
|
|
return Err(PeerError::TokenFile("too large".into()));
|
|
}
|
|
if !under_credentials(path, credentials_dir) {
|
|
let mode = {
|
|
use std::os::unix::fs::PermissionsExt;
|
|
meta.permissions().mode()
|
|
};
|
|
if mode & 0o077 != 0 {
|
|
return Err(PeerError::TokenFile("group or world readable".into()));
|
|
}
|
|
}
|
|
let mut bytes = std::fs::read(path).map_err(|_| PeerError::TokenFile("unreadable".into()))?;
|
|
if bytes.len() as u64 > MAX_TOKEN_BYTES {
|
|
return Err(PeerError::TokenFile("too large".into()));
|
|
}
|
|
if bytes.last() == Some(&b'\n') {
|
|
bytes.pop();
|
|
if bytes.last() == Some(&b'\r') {
|
|
bytes.pop();
|
|
}
|
|
}
|
|
if bytes.is_empty() {
|
|
return Err(PeerError::TokenFile("empty".into()));
|
|
}
|
|
if bytes.iter().any(|b| *b == b'\n' || *b == b'\r' || *b == 0) {
|
|
return Err(PeerError::TokenFile("invalid".into()));
|
|
}
|
|
let text = String::from_utf8(bytes).map_err(|_| PeerError::TokenFile("invalid".into()))?;
|
|
if HeaderValue::from_str(&format!("Bearer {text}")).is_err() {
|
|
return Err(PeerError::TokenFile("invalid".into()));
|
|
}
|
|
Ok(Secret::new(text))
|
|
}
|
|
|
|
fn under_credentials(path: &Path, credentials_dir: Option<&Path>) -> bool {
|
|
match credentials_dir {
|
|
Some(dir) if !dir.as_os_str().is_empty() => path.starts_with(dir),
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
fn map_io(err: std::io::Error) -> PeerError {
|
|
if err.kind() == std::io::ErrorKind::TimedOut {
|
|
return PeerError::Timeout;
|
|
}
|
|
let msg = err.to_string();
|
|
if msg.contains("Name or service not known")
|
|
|| msg.contains("nodename nor servname")
|
|
|| msg.contains("Temporary failure in name resolution")
|
|
|| msg.contains("No address associated")
|
|
|| msg.contains("Name does not resolve")
|
|
{
|
|
return PeerError::Dns;
|
|
}
|
|
PeerError::Unreachable
|
|
}
|
|
|
|
fn map_reqwest(err: reqwest::Error) -> PeerError {
|
|
if err.is_timeout() {
|
|
return PeerError::Timeout;
|
|
}
|
|
if err.is_dns() {
|
|
return PeerError::Dns;
|
|
}
|
|
if error_is_tls(&err) {
|
|
return PeerError::Tls;
|
|
}
|
|
if err.is_connect() {
|
|
return PeerError::Unreachable;
|
|
}
|
|
PeerError::Unreachable
|
|
}
|
|
|
|
fn error_is_tls(err: &(dyn std::error::Error + 'static)) -> bool {
|
|
if err.downcast_ref::<rustls::Error>().is_some() {
|
|
return true;
|
|
}
|
|
let msg = err.to_string();
|
|
if msg.contains("certificate") || msg.contains("TLS") || msg.contains("tls handshake") {
|
|
return true;
|
|
}
|
|
err.source().is_some_and(error_is_tls)
|
|
}
|
|
|
|
fn content_type(resp: &reqwest::Response) -> Option<String> {
|
|
resp.headers()
|
|
.get(axum::http::header::CONTENT_TYPE)
|
|
.and_then(|v| v.to_str().ok())
|
|
.map(str::to_owned)
|
|
}
|
|
|
|
fn wants_stream(req: &PeerRequest, resp: &reqwest::Response) -> bool {
|
|
req.accept.as_deref() == Some("text/event-stream")
|
|
|| content_type(resp).is_some_and(|ct| ct.starts_with("text/event-stream"))
|
|
}
|
|
|
|
async fn read_capped(mut resp: reqwest::Response, max: u64) -> Result<Bytes, PeerError> {
|
|
if resp.content_length().is_some_and(|n| n > max) {
|
|
return Err(PeerError::Oversize);
|
|
}
|
|
let mut buf = Vec::new();
|
|
while let Some(chunk) = resp.chunk().await.map_err(map_reqwest)? {
|
|
let next = (buf.len() as u64).saturating_add(chunk.len() as u64);
|
|
if next > max {
|
|
return Err(PeerError::Oversize);
|
|
}
|
|
buf.extend_from_slice(&chunk);
|
|
}
|
|
Ok(Bytes::from(buf))
|
|
}
|
|
|
|
struct CappedStream {
|
|
inner: Pin<Box<dyn Stream<Item = Result<Bytes, reqwest::Error>> + Send>>,
|
|
seen: u64,
|
|
max: u64,
|
|
done: bool,
|
|
_permits: Permits,
|
|
}
|
|
|
|
impl Stream for CappedStream {
|
|
type Item = Result<Bytes, PeerError>;
|
|
|
|
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
|
let this = self.get_mut();
|
|
if this.done {
|
|
return Poll::Ready(None);
|
|
}
|
|
match this.inner.as_mut().poll_next(cx) {
|
|
Poll::Pending => Poll::Pending,
|
|
Poll::Ready(None) => {
|
|
this.done = true;
|
|
Poll::Ready(None)
|
|
}
|
|
Poll::Ready(Some(Err(e))) => {
|
|
this.done = true;
|
|
Poll::Ready(Some(Err(map_reqwest(e))))
|
|
}
|
|
Poll::Ready(Some(Ok(chunk))) => {
|
|
this.seen = this.seen.saturating_add(chunk.len() as u64);
|
|
if this.seen > this.max {
|
|
this.done = true;
|
|
Poll::Ready(Some(Err(PeerError::Oversize)))
|
|
} else {
|
|
Poll::Ready(Some(Ok(chunk)))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl PeerTransport for ReqwestTransport {
|
|
async fn tcp_check(&self, peer: &PeerConfig, timeout: Duration) -> Result<Duration, PeerError> {
|
|
let deny = self.deny_list();
|
|
let (_, addr) = resolve_target(peer, &deny).await?;
|
|
let started = tokio::time::Instant::now();
|
|
let connect = tokio::net::TcpStream::connect(addr);
|
|
match tokio::time::timeout(timeout, connect).await {
|
|
Err(_) => Err(PeerError::Timeout),
|
|
Ok(Err(e)) => Err(map_io(e)),
|
|
Ok(Ok(_stream)) => Ok(started.elapsed()),
|
|
}
|
|
}
|
|
|
|
async fn get(
|
|
&self,
|
|
peer: &PeerConfig,
|
|
req: PeerRequest,
|
|
timeout: Duration,
|
|
) -> Result<PeerResponse, PeerError> {
|
|
let url = request_url(peer, &req)?;
|
|
let deny = self.deny_list();
|
|
let (host, addr) = resolve_target(peer, &deny).await?;
|
|
self.warn_plain_http(peer);
|
|
let token_path = peer.token_file.clone();
|
|
let secret = tokio::task::spawn_blocking(move || load_token(&token_path))
|
|
.await
|
|
.map_err(|_| PeerError::TokenFile("unreadable".into()))??;
|
|
let ca_path = peer.ca_file.clone();
|
|
let ca = tokio::task::spawn_blocking(move || match ca_path {
|
|
None => Ok(None),
|
|
Some(path) => load_ca_path(&path),
|
|
})
|
|
.await
|
|
.map_err(|_| PeerError::Tls)??;
|
|
let auth = HeaderValue::from_str(&format!("Bearer {}", secret.expose()))
|
|
.map_err(|_| PeerError::TokenFile("invalid".into()))?;
|
|
drop(secret);
|
|
let permits = self.acquire(&peer.name)?;
|
|
let client = self.client_for(ca, &host, addr)?;
|
|
let max = self.max_bytes();
|
|
let mut builder = client.get(url).timeout(timeout);
|
|
for (name, value) in &req.headers {
|
|
if name.as_str().eq_ignore_ascii_case("authorization") {
|
|
continue;
|
|
}
|
|
builder = builder.header(name.clone(), value.clone());
|
|
}
|
|
builder = builder.header(HeaderName::from_static("authorization"), auth);
|
|
if let Some(accept) = &req.accept {
|
|
builder = builder.header(HeaderName::from_static("accept"), accept);
|
|
}
|
|
let pending = async {
|
|
let resp = builder.send().await.map_err(map_reqwest)?;
|
|
let status = resp.status().as_u16();
|
|
let ct = content_type(&resp);
|
|
if wants_stream(&req, &resp) {
|
|
if resp.content_length().is_some_and(|n| n > max) {
|
|
return Err(PeerError::Oversize);
|
|
}
|
|
// `chunk()` rather than reqwest's `stream` feature: that
|
|
// feature pulls a wasm-streams pin this crate cannot select.
|
|
let inner = Box::pin(futures::stream::unfold(resp, |mut resp| async move {
|
|
match resp.chunk().await {
|
|
Ok(Some(chunk)) => Some((Ok(chunk), resp)),
|
|
Ok(None) => None,
|
|
Err(err) => Some((Err(err), resp)),
|
|
}
|
|
}));
|
|
return Ok(PeerResponse {
|
|
status,
|
|
content_type: ct,
|
|
body: PeerBody::Stream(Box::pin(CappedStream {
|
|
inner,
|
|
seen: 0,
|
|
max,
|
|
done: false,
|
|
_permits: permits,
|
|
})),
|
|
});
|
|
}
|
|
let body = read_capped(resp, max).await?;
|
|
drop(permits);
|
|
Ok(PeerResponse {
|
|
status,
|
|
content_type: ct,
|
|
body: PeerBody::Bytes(body),
|
|
})
|
|
};
|
|
match tokio::time::timeout(timeout, pending).await {
|
|
Err(_) => Err(PeerError::Timeout),
|
|
Ok(result) => result,
|
|
}
|
|
}
|
|
}
|
|
|
|
fn load_ca_path(path: &Path) -> Result<Option<Vec<u8>>, PeerError> {
|
|
let bytes = std::fs::read(path).map_err(|_| PeerError::Tls)?;
|
|
if bytes.is_empty() || bytes.len() > 1024 * 1024 {
|
|
return Err(PeerError::Tls);
|
|
}
|
|
Ok(Some(bytes))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::io::Write;
|
|
use std::os::unix::fs::OpenOptionsExt;
|
|
|
|
use axum::http::HeaderValue as Hv;
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
use tokio::net::TcpListener;
|
|
|
|
fn limits(per: usize, total: usize, max_bytes: u64) -> PeersConfig {
|
|
PeersConfig {
|
|
enabled: true,
|
|
max_concurrent_per_peer: per,
|
|
max_concurrent_total: total,
|
|
max_response_bytes: max_bytes,
|
|
..PeersConfig::default()
|
|
}
|
|
}
|
|
|
|
fn write_token(contents: &str, mode: u32) -> PathBuf {
|
|
static N: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
|
let path = std::env::temp_dir().join(format!(
|
|
"slinuxm-peer-token-{}-{}",
|
|
std::process::id(),
|
|
N.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
|
));
|
|
let mut opts = std::fs::OpenOptions::new();
|
|
opts.write(true).create(true).truncate(true).mode(mode);
|
|
opts.open(&path)
|
|
.unwrap()
|
|
.write_all(contents.as_bytes())
|
|
.unwrap();
|
|
path
|
|
}
|
|
|
|
fn peer(port: u16, token: &Path, ca: Option<PathBuf>) -> PeerConfig {
|
|
PeerConfig {
|
|
name: PeerName::parse("beta").unwrap(),
|
|
url: url::Url::parse(&format!("http://127.0.0.1:{port}")).unwrap(),
|
|
token_file: token.to_path_buf(),
|
|
ca_file: ca,
|
|
allow_plain_http: false,
|
|
enabled: true,
|
|
}
|
|
}
|
|
|
|
fn req(path: &str) -> PeerRequest {
|
|
PeerRequest {
|
|
path: path.into(),
|
|
query: vec![("n".into(), "1".into())],
|
|
headers: vec![(
|
|
HeaderName::from_static("x-slinuxm-hop"),
|
|
Hv::from_static("1"),
|
|
)],
|
|
accept: None,
|
|
}
|
|
}
|
|
|
|
async fn read_req(sock: &mut tokio::net::TcpStream) -> String {
|
|
let mut buf = Vec::new();
|
|
let mut tmp = [0u8; 1024];
|
|
while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
|
|
let n = tokio::time::timeout(Duration::from_secs(2), sock.read(&mut tmp))
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
if n == 0 {
|
|
break;
|
|
}
|
|
buf.extend_from_slice(&tmp[..n]);
|
|
}
|
|
String::from_utf8_lossy(&buf).into_owned()
|
|
}
|
|
|
|
async fn serve(response: Vec<u8>) -> u16 {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
tokio::spawn(async move {
|
|
let (mut sock, _) = listener.accept().await.unwrap();
|
|
let _ = read_req(&mut sock).await;
|
|
let _ = sock.write_all(&response).await;
|
|
});
|
|
port
|
|
}
|
|
|
|
#[test]
|
|
fn token_file_rules() {
|
|
let ok = write_token("abc\n", 0o600);
|
|
let secret = load_token_in(&ok, None).unwrap();
|
|
assert_eq!(secret.expose(), "abc");
|
|
assert!(!format!("{secret:?}").contains("abc"));
|
|
let _ = std::fs::remove_file(&ok);
|
|
|
|
let crlf = write_token("abc\r\n", 0o600);
|
|
assert_eq!(load_token_in(&crlf, None).unwrap().expose(), "abc");
|
|
let _ = std::fs::remove_file(&crlf);
|
|
|
|
// One trailing newline is stripped. A second one would land in the
|
|
// header, so the file is rejected rather than forwarded.
|
|
let doubled = write_token("abc\n\n", 0o600);
|
|
assert!(load_token_in(&doubled, None).is_err());
|
|
let _ = std::fs::remove_file(&doubled);
|
|
|
|
for (body, mode) in [("\n", 0o600), ("", 0o600), ("x", 0o644), ("x", 0o640)] {
|
|
let path = write_token(body, mode);
|
|
assert!(load_token_in(&path, None).is_err(), "{body:?} {mode:o}");
|
|
let _ = std::fs::remove_file(&path);
|
|
}
|
|
|
|
let big = write_token(&"a".repeat(4097), 0o600);
|
|
assert!(matches!(
|
|
load_token_in(&big, None),
|
|
Err(PeerError::TokenFile(_))
|
|
));
|
|
let _ = std::fs::remove_file(&big);
|
|
|
|
let loose = write_token("secret-value\n", 0o644);
|
|
let dir = loose.parent().unwrap();
|
|
assert!(load_token_in(&loose, Some(dir)).is_ok());
|
|
let err = load_token_in(&loose, None).unwrap_err();
|
|
assert!(matches!(err, PeerError::TokenFile(_)));
|
|
assert!(!format!("{err:?}").contains("secret-value"));
|
|
let _ = std::fs::remove_file(&loose);
|
|
|
|
assert!(load_token_in(Path::new("/no/such/token"), None).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn rejects_zero_limits_and_bad_urls() {
|
|
let mut cfg = limits(0, 1, 10);
|
|
assert!(ReqwestTransport::new(&cfg).is_err());
|
|
cfg.max_concurrent_per_peer = 1;
|
|
cfg.max_concurrent_total = 0;
|
|
assert!(ReqwestTransport::new(&cfg).is_err());
|
|
cfg.max_concurrent_total = 1;
|
|
cfg.max_response_bytes = 0;
|
|
assert!(ReqwestTransport::new(&cfg).is_err());
|
|
|
|
let token = write_token("t", 0o600);
|
|
let mut p = peer(9, &token, None);
|
|
p.url = url::Url::parse("http://user:pw@127.0.0.1:9/").unwrap();
|
|
assert!(request_url(&p, &req("/api/v1/ping")).is_err());
|
|
p.url = url::Url::parse("http://127.0.0.1:9/?q=1").unwrap();
|
|
assert!(request_url(&p, &req("/x")).is_err());
|
|
p.url = url::Url::parse("ftp://127.0.0.1/").unwrap();
|
|
assert!(request_url(&p, &req("/x")).is_err());
|
|
let _ = std::fs::remove_file(&token);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn get_is_get_and_sends_the_token_not_a_redirect() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let token = write_token("peer-token-a\n", 0o600);
|
|
tokio::spawn(async move {
|
|
let (mut sock, _) = listener.accept().await.unwrap();
|
|
let head = read_req(&mut sock).await;
|
|
assert!(head.starts_with("GET /api/v1/ping?n=1 HTTP/1.1"));
|
|
assert!(head
|
|
.to_ascii_lowercase()
|
|
.contains("authorization: bearer peer-token-a"));
|
|
assert!(head.contains("x-slinuxm-hop: 1"));
|
|
let body = "{\"status\":\"ok\",\"version\":\"9\"}";
|
|
let resp = format!(
|
|
"HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:1/gone\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
|
body.len()
|
|
);
|
|
let _ = sock.write_all(resp.as_bytes()).await;
|
|
});
|
|
let transport = ReqwestTransport::new(&limits(4, 8, 1024)).unwrap();
|
|
let resp = transport
|
|
.get(
|
|
&peer(port, &token, None),
|
|
req("/api/v1/ping"),
|
|
Duration::from_secs(2),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status, 302, "redirects are not followed");
|
|
match resp.body {
|
|
PeerBody::Bytes(b) => assert!(b.windows(3).any(|w| w == b"9\"}")),
|
|
PeerBody::Stream(_) => panic!("buffered"),
|
|
}
|
|
let _ = std::fs::remove_file(&token);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn rereads_the_token_file() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let token = write_token("first-token\n", 0o600);
|
|
tokio::spawn(async move {
|
|
for _ in 0..2 {
|
|
let (mut sock, _) = listener.accept().await.unwrap();
|
|
let head = read_req(&mut sock).await;
|
|
let line = head
|
|
.lines()
|
|
.find(|l| l.to_ascii_lowercase().starts_with("authorization:"))
|
|
.unwrap()
|
|
.to_owned();
|
|
let body = line;
|
|
let resp = format!(
|
|
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
|
body.len()
|
|
);
|
|
let _ = sock.write_all(resp.as_bytes()).await;
|
|
}
|
|
});
|
|
let transport = ReqwestTransport::new(&limits(4, 8, 4096)).unwrap();
|
|
let p = peer(port, &token, None);
|
|
let first = transport
|
|
.get(&p, req("/api/v1/ping"), Duration::from_secs(2))
|
|
.await
|
|
.unwrap();
|
|
std::fs::OpenOptions::new()
|
|
.write(true)
|
|
.truncate(true)
|
|
.mode(0o600)
|
|
.open(&token)
|
|
.unwrap()
|
|
.write_all(b"second-token\n")
|
|
.unwrap();
|
|
let second = transport
|
|
.get(&p, req("/api/v1/ping"), Duration::from_secs(2))
|
|
.await
|
|
.unwrap();
|
|
let text = |r: PeerResponse| match r.body {
|
|
PeerBody::Bytes(b) => String::from_utf8(b.to_vec()).unwrap(),
|
|
PeerBody::Stream(_) => panic!("buffered"),
|
|
};
|
|
assert!(text(first).contains("first-token"));
|
|
assert!(text(second).contains("second-token"));
|
|
let _ = std::fs::remove_file(&token);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn timeout_and_closed_port() {
|
|
let open = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let open_port = open.local_addr().unwrap().port();
|
|
tokio::spawn(async move {
|
|
let (_sock, _) = open.accept().await.unwrap();
|
|
tokio::time::sleep(Duration::from_secs(2)).await;
|
|
});
|
|
let hang = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let hang_port = hang.local_addr().unwrap().port();
|
|
tokio::spawn(async move {
|
|
let (mut sock, _) = hang.accept().await.unwrap();
|
|
let _ = read_req(&mut sock).await;
|
|
tokio::time::sleep(Duration::from_secs(5)).await;
|
|
});
|
|
let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap();
|
|
let token = write_token("t", 0o600);
|
|
assert!(transport
|
|
.tcp_check(&peer(open_port, &token, None), Duration::from_secs(1))
|
|
.await
|
|
.is_ok());
|
|
|
|
let started = tokio::time::Instant::now();
|
|
let err = transport
|
|
.get(
|
|
&peer(hang_port, &token, None),
|
|
req("/api/v1/ping"),
|
|
Duration::from_millis(200),
|
|
)
|
|
.await
|
|
.unwrap_err();
|
|
assert!(matches!(err, PeerError::Timeout), "{err:?}");
|
|
assert!(started.elapsed() < Duration::from_secs(2));
|
|
|
|
let closed = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let closed_port = closed.local_addr().unwrap().port();
|
|
drop(closed);
|
|
let err = transport
|
|
.tcp_check(&peer(closed_port, &token, None), Duration::from_secs(1))
|
|
.await
|
|
.unwrap_err();
|
|
assert!(matches!(err, PeerError::Unreachable), "{err:?}");
|
|
let _ = std::fs::remove_file(&token);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn byte_cap_ignores_a_lying_content_length_and_counts_chunks() {
|
|
let transport = ReqwestTransport::new(&limits(2, 2, 32)).unwrap();
|
|
let token = write_token("t", 0o600);
|
|
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
tokio::spawn(async move {
|
|
let (mut sock, _) = listener.accept().await.unwrap();
|
|
let _ = read_req(&mut sock).await;
|
|
let _ = sock
|
|
.write_all(
|
|
b"HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\nConnection: close\r\n\r\n",
|
|
)
|
|
.await;
|
|
tokio::time::sleep(Duration::from_secs(5)).await;
|
|
});
|
|
let started = tokio::time::Instant::now();
|
|
let err = transport
|
|
.get(
|
|
&peer(port, &token, None),
|
|
req("/big"),
|
|
Duration::from_secs(2),
|
|
)
|
|
.await
|
|
.unwrap_err();
|
|
assert!(matches!(err, PeerError::Oversize), "{err:?}");
|
|
assert!(started.elapsed() < Duration::from_secs(1));
|
|
|
|
let body = vec![b'x'; 200];
|
|
let mut bytes =
|
|
b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n".to_vec();
|
|
bytes.extend(format!("{:x}\r\n", body.len()).into_bytes());
|
|
bytes.extend_from_slice(&body);
|
|
bytes.extend(b"\r\n0\r\n\r\n");
|
|
let port = serve(bytes).await;
|
|
let err = transport
|
|
.get(
|
|
&peer(port, &token, None),
|
|
req("/chunked"),
|
|
Duration::from_secs(2),
|
|
)
|
|
.await
|
|
.unwrap_err();
|
|
assert!(matches!(err, PeerError::Oversize), "{err:?}");
|
|
let _ = std::fs::remove_file(&token);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn semaphores_are_per_peer_and_global() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
|
tokio::spawn(async move {
|
|
let (mut sock, _) = listener.accept().await.unwrap();
|
|
let _ = read_req(&mut sock).await;
|
|
let _ = ready_tx.send(());
|
|
tokio::time::sleep(Duration::from_secs(5)).await;
|
|
});
|
|
let transport = Arc::new(ReqwestTransport::new(&limits(1, 4, 1024)).unwrap());
|
|
let token = write_token("t", 0o600);
|
|
let p = peer(port, &token, None);
|
|
let t2 = transport.clone();
|
|
let p2 = p.clone();
|
|
let hung =
|
|
tokio::spawn(async move { t2.get(&p2, req("/hang"), Duration::from_secs(4)).await });
|
|
ready_rx.await.unwrap();
|
|
let started = tokio::time::Instant::now();
|
|
let err = transport
|
|
.get(&p, req("/other"), Duration::from_millis(500))
|
|
.await
|
|
.unwrap_err();
|
|
assert!(matches!(err, PeerError::Saturated), "{err:?}");
|
|
assert!(started.elapsed() < Duration::from_millis(400));
|
|
|
|
let mut gamma = p.clone();
|
|
gamma.name = PeerName::parse("gamma").unwrap();
|
|
// gamma is not blocked by beta's permit; it fails because nothing is listening twice.
|
|
// Use a fresh accepting server so the call is not Saturated.
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let gport = listener.local_addr().unwrap().port();
|
|
tokio::spawn(async move {
|
|
let (mut sock, _) = listener.accept().await.unwrap();
|
|
let _ = read_req(&mut sock).await;
|
|
let _ = sock
|
|
.write_all(
|
|
b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
|
|
)
|
|
.await;
|
|
});
|
|
gamma.url = url::Url::parse(&format!("http://127.0.0.1:{gport}")).unwrap();
|
|
let resp = transport
|
|
.get(&gamma, req("/ok"), Duration::from_secs(2))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status, 204);
|
|
hung.abort();
|
|
let _ = std::fs::remove_file(&token);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn global_semaphore() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
|
tokio::spawn(async move {
|
|
let (mut sock, _) = listener.accept().await.unwrap();
|
|
let _ = read_req(&mut sock).await;
|
|
let _ = ready_tx.send(());
|
|
tokio::time::sleep(Duration::from_secs(5)).await;
|
|
});
|
|
let transport = Arc::new(ReqwestTransport::new(&limits(4, 1, 1024)).unwrap());
|
|
let token = write_token("t", 0o600);
|
|
let beta = peer(port, &token, None);
|
|
let mut gamma = beta.clone();
|
|
gamma.name = PeerName::parse("gamma").unwrap();
|
|
let t2 = transport.clone();
|
|
let b2 = beta.clone();
|
|
let hung =
|
|
tokio::spawn(async move { t2.get(&b2, req("/hang"), Duration::from_secs(4)).await });
|
|
ready_rx.await.unwrap();
|
|
let err = transport
|
|
.get(&gamma, req("/x"), Duration::from_millis(500))
|
|
.await
|
|
.unwrap_err();
|
|
assert!(matches!(err, PeerError::Saturated), "{err:?}");
|
|
hung.abort();
|
|
let _ = std::fs::remove_file(&token);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn custom_ca_is_required_for_a_private_cert() {
|
|
let ck = rcgen::generate_simple_self_signed(vec!["localhost".into(), "127.0.0.1".into()])
|
|
.unwrap();
|
|
let cert_pem = ck.cert.pem();
|
|
let key_pem = ck.signing_key.serialize_pem();
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let certs: Vec<_> = CertificateDer::pem_slice_iter(cert_pem.as_bytes())
|
|
.map(|c| c.unwrap())
|
|
.collect();
|
|
let key = rustls::pki_types::PrivateKeyDer::from_pem_slice(key_pem.as_bytes()).unwrap();
|
|
let provider = Arc::new(rustls::crypto::ring::default_provider());
|
|
let mut scfg = rustls::ServerConfig::builder_with_provider(provider)
|
|
.with_safe_default_protocol_versions()
|
|
.unwrap()
|
|
.with_no_client_auth()
|
|
.with_single_cert(certs, key)
|
|
.unwrap();
|
|
scfg.alpn_protocols = vec![b"http/1.1".to_vec()];
|
|
let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(scfg));
|
|
tokio::spawn(async move {
|
|
for _ in 0..2 {
|
|
let (sock, _) = listener.accept().await.unwrap();
|
|
let Ok(mut tls) = acceptor.accept(sock).await else {
|
|
continue;
|
|
};
|
|
let mut buf = [0u8; 2048];
|
|
let _ = tokio::time::timeout(Duration::from_secs(2), tls.read(&mut buf)).await;
|
|
let _ = tls
|
|
.write_all(
|
|
b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok",
|
|
)
|
|
.await;
|
|
}
|
|
});
|
|
|
|
let token = write_token("t", 0o600);
|
|
let ca_path =
|
|
std::env::temp_dir().join(format!("slinuxm-ca-{}-{}", std::process::id(), port));
|
|
std::fs::write(&ca_path, &cert_pem).unwrap();
|
|
let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap();
|
|
let mut https = peer(port, &token, None);
|
|
https.url = url::Url::parse(&format!("https://127.0.0.1:{port}")).unwrap();
|
|
let err = transport
|
|
.get(&https, req("/api/v1/ping"), Duration::from_secs(2))
|
|
.await
|
|
.unwrap_err();
|
|
assert!(matches!(err, PeerError::Tls), "{err:?}");
|
|
|
|
https.ca_file = Some(ca_path.clone());
|
|
let resp = transport
|
|
.get(&https, req("/api/v1/ping"), Duration::from_secs(2))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status, 200);
|
|
let _ = std::fs::remove_file(&token);
|
|
let _ = std::fs::remove_file(&ca_path);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn deny_cidrs_are_enforced_at_connect_time() {
|
|
let mut cfg = limits(2, 2, 1024);
|
|
cfg.deny_cidrs = vec!["127.0.0.0/8".into()];
|
|
let transport = ReqwestTransport::new(&cfg).unwrap();
|
|
let token = write_token("t", 0o600);
|
|
let p = peer(9, &token, None);
|
|
assert!(matches!(
|
|
transport.tcp_check(&p, Duration::from_secs(1)).await,
|
|
Err(PeerError::Unreachable)
|
|
));
|
|
assert!(matches!(
|
|
transport
|
|
.get(&p, req("/api/v1/ping"), Duration::from_secs(1))
|
|
.await,
|
|
Err(PeerError::Unreachable)
|
|
));
|
|
let _ = std::fs::remove_file(&token);
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_deny_cidr_and_plain_http_opt_in() {
|
|
let mut cfg = limits(2, 2, 1024);
|
|
cfg.deny_cidrs = vec!["not-a-cidr".into()];
|
|
assert!(ReqwestTransport::new(&cfg).is_err());
|
|
|
|
let transport = ReqwestTransport::new(&limits(2, 2, 1024)).unwrap();
|
|
let token = write_token("t", 0o600);
|
|
let mut p = peer(9, &token, None);
|
|
p.url = url::Url::parse("http://10.0.0.5:1984").unwrap();
|
|
assert!(!p.allow_plain_http);
|
|
assert!(validate_target(&p).is_err());
|
|
p.allow_plain_http = true;
|
|
assert!(validate_target(&p).is_ok());
|
|
assert!(transport
|
|
.deny_list()
|
|
.iter()
|
|
.any(|n| n.to_string() == "169.254.0.0/16"));
|
|
let _ = std::fs::remove_file(&token);
|
|
}
|
|
|
|
#[test]
|
|
fn loopback_names() {
|
|
assert!(host_is_loopback(Some("127.0.0.1")));
|
|
assert!(host_is_loopback(Some("::1")));
|
|
assert!(host_is_loopback(Some("localhost")));
|
|
assert!(!host_is_loopback(Some("10.0.0.1")));
|
|
assert!(!host_is_loopback(Some("beta")));
|
|
}
|
|
}
|