Serve plain HTTP through the shared hyper accept loop with connection cap and header timeout
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
1 parent
d5a0373a75
commit
f1d659965b
6 files changed
+245
-136
No files matched your search
+5
-7
@@ -168,17 +168,15 @@ async fn serve(config: Config) -> Result<()> {
|
||||
// `serve_listener` returns once the shutdown token fires and connections
|
||||
// drain. If the token has not fired it is a server error, so return
|
||||
// immediately. Bound the drain so a stuck client cannot hang SIGTERM.
|
||||
let serving: std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send>> = match tls
|
||||
{
|
||||
Some((acceptor, _)) => Box::pin(tls::serve_tls(
|
||||
let acceptor = tls.map(|(acceptor, _)| acceptor);
|
||||
let serving: std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send>> =
|
||||
Box::pin(server::serve_listener(
|
||||
listener,
|
||||
acceptor,
|
||||
app,
|
||||
shutdown.clone(),
|
||||
tls::Limits::default(),
|
||||
)),
|
||||
None => Box::pin(server::serve_listener(listener, app, shutdown.clone())),
|
||||
};
|
||||
server::Limits::default(),
|
||||
));
|
||||
tokio::pin!(serving);
|
||||
let result = tokio::select! {
|
||||
r = &mut serving => r,
|
||||
|
||||
+120
-6
@@ -8,6 +8,15 @@ use tokio_util::sync::CancellationToken;
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer};
|
||||
use hyper_util::server::conn::auto::Builder;
|
||||
use hyper_util::service::TowerToHyperService;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::Instant;
|
||||
use tokio_rustls::TlsAcceptor;
|
||||
|
||||
use axum::extract::{Request, State};
|
||||
use axum::http::{header, StatusCode};
|
||||
@@ -42,18 +51,123 @@ pub fn build_router(state: AppState) -> Router {
|
||||
.layer(TraceLayer::new_for_http())
|
||||
}
|
||||
|
||||
/// Serve `app` on `listener` until `shutdown` is cancelled.
|
||||
/// Connection limits shared by the plain and TLS listeners.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct Limits {
|
||||
/// Maximum concurrent connections (handshaking or established). At the
|
||||
/// cap, accepting pauses until a connection closes.
|
||||
pub max_conns: usize,
|
||||
/// Time allowed to receive a request header (also covers idle
|
||||
/// keep-alive and post-handshake silence).
|
||||
pub header_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for Limits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_conns: 1024,
|
||||
header_timeout: Duration::from_secs(30),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Serve `app` on `listener` (over TLS when `acceptor` is set) until
|
||||
/// `shutdown` is cancelled, then drain: stop accepting, ask each connection
|
||||
/// to finish gracefully, and wait. The caller bounds the drain by dropping
|
||||
/// this future (which aborts the connections).
|
||||
pub async fn serve_listener(
|
||||
listener: tokio::net::TcpListener,
|
||||
acceptor: Option<TlsAcceptor>,
|
||||
app: Router,
|
||||
shutdown: CancellationToken,
|
||||
limits: Limits,
|
||||
) -> Result<()> {
|
||||
let addr = listener.local_addr().context("local_addr")?;
|
||||
tracing::info!(%addr, "listening");
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move { shutdown.cancelled().await })
|
||||
.await
|
||||
.context("server error")
|
||||
tracing::info!(%addr, tls = acceptor.is_some(), "listening");
|
||||
let mut conns: JoinSet<()> = JoinSet::new();
|
||||
let slots = Arc::new(Semaphore::new(limits.max_conns));
|
||||
let mut last_accept_warn: Option<Instant> = None;
|
||||
loop {
|
||||
// Wait for a free slot first, so at the cap we stop accepting.
|
||||
let permit = tokio::select! {
|
||||
_ = shutdown.cancelled() => break,
|
||||
Some(_) = conns.join_next(), if !conns.is_empty() => continue,
|
||||
p = slots.clone().acquire_owned() => match p {
|
||||
Ok(p) => p,
|
||||
Err(_) => break,
|
||||
},
|
||||
};
|
||||
tokio::select! {
|
||||
_ = shutdown.cancelled() => break,
|
||||
Some(_) = conns.join_next(), if !conns.is_empty() => {}
|
||||
r = listener.accept() => {
|
||||
let (tcp, peer) = match r {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
if last_accept_warn.is_none_or(|t| t.elapsed() > Duration::from_secs(10)) {
|
||||
tracing::warn!(error = %e, "accept failed (further failures suppressed for 10s)");
|
||||
last_accept_warn = Some(Instant::now());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let acceptor = acceptor.clone();
|
||||
let svc = TowerToHyperService::new(app.clone());
|
||||
let stop = shutdown.clone();
|
||||
conns.spawn(async move {
|
||||
let _permit = permit;
|
||||
match acceptor {
|
||||
None => serve_conn(tcp, peer, svc, stop, limits).await,
|
||||
Some(acceptor) => {
|
||||
match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(tcp)).await {
|
||||
Ok(Ok(tls)) => serve_conn(tls, peer, svc, stop, limits).await,
|
||||
Ok(Err(e)) => tracing::debug!(%peer, error = %e, "TLS handshake failed"),
|
||||
Err(_) => tracing::debug!(%peer, "TLS handshake timed out"),
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(listener);
|
||||
while conns.join_next().await.is_some() {}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn serve_conn<I>(
|
||||
io: I,
|
||||
peer: std::net::SocketAddr,
|
||||
svc: TowerToHyperService<Router>,
|
||||
stop: CancellationToken,
|
||||
limits: Limits,
|
||||
) where
|
||||
I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let mut builder = Builder::new(TokioExecutor::new()).http1_only();
|
||||
builder
|
||||
.http1()
|
||||
.timer(TokioTimer::new())
|
||||
.header_read_timeout(limits.header_timeout);
|
||||
// Plain serve_connection: the auto builder's _with_upgrades variant
|
||||
// sniffs the protocol first, which would defeat the header timeout for
|
||||
// silent clients.
|
||||
let conn = builder.serve_connection(TokioIo::new(io), svc);
|
||||
tokio::pin!(conn);
|
||||
tokio::select! {
|
||||
r = conn.as_mut() => {
|
||||
if let Err(e) = r { tracing::debug!(%peer, error = %e, "connection error"); }
|
||||
}
|
||||
_ = stop.cancelled() => {
|
||||
conn.as_mut().graceful_shutdown();
|
||||
if let Err(e) = conn.as_mut().await {
|
||||
tracing::debug!(%peer, error = %e, "connection error");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn allowed_hosts(c: &Config) -> Arc<Vec<String>> {
|
||||
|
||||
+2
-117
@@ -1,7 +1,8 @@
|
||||
//! Native TLS: rustls with the `ring` provider passed explicitly to
|
||||
//! the config builder (nothing is installed process-wide, so the OTLP
|
||||
//! clients' provider choice is unaffected). Only used when
|
||||
//! `server.tls.enabled`; the plain path never touches this module.
|
||||
//! `server.tls.enabled`; the plain path never touches this module. The accept
|
||||
//! loop itself lives in [`crate::server`].
|
||||
//!
|
||||
//! The certificate lives behind a [`ResolvesServerCert`] that can be swapped
|
||||
//! at runtime, so SIGHUP reload leaves established connections untouched and
|
||||
@@ -9,29 +10,19 @@
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use axum::Router;
|
||||
use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer};
|
||||
use hyper_util::server::conn::auto::Builder;
|
||||
use hyper_util::service::TowerToHyperService;
|
||||
use rustls::crypto::ring as provider;
|
||||
use rustls::pki_types::pem::PemObject;
|
||||
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
|
||||
use rustls::server::{ClientHello, ResolvesServerCert};
|
||||
use rustls::sign::CertifiedKey;
|
||||
use rustls::ServerConfig;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::Instant;
|
||||
use tokio_rustls::TlsAcceptor;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::config::TlsConfig;
|
||||
|
||||
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Hot-swappable server certificate.
|
||||
#[derive(Debug)]
|
||||
pub struct ReloadableCert {
|
||||
@@ -131,109 +122,3 @@ pub fn spawn_sighup_reload(
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Connection limits for the TLS listener.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct Limits {
|
||||
/// Maximum concurrent connections (handshaking or established). At the
|
||||
/// cap, accepting pauses until a connection closes.
|
||||
pub max_conns: usize,
|
||||
/// Time allowed to receive a request header (also covers idle
|
||||
/// keep-alive and post-handshake silence).
|
||||
pub header_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for Limits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_conns: 1024,
|
||||
header_timeout: Duration::from_secs(30),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Serve `app` over TLS until `shutdown` is cancelled, then drain: stop
|
||||
/// accepting, ask each connection to finish gracefully, and wait. The caller
|
||||
/// bounds the drain by dropping this future (which aborts the connections).
|
||||
pub async fn serve_tls(
|
||||
listener: tokio::net::TcpListener,
|
||||
acceptor: TlsAcceptor,
|
||||
app: Router,
|
||||
shutdown: CancellationToken,
|
||||
limits: Limits,
|
||||
) -> Result<()> {
|
||||
let addr = listener.local_addr().context("local_addr")?;
|
||||
tracing::info!(%addr, tls = true, "listening");
|
||||
let mut conns: JoinSet<()> = JoinSet::new();
|
||||
let slots = Arc::new(Semaphore::new(limits.max_conns));
|
||||
let mut last_accept_warn: Option<Instant> = None;
|
||||
loop {
|
||||
// Wait for a free slot first, so at the cap we stop accepting.
|
||||
let permit = tokio::select! {
|
||||
_ = shutdown.cancelled() => break,
|
||||
Some(_) = conns.join_next(), if !conns.is_empty() => continue,
|
||||
p = slots.clone().acquire_owned() => match p {
|
||||
Ok(p) => p,
|
||||
Err(_) => break,
|
||||
},
|
||||
};
|
||||
tokio::select! {
|
||||
_ = shutdown.cancelled() => break,
|
||||
Some(_) = conns.join_next(), if !conns.is_empty() => {}
|
||||
r = listener.accept() => {
|
||||
let (tcp, peer) = match r {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
if last_accept_warn.is_none_or(|t| t.elapsed() > Duration::from_secs(10)) {
|
||||
tracing::warn!(error = %e, "accept failed (further failures suppressed for 10s)");
|
||||
last_accept_warn = Some(Instant::now());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let acceptor = acceptor.clone();
|
||||
let svc = TowerToHyperService::new(app.clone());
|
||||
let stop = shutdown.clone();
|
||||
conns.spawn(async move {
|
||||
let _permit = permit;
|
||||
let tls = match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(tcp)).await {
|
||||
Ok(Ok(t)) => t,
|
||||
Ok(Err(e)) => {
|
||||
tracing::debug!(%peer, error = %e, "TLS handshake failed");
|
||||
return;
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::debug!(%peer, "TLS handshake timed out");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let mut builder = Builder::new(TokioExecutor::new()).http1_only();
|
||||
builder
|
||||
.http1()
|
||||
.timer(TokioTimer::new())
|
||||
.header_read_timeout(limits.header_timeout);
|
||||
// Plain serve_connection: the auto builder's _with_upgrades
|
||||
// variant sniffs the protocol first, which would defeat the
|
||||
// header timeout for silent clients.
|
||||
let conn = builder.serve_connection(TokioIo::new(tls), svc);
|
||||
tokio::pin!(conn);
|
||||
tokio::select! {
|
||||
r = conn.as_mut() => {
|
||||
if let Err(e) = r { tracing::debug!(%peer, error = %e, "connection error"); }
|
||||
}
|
||||
_ = stop.cancelled() => {
|
||||
conn.as_mut().graceful_shutdown();
|
||||
if let Err(e) = conn.as_mut().await {
|
||||
tracing::debug!(%peer, error = %e, "connection error");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(listener);
|
||||
while conns.join_next().await.is_some() {}
|
||||
Ok(())
|
||||
}
|
||||
@@ -43,7 +43,13 @@ async fn start(auth: bool) -> Harness {
|
||||
let shutdown = state.shutdown.clone();
|
||||
let state_db = state.db.clone();
|
||||
let app = server::build_router(state);
|
||||
let join = tokio::spawn(server::serve_listener(listener, app, shutdown.clone()));
|
||||
let join = tokio::spawn(server::serve_listener(
|
||||
listener,
|
||||
None,
|
||||
app,
|
||||
shutdown.clone(),
|
||||
server::Limits::default(),
|
||||
));
|
||||
Harness {
|
||||
state_db,
|
||||
port,
|
||||
@@ -212,8 +218,10 @@ async fn docker_unavailable_does_not_affect_systemd() {
|
||||
let shutdown = state.shutdown.clone();
|
||||
tokio::spawn(server::serve_listener(
|
||||
listener,
|
||||
None,
|
||||
server::build_router(state),
|
||||
shutdown.clone(),
|
||||
server::Limits::default(),
|
||||
));
|
||||
let (st, _, _) = http(port, "GET", "/api/v1/docker/ps", &[], None, SHORT).await;
|
||||
assert_eq!(st, 503);
|
||||
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
//! Plain-HTTP listener limits: the same accept loop as TLS, so the same
|
||||
//! connection cap, header timeout and graceful drain apply.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use skyg_linux_tracker::config::Config;
|
||||
use skyg_linux_tracker::server::{self, Limits};
|
||||
use skyg_linux_tracker::sources::fake::fake_state;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
async fn plain(limits: Limits) -> (u16, CancellationToken, JoinHandle<anyhow::Result<()>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
let mut config = Config::default();
|
||||
config.auth.enabled = false;
|
||||
let state = fake_state(config);
|
||||
let shutdown = state.shutdown.clone();
|
||||
let join = tokio::spawn(server::serve_listener(
|
||||
listener,
|
||||
None,
|
||||
server::build_router(state),
|
||||
shutdown.clone(),
|
||||
limits,
|
||||
));
|
||||
(port, shutdown, join)
|
||||
}
|
||||
|
||||
/// Connect and complete one `/healthz` exchange, keeping the connection open.
|
||||
async fn served(port: u16) -> TcpStream {
|
||||
let mut c = TcpStream::connect(("127.0.0.1", port)).await.unwrap();
|
||||
c.write_all(b"GET /healthz HTTP/1.1\r\nHost: localhost\r\n\r\n")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut buf = [0u8; 1024];
|
||||
let n = c.read(&mut buf).await.unwrap();
|
||||
assert!(String::from_utf8_lossy(&buf[..n]).contains("200 OK"));
|
||||
c
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn slowloris_header_is_closed_after_timeout() {
|
||||
let (port, shutdown, _j) = plain(Limits {
|
||||
max_conns: 16,
|
||||
header_timeout: Duration::from_millis(500),
|
||||
})
|
||||
.await;
|
||||
let mut c = TcpStream::connect(("127.0.0.1", port)).await.unwrap();
|
||||
// A partial request line, then silence.
|
||||
c.write_all(b"GET /healthz HTTP/1.1\r\nHost: loc")
|
||||
.await
|
||||
.unwrap();
|
||||
let start = std::time::Instant::now();
|
||||
let mut buf = [0u8; 256];
|
||||
let r = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
match c.read(&mut buf).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(_) => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(r.is_ok(), "slowloris connection was not closed");
|
||||
assert!(start.elapsed() < Duration::from_secs(4));
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_cap_is_enforced() {
|
||||
let (port, shutdown, _j) = plain(Limits {
|
||||
max_conns: 2,
|
||||
header_timeout: Duration::from_secs(30),
|
||||
})
|
||||
.await;
|
||||
let c1 = served(port).await;
|
||||
let c2 = served(port).await;
|
||||
let third = tokio::time::timeout(Duration::from_millis(800), served(port)).await;
|
||||
assert!(third.is_err(), "third connection was served beyond the cap");
|
||||
drop(c1);
|
||||
let c3 = tokio::time::timeout(Duration::from_secs(5), served(port)).await;
|
||||
assert!(c3.is_ok(), "slot was not released after a close");
|
||||
drop(c2);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn graceful_shutdown_drains_idle_connections() {
|
||||
let (port, shutdown, join) = plain(Limits::default()).await;
|
||||
let mut c = served(port).await;
|
||||
// The idle keep-alive connection is still open; shutdown must complete.
|
||||
shutdown.cancel();
|
||||
let r = tokio::time::timeout(Duration::from_secs(5), join)
|
||||
.await
|
||||
.expect("serve did not return after shutdown");
|
||||
r.unwrap().unwrap();
|
||||
let mut buf = [0u8; 64];
|
||||
let n = tokio::time::timeout(Duration::from_secs(2), c.read(&mut buf))
|
||||
.await
|
||||
.expect("idle connection left open");
|
||||
assert!(matches!(n, Ok(0) | Err(_)));
|
||||
}
|
||||
+5
-5
@@ -567,7 +567,7 @@ async fn sigterm_with_open_tls_connections_exits_promptly() {
|
||||
/// In-process TLS listener with explicit limits.
|
||||
async fn inproc(
|
||||
pair: &Pair,
|
||||
limits: skyg_linux_tracker::tls::Limits,
|
||||
limits: skyg_linux_tracker::server::Limits,
|
||||
) -> (u16, tokio_util::sync::CancellationToken, tempdir::Dir) {
|
||||
use skyg_linux_tracker::{config::Config, server, sources::fake::fake_state, tls};
|
||||
let dir = tempdir::Dir::new("inproc");
|
||||
@@ -584,9 +584,9 @@ async fn inproc(
|
||||
let app = server::build_router(state);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
tokio::spawn(tls::serve_tls(
|
||||
tokio::spawn(server::serve_listener(
|
||||
listener,
|
||||
acceptor,
|
||||
Some(acceptor),
|
||||
app,
|
||||
shutdown.clone(),
|
||||
limits,
|
||||
@@ -609,7 +609,7 @@ async fn silent_connection_closed_after_header_timeout() {
|
||||
let pair = gen_pair();
|
||||
let (port, shutdown, _d) = inproc(
|
||||
&pair,
|
||||
skyg_linux_tracker::tls::Limits {
|
||||
skyg_linux_tracker::server::Limits {
|
||||
max_conns: 16,
|
||||
header_timeout: Duration::from_millis(500),
|
||||
},
|
||||
@@ -637,7 +637,7 @@ async fn connection_cap_is_enforced() {
|
||||
let pair = gen_pair();
|
||||
let (port, shutdown, _d) = inproc(
|
||||
&pair,
|
||||
skyg_linux_tracker::tls::Limits {
|
||||
skyg_linux_tracker::server::Limits {
|
||||
max_conns: 2,
|
||||
header_timeout: Duration::from_secs(30),
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user