diff --git a/src/main.rs b/src/main.rs index 929dadf..308520e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -168,17 +168,15 @@ async fn serve(config: Config) -> Result<()> { // `serve_listener` returns once the shutdown token fires and connections // drain. If the token has not fired it is a server error, so return // immediately. Bound the drain so a stuck client cannot hang SIGTERM. - let serving: std::pin::Pin> + Send>> = match tls - { - Some((acceptor, _)) => Box::pin(tls::serve_tls( + let acceptor = tls.map(|(acceptor, _)| acceptor); + let serving: std::pin::Pin> + Send>> = + Box::pin(server::serve_listener( listener, acceptor, app, shutdown.clone(), - tls::Limits::default(), - )), - None => Box::pin(server::serve_listener(listener, app, shutdown.clone())), - }; + server::Limits::default(), + )); tokio::pin!(serving); let result = tokio::select! { r = &mut serving => r, diff --git a/src/server.rs b/src/server.rs index 99048ae..4a42c0f 100644 --- a/src/server.rs +++ b/src/server.rs @@ -8,6 +8,15 @@ use tokio_util::sync::CancellationToken; use tower_http::trace::TraceLayer; use std::sync::Arc; +use std::time::Duration; + +use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer}; +use hyper_util::server::conn::auto::Builder; +use hyper_util::service::TowerToHyperService; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; +use tokio::time::Instant; +use tokio_rustls::TlsAcceptor; use axum::extract::{Request, State}; use axum::http::{header, StatusCode}; @@ -42,18 +51,123 @@ pub fn build_router(state: AppState) -> Router { .layer(TraceLayer::new_for_http()) } -/// Serve `app` on `listener` until `shutdown` is cancelled. +/// Connection limits shared by the plain and TLS listeners. +#[derive(Debug, Clone, Copy)] +pub struct Limits { + /// Maximum concurrent connections (handshaking or established). At the + /// cap, accepting pauses until a connection closes. + pub max_conns: usize, + /// Time allowed to receive a request header (also covers idle + /// keep-alive and post-handshake silence). + pub header_timeout: Duration, +} + +impl Default for Limits { + fn default() -> Self { + Self { + max_conns: 1024, + header_timeout: Duration::from_secs(30), + } + } +} + +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); + +/// Serve `app` on `listener` (over TLS when `acceptor` is set) until +/// `shutdown` is cancelled, then drain: stop accepting, ask each connection +/// to finish gracefully, and wait. The caller bounds the drain by dropping +/// this future (which aborts the connections). pub async fn serve_listener( listener: tokio::net::TcpListener, + acceptor: Option, app: Router, shutdown: CancellationToken, + limits: Limits, ) -> Result<()> { let addr = listener.local_addr().context("local_addr")?; - tracing::info!(%addr, "listening"); - axum::serve(listener, app) - .with_graceful_shutdown(async move { shutdown.cancelled().await }) - .await - .context("server error") + tracing::info!(%addr, tls = acceptor.is_some(), "listening"); + let mut conns: JoinSet<()> = JoinSet::new(); + let slots = Arc::new(Semaphore::new(limits.max_conns)); + let mut last_accept_warn: Option = None; + loop { + // Wait for a free slot first, so at the cap we stop accepting. + let permit = tokio::select! { + _ = shutdown.cancelled() => break, + Some(_) = conns.join_next(), if !conns.is_empty() => continue, + p = slots.clone().acquire_owned() => match p { + Ok(p) => p, + Err(_) => break, + }, + }; + tokio::select! { + _ = shutdown.cancelled() => break, + Some(_) = conns.join_next(), if !conns.is_empty() => {} + r = listener.accept() => { + let (tcp, peer) = match r { + Ok(x) => x, + Err(e) => { + if last_accept_warn.is_none_or(|t| t.elapsed() > Duration::from_secs(10)) { + tracing::warn!(error = %e, "accept failed (further failures suppressed for 10s)"); + last_accept_warn = Some(Instant::now()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + }; + let acceptor = acceptor.clone(); + let svc = TowerToHyperService::new(app.clone()); + let stop = shutdown.clone(); + conns.spawn(async move { + let _permit = permit; + match acceptor { + None => serve_conn(tcp, peer, svc, stop, limits).await, + Some(acceptor) => { + match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(tcp)).await { + Ok(Ok(tls)) => serve_conn(tls, peer, svc, stop, limits).await, + Ok(Err(e)) => tracing::debug!(%peer, error = %e, "TLS handshake failed"), + Err(_) => tracing::debug!(%peer, "TLS handshake timed out"), + } + } + } + }); + } + } + } + drop(listener); + while conns.join_next().await.is_some() {} + Ok(()) +} + +async fn serve_conn( + io: I, + peer: std::net::SocketAddr, + svc: TowerToHyperService, + stop: CancellationToken, + limits: Limits, +) where + I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + let mut builder = Builder::new(TokioExecutor::new()).http1_only(); + builder + .http1() + .timer(TokioTimer::new()) + .header_read_timeout(limits.header_timeout); + // Plain serve_connection: the auto builder's _with_upgrades variant + // sniffs the protocol first, which would defeat the header timeout for + // silent clients. + let conn = builder.serve_connection(TokioIo::new(io), svc); + tokio::pin!(conn); + tokio::select! { + r = conn.as_mut() => { + if let Err(e) = r { tracing::debug!(%peer, error = %e, "connection error"); } + } + _ = stop.cancelled() => { + conn.as_mut().graceful_shutdown(); + if let Err(e) = conn.as_mut().await { + tracing::debug!(%peer, error = %e, "connection error"); + } + } + } } fn allowed_hosts(c: &Config) -> Arc> { diff --git a/src/tls.rs b/src/tls.rs index 2ea4a94..e9ea12a 100644 --- a/src/tls.rs +++ b/src/tls.rs @@ -1,7 +1,8 @@ //! Native TLS: rustls with the `ring` provider passed explicitly to //! the config builder (nothing is installed process-wide, so the OTLP //! clients' provider choice is unaffected). Only used when -//! `server.tls.enabled`; the plain path never touches this module. +//! `server.tls.enabled`; the plain path never touches this module. The accept +//! loop itself lives in [`crate::server`]. //! //! The certificate lives behind a [`ResolvesServerCert`] that can be swapped //! at runtime, so SIGHUP reload leaves established connections untouched and @@ -9,29 +10,19 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; -use std::time::Duration; use anyhow::{anyhow, Context, Result}; -use axum::Router; -use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer}; -use hyper_util::server::conn::auto::Builder; -use hyper_util::service::TowerToHyperService; use rustls::crypto::ring as provider; use rustls::pki_types::pem::PemObject; use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use rustls::server::{ClientHello, ResolvesServerCert}; use rustls::sign::CertifiedKey; use rustls::ServerConfig; -use tokio::sync::Semaphore; -use tokio::task::JoinSet; -use tokio::time::Instant; use tokio_rustls::TlsAcceptor; use tokio_util::sync::CancellationToken; use crate::config::TlsConfig; -const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); - /// Hot-swappable server certificate. #[derive(Debug)] pub struct ReloadableCert { @@ -131,109 +122,3 @@ pub fn spawn_sighup_reload( } }); } - -/// Connection limits for the TLS listener. -#[derive(Debug, Clone, Copy)] -pub struct Limits { - /// Maximum concurrent connections (handshaking or established). At the - /// cap, accepting pauses until a connection closes. - pub max_conns: usize, - /// Time allowed to receive a request header (also covers idle - /// keep-alive and post-handshake silence). - pub header_timeout: Duration, -} - -impl Default for Limits { - fn default() -> Self { - Self { - max_conns: 1024, - header_timeout: Duration::from_secs(30), - } - } -} - -/// Serve `app` over TLS until `shutdown` is cancelled, then drain: stop -/// accepting, ask each connection to finish gracefully, and wait. The caller -/// bounds the drain by dropping this future (which aborts the connections). -pub async fn serve_tls( - listener: tokio::net::TcpListener, - acceptor: TlsAcceptor, - app: Router, - shutdown: CancellationToken, - limits: Limits, -) -> Result<()> { - let addr = listener.local_addr().context("local_addr")?; - tracing::info!(%addr, tls = true, "listening"); - let mut conns: JoinSet<()> = JoinSet::new(); - let slots = Arc::new(Semaphore::new(limits.max_conns)); - let mut last_accept_warn: Option = None; - loop { - // Wait for a free slot first, so at the cap we stop accepting. - let permit = tokio::select! { - _ = shutdown.cancelled() => break, - Some(_) = conns.join_next(), if !conns.is_empty() => continue, - p = slots.clone().acquire_owned() => match p { - Ok(p) => p, - Err(_) => break, - }, - }; - tokio::select! { - _ = shutdown.cancelled() => break, - Some(_) = conns.join_next(), if !conns.is_empty() => {} - r = listener.accept() => { - let (tcp, peer) = match r { - Ok(x) => x, - Err(e) => { - if last_accept_warn.is_none_or(|t| t.elapsed() > Duration::from_secs(10)) { - tracing::warn!(error = %e, "accept failed (further failures suppressed for 10s)"); - last_accept_warn = Some(Instant::now()); - } - tokio::time::sleep(Duration::from_millis(100)).await; - continue; - } - }; - let acceptor = acceptor.clone(); - let svc = TowerToHyperService::new(app.clone()); - let stop = shutdown.clone(); - conns.spawn(async move { - let _permit = permit; - let tls = match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(tcp)).await { - Ok(Ok(t)) => t, - Ok(Err(e)) => { - tracing::debug!(%peer, error = %e, "TLS handshake failed"); - return; - } - Err(_) => { - tracing::debug!(%peer, "TLS handshake timed out"); - return; - } - }; - let mut builder = Builder::new(TokioExecutor::new()).http1_only(); - builder - .http1() - .timer(TokioTimer::new()) - .header_read_timeout(limits.header_timeout); - // Plain serve_connection: the auto builder's _with_upgrades - // variant sniffs the protocol first, which would defeat the - // header timeout for silent clients. - let conn = builder.serve_connection(TokioIo::new(tls), svc); - tokio::pin!(conn); - tokio::select! { - r = conn.as_mut() => { - if let Err(e) = r { tracing::debug!(%peer, error = %e, "connection error"); } - } - _ = stop.cancelled() => { - conn.as_mut().graceful_shutdown(); - if let Err(e) = conn.as_mut().await { - tracing::debug!(%peer, error = %e, "connection error"); - } - } - } - }); - } - } - } - drop(listener); - while conns.join_next().await.is_some() {} - Ok(()) -} diff --git a/tests/integration.rs b/tests/integration.rs index 2fe082f..e93ebc6 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -43,7 +43,13 @@ async fn start(auth: bool) -> Harness { let shutdown = state.shutdown.clone(); let state_db = state.db.clone(); let app = server::build_router(state); - let join = tokio::spawn(server::serve_listener(listener, app, shutdown.clone())); + let join = tokio::spawn(server::serve_listener( + listener, + None, + app, + shutdown.clone(), + server::Limits::default(), + )); Harness { state_db, port, @@ -212,8 +218,10 @@ async fn docker_unavailable_does_not_affect_systemd() { let shutdown = state.shutdown.clone(); tokio::spawn(server::serve_listener( listener, + None, server::build_router(state), shutdown.clone(), + server::Limits::default(), )); let (st, _, _) = http(port, "GET", "/api/v1/docker/ps", &[], None, SHORT).await; assert_eq!(st, 503); diff --git a/tests/plain.rs b/tests/plain.rs new file mode 100644 index 0000000..247b72b --- /dev/null +++ b/tests/plain.rs @@ -0,0 +1,104 @@ +//! Plain-HTTP listener limits: the same accept loop as TLS, so the same +//! connection cap, header timeout and graceful drain apply. + +use std::time::Duration; + +use skyg_linux_tracker::config::Config; +use skyg_linux_tracker::server::{self, Limits}; +use skyg_linux_tracker::sources::fake::fake_state; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +async fn plain(limits: Limits) -> (u16, CancellationToken, JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let mut config = Config::default(); + config.auth.enabled = false; + let state = fake_state(config); + let shutdown = state.shutdown.clone(); + let join = tokio::spawn(server::serve_listener( + listener, + None, + server::build_router(state), + shutdown.clone(), + limits, + )); + (port, shutdown, join) +} + +/// Connect and complete one `/healthz` exchange, keeping the connection open. +async fn served(port: u16) -> TcpStream { + let mut c = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + c.write_all(b"GET /healthz HTTP/1.1\r\nHost: localhost\r\n\r\n") + .await + .unwrap(); + let mut buf = [0u8; 1024]; + let n = c.read(&mut buf).await.unwrap(); + assert!(String::from_utf8_lossy(&buf[..n]).contains("200 OK")); + c +} + +#[tokio::test] +async fn slowloris_header_is_closed_after_timeout() { + let (port, shutdown, _j) = plain(Limits { + max_conns: 16, + header_timeout: Duration::from_millis(500), + }) + .await; + let mut c = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + // A partial request line, then silence. + c.write_all(b"GET /healthz HTTP/1.1\r\nHost: loc") + .await + .unwrap(); + let start = std::time::Instant::now(); + let mut buf = [0u8; 256]; + let r = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match c.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + }) + .await; + assert!(r.is_ok(), "slowloris connection was not closed"); + assert!(start.elapsed() < Duration::from_secs(4)); + shutdown.cancel(); +} + +#[tokio::test] +async fn connection_cap_is_enforced() { + let (port, shutdown, _j) = plain(Limits { + max_conns: 2, + header_timeout: Duration::from_secs(30), + }) + .await; + let c1 = served(port).await; + let c2 = served(port).await; + let third = tokio::time::timeout(Duration::from_millis(800), served(port)).await; + assert!(third.is_err(), "third connection was served beyond the cap"); + drop(c1); + let c3 = tokio::time::timeout(Duration::from_secs(5), served(port)).await; + assert!(c3.is_ok(), "slot was not released after a close"); + drop(c2); + shutdown.cancel(); +} + +#[tokio::test] +async fn graceful_shutdown_drains_idle_connections() { + let (port, shutdown, join) = plain(Limits::default()).await; + let mut c = served(port).await; + // The idle keep-alive connection is still open; shutdown must complete. + shutdown.cancel(); + let r = tokio::time::timeout(Duration::from_secs(5), join) + .await + .expect("serve did not return after shutdown"); + r.unwrap().unwrap(); + let mut buf = [0u8; 64]; + let n = tokio::time::timeout(Duration::from_secs(2), c.read(&mut buf)) + .await + .expect("idle connection left open"); + assert!(matches!(n, Ok(0) | Err(_))); +} diff --git a/tests/tls.rs b/tests/tls.rs index 247dc17..57eabcf 100644 --- a/tests/tls.rs +++ b/tests/tls.rs @@ -567,7 +567,7 @@ async fn sigterm_with_open_tls_connections_exits_promptly() { /// In-process TLS listener with explicit limits. async fn inproc( pair: &Pair, - limits: skyg_linux_tracker::tls::Limits, + limits: skyg_linux_tracker::server::Limits, ) -> (u16, tokio_util::sync::CancellationToken, tempdir::Dir) { use skyg_linux_tracker::{config::Config, server, sources::fake::fake_state, tls}; let dir = tempdir::Dir::new("inproc"); @@ -584,9 +584,9 @@ async fn inproc( let app = server::build_router(state); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); - tokio::spawn(tls::serve_tls( + tokio::spawn(server::serve_listener( listener, - acceptor, + Some(acceptor), app, shutdown.clone(), limits, @@ -609,7 +609,7 @@ async fn silent_connection_closed_after_header_timeout() { let pair = gen_pair(); let (port, shutdown, _d) = inproc( &pair, - skyg_linux_tracker::tls::Limits { + skyg_linux_tracker::server::Limits { max_conns: 16, header_timeout: Duration::from_millis(500), }, @@ -637,7 +637,7 @@ async fn connection_cap_is_enforced() { let pair = gen_pair(); let (port, shutdown, _d) = inproc( &pair, - skyg_linux_tracker::tls::Limits { + skyg_linux_tracker::server::Limits { max_conns: 2, header_timeout: Duration::from_secs(30), },