H: make the SIGTERM tests immune to a stolen port Both helpers verify the socket answers as our server and retry on a fresh port when the child lost the bind race; the SSE exit bound is 10 s

This commit is contained in:
ari committed 2026-10-20 16:16:13 -04:00
1 parent 3f017725f8
commit fea6a52bb0
2 files changed
+136 -29

No files matched your search

+76 -8
View File
@@ -390,12 +390,33 @@ fn nonloopback_without_auth_exits_nonzero() {
#[test] #[test]
fn sigterm_with_open_sse_client_exits_promptly() { fn sigterm_with_open_sse_client_exits_promptly() {
use std::io::{Read, Write}; // The port is probed before the child binds it. If something else takes
use std::net::TcpStream as StdStream; // it in between, our child exits and the socket belongs to a stranger, so
// a fresh port is tried instead of signalling the wrong process.
let mut started = false;
for _ in 0..3 {
if sse_stream_then_sigterm() {
started = true;
break;
}
}
assert!(started, "server did not stay up on any of three ports");
}
/// One attempt on a fresh port. `false` means the child exited before it
/// served, which only happens when the probed port was taken by someone else.
fn sse_stream_then_sigterm() -> bool {
let port = { let port = {
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
l.local_addr().unwrap().port() l.local_addr().unwrap().port()
}; };
sse_stream_then_sigterm_on(port)
}
fn sse_stream_then_sigterm_on(port: u16) -> bool {
use std::io::{Read, Write};
use std::net::TcpStream as StdStream;
let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_slinuxm")) let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_slinuxm"))
.args(["serve", "--auth-enabled", "false", "--bind", "127.0.0.1"]) .args(["serve", "--auth-enabled", "false", "--bind", "127.0.0.1"])
.args(["--port", &port.to_string()]) .args(["--port", &port.to_string()])
@@ -403,15 +424,29 @@ fn sigterm_with_open_sse_client_exits_promptly() {
.stderr(std::process::Stdio::null()) .stderr(std::process::Stdio::null())
.spawn() .spawn()
.unwrap(); .unwrap();
let mut conn = None; // Only a server that answers proves the socket is ours; a listener that
// answers nothing is a squatter, and the child that lost the race exits.
let mut ready = false;
for _ in 0..50 { for _ in 0..50 {
if let Ok(c) = StdStream::connect(("127.0.0.1", port)) { if child.try_wait().unwrap().is_some() {
conn = Some(c); return false;
}
if healthz_answers(port) {
ready = true;
break; break;
} }
std::thread::sleep(Duration::from_millis(100)); std::thread::sleep(Duration::from_millis(100));
} }
let mut conn = conn.expect("server did not start"); if !ready || child.try_wait().unwrap().is_some() {
// Alive but never answered: a real startup failure, not a lost race.
let hung = !ready && child.try_wait().unwrap().is_none();
let _ = child.kill();
let _ = child.wait();
assert!(!hung, "server did not start");
return false;
}
let mut conn = StdStream::connect(("127.0.0.1", port)).unwrap();
conn.set_read_timeout(Some(Duration::from_millis(500))) conn.set_read_timeout(Some(Duration::from_millis(500)))
.unwrap(); .unwrap();
write!( write!(
@@ -433,12 +468,45 @@ fn sigterm_with_open_sse_client_exits_promptly() {
assert!(st.success(), "exit status {st}"); assert!(st.success(), "exit status {st}");
break; break;
} }
if start.elapsed() > Duration::from_secs(5) { if start.elapsed() > Duration::from_secs(10) {
let _ = child.kill(); let _ = child.kill();
panic!("process did not exit within 5s of SIGTERM"); panic!("process did not exit within 10s of SIGTERM");
} }
std::thread::sleep(Duration::from_millis(50)); std::thread::sleep(Duration::from_millis(50));
} }
true
}
/// A port already held by a listener that never answers is not ours: the
/// helper reports "retry" instead of signalling the wrong server.
#[test]
fn a_probed_port_taken_by_someone_else_is_retried() {
let squatter = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = squatter.local_addr().unwrap().port();
assert!(!sse_stream_then_sigterm_on(port));
drop(squatter);
}
fn healthz_answers(port: u16) -> bool {
use std::io::{Read, Write};
let Ok(mut conn) = std::net::TcpStream::connect(("127.0.0.1", port)) else {
return false;
};
conn.set_read_timeout(Some(Duration::from_millis(500)))
.unwrap();
if write!(
conn,
"GET /healthz HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n"
)
.is_err()
{
return false;
}
let mut buf = [0u8; 32];
match conn.read(&mut buf) {
Ok(n) => String::from_utf8_lossy(&buf[..n]).starts_with("HTTP/1.1 200"),
Err(_) => false,
}
} }
/// Real binary: an open MCP session with an in-flight `tail_journal` call /// Real binary: an open MCP session with an in-flight `tail_journal` call
+60 -21
View File
@@ -109,34 +109,73 @@ fn mint_token(cfg: &Path) -> String {
String::from_utf8(out.stdout).unwrap().trim().to_owned() String::from_utf8(out.stdout).unwrap().trim().to_owned()
} }
fn start(pair: &Pair) -> Server { async fn start(pair: &Pair) -> Server {
// The port is probed before the child binds it. If something else takes
// it in between, our child exits and the socket belongs to a stranger, so
// a fresh port is tried instead of testing the wrong server.
for _ in 0..3 {
if let Some(s) = try_start_on(pair, free_port()).await {
return s;
}
}
panic!("server did not start");
}
/// Ready means the port completes *our* TLS handshake while the child is
/// still running. `None` means the child was gone before it served, which is
/// only possible when the probed port was taken by someone else.
async fn try_start_on(pair: &Pair, port: u16) -> Option<Server> {
let dir = tempdir::Dir::new("srv"); let dir = tempdir::Dir::new("srv");
let (c, k) = write_pair(&dir.0, pair); let (c, k) = write_pair(&dir.0, pair);
let port = free_port();
let cfg = config_toml(&dir.0, port, &c, &k); let cfg = config_toml(&dir.0, port, &c, &k);
let token = mint_token(&cfg); let token = mint_token(&cfg);
let child = Command::new(BIN) let mut s = Server {
.args(["serve", "--config"]) child: Command::new(BIN)
.arg(&cfg) .args(["serve", "--config"])
.env_clear() .arg(&cfg)
.stderr(std::fs::File::create(dir.0.join("log.txt")).unwrap()) .env_clear()
.spawn() .stderr(std::fs::File::create(dir.0.join("log.txt")).unwrap())
.unwrap(); .spawn()
let s = Server { .unwrap(),
child,
port, port,
token, token,
dir, dir,
}; };
for _ in 0..100 { for _ in 0..100 {
if std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() { if s.child.try_wait().unwrap().is_some() {
return s; return None;
} }
std::thread::sleep(Duration::from_millis(100)); if handshake_succeeds(pair, port).await {
return Some(s);
}
tokio::time::sleep(Duration::from_millis(100)).await;
} }
panic!("server did not start"); panic!("server did not start");
} }
async fn handshake_succeeds(pair: &Pair, port: u16) -> bool {
let Ok(tcp) = TcpStream::connect(("127.0.0.1", port)).await else {
return false;
};
// A listener that never speaks TLS must not hang the probe.
let probe = connector(pair).connect(ServerName::try_from("localhost").unwrap(), tcp);
matches!(
tokio::time::timeout(Duration::from_millis(500), probe).await,
Ok(Ok(_))
)
}
/// A port already held by a listener that never speaks TLS is not ours: the
/// helper reports "retry" instead of testing the wrong server.
#[tokio::test]
async fn a_probed_port_taken_by_someone_else_is_retried() {
let pair = gen_pair();
let squatter = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = squatter.local_addr().unwrap().port();
assert!(try_start_on(&pair, port).await.is_none());
drop(squatter);
}
fn connector(trusted: &Pair) -> TlsConnector { fn connector(trusted: &Pair) -> TlsConnector {
let mut roots = rustls::RootCertStore::empty(); let mut roots = rustls::RootCertStore::empty();
for c in CertificateDer::pem_slice_iter(trusted.cert_pem.as_bytes()) { for c in CertificateDer::pem_slice_iter(trusted.cert_pem.as_bytes()) {
@@ -213,7 +252,7 @@ fn signal(child: &Child, sig: &str) {
#[tokio::test] #[tokio::test]
async fn healthz_over_https() { async fn healthz_over_https() {
let pair = gen_pair(); let pair = gen_pair();
let s = start(&pair); let s = start(&pair).await;
let r = https(s.port, &pair, "GET", "/healthz", &[], None).await; let r = https(s.port, &pair, "GET", "/healthz", &[], None).await;
assert_eq!(r.status, 200, "{}", r.head); assert_eq!(r.status, 200, "{}", r.head);
} }
@@ -221,7 +260,7 @@ async fn healthz_over_https() {
#[tokio::test] #[tokio::test]
async fn auth_over_tls() { async fn auth_over_tls() {
let pair = gen_pair(); let pair = gen_pair();
let s = start(&pair); let s = start(&pair).await;
let r = https(s.port, &pair, "GET", "/api/v1/nope", &[], None).await; let r = https(s.port, &pair, "GET", "/api/v1/nope", &[], None).await;
assert_eq!(r.status, 401); assert_eq!(r.status, 401);
let auth = format!("Bearer {}", s.token); let auth = format!("Bearer {}", s.token);
@@ -250,7 +289,7 @@ async fn auth_over_tls() {
#[tokio::test] #[tokio::test]
async fn plain_http_to_tls_port_fails() { async fn plain_http_to_tls_port_fails() {
let pair = gen_pair(); let pair = gen_pair();
let s = start(&pair); let s = start(&pair).await;
let mut c = TcpStream::connect(("127.0.0.1", s.port)).await.unwrap(); let mut c = TcpStream::connect(("127.0.0.1", s.port)).await.unwrap();
c.write_all(b"GET /healthz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") c.write_all(b"GET /healthz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")
.await .await
@@ -266,7 +305,7 @@ async fn plain_http_to_tls_port_fails() {
#[tokio::test] #[tokio::test]
async fn mcp_initialize_over_https() { async fn mcp_initialize_over_https() {
let pair = gen_pair(); let pair = gen_pair();
let s = start(&pair); let s = start(&pair).await;
let auth = format!("Bearer {}", s.token); let auth = format!("Bearer {}", s.token);
let origin = format!("https://localhost:{}", s.port); let origin = format!("https://localhost:{}", s.port);
let r = https( let r = https(
@@ -397,7 +436,7 @@ async fn wait_fp(port: u16, want: impl Fn(&str) -> bool) -> String {
async fn sighup_serves_new_cert_and_bad_cert_keeps_old() { async fn sighup_serves_new_cert_and_bad_cert_keeps_old() {
let a = gen_pair(); let a = gen_pair();
let b = gen_pair(); let b = gen_pair();
let s = start(&a); let s = start(&a).await;
let fp_a = https(s.port, &a, "GET", "/healthz", &[], None) let fp_a = https(s.port, &a, "GET", "/healthz", &[], None)
.await .await
.peer_fp; .peer_fp;
@@ -524,7 +563,7 @@ fn enabled_without_paths_fails_validation_and_disabled_ignores_paths() {
#[tokio::test] #[tokio::test]
async fn sigterm_with_open_tls_connections_exits_promptly() { async fn sigterm_with_open_tls_connections_exits_promptly() {
let pair = gen_pair(); let pair = gen_pair();
let mut s = start(&pair); let mut s = start(&pair).await;
let tcp = TcpStream::connect(("127.0.0.1", s.port)).await.unwrap(); let tcp = TcpStream::connect(("127.0.0.1", s.port)).await.unwrap();
let mut idle = connector(&pair) let mut idle = connector(&pair)
.connect(ServerName::try_from("localhost").unwrap(), tcp) .connect(ServerName::try_from("localhost").unwrap(), tcp)
@@ -704,7 +743,7 @@ fn sighup_with_tls_disabled_keeps_serving() {
async fn mismatched_pair_on_sighup_keeps_old_cert() { async fn mismatched_pair_on_sighup_keeps_old_cert() {
let a = gen_pair(); let a = gen_pair();
let b = gen_pair(); let b = gen_pair();
let s = start(&a); let s = start(&a).await;
let fp_a = https(s.port, &a, "GET", "/healthz", &[], None) let fp_a = https(s.port, &a, "GET", "/healthz", &[], None)
.await .await
.peer_fp; .peer_fp;