H: make the SIGTERM tests immune to a stolen port Both helpers verify the socket answers as our server and retry on a fresh port when the child lost the bind race; the SSE exit bound is 10 s
This commit is contained in:
1 parent
3f017725f8
commit
fea6a52bb0
2 files changed
+136
-29
No files matched your search
+76
-8
@@ -390,12 +390,33 @@ fn nonloopback_without_auth_exits_nonzero() {
|
||||
|
||||
#[test]
|
||||
fn sigterm_with_open_sse_client_exits_promptly() {
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpStream as StdStream;
|
||||
// The port is probed before the child binds it. If something else takes
|
||||
// it in between, our child exits and the socket belongs to a stranger, so
|
||||
// a fresh port is tried instead of signalling the wrong process.
|
||||
let mut started = false;
|
||||
for _ in 0..3 {
|
||||
if sse_stream_then_sigterm() {
|
||||
started = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(started, "server did not stay up on any of three ports");
|
||||
}
|
||||
|
||||
/// One attempt on a fresh port. `false` means the child exited before it
|
||||
/// served, which only happens when the probed port was taken by someone else.
|
||||
fn sse_stream_then_sigterm() -> bool {
|
||||
let port = {
|
||||
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
l.local_addr().unwrap().port()
|
||||
};
|
||||
sse_stream_then_sigterm_on(port)
|
||||
}
|
||||
|
||||
fn sse_stream_then_sigterm_on(port: u16) -> bool {
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpStream as StdStream;
|
||||
|
||||
let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_slinuxm"))
|
||||
.args(["serve", "--auth-enabled", "false", "--bind", "127.0.0.1"])
|
||||
.args(["--port", &port.to_string()])
|
||||
@@ -403,15 +424,29 @@ fn sigterm_with_open_sse_client_exits_promptly() {
|
||||
.stderr(std::process::Stdio::null())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let mut conn = None;
|
||||
// Only a server that answers proves the socket is ours; a listener that
|
||||
// answers nothing is a squatter, and the child that lost the race exits.
|
||||
let mut ready = false;
|
||||
for _ in 0..50 {
|
||||
if let Ok(c) = StdStream::connect(("127.0.0.1", port)) {
|
||||
conn = Some(c);
|
||||
if child.try_wait().unwrap().is_some() {
|
||||
return false;
|
||||
}
|
||||
if healthz_answers(port) {
|
||||
ready = true;
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
let mut conn = conn.expect("server did not start");
|
||||
if !ready || child.try_wait().unwrap().is_some() {
|
||||
// Alive but never answered: a real startup failure, not a lost race.
|
||||
let hung = !ready && child.try_wait().unwrap().is_none();
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
assert!(!hung, "server did not start");
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut conn = StdStream::connect(("127.0.0.1", port)).unwrap();
|
||||
conn.set_read_timeout(Some(Duration::from_millis(500)))
|
||||
.unwrap();
|
||||
write!(
|
||||
@@ -433,12 +468,45 @@ fn sigterm_with_open_sse_client_exits_promptly() {
|
||||
assert!(st.success(), "exit status {st}");
|
||||
break;
|
||||
}
|
||||
if start.elapsed() > Duration::from_secs(5) {
|
||||
if start.elapsed() > Duration::from_secs(10) {
|
||||
let _ = child.kill();
|
||||
panic!("process did not exit within 5s of SIGTERM");
|
||||
panic!("process did not exit within 10s of SIGTERM");
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// A port already held by a listener that never answers is not ours: the
|
||||
/// helper reports "retry" instead of signalling the wrong server.
|
||||
#[test]
|
||||
fn a_probed_port_taken_by_someone_else_is_retried() {
|
||||
let squatter = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = squatter.local_addr().unwrap().port();
|
||||
assert!(!sse_stream_then_sigterm_on(port));
|
||||
drop(squatter);
|
||||
}
|
||||
|
||||
fn healthz_answers(port: u16) -> bool {
|
||||
use std::io::{Read, Write};
|
||||
let Ok(mut conn) = std::net::TcpStream::connect(("127.0.0.1", port)) else {
|
||||
return false;
|
||||
};
|
||||
conn.set_read_timeout(Some(Duration::from_millis(500)))
|
||||
.unwrap();
|
||||
if write!(
|
||||
conn,
|
||||
"GET /healthz HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.is_err()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut buf = [0u8; 32];
|
||||
match conn.read(&mut buf) {
|
||||
Ok(n) => String::from_utf8_lossy(&buf[..n]).starts_with("HTTP/1.1 200"),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Real binary: an open MCP session with an in-flight `tail_journal` call
|
||||
|
||||
+60
-21
@@ -109,34 +109,73 @@ fn mint_token(cfg: &Path) -> String {
|
||||
String::from_utf8(out.stdout).unwrap().trim().to_owned()
|
||||
}
|
||||
|
||||
fn start(pair: &Pair) -> Server {
|
||||
async fn start(pair: &Pair) -> Server {
|
||||
// The port is probed before the child binds it. If something else takes
|
||||
// it in between, our child exits and the socket belongs to a stranger, so
|
||||
// a fresh port is tried instead of testing the wrong server.
|
||||
for _ in 0..3 {
|
||||
if let Some(s) = try_start_on(pair, free_port()).await {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
panic!("server did not start");
|
||||
}
|
||||
|
||||
/// Ready means the port completes *our* TLS handshake while the child is
|
||||
/// still running. `None` means the child was gone before it served, which is
|
||||
/// only possible when the probed port was taken by someone else.
|
||||
async fn try_start_on(pair: &Pair, port: u16) -> Option<Server> {
|
||||
let dir = tempdir::Dir::new("srv");
|
||||
let (c, k) = write_pair(&dir.0, pair);
|
||||
let port = free_port();
|
||||
let cfg = config_toml(&dir.0, port, &c, &k);
|
||||
let token = mint_token(&cfg);
|
||||
let child = Command::new(BIN)
|
||||
.args(["serve", "--config"])
|
||||
.arg(&cfg)
|
||||
.env_clear()
|
||||
.stderr(std::fs::File::create(dir.0.join("log.txt")).unwrap())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let s = Server {
|
||||
child,
|
||||
let mut s = Server {
|
||||
child: Command::new(BIN)
|
||||
.args(["serve", "--config"])
|
||||
.arg(&cfg)
|
||||
.env_clear()
|
||||
.stderr(std::fs::File::create(dir.0.join("log.txt")).unwrap())
|
||||
.spawn()
|
||||
.unwrap(),
|
||||
port,
|
||||
token,
|
||||
dir,
|
||||
};
|
||||
for _ in 0..100 {
|
||||
if std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() {
|
||||
return s;
|
||||
if s.child.try_wait().unwrap().is_some() {
|
||||
return None;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
if handshake_succeeds(pair, port).await {
|
||||
return Some(s);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
panic!("server did not start");
|
||||
}
|
||||
|
||||
async fn handshake_succeeds(pair: &Pair, port: u16) -> bool {
|
||||
let Ok(tcp) = TcpStream::connect(("127.0.0.1", port)).await else {
|
||||
return false;
|
||||
};
|
||||
// A listener that never speaks TLS must not hang the probe.
|
||||
let probe = connector(pair).connect(ServerName::try_from("localhost").unwrap(), tcp);
|
||||
matches!(
|
||||
tokio::time::timeout(Duration::from_millis(500), probe).await,
|
||||
Ok(Ok(_))
|
||||
)
|
||||
}
|
||||
|
||||
/// A port already held by a listener that never speaks TLS is not ours: the
|
||||
/// helper reports "retry" instead of testing the wrong server.
|
||||
#[tokio::test]
|
||||
async fn a_probed_port_taken_by_someone_else_is_retried() {
|
||||
let pair = gen_pair();
|
||||
let squatter = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = squatter.local_addr().unwrap().port();
|
||||
assert!(try_start_on(&pair, port).await.is_none());
|
||||
drop(squatter);
|
||||
}
|
||||
|
||||
fn connector(trusted: &Pair) -> TlsConnector {
|
||||
let mut roots = rustls::RootCertStore::empty();
|
||||
for c in CertificateDer::pem_slice_iter(trusted.cert_pem.as_bytes()) {
|
||||
@@ -213,7 +252,7 @@ fn signal(child: &Child, sig: &str) {
|
||||
#[tokio::test]
|
||||
async fn healthz_over_https() {
|
||||
let pair = gen_pair();
|
||||
let s = start(&pair);
|
||||
let s = start(&pair).await;
|
||||
let r = https(s.port, &pair, "GET", "/healthz", &[], None).await;
|
||||
assert_eq!(r.status, 200, "{}", r.head);
|
||||
}
|
||||
@@ -221,7 +260,7 @@ async fn healthz_over_https() {
|
||||
#[tokio::test]
|
||||
async fn auth_over_tls() {
|
||||
let pair = gen_pair();
|
||||
let s = start(&pair);
|
||||
let s = start(&pair).await;
|
||||
let r = https(s.port, &pair, "GET", "/api/v1/nope", &[], None).await;
|
||||
assert_eq!(r.status, 401);
|
||||
let auth = format!("Bearer {}", s.token);
|
||||
@@ -250,7 +289,7 @@ async fn auth_over_tls() {
|
||||
#[tokio::test]
|
||||
async fn plain_http_to_tls_port_fails() {
|
||||
let pair = gen_pair();
|
||||
let s = start(&pair);
|
||||
let s = start(&pair).await;
|
||||
let mut c = TcpStream::connect(("127.0.0.1", s.port)).await.unwrap();
|
||||
c.write_all(b"GET /healthz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")
|
||||
.await
|
||||
@@ -266,7 +305,7 @@ async fn plain_http_to_tls_port_fails() {
|
||||
#[tokio::test]
|
||||
async fn mcp_initialize_over_https() {
|
||||
let pair = gen_pair();
|
||||
let s = start(&pair);
|
||||
let s = start(&pair).await;
|
||||
let auth = format!("Bearer {}", s.token);
|
||||
let origin = format!("https://localhost:{}", s.port);
|
||||
let r = https(
|
||||
@@ -397,7 +436,7 @@ async fn wait_fp(port: u16, want: impl Fn(&str) -> bool) -> String {
|
||||
async fn sighup_serves_new_cert_and_bad_cert_keeps_old() {
|
||||
let a = gen_pair();
|
||||
let b = gen_pair();
|
||||
let s = start(&a);
|
||||
let s = start(&a).await;
|
||||
let fp_a = https(s.port, &a, "GET", "/healthz", &[], None)
|
||||
.await
|
||||
.peer_fp;
|
||||
@@ -524,7 +563,7 @@ fn enabled_without_paths_fails_validation_and_disabled_ignores_paths() {
|
||||
#[tokio::test]
|
||||
async fn sigterm_with_open_tls_connections_exits_promptly() {
|
||||
let pair = gen_pair();
|
||||
let mut s = start(&pair);
|
||||
let mut s = start(&pair).await;
|
||||
let tcp = TcpStream::connect(("127.0.0.1", s.port)).await.unwrap();
|
||||
let mut idle = connector(&pair)
|
||||
.connect(ServerName::try_from("localhost").unwrap(), tcp)
|
||||
@@ -704,7 +743,7 @@ fn sighup_with_tls_disabled_keeps_serving() {
|
||||
async fn mismatched_pair_on_sighup_keeps_old_cert() {
|
||||
let a = gen_pair();
|
||||
let b = gen_pair();
|
||||
let s = start(&a);
|
||||
let s = start(&a).await;
|
||||
let fp_a = https(s.port, &a, "GET", "/healthz", &[], None)
|
||||
.await
|
||||
.peer_fp;
|
||||
|
||||
Reference in new issue
Block a user