H: make the SIGTERM tests immune to a stolen port Both helpers verify the socket answers as our server and retry on a fresh port when the child lost the bind race; the SSE exit bound is 10 s
This commit is contained in:
1 parent
3f017725f8
commit
fea6a52bb0
2 files changed
+136
-29
No files matched your search
+76
-8
@@ -390,12 +390,33 @@ fn nonloopback_without_auth_exits_nonzero() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn sigterm_with_open_sse_client_exits_promptly() {
|
fn sigterm_with_open_sse_client_exits_promptly() {
|
||||||
use std::io::{Read, Write};
|
// The port is probed before the child binds it. If something else takes
|
||||||
use std::net::TcpStream as StdStream;
|
// it in between, our child exits and the socket belongs to a stranger, so
|
||||||
|
// a fresh port is tried instead of signalling the wrong process.
|
||||||
|
let mut started = false;
|
||||||
|
for _ in 0..3 {
|
||||||
|
if sse_stream_then_sigterm() {
|
||||||
|
started = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(started, "server did not stay up on any of three ports");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One attempt on a fresh port. `false` means the child exited before it
|
||||||
|
/// served, which only happens when the probed port was taken by someone else.
|
||||||
|
fn sse_stream_then_sigterm() -> bool {
|
||||||
let port = {
|
let port = {
|
||||||
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
l.local_addr().unwrap().port()
|
l.local_addr().unwrap().port()
|
||||||
};
|
};
|
||||||
|
sse_stream_then_sigterm_on(port)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sse_stream_then_sigterm_on(port: u16) -> bool {
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::net::TcpStream as StdStream;
|
||||||
|
|
||||||
let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_slinuxm"))
|
let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_slinuxm"))
|
||||||
.args(["serve", "--auth-enabled", "false", "--bind", "127.0.0.1"])
|
.args(["serve", "--auth-enabled", "false", "--bind", "127.0.0.1"])
|
||||||
.args(["--port", &port.to_string()])
|
.args(["--port", &port.to_string()])
|
||||||
@@ -403,15 +424,29 @@ fn sigterm_with_open_sse_client_exits_promptly() {
|
|||||||
.stderr(std::process::Stdio::null())
|
.stderr(std::process::Stdio::null())
|
||||||
.spawn()
|
.spawn()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let mut conn = None;
|
// Only a server that answers proves the socket is ours; a listener that
|
||||||
|
// answers nothing is a squatter, and the child that lost the race exits.
|
||||||
|
let mut ready = false;
|
||||||
for _ in 0..50 {
|
for _ in 0..50 {
|
||||||
if let Ok(c) = StdStream::connect(("127.0.0.1", port)) {
|
if child.try_wait().unwrap().is_some() {
|
||||||
conn = Some(c);
|
return false;
|
||||||
|
}
|
||||||
|
if healthz_answers(port) {
|
||||||
|
ready = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
std::thread::sleep(Duration::from_millis(100));
|
std::thread::sleep(Duration::from_millis(100));
|
||||||
}
|
}
|
||||||
let mut conn = conn.expect("server did not start");
|
if !ready || child.try_wait().unwrap().is_some() {
|
||||||
|
// Alive but never answered: a real startup failure, not a lost race.
|
||||||
|
let hung = !ready && child.try_wait().unwrap().is_none();
|
||||||
|
let _ = child.kill();
|
||||||
|
let _ = child.wait();
|
||||||
|
assert!(!hung, "server did not start");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut conn = StdStream::connect(("127.0.0.1", port)).unwrap();
|
||||||
conn.set_read_timeout(Some(Duration::from_millis(500)))
|
conn.set_read_timeout(Some(Duration::from_millis(500)))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
write!(
|
write!(
|
||||||
@@ -433,12 +468,45 @@ fn sigterm_with_open_sse_client_exits_promptly() {
|
|||||||
assert!(st.success(), "exit status {st}");
|
assert!(st.success(), "exit status {st}");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if start.elapsed() > Duration::from_secs(5) {
|
if start.elapsed() > Duration::from_secs(10) {
|
||||||
let _ = child.kill();
|
let _ = child.kill();
|
||||||
panic!("process did not exit within 5s of SIGTERM");
|
panic!("process did not exit within 10s of SIGTERM");
|
||||||
}
|
}
|
||||||
std::thread::sleep(Duration::from_millis(50));
|
std::thread::sleep(Duration::from_millis(50));
|
||||||
}
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A port already held by a listener that never answers is not ours: the
|
||||||
|
/// helper reports "retry" instead of signalling the wrong server.
|
||||||
|
#[test]
|
||||||
|
fn a_probed_port_taken_by_someone_else_is_retried() {
|
||||||
|
let squatter = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let port = squatter.local_addr().unwrap().port();
|
||||||
|
assert!(!sse_stream_then_sigterm_on(port));
|
||||||
|
drop(squatter);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn healthz_answers(port: u16) -> bool {
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
let Ok(mut conn) = std::net::TcpStream::connect(("127.0.0.1", port)) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
conn.set_read_timeout(Some(Duration::from_millis(500)))
|
||||||
|
.unwrap();
|
||||||
|
if write!(
|
||||||
|
conn,
|
||||||
|
"GET /healthz HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n"
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let mut buf = [0u8; 32];
|
||||||
|
match conn.read(&mut buf) {
|
||||||
|
Ok(n) => String::from_utf8_lossy(&buf[..n]).starts_with("HTTP/1.1 200"),
|
||||||
|
Err(_) => false,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Real binary: an open MCP session with an in-flight `tail_journal` call
|
/// Real binary: an open MCP session with an in-flight `tail_journal` call
|
||||||
|
|||||||
+60
-21
@@ -109,34 +109,73 @@ fn mint_token(cfg: &Path) -> String {
|
|||||||
String::from_utf8(out.stdout).unwrap().trim().to_owned()
|
String::from_utf8(out.stdout).unwrap().trim().to_owned()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn start(pair: &Pair) -> Server {
|
async fn start(pair: &Pair) -> Server {
|
||||||
|
// The port is probed before the child binds it. If something else takes
|
||||||
|
// it in between, our child exits and the socket belongs to a stranger, so
|
||||||
|
// a fresh port is tried instead of testing the wrong server.
|
||||||
|
for _ in 0..3 {
|
||||||
|
if let Some(s) = try_start_on(pair, free_port()).await {
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
panic!("server did not start");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ready means the port completes *our* TLS handshake while the child is
|
||||||
|
/// still running. `None` means the child was gone before it served, which is
|
||||||
|
/// only possible when the probed port was taken by someone else.
|
||||||
|
async fn try_start_on(pair: &Pair, port: u16) -> Option<Server> {
|
||||||
let dir = tempdir::Dir::new("srv");
|
let dir = tempdir::Dir::new("srv");
|
||||||
let (c, k) = write_pair(&dir.0, pair);
|
let (c, k) = write_pair(&dir.0, pair);
|
||||||
let port = free_port();
|
|
||||||
let cfg = config_toml(&dir.0, port, &c, &k);
|
let cfg = config_toml(&dir.0, port, &c, &k);
|
||||||
let token = mint_token(&cfg);
|
let token = mint_token(&cfg);
|
||||||
let child = Command::new(BIN)
|
let mut s = Server {
|
||||||
.args(["serve", "--config"])
|
child: Command::new(BIN)
|
||||||
.arg(&cfg)
|
.args(["serve", "--config"])
|
||||||
.env_clear()
|
.arg(&cfg)
|
||||||
.stderr(std::fs::File::create(dir.0.join("log.txt")).unwrap())
|
.env_clear()
|
||||||
.spawn()
|
.stderr(std::fs::File::create(dir.0.join("log.txt")).unwrap())
|
||||||
.unwrap();
|
.spawn()
|
||||||
let s = Server {
|
.unwrap(),
|
||||||
child,
|
|
||||||
port,
|
port,
|
||||||
token,
|
token,
|
||||||
dir,
|
dir,
|
||||||
};
|
};
|
||||||
for _ in 0..100 {
|
for _ in 0..100 {
|
||||||
if std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() {
|
if s.child.try_wait().unwrap().is_some() {
|
||||||
return s;
|
return None;
|
||||||
}
|
}
|
||||||
std::thread::sleep(Duration::from_millis(100));
|
if handshake_succeeds(pair, port).await {
|
||||||
|
return Some(s);
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||||
}
|
}
|
||||||
panic!("server did not start");
|
panic!("server did not start");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn handshake_succeeds(pair: &Pair, port: u16) -> bool {
|
||||||
|
let Ok(tcp) = TcpStream::connect(("127.0.0.1", port)).await else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
// A listener that never speaks TLS must not hang the probe.
|
||||||
|
let probe = connector(pair).connect(ServerName::try_from("localhost").unwrap(), tcp);
|
||||||
|
matches!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(500), probe).await,
|
||||||
|
Ok(Ok(_))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A port already held by a listener that never speaks TLS is not ours: the
|
||||||
|
/// helper reports "retry" instead of testing the wrong server.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_probed_port_taken_by_someone_else_is_retried() {
|
||||||
|
let pair = gen_pair();
|
||||||
|
let squatter = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let port = squatter.local_addr().unwrap().port();
|
||||||
|
assert!(try_start_on(&pair, port).await.is_none());
|
||||||
|
drop(squatter);
|
||||||
|
}
|
||||||
|
|
||||||
fn connector(trusted: &Pair) -> TlsConnector {
|
fn connector(trusted: &Pair) -> TlsConnector {
|
||||||
let mut roots = rustls::RootCertStore::empty();
|
let mut roots = rustls::RootCertStore::empty();
|
||||||
for c in CertificateDer::pem_slice_iter(trusted.cert_pem.as_bytes()) {
|
for c in CertificateDer::pem_slice_iter(trusted.cert_pem.as_bytes()) {
|
||||||
@@ -213,7 +252,7 @@ fn signal(child: &Child, sig: &str) {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn healthz_over_https() {
|
async fn healthz_over_https() {
|
||||||
let pair = gen_pair();
|
let pair = gen_pair();
|
||||||
let s = start(&pair);
|
let s = start(&pair).await;
|
||||||
let r = https(s.port, &pair, "GET", "/healthz", &[], None).await;
|
let r = https(s.port, &pair, "GET", "/healthz", &[], None).await;
|
||||||
assert_eq!(r.status, 200, "{}", r.head);
|
assert_eq!(r.status, 200, "{}", r.head);
|
||||||
}
|
}
|
||||||
@@ -221,7 +260,7 @@ async fn healthz_over_https() {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn auth_over_tls() {
|
async fn auth_over_tls() {
|
||||||
let pair = gen_pair();
|
let pair = gen_pair();
|
||||||
let s = start(&pair);
|
let s = start(&pair).await;
|
||||||
let r = https(s.port, &pair, "GET", "/api/v1/nope", &[], None).await;
|
let r = https(s.port, &pair, "GET", "/api/v1/nope", &[], None).await;
|
||||||
assert_eq!(r.status, 401);
|
assert_eq!(r.status, 401);
|
||||||
let auth = format!("Bearer {}", s.token);
|
let auth = format!("Bearer {}", s.token);
|
||||||
@@ -250,7 +289,7 @@ async fn auth_over_tls() {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn plain_http_to_tls_port_fails() {
|
async fn plain_http_to_tls_port_fails() {
|
||||||
let pair = gen_pair();
|
let pair = gen_pair();
|
||||||
let s = start(&pair);
|
let s = start(&pair).await;
|
||||||
let mut c = TcpStream::connect(("127.0.0.1", s.port)).await.unwrap();
|
let mut c = TcpStream::connect(("127.0.0.1", s.port)).await.unwrap();
|
||||||
c.write_all(b"GET /healthz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")
|
c.write_all(b"GET /healthz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")
|
||||||
.await
|
.await
|
||||||
@@ -266,7 +305,7 @@ async fn plain_http_to_tls_port_fails() {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn mcp_initialize_over_https() {
|
async fn mcp_initialize_over_https() {
|
||||||
let pair = gen_pair();
|
let pair = gen_pair();
|
||||||
let s = start(&pair);
|
let s = start(&pair).await;
|
||||||
let auth = format!("Bearer {}", s.token);
|
let auth = format!("Bearer {}", s.token);
|
||||||
let origin = format!("https://localhost:{}", s.port);
|
let origin = format!("https://localhost:{}", s.port);
|
||||||
let r = https(
|
let r = https(
|
||||||
@@ -397,7 +436,7 @@ async fn wait_fp(port: u16, want: impl Fn(&str) -> bool) -> String {
|
|||||||
async fn sighup_serves_new_cert_and_bad_cert_keeps_old() {
|
async fn sighup_serves_new_cert_and_bad_cert_keeps_old() {
|
||||||
let a = gen_pair();
|
let a = gen_pair();
|
||||||
let b = gen_pair();
|
let b = gen_pair();
|
||||||
let s = start(&a);
|
let s = start(&a).await;
|
||||||
let fp_a = https(s.port, &a, "GET", "/healthz", &[], None)
|
let fp_a = https(s.port, &a, "GET", "/healthz", &[], None)
|
||||||
.await
|
.await
|
||||||
.peer_fp;
|
.peer_fp;
|
||||||
@@ -524,7 +563,7 @@ fn enabled_without_paths_fails_validation_and_disabled_ignores_paths() {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn sigterm_with_open_tls_connections_exits_promptly() {
|
async fn sigterm_with_open_tls_connections_exits_promptly() {
|
||||||
let pair = gen_pair();
|
let pair = gen_pair();
|
||||||
let mut s = start(&pair);
|
let mut s = start(&pair).await;
|
||||||
let tcp = TcpStream::connect(("127.0.0.1", s.port)).await.unwrap();
|
let tcp = TcpStream::connect(("127.0.0.1", s.port)).await.unwrap();
|
||||||
let mut idle = connector(&pair)
|
let mut idle = connector(&pair)
|
||||||
.connect(ServerName::try_from("localhost").unwrap(), tcp)
|
.connect(ServerName::try_from("localhost").unwrap(), tcp)
|
||||||
@@ -704,7 +743,7 @@ fn sighup_with_tls_disabled_keeps_serving() {
|
|||||||
async fn mismatched_pair_on_sighup_keeps_old_cert() {
|
async fn mismatched_pair_on_sighup_keeps_old_cert() {
|
||||||
let a = gen_pair();
|
let a = gen_pair();
|
||||||
let b = gen_pair();
|
let b = gen_pair();
|
||||||
let s = start(&a);
|
let s = start(&a).await;
|
||||||
let fp_a = https(s.port, &a, "GET", "/healthz", &[], None)
|
let fp_a = https(s.port, &a, "GET", "/healthz", &[], None)
|
||||||
.await
|
.await
|
||||||
.peer_fp;
|
.peer_fp;
|
||||||
|
|||||||
Reference in new issue
Block a user