P3 review follow-ups: bounded shutdown, unit hardening, host guard, more tests
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YShymaPrXmc3KYRUg2HMSs
This commit is contained in:
1 parent
acedb9522e
commit
533659dfc1
8 files changed
+355
-14
No files matched your search
+32
-3
@@ -99,6 +99,11 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Upper bounds for shutdown phases (the OTel flush adds up to 5s per
|
||||
/// provider, so the unit uses TimeoutStopSec=30).
|
||||
const DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
const EXPORTER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
async fn serve(config: Config) -> Result<()> {
|
||||
let guard = telemetry::init(&config).context("initialising telemetry")?;
|
||||
let db = if config.auth.enabled {
|
||||
@@ -125,11 +130,35 @@ async fn serve(config: Config) -> Result<()> {
|
||||
signal_token.cancel();
|
||||
});
|
||||
|
||||
let result = server::serve_listener(listener, app, shutdown.clone()).await;
|
||||
// `serve_listener` returns once the shutdown token fires and connections
|
||||
// drain. If the token has not fired it is a server error, so return
|
||||
// immediately. Bound the drain so a stuck client cannot hang SIGTERM.
|
||||
let serving = server::serve_listener(listener, app, shutdown.clone());
|
||||
tokio::pin!(serving);
|
||||
let result = tokio::select! {
|
||||
r = &mut serving => r,
|
||||
_ = shutdown.cancelled() => {
|
||||
match tokio::time::timeout(DRAIN_TIMEOUT, &mut serving).await {
|
||||
Ok(r) => r,
|
||||
Err(_) => {
|
||||
tracing::warn!("graceful drain timed out; closing remaining connections");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
// Reached on signal or server error: stop everything, then flush OTel.
|
||||
shutdown.cancel();
|
||||
for h in exporters {
|
||||
let _ = h.await;
|
||||
let join_all = async {
|
||||
for h in exporters {
|
||||
let _ = h.await;
|
||||
}
|
||||
};
|
||||
if tokio::time::timeout(EXPORTER_TIMEOUT, join_all)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!("exporter tasks did not stop in time");
|
||||
}
|
||||
guard.shutdown().await;
|
||||
if let Some(pool) = pool {
|
||||
|
||||
Reference in new issue
Block a user