From 7a81f9f334cdcd53600d03f149efde10be559e24 Mon Sep 17 00:00:00 2001 From: Hermenix Date: Thu, 1 Oct 2026 12:49:18 +0000 Subject: [PATCH] checks/vm-sessions: size the negative control's window for its own cleanup `a session touched inside the window is not reaped` was racing itself. The idle window restarts at the *last* touch, so everything the subtest does after the loop -- the `status --json` round trip, `kill -0`, and the `kill` that clears the session away -- has to finish inside one whole window. With `--idle-timeout 6s` and a `systemd-run --wait` round trip costing several seconds in this guest, it did not: the session was reaped 6.3 s after the last touch, legitimately, and the cleanup `kill` arrived ~13.5 s after it and returned 4 ("not in a usable state"). The daemon was right and the test was wrong. The window is now 45 s, which is the margin the cleanup gets. The assertion is unchanged in meaning and stricter in degree: nine touches 15 s apart -- a third of a window -- keep the session alive for 135 s, three whole windows of real time, so a `touch` that did not reset the deadline would have let the reaper fire two windows before the loop ended. Nothing in the cleanup stamps activity (`registry.touch` has exactly one caller, `manager.touch`), so the margin is slack and not a hidden touch. Two details keep it honest rather than lucky: the loop ends with a touch instead of a trailing sleep, so the cleanup gets a whole window and not a window less one gap; and it runs under `set -e`, so a `touch` that fails -- exit 4 against an already-reaped session -- fails the loop where it happens instead of being swallowed by the sleep that used to end each iteration. Verified both ways: the check passes (258 s; cleanup finished 4.1 s after the last touch, against the 45 s window), and with `manager.touch` temporarily not calling `registry.touch` it goes red here, on the touch that follows the reap, with the diagnostics dump naming the failure mode. That break is reverted. The positive control (`the idle reaper fires for real`, 5 s) needs no such margin and is unchanged: it waits for `reaped`, which is terminal and held for the tombstone TTL, so time passing only confirms it, and it runs no cleanup that needs a usable session. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01F8R9yNgHeHMnY4nDQwaLtu --- checks/vm-sessions.nix | 59 +++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/checks/vm-sessions.nix b/checks/vm-sessions.nix index 5740ce0..fca0bd6 100644 --- a/checks/vm-sessions.nix +++ b/checks/vm-sessions.nix @@ -25,6 +25,11 @@ # The two real-time sleeps are the idle-reaper subtests, and they are the point # of those subtests: no clock is injected anywhere, because an injected clock # would prove the arithmetic (the unit tests already do) and not the daemon. +# Those two sleeps are sized against the cost of a `systemd-run` round trip in +# this guest and not against the arithmetic, because an idle window has to +# outlast the subtest's own cleanup as well as its loop -- see the negative +# control ("a session touched inside the window is not reaped") for the +# measurement that fixes the numbers. # # ───────────────────────────────────────────────────────────────────────────── # KNOWN DEFECTS this check is pointed at, as of 2026-10-01. @@ -130,8 +135,10 @@ testers.runNixOSTest { name = "hhstream-vm-sessions"; - # Boot plus compositor startup plus ~30 s of real idle-clock waiting. Capped - # so a hang is a failure rather than a wedged builder. + # Boot plus compositor startup plus ~2.5 min of real idle-clock waiting (five + # seconds for the reaper to fire, three 45 s windows for the touched session + # that must outlive them). Capped so a hang is a failure rather than a wedged + # builder. globalTimeout = 1800; nodes.machine = { @@ -578,25 +585,53 @@ testers.runNixOSTest { # satisfied by a daemon that killed everything after five seconds # whatever the caller did, which proves nothing about `touch`. # - # Six seconds of idle window, a touch every two seconds, for fourteen - # seconds -- more than twice the window, so a `touch` that did not - # reset the clock would have let the reaper fire at t=6. The assertion - # itself runs once, after the loop. + # The numbers below are sized so that the subtest's own cleanup cannot + # race the deadline it is asserting about. The window restarts at the + # *last* touch, so everything after the loop -- the status round trip, + # `kill -0`, and the `kill` that clears the session away -- has to + # finish inside one whole window. A `systemd-run --wait` round trip + # costs several seconds in this guest, and three of them do not fit in + # six seconds: with a six-second window the session was legitimately + # reaped underneath its own cleanup and `kill` returned 4 ("not in a + # usable state"). The window is therefore tens of seconds, and that + # whole window is the margin the cleanup gets. + # + # What is asserted is unchanged, and stricter in degree. Touches are + # KEPT_INTERVAL apart -- a third of the window -- and the loop runs for + # KEPT_TOUCHES * KEPT_INTERVAL seconds, three whole windows of real + # time: a `touch` that did not reset the deadline would have let the + # reaper fire two windows before the loop ended, and the assertion + # below would read `reaped`. Two details keep it honest rather than + # lucky: the loop ends with a touch instead of a sleep, so the cleanup + # gets a whole window and not a window less one gap; and it runs under + # `set -e`, so a `touch` that fails fails the loop here instead of + # being discovered later as a reap. + KEPT_WINDOW = 45 + KEPT_INTERVAL = 15 + KEPT_TOUCHES = 9 try: - kept = json.loads(machine.succeed(cli("start --json --idle-timeout 6s foot"), timeout=180)) + kept = json.loads( + machine.succeed( + cli(f"start --json --idle-timeout {KEPT_WINDOW}s foot"), timeout=180 + ) + ) kept_id, kept_pid = kept["id"], kept["pid"] print(f"touched session {kept_id}: pid {kept_pid}, timeout {kept['idle_timeout_s']}s") - assert kept["idle_timeout_s"] == 6.0, kept["idle_timeout_s"] + assert kept["idle_timeout_s"] == float(KEPT_WINDOW), kept["idle_timeout_s"] machine.succeed( - "for i in 1 2 3 4 5 6 7; do " + "set -e; for i in " + + " ".join(str(n) for n in range(1, KEPT_TOUCHES + 1)) + + f"; do sleep {KEPT_INTERVAL}; " + cli(f"touch {kept_id}") - + " >/dev/null; sleep 2; done", - timeout=180, + + " >/dev/null; done", + timeout=KEPT_TOUCHES * (KEPT_INTERVAL + 30) + 60, ) record = json.loads(machine.succeed(cli(f"status --json {kept_id}"), timeout=120))["session"] print(json.dumps(record, indent=2, sort_keys=True)) assert record["state"] == "ready", ( - f"a session touched every 2s inside a 6s window is {record['state']!r}" + f"a session touched every {KEPT_INTERVAL}s for " + f"{KEPT_TOUCHES * KEPT_INTERVAL}s, inside a {KEPT_WINDOW}s window, " + f"is {record['state']!r}" ) assert record["live"] is True, record machine.succeed(f"kill -0 {kept_pid}")