# checks..vm-sessions -- the session broker, on a real machine, against # the real compositor. # # Same machine as `vm-hyprland` (compositor + gpu-session.nix), plus # `sessions.enable = true` and one allowlisted victim application: `foot`, a # real Wayland client with no GPU demands. Nothing here reads the frame socket: # the source of truth for "the application is on the screen" is **the compositor # itself**, asked over its own IPC, because the frame publisher could be happy # about a screen with nothing on it. # # Habits copied from vm-hyprland, deliberately: every wait is bounded, each # subtest makes its assertions once, and a failure dumps the compositor's # diagnostics *and* the broker's journal, registry and the application's own # captured output before it propagates. # # Every wait in the script is one of five kinds, and each waits for an *event # that must happen* rather than for an assertion to come good: # # 1. the compositor's unit and the broker's unit coming up (wait_for_unit); # 2. the compositor's and the broker's sockets appearing (wait_for_file); # 3. the compositor mapping the client foot just connected with; # 4. a process finishing dying after a kill; # 5. the idle reaper firing, and the restarted broker answering again. # # The two real-time sleeps are the idle-reaper subtests, and they are the point # of those subtests: no clock is injected anywhere, because an injected clock # would prove the arithmetic (the unit tests already do) and not the daemon. # Those two sleeps are sized against the cost of a `systemd-run` round trip in # this guest and not against the arithmetic, because an idle window has to # outlast the subtest's own cleanup as well as its loop -- see the negative # control ("a session touched inside the window is not reaped") for the # measurement that fixes the numbers. # # ───────────────────────────────────────────────────────────────────────────── # KNOWN DEFECTS this check is pointed at, as of 2026-10-01. # # Nothing below is worked around here. No WAYLAND_DISPLAY is injected into the # unit, no ExecStart is overridden, and no assertion is softened to let a # known-broken path through: a check that accommodated the daemon would go # green while the capability it exists to prove stayed broken. # # * BUG-01 and the unknown `--compositor-runtime-dir` flag -- **fixed**. This # check found both on its first run. The module passed the daemon a flag its # parser did not define, so hhstreamd exited 2/INVALIDARGUMENT before it # ever listened (restart counter past 40 by the time the test gave up), and # even past that, a launched application was handed the daemon's own private # per-session XDG_RUNTIME_DIR with no WAYLAND_DISPLAY and died inside a # second. `hhstreamd/compositor.py` and `--compositor-runtime-dir` are the # fix, and this check is what holds them honest. # # Worth keeping from that run: `wait_for_unit` *passed* against the crash # loop, because the unit is `Type=simple` and systemd calls it active as # soon as it has forked, and each attempt was slow enough (~3 s) that the # default start limit never tripped. That is why the unit subtest asserts on # the socket and on `NRestarts` rather than trusting `wait_for_unit` alone. # # * BUG-02 -- **open**, being fixed separately. `procmon.verify` reports *any* # cmdline change as `pid_reuse_detected`, and `SessionManager.teardown` # treats that name as "do not signal, and do not sweep either", so a session # whose application exec-replaced its own argv -- a wrapper script, most of # them -- is tombstoned `killed` while the process keeps running. The kill # and reaper subtests below assert that the pid, its named children, its # process group and its session id are all gone, and they stay that way: if # BUG-02 reaches this check's victim, the right outcome is a red test, not a # softer assertion. (The victim here is `${foot}/bin/foot`, a plain ELF in # nixpkgs rather than a wrapper, so the trigger may not fire through it -- # which is a fact about foot's packaging and not a reason to trust the path.) # # * The `/proc` session-id sweep cannot reach a PTY child -- **found by this # check, open**. DESIGN.md §7 #7 says a child that called `setsid()` once is # still caught, because it keeps the session id, and only a double # `setsid()` escapes. Measured in this guest, that is not so: foot is # pid/pgid/sid 1042 and the shell it runs is pid/pgid/sid **1056**. A # terminal emulator must `setsid()` to give its child a controlling # terminal, and `setsid()` changes the session id -- the very key the sweep # searches on. So one `setsid()` is enough to leave both the process group # `killpg` signals and the session the sweep walks, and any app that opens a # PTY (a terminal, `script`, `tmux`, `ssh`) leaks its child by construction. # # In practice foot's shell still dies, because closing the PTY master sends # it SIGHUP -- but that is the kernel's tty semantics, not this daemon's # containment, and a child that ignores SIGHUP or holds no tty would # survive. That is why the subtest below walks the **process tree** and # names the pids it expects to be gone, rather than trusting an empty # process group to mean "nothing survived". # # * A broker restart kills every session it was managing -- **found by this # check, open, and it is a one-line module fix**. `systemctl restart # hhstream-sessions` tombstones every live session as `exited` # (`exit_detected_by: "proc_check"`, `last_error: "gone"`), and the new # daemon logs `adopted 0, tombstoned 1`. The application is not gone because # the daemon let go of it; it is gone because the *unit* killed it. # `systemd.services.hhstream-sessions` sets no `KillMode=`, so it gets the # default `control-group`: on stop systemd signals every process still in # the unit's cgroup, and a launched app is a child of the daemon and so # lives in that cgroup. # # This defeats DESIGN.md §2.4 and the whole adopt-on-startup path. # `hhstreamd/launcher.py` explains at length why it uses a plain `Popen` # rather than asyncio's subprocess transport -- precisely so that a daemon # restart is not destructive -- and notes that "no unit file setting can fix # it because the kill comes from inside this process". The converse is what # bites here: the daemon's own care is undone from outside by the unit's # default kill mode. `KillMode = "process"` on that unit is the fix. # systemd.kill(5) warns that `process` lets children escape the service # manager's lifecycle, which is exactly the trade this design already made: # the registry, `killpg` and the `/proc` sweep are the lifecycle here. # # The restart subtests below are left asserting survival, and therefore red, # until that lands. # ───────────────────────────────────────────────────────────────────────────── { testers, hhstream, testTools, hhsModule, hyprland, foot, jq, procps, coreutils, dejavu_fonts, width ? 1280, height ? 720, fps ? 30, # The account the broker and every application it launches run as. The module # refuses to create it (launched applications run as a real account), so the # guest below declares it. sessionUser ? "hermes", # Long enough that nothing in this test is reaped by surprise, short enough # that the number is visibly this test's and not the module's 3 h default. idleTimeout ? 600, controlSocket ? "/run/hhstream/control.sock", }: testers.runNixOSTest { name = "hhstream-vm-sessions"; # Boot plus compositor startup plus ~2.5 min of real idle-clock waiting (five # seconds for the reaper to fire, three 45 s windows for the touched session # that must outlive them). Capped so a hang is a failure rather than a wedged # builder. globalTimeout = 1800; nodes.machine = { imports = [ hhsModule ./lib/gpu-session.nix ]; services.hyprland-headless-stream = { enable = true; package = hhstream; hyprlandPackage = hyprland; # No `name`: the wildcard rule applies to whatever connector the virtio # GPU gives this guest (see vm-hyprland.nix for why the check must not # know that name). outputs = [ { inherit width height; refresh = fps; } ]; sessions = { enable = true; user = sessionUser; inherit idleTimeout; apps.foot = { exec = "${foot}/bin/foot"; # The daemon refuses unlisted arguments, and this check never passes # any: `foot` with no argv opens a window running the account's shell, # which is all the victim has to do. argsAllow = [ ]; }; }; }; # The module deliberately does not create this user. `video` and `render` # are what any client of this session needs to reach the GPU; the broker's # unit adds the compositor's own group on top (SupplementaryGroups=), which # is what lets a launched client open the Wayland socket. users.users.${sessionUser} = { isNormalUser = true; extraGroups = [ "video" "render" ]; }; # foot resolves "monospace" through fontconfig and refuses to start with no # usable font. A NixOS test machine has `fonts.enableDefaultPackages = # false`, so without this the victim application fails for a reason that has # nothing to do with the broker. One font, not the default set. fonts.packages = [ dejavu_fonts ]; environment.systemPackages = [ testTools # hhs-hyprland-diagnostics hhstream # the CLI under test hyprland # hyprctl: the compositor is the source of truth here foot jq # the in-guest predicates the bounded waits poll procps # ps, for "the process group and the session id are gone" ]; virtualisation = { memorySize = 2048; cores = 2; diskSize = 4096; graphics = false; }; }; testScript = '' import json USER = "${sessionUser}" CONTROL = "${controlSocket}" CONTROL_DIR = "/run/hhstream" IDLE_TIMEOUT = ${toString idleTimeout}.0 COMPOSITOR_RUNTIME = "/run/hypr-stream" # The module pins the compositor's XDG_RUNTIME_DIR; the instance signature # under it is the only part not known ahead of time. Root may connect to # the IPC socket even though it belongs to `hhstream`. Same shape as # vm-hyprland.nix, which asks this for monitors. HYPRCTL = ( "set -eu; " f"sig=$(ls -1 {COMPOSITOR_RUNTIME}/hypr | head -n1); " f"XDG_RUNTIME_DIR={COMPOSITOR_RUNTIME}" " HYPRLAND_INSTANCE_SIGNATURE=$sig hyprctl -j" ) HHSTREAM = "${hhstream}/bin/hhstream" TRUE = "${coreutils}/bin/true" FALSE = "${coreutils}/bin/false" ID = "${coreutils}/bin/id" def as_user(command): """`command` (absolute path plus argv, never a shell) run as USER. A transient unit rather than `su`, because that is what the existing checks already do for the compositor's account, and because it gives a clean argv with no login shell in the middle. `--pipe --wait` propagates the command's own exit status, which the first subtest proves before anything depends on it. """ return ( "systemd-run --quiet --collect --pipe --wait" f" --uid={USER} --setenv=HOME=/home/{USER} -- {command}" ) def cli(args): """`hhstream session …` as USER, against the control socket.""" return as_user(f"{HHSTREAM} session --socket {CONTROL} {args}") def gone(pid, pgid, sid, children=()): """A shell predicate: this pid, each pid in `children`, the process group and the session id are all absent from /proc. `children` is named explicitly rather than left to the process-group check because "kill really kills, including children" is the assertion, and a predicate that only looked at pgid/sid would also pass against a teardown that never signalled anything if the group happened to be empty. Built by concatenation so awk's braces do not have to be escaped through an f-string. """ awk = "awk '$2 == " + str(pgid) + " || $3 == " + str(sid) + "'" parts = [ "! kill -0 " + str(victim) + " 2>/dev/null" for victim in (pid, *children) ] parts.append('[ -z "$(ps -eo pid=,pgid=,sid= | ' + awk + ')" ]') return " && ".join(parts) def dump(label, session_id=None): print(f"########## diagnostics: {label} ##########") status, out = machine.execute("hhs-hyprland-diagnostics", timeout=300) print(out) print(f"----- hhs-hyprland-diagnostics exit {status} -----") commands = [ "systemctl --no-pager --full status hhstream-sessions.service", "journalctl --no-pager --boot --lines=200 --unit hhstream-sessions.service", "systemctl show -p ExecStart --value hhstream-sessions.service", f"ls -la {CONTROL_DIR} {CONTROL_DIR}/sessions /var/lib/hhstream" " /var/lib/hhstream/sessions /var/lib/hhstream/logs", "cat /var/lib/hhstream/sessions/*.json", "ps -eo pid,ppid,pgid,sid,user,stat,comm,args --sort=pid", cli("list --all"), ] if session_id: commands += [ cli(f"status {session_id}"), # The application's own captured output: the only place a # failed launch explains itself. cli(f"logs {session_id} --lines 200"), ] for command in commands: status, out = machine.execute(f"{command} 2>&1", timeout=180) print(f"----- {command} (exit {status}) -----") print(out) status, out = machine.execute(f"{HYPRCTL} clients 2>&1", timeout=180) print(f"----- hyprctl clients (exit {status}) -----") print(out) print("########## end diagnostics ##########") start_all() with subtest("the as-user wrapper propagates exit status"): # Everything below reads `machine.succeed`/`machine.fail` through # `as_user`. If the wrapper swallowed the exit status, every one of # those assertions would be vacuous, so prove it here first -- the same # reason checks/lib/test-tools.nix self-tests its helpers at build time. machine.succeed(as_user(TRUE), timeout=120) machine.fail(as_user(FALSE), timeout=120) print(machine.succeed(as_user(ID), timeout=120)) with subtest("the compositor comes up"): # Not the thing under test, but every assertion below needs a session # for the applications to be launched into. try: machine.wait_for_unit("hyprland-headless.service", timeout=300) machine.wait_for_file(f"{COMPOSITOR_RUNTIME}/hypr", timeout=120) except Exception: dump("the compositor never started") raise with subtest("the broker's unit comes up and listens"): # `wait_for_unit` on its own is not enough here, and this is measured # rather than assumed: the unit is `Type=simple` with no readiness # protocol, so systemd calls it active the instant it has forked. A # daemon that exits two milliseconds later and is restarted every # second therefore passes `wait_for_unit` -- which is exactly what the # ExecStart below does today. The honest test of "it came up" is the # socket it exists to listen on, plus a restart count of zero. try: machine.wait_for_unit("hhstream-sessions.service", timeout=180) machine.wait_for_file(CONTROL, timeout=120) machine.succeed("systemctl is-active hhstream-sessions.service") restarts = machine.succeed( "systemctl show -p NRestarts --value hhstream-sessions.service" ).strip() assert restarts == "0", ( f"the broker has restarted {restarts} time(s) already: it is " "crash-looping, not running" ) except Exception: dump("hhstream-sessions.service did not come up and listen") raise with subtest("the control socket has the expected owner and mode"): try: print(machine.succeed(f"stat -c 'owner=%U group=%G mode=%a type=%F' {CONTROL}")) print(machine.succeed(f"stat -c 'owner=%U group=%G mode=%a' {CONTROL_DIR}")) machine.succeed(f'[ "$(stat -c %F {CONTROL})" = socket ]') machine.succeed(f'[ "$(stat -c %U {CONTROL})" = {USER} ]') # The unit sets no Group=, so the socket's group is the account's # own primary group. Asked of the guest rather than hardcoded: the # module's contract is "the account that drives the CLI owns it", # not "the group is called users". machine.succeed(f'[ "$(stat -c %G {CONTROL})" = "$(id -gn {USER})" ]') machine.succeed(f'[ "$(stat -c %a {CONTROL})" = 660 ]') machine.succeed(f'[ "$(stat -c %a {CONTROL_DIR})" = 750 ]') machine.succeed(f'[ "$(stat -c %U {CONTROL_DIR})" = {USER} ]') except Exception: dump("the control socket is missing or not as specified") raise with subtest("start prints a bare session id on stdout"): # The contract: stdout is the id and nothing else, so $(...) is usable. # Human detail goes to stderr, which the driver sends to the test log. try: out = machine.succeed(cli("start foot"), timeout=180) assert out.endswith("\n"), f"start's stdout is not one line: {out!r}" session_id = out.strip() assert session_id.startswith("foot-"), ( f"stdout is not a session id for the app that was started: {out!r}" ) assert len(session_id.split()) == 1, ( f"stdout carries more than the bare id: {out!r}" ) print(f"session id: {session_id}") except Exception: dump("start did not print a session id") raise with subtest("the session reports ready, with a live pid"): try: reply = json.loads(machine.succeed(cli(f"status --json {session_id}"), timeout=120)) session = reply["session"] print(json.dumps(session, indent=2, sort_keys=True)) assert session["state"] == "ready", f"state is {session['state']!r}, not ready" pid, pgid, sid = session["pid"], session["pgid"], session["sid"] assert isinstance(pid, int) and pid > 1, f"implausible pid {pid!r}" # start_new_session(): the kill handle is sid == pgid == pid. assert pgid == pid and sid == pid, f"pid {pid}, pgid {pgid}, sid {sid} disagree" assert session["idle_timeout_s"] == IDLE_TIMEOUT, ( f"idle timeout is {session['idle_timeout_s']!r}, not the module's " f"sessions.idleTimeout ({IDLE_TIMEOUT})" ) machine.succeed(f"kill -0 {pid}") except Exception: dump("the session did not report ready", locals().get("session_id")) raise with subtest("the compositor reports a client whose pid is the session's"): # THE assertion of this check: the compositor, over its own IPC, is the # only thing that can say an application is really on this screen. The # frame publisher is not consulted, and neither class nor title is # asserted on -- only the pid, which is the one fact that ties the # client on the screen to the session the broker handed back. # # One bounded wait, for the compositor to map a client that has only # just connected: `ready` means "the process is alive", never "a window # exists". Everything after it is a single assertion on a session that # has finished coming up. try: machine.wait_until_succeeds( f"{HYPRCTL} clients | jq -e --argjson pid {pid} 'any(.pid == $pid)' >/dev/null", timeout=120, ) clients = json.loads(machine.succeed(f"{HYPRCTL} clients", timeout=120)) print(f"compositor clients by pid: {[c.get('pid') for c in clients]}") assert [c.get("pid") for c in clients].count(pid) == 1, ( f"the compositor does not report exactly one client with pid {pid}" ) except Exception: dump("the compositor never saw the launched application", session_id) raise with subtest("list reports the session with the right pid and state"): try: table = machine.succeed(cli("list"), timeout=120) print(table) rows = [line.split() for line in table.splitlines()[1:] if line.strip()] mine = [row for row in rows if row[0] == session_id] assert len(mine) == 1, f"list does not show {session_id} exactly once" app, state, listed_pid = mine[0][1], mine[0][2], mine[0][3] assert app == "foot", f"list calls the app {app!r}" assert state == "ready", f"list calls the state {state!r}" assert listed_pid == str(pid), f"list reports pid {listed_pid}, not {pid}" except Exception: dump("list did not report the session", session_id) raise with subtest("status --json reports the same pid and state"): try: reply = json.loads(machine.succeed(cli(f"status --json {session_id}"), timeout=120)) assert reply["ok"] is True, reply assert reply["id"] == session_id, reply assert reply["state"] == "ready", reply assert reply["session"]["pid"] == pid, reply["session"] assert reply["session"]["app"] == "foot", reply["session"] assert reply["session"]["live"] is True, reply["session"] assert reply["session"]["ready_signal"] == "spawned", reply["session"] except Exception: dump("status --json did not report the session", session_id) raise with subtest("the application really has a child of its own"): # Captured before the kill, so "including children" is an assertion # about named pids afterwards and not an inference from an empty # process group. The window being mapped (above) means the child # already exists, so this needs no wait. # # The walk is over **ppid**, not over the session id, and that is the # whole point. Measured in this guest: foot is pid/pgid/sid 1042 and # the shell it runs is pid/pgid/sid 1056 -- a terminal emulator must # call setsid() to get a controlling terminal for its child, so that # child leaves both the process group the daemon signals *and* the # session id the daemon sweeps. So a pgid/sid-only check would pass # with the child still running -- it is absent from both keys whether # it is alive or dead -- and capturing "children" by session id finds # nothing to name. The process tree is the only honest source. try: table = machine.succeed("ps -eo pid=,ppid=", timeout=120) by_parent: dict[int, list[int]] = {} for line in table.splitlines(): bits = line.split() if len(bits) == 2: by_parent.setdefault(int(bits[1]), []).append(int(bits[0])) children: list[int] = [] frontier = [pid] while frontier: for child in by_parent.get(frontier.pop(), []): if child not in children: children.append(child) frontier.append(child) print(f"descendants of {pid}: {children}") assert children, ( f"{pid} has no descendant at all: the victim never started its " "child, so this check cannot say anything about killing children" ) print( machine.succeed( "ps -o pid,ppid,pgid,sid,comm " + " ".join(f"-p {p}" for p in (pid, *children)), timeout=120, ) ) except Exception: dump("the launched application never had a child", session_id) raise with subtest("kill ends the session"): try: reply = json.loads(machine.succeed(cli(f"kill --json {session_id}"), timeout=180)) print(json.dumps(reply, indent=2, sort_keys=True)) assert reply["ok"] is True, reply assert reply["session"]["state"] == "killed", reply["session"] # Not asserted: `stragglers`. It counts what the /proc sweep had to # SIGKILL after the process group died, which is the sweep working # rather than the kill failing. The next subtest is the assertion. print(f"stragglers swept by session id: {reply['session'].get('stragglers')}") except Exception: dump("kill did not end the session", session_id) raise with subtest("the pid, its children, its process group and its session id are gone"): try: # Bounded, and only for the kernel to finish a teardown the daemon # has already performed: `kill` returns after the process group was # signalled, the grace spent and the session id swept. It is not a # retry of the assertion -- if anything is still there after this, # something escaped. machine.wait_until_succeeds(gone(pid, pgid, sid, children), timeout=60) print(machine.succeed("ps -eo pid,pgid,sid,stat,comm --sort=pid")) except Exception: dump("something survived the kill", session_id) raise with subtest("the compositor no longer lists the client"): try: machine.wait_until_succeeds( f"{HYPRCTL} clients | jq -e --argjson pid {pid} 'all(.pid != $pid)' >/dev/null", timeout=60, ) clients = json.loads(machine.succeed(f"{HYPRCTL} clients", timeout=120)) print(f"compositor clients by pid: {[c.get('pid') for c in clients]}") assert pid not in [c.get("pid") for c in clients] except Exception: dump("the compositor still lists the killed client", session_id) raise with subtest("the idle reaper fires for real"): # No injected clock anywhere: the daemon's own CLOCK_BOOTTIME deadline, # a five-second timeout, and real time passing in the guest. # # `start --json` rather than start-then-status, so the pid is captured # in the same call that created the session: with a five-second window # a second round trip could land after the reaper had already fired. try: reaped = json.loads(machine.succeed(cli("start --json --idle-timeout 5s foot"), timeout=180)) reaped_id, reaped_pid = reaped["id"], reaped["pid"] reaped_pgid, reaped_sid = reaped["pgid"], reaped["sid"] print(f"short-lived session {reaped_id}: pid {reaped_pid}, timeout " f"{reaped['idle_timeout_s']}s") assert reaped["idle_timeout_s"] == 5.0, reaped["idle_timeout_s"] assert reaped["state"] == "ready", reaped["state"] # Nothing touches it. The only wait is for the deadline to arrive. machine.wait_until_succeeds( cli(f"status --json {reaped_id}") + " | jq -e '.session.state == \"reaped\"' >/dev/null", timeout=90, ) record = json.loads(machine.succeed(cli(f"status --json {reaped_id}"), timeout=120))["session"] assert record["state"] == "reaped", record["state"] assert record["live"] is False, record except Exception: dump("the idle reaper never fired", locals().get("reaped_id")) raise with subtest("the reaped session's application is gone"): try: machine.wait_until_succeeds(gone(reaped_pid, reaped_pgid, reaped_sid), timeout=60) except Exception: dump("the reaper left the application running", reaped_id) raise with subtest("a session touched inside the window is not reaped"): # The other half of the clock. Without this the subtest above would be # satisfied by a daemon that killed everything after five seconds # whatever the caller did, which proves nothing about `touch`. # # The numbers below are sized so that the subtest's own cleanup cannot # race the deadline it is asserting about. The window restarts at the # *last* touch, so everything after the loop -- the status round trip, # `kill -0`, and the `kill` that clears the session away -- has to # finish inside one whole window. A `systemd-run --wait` round trip # costs several seconds in this guest, and three of them do not fit in # six seconds: with a six-second window the session was legitimately # reaped underneath its own cleanup and `kill` returned 4 ("not in a # usable state"). The window is therefore tens of seconds, and that # whole window is the margin the cleanup gets. # # What is asserted is unchanged, and stricter in degree. Touches are # KEPT_INTERVAL apart -- a third of the window -- and the loop runs for # KEPT_TOUCHES * KEPT_INTERVAL seconds, three whole windows of real # time: a `touch` that did not reset the deadline would have let the # reaper fire two windows before the loop ended, and the assertion # below would read `reaped`. Two details keep it honest rather than # lucky: the loop ends with a touch instead of a sleep, so the cleanup # gets a whole window and not a window less one gap; and it runs under # `set -e`, so a `touch` that fails fails the loop here instead of # being discovered later as a reap. KEPT_WINDOW = 45 KEPT_INTERVAL = 15 KEPT_TOUCHES = 9 try: kept = json.loads( machine.succeed( cli(f"start --json --idle-timeout {KEPT_WINDOW}s foot"), timeout=180 ) ) kept_id, kept_pid = kept["id"], kept["pid"] print(f"touched session {kept_id}: pid {kept_pid}, timeout {kept['idle_timeout_s']}s") assert kept["idle_timeout_s"] == float(KEPT_WINDOW), kept["idle_timeout_s"] machine.succeed( "set -e; for i in " + " ".join(str(n) for n in range(1, KEPT_TOUCHES + 1)) + f"; do sleep {KEPT_INTERVAL}; " + cli(f"touch {kept_id}") + " >/dev/null; done", timeout=KEPT_TOUCHES * (KEPT_INTERVAL + 30) + 60, ) record = json.loads(machine.succeed(cli(f"status --json {kept_id}"), timeout=120))["session"] print(json.dumps(record, indent=2, sort_keys=True)) assert record["state"] == "ready", ( f"a session touched every {KEPT_INTERVAL}s for " f"{KEPT_TOUCHES * KEPT_INTERVAL}s, inside a {KEPT_WINDOW}s window, " f"is {record['state']!r}" ) assert record["live"] is True, record machine.succeed(f"kill -0 {kept_pid}") # Leave nothing behind for the restart subtest to trip over. machine.succeed(cli(f"kill {kept_id}"), timeout=180) except Exception: dump("a touched session was reaped anyway", locals().get("kept_id")) raise with subtest("a session survives a restart of the broker"): try: survivor = json.loads(machine.succeed(cli("start --json foot"), timeout=180)) survivor_id, survivor_pid = survivor["id"], survivor["pid"] print(f"survivor {survivor_id}: pid {survivor_pid}") machine.succeed("systemctl restart hhstream-sessions.service", timeout=180) machine.wait_for_unit("hhstream-sessions.service", timeout=180) # The new daemon rebinds the socket and adopts the live records # before it answers; one bounded wait for it to answer at all, # rather than for the answer we want. machine.wait_until_succeeds(cli("list") + " >/dev/null", timeout=120) record = json.loads( machine.succeed(cli(f"status --json {survivor_id}"), timeout=120) )["session"] print(json.dumps(record, indent=2, sort_keys=True)) assert record["pid"] == survivor_pid, ( f"re-adopted as pid {record['pid']}, not {survivor_pid}" ) assert record["state"] == "ready", f"re-adopted as {record['state']!r}" assert record["adopted"] is True, "the record does not say it was adopted" assert record["pgid"] == survivor["pgid"], record assert record["sid"] == survivor["sid"], record machine.succeed(f"kill -0 {survivor_pid}") except Exception: dump("the session did not survive the restart", locals().get("survivor_id")) raise with subtest("the re-adopted session is still killable"): try: reply = json.loads(machine.succeed(cli(f"kill --json {survivor_id}"), timeout=180)) assert reply["ok"] is True, reply assert reply["session"]["state"] == "killed", reply["session"] machine.wait_until_succeeds( gone(survivor_pid, survivor["pgid"], survivor["sid"]), timeout=60 ) machine.succeed("systemctl is-active hhstream-sessions.service") except Exception: dump("the re-adopted session could not be killed", locals().get("survivor_id")) raise ''; }