integrate the session broker: one CLI, a module unit, and a VM check
M0 of the sessions design, wired into the flake and gated by a real check. What is here now: - `src/hhstreamd/` — the broker: session registry, lifecycle verbs, process-group teardown, app allowlist, audit log, idle reaper on CLOCK_BOOTTIME. - `src/hhstream_session/` — the CLI handlers, registered into the one front door. - `src/hhstream/cli.py` — `hhstream status|shot|run|record|session …`, so the frame verbs and the session namespace are one binary. - `nix/module.nix` — `sessions.*` options and the `hhstream-sessions.service` unit: the operator's own account (never a service account, because launched applications should see that account's home), the compositor's group as a supplementary group (without it a launched GUI client cannot open the Wayland socket), the generated app allowlist, and `--compositor-runtime-dir` so applications are handed the compositor's session rather than the broker's. - `checks/vm-sessions.nix` — 18 subtests on a real compositor in a VM. Two bugs found by driving it, both fixed: - BUG-01: launched applications could not reach the compositor at all — the broker spawned them with its own inherited environment, which names the login session. `hhstreamd/compositor.py` discovers the socket and builds the child environment; a directory with no usable socket is a startup refusal, not a warning. - BUG-02: `kill` refused to signal a live application and said nothing was wrong. The identity gate hashed `/proc/<pid>/cmdline`, which every `execve` replaces — and on NixOS applications are reached through wrapper scripts that do exactly that. The gate now rests on `starttime` (+ boot id, + session id where it is immutable) and records command-line drift as evidence of nothing. KillMode=process is set on the unit on purpose: the default control-group kill would take the applications down with the broker, which would make adoption and a broker restart pointless.
This commit is contained in:
1 parent
9dc9e0da9a
commit
30f456c07e
37 files changed
+9118
-1
No files matched your search
@@ -0,0 +1,345 @@
|
||||
"""``hhstream`` — the one front door to this machine's headless Hyprland session.
|
||||
|
||||
Four verbs that work whether or not the session daemon is running, and one
|
||||
namespace that needs it:
|
||||
|
||||
hhstream status is the session up, and can I reach it?
|
||||
hhstream shot one frame of the screen
|
||||
hhstream run -- CMD... launch a GUI app into the session (unmanaged)
|
||||
hhstream record film the screen for N seconds
|
||||
hhstream session ... managed application sessions (start/list/touch/kill)
|
||||
|
||||
``run`` and ``session start`` both put an application on the screen; they are
|
||||
deliberately different. ``run`` is the primitive: it execs and forgets, nothing
|
||||
tracks the process, and stopping it is the caller's problem. ``session start``
|
||||
registers it, reports its pid, keeps its own clock, can be killed by name, and is
|
||||
reaped if it goes quiet for too long.
|
||||
|
||||
Access: the frame socket lives in a directory owned by the compositor's group
|
||||
(mode 0750). If this process cannot read it, the CLI re-execs itself through the
|
||||
*setuid* ``sg`` wrapper — deliberately, because a non-setuid ``sg`` earlier on
|
||||
PATH fails with "setgroups: Operation not permitted", which looks like every
|
||||
other failure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Iterator
|
||||
|
||||
from . import client as frames
|
||||
from . import protocol
|
||||
|
||||
#: The frame socket the compositor's publisher serves.
|
||||
DEFAULT_FRAME_SOCKET = frames.DEFAULT_SOCKET
|
||||
ENV_FRAME_SOCKET = "HHSTREAM_FRAME_SOCKET"
|
||||
#: The group that owns the session's runtime directory.
|
||||
DEFAULT_GROUP = "hhstream"
|
||||
ENV_GROUP = "HHSTREAM_GROUP"
|
||||
#: The units `status` reports on. Overridable so the CLI is not welded to this
|
||||
#: machine's naming, but the defaults are the ones the module ships.
|
||||
DEFAULT_UNITS = ("hyprland-headless.service", "hyprland-headless-stream.service")
|
||||
ENV_UNITS = "HHSTREAM_UNITS"
|
||||
|
||||
FFMPEG = "ffmpeg"
|
||||
|
||||
|
||||
# ── access ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def setuid_sg() -> str | None:
|
||||
"""The ``sg`` that can actually switch groups.
|
||||
|
||||
Two exist on a NixOS box: the setuid wrapper in /run/wrappers/bin and a
|
||||
plain symlink to shadow's copy in the system profile. Only the wrapper
|
||||
works; a login shell happens to find it first, a cron job does not.
|
||||
"""
|
||||
|
||||
candidates = ["/run/wrappers/bin/sg", shutil.which("sg")]
|
||||
for path in candidates:
|
||||
if not path or not os.path.isfile(path) or not os.access(path, os.X_OK):
|
||||
continue
|
||||
if os.stat(path).st_mode & 0o4000:
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def have_access(socket_path: str) -> bool:
|
||||
directory = pathlib.Path(socket_path).parent
|
||||
return os.access(directory, os.X_OK) and os.access(socket_path, os.R_OK)
|
||||
|
||||
|
||||
def ensure_access(socket_path: str, group: str, argv: list[str]) -> None:
|
||||
"""Re-exec through ``sg`` when the socket is out of reach. Returns if fine."""
|
||||
|
||||
if have_access(socket_path):
|
||||
return
|
||||
sg = setuid_sg()
|
||||
if sg is None:
|
||||
sys.exit(
|
||||
f"hhstream: cannot reach {socket_path} and no setuid 'sg' was found "
|
||||
f"(looked in /run/wrappers/bin) to join group {group!r}"
|
||||
)
|
||||
os.execv(sg, [sg, group, "-c", " ".join(shlex.quote(a) for a in argv)])
|
||||
|
||||
|
||||
def wayland_socket(runtime_dir: str) -> pathlib.Path | None:
|
||||
"""Whichever ``wayland-N`` the compositor bound — never assume ``-0``."""
|
||||
|
||||
root = pathlib.Path(runtime_dir)
|
||||
for candidate in sorted(root.glob("wayland-*")):
|
||||
if candidate.name.endswith(".lock"):
|
||||
continue
|
||||
if candidate.is_socket():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
# ── helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def read_frames(socket_path: str, *, count: int, timeout: float) -> tuple[dict, list[bytes]]:
|
||||
return frames.read_stream(socket_path, count=count, timeout=timeout)
|
||||
|
||||
|
||||
def frame_extension(hello: dict) -> str:
|
||||
return "jpg" if str(hello.get("format", "")).endswith("jpeg") else "png"
|
||||
|
||||
|
||||
def stream_frames(socket_path: str, seconds: float, timeout: float = 10.0) -> Iterator[bytes]:
|
||||
"""Yield frames off one connection until `seconds` of wall time have passed.
|
||||
|
||||
One connection on purpose: the publisher drops frames for a slow reader
|
||||
rather than queueing them, and reconnecting per frame would make us the slow
|
||||
reader.
|
||||
"""
|
||||
|
||||
import socket as _socket
|
||||
|
||||
conn = _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM)
|
||||
conn.settimeout(timeout)
|
||||
try:
|
||||
conn.connect(socket_path)
|
||||
with conn.makefile("rb") as stream:
|
||||
protocol.read_hello(stream)
|
||||
started = time.monotonic()
|
||||
while time.monotonic() - started < seconds:
|
||||
payload = protocol.read_frame(stream)
|
||||
if payload is None:
|
||||
return
|
||||
yield payload
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def env_for_session(runtime_dir: str) -> dict[str, str]:
|
||||
"""The environment a GUI client needs to attach to the running compositor."""
|
||||
|
||||
wayland = wayland_socket(runtime_dir)
|
||||
if wayland is None:
|
||||
sys.exit(f"hhstream: no wayland socket in {runtime_dir} — is the compositor up?")
|
||||
return {
|
||||
"XDG_RUNTIME_DIR": runtime_dir,
|
||||
"WAYLAND_DISPLAY": wayland.name,
|
||||
"XDG_SESSION_TYPE": "wayland",
|
||||
}
|
||||
|
||||
|
||||
# ── verbs ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def cmd_status(args: argparse.Namespace) -> int:
|
||||
units = [u for u in (os.environ.get(ENV_UNITS) or "").split(",") if u] or list(DEFAULT_UNITS)
|
||||
print("services:")
|
||||
for unit in units:
|
||||
state = subprocess.run(
|
||||
["systemctl", "is-active", unit], capture_output=True, text=True
|
||||
).stdout.strip() or "unknown"
|
||||
print(f" {unit:<34} {state}")
|
||||
print(f"frame: {args.frame_socket}")
|
||||
print(f"access: {'readable' if have_access(args.frame_socket) else 'DENIED'}")
|
||||
runtime_dir = str(pathlib.Path(args.frame_socket).parent)
|
||||
wayland = wayland_socket(runtime_dir)
|
||||
print(f"wayland: {wayland if wayland else 'none found'}")
|
||||
print("probe: ", end="")
|
||||
try:
|
||||
hello, payloads = read_frames(args.frame_socket, count=1, timeout=args.timeout)
|
||||
except OSError as exc:
|
||||
print(f"no frame — {exc}")
|
||||
return 1
|
||||
if not payloads:
|
||||
print("no frame — the publisher closed without sending one")
|
||||
return 1
|
||||
print(f"{len(payloads[0])} bytes, {hello['width']}x{hello['height']} {hello['format']}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_shot(args: argparse.Namespace) -> int:
|
||||
hello, payloads = read_frames(args.frame_socket, count=1, timeout=args.timeout)
|
||||
if not payloads:
|
||||
print("hhstream: the publisher sent no frame", file=sys.stderr)
|
||||
return 1
|
||||
out = pathlib.Path(args.out or f"hhstream-frame.{frame_extension(hello)}")
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_bytes(payloads[0])
|
||||
print(out)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_run(args: argparse.Namespace) -> int:
|
||||
argv = list(args.argv or [])
|
||||
if not argv:
|
||||
print("hhstream: run needs a command, e.g. hhstream run -- myapp", file=sys.stderr)
|
||||
return 2
|
||||
env = dict(os.environ)
|
||||
env.update(env_for_session(str(pathlib.Path(args.frame_socket).parent)))
|
||||
os.execvpe(argv[0], argv, env)
|
||||
return 0 # unreachable with execvpe
|
||||
|
||||
|
||||
def cmd_record(args: argparse.Namespace) -> int:
|
||||
if shutil.which(FFMPEG) is None:
|
||||
print(f"hhstream: {FFMPEG} is not on PATH", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
out = pathlib.Path(args.out or "hhstream-recording.mp4")
|
||||
work = pathlib.Path(args.work_dir) if args.work_dir else pathlib.Path(
|
||||
os.environ.get("TMPDIR", "/tmp")
|
||||
) / f"hhstream-rec-{os.getpid()}"
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
hello, first = read_frames(args.frame_socket, count=1, timeout=args.timeout)
|
||||
ext = frame_extension(hello)
|
||||
if first:
|
||||
(work / f"frame-{1:05d}.{ext}").write_bytes(first[0])
|
||||
|
||||
count = len(first)
|
||||
started = time.monotonic()
|
||||
for payload in stream_frames(args.frame_socket, max(0.0, args.seconds - 0.05)):
|
||||
count += 1
|
||||
(work / f"frame-{count:05d}.{ext}").write_bytes(payload)
|
||||
elapsed = max(time.monotonic() - started, 0.001)
|
||||
|
||||
if count == 0:
|
||||
print(f"hhstream: read no frames in {args.seconds}s", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Encode at the *measured* rate: a software-rendered VM tops out well under
|
||||
# the advertised fps, so anything else would play back at the wrong speed.
|
||||
fps = max(1, round(count / elapsed))
|
||||
subprocess.run(
|
||||
[
|
||||
FFMPEG, "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-framerate", str(fps),
|
||||
"-i", str(work / f"frame-%05d.{ext}"),
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p",
|
||||
"-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2",
|
||||
# Media stores reject files carrying metadata; strip it here so a
|
||||
# recording is uploadable without a second pass.
|
||||
"-map_metadata", "-1", "-fflags", "+bitexact", "-flags:v", "+bitexact",
|
||||
str(out),
|
||||
]
|
||||
)
|
||||
print(f"{out} ({count} frames in {elapsed:.2f}s -> {fps} fps)")
|
||||
if not args.keep:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
return 0
|
||||
|
||||
|
||||
# ── parser ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="hhstream",
|
||||
description=(
|
||||
"Work this machine's headless Hyprland session: look at it, put an "
|
||||
"application on it, manage that application as a session."
|
||||
),
|
||||
epilog=(
|
||||
"'run' launches and forgets. 'session start' launches and tracks — "
|
||||
"use it when something must be found, kept alive and killed later."
|
||||
),
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--frame-socket",
|
||||
default=os.environ.get(ENV_FRAME_SOCKET, DEFAULT_FRAME_SOCKET),
|
||||
metavar="PATH",
|
||||
help=f"the compositor's frame socket (default: ${ENV_FRAME_SOCKET} or "
|
||||
f"{DEFAULT_FRAME_SOCKET})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout", type=float, default=10.0, metavar="SECONDS", help="per-read socket timeout"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--group",
|
||||
default=os.environ.get(ENV_GROUP, DEFAULT_GROUP),
|
||||
metavar="NAME",
|
||||
help=f"group that owns the session's runtime directory (default: {DEFAULT_GROUP})",
|
||||
)
|
||||
|
||||
verbs = parser.add_subparsers(dest="command", metavar="COMMAND")
|
||||
|
||||
verbs.add_parser("status", help="are the units up, and can this process reach the socket?")
|
||||
|
||||
shot = verbs.add_parser("shot", help="write one frame of the screen")
|
||||
shot.add_argument("--out", metavar="FILE", help="where to write it (default: ./hhstream-frame.*)")
|
||||
|
||||
run = verbs.add_parser("run", help="launch a GUI app into the session (unmanaged)")
|
||||
run.add_argument("argv", nargs="*", metavar="-- CMD...", help="the command, after a literal --")
|
||||
|
||||
record = verbs.add_parser("record", help="film the screen for N seconds")
|
||||
record.add_argument("--seconds", type=float, default=10.0, metavar="N")
|
||||
record.add_argument("--out", metavar="FILE", help="output mp4 (default: ./hhstream-recording.mp4)")
|
||||
record.add_argument("--work-dir", metavar="DIR", help="where frames go (default: a temp dir)")
|
||||
record.add_argument("--keep", action="store_true", help="keep the frames after encoding")
|
||||
|
||||
# The managed namespace. It brings its own --socket (the *control* socket,
|
||||
# which is a different socket from the frame one) and its own exit codes.
|
||||
from hhstream_session import register as register_session
|
||||
|
||||
register_session(verbs)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.command is None:
|
||||
parser.print_help(sys.stderr)
|
||||
return 2
|
||||
|
||||
# Every verb except the session namespace speaks to the frame socket, so the
|
||||
# group check happens here, once. The session handlers do their own.
|
||||
if args.command in ("status", "shot", "run", "record"):
|
||||
# Re-exec the ORIGINAL command line, not a reconstruction: `sys.executable`
|
||||
# plus the arguments would drop this script's own path and hand Python a
|
||||
# verb as a filename ("can't open file 'status'").
|
||||
argv = list(getattr(sys, "orig_argv", None) or [sys.executable, *sys.argv])
|
||||
ensure_access(args.frame_socket, args.group, argv)
|
||||
|
||||
handler = {
|
||||
"status": cmd_status,
|
||||
"shot": cmd_shot,
|
||||
"run": cmd_run,
|
||||
"record": cmd_record,
|
||||
}.get(args.command)
|
||||
if handler is not None:
|
||||
return handler(args)
|
||||
|
||||
# session … — argparse already selected the handler when it parsed
|
||||
return int(getattr(args, "func")(args))
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,508 @@
|
||||
"""``hhstream session …`` — the CLI handlers, as a drop-in for someone else's CLI.
|
||||
|
||||
The integration contract is one function::
|
||||
|
||||
from hhstream_session import register
|
||||
register(subparsers) # subparsers from argparse's add_subparsers()
|
||||
...
|
||||
args = parser.parse_args()
|
||||
raise SystemExit(args.func(args))
|
||||
|
||||
``register`` adds a single ``session`` subcommand with its own verb tree, sets
|
||||
``func`` (and ``hhstream_session_handler``, for hosts that dispatch on their own
|
||||
attribute) to a callable taking the parsed namespace and returning an **exit
|
||||
code from the frozen table** in ``DESIGN.md`` §1.6. The registrant needs to
|
||||
know nothing about the daemon beyond the socket path, which comes from
|
||||
``--socket`` or ``$HHSTREAM_SOCKET``.
|
||||
|
||||
Nothing here is imported until a ``session`` verb actually runs: the module
|
||||
imports only :mod:`hhstreamd.protocol` (framing and exit codes) and the
|
||||
standard library, so a host CLI pays no startup cost and gains no new failure
|
||||
mode by registering it.
|
||||
|
||||
**The one thing to read before using this:** ``touch`` is the only mark of
|
||||
activity that exists. ``list``, ``status``, ``logs`` and ``gc`` deliberately do
|
||||
not reset a session's idle clock, so an agent watching a long-running app must
|
||||
call ``touch`` or the session will be reaped on schedule.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
from typing import Any, Callable
|
||||
|
||||
from hhstreamd.clock import format_duration, parse_duration
|
||||
from hhstreamd.protocol import ExitCode
|
||||
|
||||
from . import format as fmt
|
||||
from .client import DEFAULT_SOCKET, ENV_SOCKET, ClientError, call, reply_exit_code
|
||||
|
||||
__all__ = ["register", "build_parser", "dispatch", "main", "ExitCode"]
|
||||
|
||||
#: Printed under `session --help` and under `start --help`. It is here, in two
|
||||
#: places, because it is the single fact an agent gets wrong.
|
||||
ACTIVITY_NOTE = (
|
||||
"ACTIVITY: 'touch' is the ONLY thing that resets a session's idle clock. "
|
||||
"'list', 'status', 'logs' and 'gc' do NOT count — if polling kept sessions "
|
||||
"alive, an abandoned session would never be reaped. An agent that leaves an "
|
||||
"app running must call 'hhstream session touch <id>' before the idle timeout "
|
||||
"expires."
|
||||
)
|
||||
|
||||
_FOLLOW_INTERVAL_S = 0.5
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# registration
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def register(subparsers: Any, *, name: str = "session") -> argparse.ArgumentParser:
|
||||
"""Add the ``session`` namespace to an existing argparse subparser set.
|
||||
|
||||
Returns the namespace's parser, so a host that wants to add to it can.
|
||||
"""
|
||||
|
||||
parser = subparsers.add_parser(
|
||||
name,
|
||||
help="manage tracked application sessions (needs the hhstream session daemon)",
|
||||
description="Start, inspect, keep alive and kill tracked application sessions.",
|
||||
epilog=ACTIVITY_NOTE,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
_build_namespace(parser)
|
||||
return parser
|
||||
|
||||
|
||||
def _socket_parent() -> argparse.ArgumentParser:
|
||||
"""Shared ``--socket`` so it may be given before *or* after the verb.
|
||||
|
||||
``SUPPRESS`` matters: without it the verb-level default would overwrite a
|
||||
value given at the namespace level.
|
||||
"""
|
||||
|
||||
common = argparse.ArgumentParser(add_help=False)
|
||||
common.add_argument(
|
||||
"--socket",
|
||||
default=argparse.SUPPRESS,
|
||||
metavar="PATH",
|
||||
help=f"control socket (default: ${ENV_SOCKET} or {DEFAULT_SOCKET})",
|
||||
)
|
||||
return common
|
||||
|
||||
|
||||
def _build_namespace(parser: argparse.ArgumentParser) -> None:
|
||||
common = _socket_parent()
|
||||
parser.add_argument(
|
||||
"--socket",
|
||||
default=None,
|
||||
metavar="PATH",
|
||||
help=f"control socket (default: ${ENV_SOCKET} or {DEFAULT_SOCKET})",
|
||||
)
|
||||
# `hhstream session` with no verb lists, because that is what everyone
|
||||
# types first.
|
||||
parser.set_defaults(func=_cmd_list, hhstream_session_handler=_cmd_list, all=False, json=False)
|
||||
|
||||
verbs = parser.add_subparsers(dest="session_verb", metavar="VERB")
|
||||
|
||||
def add(name: str, handler: Callable[[argparse.Namespace], int], **kwargs: Any):
|
||||
sub = verbs.add_parser(name, parents=[common], **kwargs)
|
||||
sub.set_defaults(func=handler, hhstream_session_handler=handler)
|
||||
sub.add_argument("--json", action="store_true", help="emit the protocol object on stdout")
|
||||
return sub
|
||||
|
||||
# ---- start -------------------------------------------------------
|
||||
start = add(
|
||||
"start",
|
||||
_cmd_start,
|
||||
help="launch an allowlisted application as a tracked session",
|
||||
description=(
|
||||
"Launch an allowlisted application. Prints the bare session id on "
|
||||
"stdout and human-readable detail on stderr, so $(hhstream session "
|
||||
"start foo) is the id and nothing else.\n\n"
|
||||
"Everything after a literal -- is passed to the application as an "
|
||||
"argv list: 'session start chromium -- --new-window URL'."
|
||||
),
|
||||
epilog=ACTIVITY_NOTE,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
start.add_argument("app", help="an app name from the daemon's allowlist, never a path")
|
||||
start.add_argument(
|
||||
"argv",
|
||||
nargs="*",
|
||||
metavar="-- ARGV...",
|
||||
help="arguments for the application, after a literal --. Passed as an "
|
||||
"argv list; there is never a shell.",
|
||||
)
|
||||
start.add_argument(
|
||||
"--idle-timeout",
|
||||
type=_duration,
|
||||
metavar="DURATION",
|
||||
help="reap the session after this long with no 'touch' (e.g. 30s, 5m, 3h)",
|
||||
)
|
||||
start.add_argument(
|
||||
"--wait-ready",
|
||||
type=_duration,
|
||||
metavar="DURATION",
|
||||
help="how long to wait for the app to signal readiness",
|
||||
)
|
||||
home = start.add_mutually_exclusive_group()
|
||||
home.add_argument(
|
||||
"--home", metavar="PATH", help="run the app with this HOME instead of the inherited one"
|
||||
)
|
||||
home.add_argument(
|
||||
"--ephemeral-home",
|
||||
action="store_true",
|
||||
help="give the session a private HOME and delete it when the session ends "
|
||||
"(opt-in; by default the app inherits the real home directory)",
|
||||
)
|
||||
start.add_argument(
|
||||
"--request-id",
|
||||
metavar="ID",
|
||||
help="deduplication token: retrying a start with the same id returns the "
|
||||
"original reply instead of launching a second application",
|
||||
)
|
||||
|
||||
# ---- list --------------------------------------------------------
|
||||
lst = add("list", _cmd_list, help="list sessions (does NOT count as activity)")
|
||||
lst.add_argument("--all", action="store_true", help="include tombstones of ended sessions")
|
||||
|
||||
# ---- status ------------------------------------------------------
|
||||
status = add("status", _cmd_status, help="show one session (does NOT count as activity)")
|
||||
status.add_argument("id", help="session id")
|
||||
status.add_argument(
|
||||
"--events", type=int, default=20, metavar="N", help="recent audit events to include"
|
||||
)
|
||||
|
||||
# ---- touch -------------------------------------------------------
|
||||
add(
|
||||
"touch",
|
||||
_cmd_touch,
|
||||
help="reset the idle clock — THE ONLY MARK OF ACTIVITY",
|
||||
description=(
|
||||
"Reset the session's idle clock. Injects nothing, changes nothing "
|
||||
"else, and is the only verb that counts as activity: an agent "
|
||||
"observing a long-running application has no other way to say "
|
||||
"'still working'."
|
||||
),
|
||||
epilog=ACTIVITY_NOTE,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
).add_argument("id", help="session id")
|
||||
|
||||
# ---- kill --------------------------------------------------------
|
||||
kill = add("kill", _cmd_kill, help="terminate a session and everything it spawned")
|
||||
kill.add_argument("id", help="session id")
|
||||
kill.add_argument(
|
||||
"--grace",
|
||||
type=_duration,
|
||||
metavar="DURATION",
|
||||
help="how long to wait after the first signal before SIGKILL (default: the daemon's)",
|
||||
)
|
||||
kill.add_argument(
|
||||
"--signal", default="TERM", metavar="SIG", help="first signal to send (default: TERM)"
|
||||
)
|
||||
kill.add_argument("--request-id", metavar="ID", help="deduplication token")
|
||||
|
||||
# ---- gc ----------------------------------------------------------
|
||||
add(
|
||||
"gc",
|
||||
_cmd_gc,
|
||||
help="prune tombstones and reconcile the registry with /proc "
|
||||
"(does NOT count as activity)",
|
||||
).add_argument("--request-id", metavar="ID", help="deduplication token")
|
||||
|
||||
# ---- adopt -------------------------------------------------------
|
||||
adopt = add("adopt", _cmd_adopt, help="bring an already-running process under management")
|
||||
adopt.add_argument("--pid", type=int, required=True, help="pid to adopt")
|
||||
adopt.add_argument("--app", help="name to record (default: the executable's basename)")
|
||||
adopt.add_argument("--idle-timeout", type=_duration, metavar="DURATION")
|
||||
adopt.add_argument("--request-id", metavar="ID", help="deduplication token")
|
||||
|
||||
# ---- logs --------------------------------------------------------
|
||||
logs = add(
|
||||
"logs",
|
||||
_cmd_logs,
|
||||
help="the application's captured stdout/stderr (does NOT count as activity)",
|
||||
)
|
||||
logs.add_argument("id", help="session id")
|
||||
logs.add_argument("--lines", type=int, default=100, metavar="N", help="tail this many lines")
|
||||
logs.add_argument(
|
||||
"--follow", "-f", action="store_true", help="keep printing new output until interrupted"
|
||||
)
|
||||
|
||||
|
||||
def _duration(text: str) -> float:
|
||||
try:
|
||||
return parse_duration(text)
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(str(exc)) from exc
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# handlers
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _socket_of(args: argparse.Namespace) -> str | None:
|
||||
return getattr(args, "socket", None)
|
||||
|
||||
|
||||
def _request(args: argparse.Namespace, op: str, **fields: Any) -> dict[str, Any]:
|
||||
return call(op, socket_path=_socket_of(args), **fields)
|
||||
|
||||
|
||||
def _finish(args: argparse.Namespace, reply: dict[str, Any], render) -> int:
|
||||
"""Print a reply and return its exit code.
|
||||
|
||||
Errors always go to stderr and always carry the machine field, whatever the
|
||||
output mode, because that is what an agent branches on.
|
||||
"""
|
||||
|
||||
code = reply_exit_code(reply)
|
||||
if getattr(args, "json", False):
|
||||
fmt.dump_json(reply, stream=sys.stdout if reply.get("ok") else sys.stderr)
|
||||
elif reply.get("ok"):
|
||||
text = render(reply)
|
||||
if text:
|
||||
print(text)
|
||||
else:
|
||||
print(fmt.render_error(reply), file=sys.stderr)
|
||||
return int(code)
|
||||
|
||||
|
||||
def _guard(handler: Callable[[argparse.Namespace], int]) -> Callable[[argparse.Namespace], int]:
|
||||
def wrapper(args: argparse.Namespace) -> int:
|
||||
try:
|
||||
return handler(args)
|
||||
except ClientError as exc:
|
||||
if getattr(args, "json", False):
|
||||
fmt.dump_json(
|
||||
{"ok": False, "error": exc.error, "reason": exc.reason, "hint": exc.hint},
|
||||
stream=sys.stderr,
|
||||
)
|
||||
else:
|
||||
print(f"error: {exc.reason}", file=sys.stderr)
|
||||
if exc.hint:
|
||||
print(f"hint: {exc.hint}", file=sys.stderr)
|
||||
return int(exc.exit_code)
|
||||
except KeyboardInterrupt:
|
||||
return int(ExitCode.USAGE)
|
||||
|
||||
wrapper.__name__ = handler.__name__
|
||||
wrapper.__doc__ = handler.__doc__
|
||||
return wrapper
|
||||
|
||||
|
||||
@_guard
|
||||
def _cmd_start(args: argparse.Namespace) -> int:
|
||||
# argparse has already removed the separating `--`, so what is left is the
|
||||
# application's own argv, as a list. It is never joined into a string.
|
||||
argv = list(args.argv or [])
|
||||
reply = _request(
|
||||
args,
|
||||
"start",
|
||||
app=args.app,
|
||||
argv=argv,
|
||||
idle_timeout_s=args.idle_timeout,
|
||||
wait_ready_s=args.wait_ready,
|
||||
home=args.home,
|
||||
ephemeral_home=args.ephemeral_home or None,
|
||||
request_id=args.request_id,
|
||||
)
|
||||
if not reply.get("ok"):
|
||||
# A failed start may still have created a session (it exited at once, or
|
||||
# it is slow to become ready). Print the id so it can be inspected.
|
||||
if reply.get("id") and not args.json:
|
||||
print(reply["id"])
|
||||
return _finish(args, reply, lambda r: "")
|
||||
session = reply.get("session") or {}
|
||||
if args.json:
|
||||
fmt.dump_json(session)
|
||||
return int(ExitCode.OK)
|
||||
session_id = reply.get("id") or session.get("id") or ""
|
||||
# stdout is the bare id and nothing else; detail goes to stderr.
|
||||
print(session_id)
|
||||
print(
|
||||
f"started {session.get('app')} → {session_id} "
|
||||
f"(pid {session.get('pid')}, pgid {session.get('pgid')}, sid {session.get('sid')})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
f"state {session.get('state')} via {session.get('ready_signal')}; "
|
||||
f"idle timeout {_fmt_timeout(session.get('idle_timeout_s'))}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
f"keep it alive with: hhstream session touch {session_id} "
|
||||
"(the only mark of activity; list/status/logs/gc do not count)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return int(ExitCode.OK)
|
||||
|
||||
|
||||
def _fmt_timeout(seconds: Any) -> str:
|
||||
return "disabled" if not seconds else format_duration(seconds)
|
||||
|
||||
|
||||
@_guard
|
||||
def _cmd_list(args: argparse.Namespace) -> int:
|
||||
reply = _request(args, "list", all=bool(getattr(args, "all", False)) or None)
|
||||
return _finish(args, reply, lambda r: fmt.render_list(r.get("sessions") or []))
|
||||
|
||||
|
||||
@_guard
|
||||
def _cmd_status(args: argparse.Namespace) -> int:
|
||||
reply = _request(args, "status", id=args.id, events=args.events)
|
||||
return _finish(args, reply, lambda r: fmt.render_status(r.get("session") or {}))
|
||||
|
||||
|
||||
@_guard
|
||||
def _cmd_touch(args: argparse.Namespace) -> int:
|
||||
reply = _request(args, "touch", id=args.id)
|
||||
|
||||
def render(r: dict[str, Any]) -> str:
|
||||
session = r.get("session") or {}
|
||||
return (
|
||||
f"{session.get('id')}: idle clock reset; "
|
||||
f"expires in {format_duration(session.get('expires_in_s'))}"
|
||||
)
|
||||
|
||||
return _finish(args, reply, render)
|
||||
|
||||
|
||||
@_guard
|
||||
def _cmd_kill(args: argparse.Namespace) -> int:
|
||||
reply = _request(
|
||||
args,
|
||||
"kill",
|
||||
id=args.id,
|
||||
grace_s=args.grace,
|
||||
signal=args.signal,
|
||||
request_id=args.request_id,
|
||||
)
|
||||
|
||||
def render(r: dict[str, Any]) -> str:
|
||||
"""Say what the daemon actually did, not merely that it finished.
|
||||
|
||||
The old rendering could only distinguish "signalled" from "not
|
||||
signalled", and printed *the process was already gone* for both the
|
||||
routine case and the case where the daemon refused to signal a live
|
||||
application (BUG-02). A refusal is now an error reply and never reaches
|
||||
here; what is left is reported with the signal and the process group it
|
||||
went to, so "killed" cannot be read as more than it says.
|
||||
"""
|
||||
|
||||
session = r.get("session") or {}
|
||||
bits = [f"{session.get('id')}: {session.get('state')}"]
|
||||
sent = session.get("signal_sent")
|
||||
if sent:
|
||||
bits.append(f"sent {sent} to process group {session.get('pgid')}")
|
||||
if session.get("escalated_to_sigkill"):
|
||||
bits.append("escalated to SIGKILL")
|
||||
stragglers = session.get("stragglers") or 0
|
||||
if stragglers:
|
||||
bits.append(f"{stragglers} straggler(s) swept by session id")
|
||||
if session.get("signalled") is False and not sent:
|
||||
bits.append("nothing signalled (the process was already gone)")
|
||||
survivors = session.get("survivors") or []
|
||||
if survivors:
|
||||
# The loudest thing this command can say: the kill did not work.
|
||||
bits.append(
|
||||
f"WARNING: {len(survivors)} process(es) SURVIVED "
|
||||
f"({', '.join(str(p) for p in survivors)})"
|
||||
)
|
||||
return "; ".join(bits)
|
||||
|
||||
return _finish(args, reply, render)
|
||||
|
||||
|
||||
@_guard
|
||||
def _cmd_gc(args: argparse.Namespace) -> int:
|
||||
reply = _request(args, "gc", request_id=args.request_id)
|
||||
return _finish(args, reply, fmt.render_gc)
|
||||
|
||||
|
||||
@_guard
|
||||
def _cmd_adopt(args: argparse.Namespace) -> int:
|
||||
reply = _request(
|
||||
args,
|
||||
"adopt",
|
||||
pid=args.pid,
|
||||
app=args.app,
|
||||
idle_timeout_s=args.idle_timeout,
|
||||
request_id=args.request_id,
|
||||
)
|
||||
if reply.get("ok") and not args.json:
|
||||
print(reply.get("id", ""))
|
||||
session = reply.get("session") or {}
|
||||
print(
|
||||
f"adopted pid {session.get('pid')} as {reply['id']}; "
|
||||
"exit code will be unavailable (not a child of the daemon)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return int(ExitCode.OK)
|
||||
return _finish(args, reply, lambda r: "")
|
||||
|
||||
|
||||
@_guard
|
||||
def _cmd_logs(args: argparse.Namespace) -> int:
|
||||
reply = _request(args, "logs", id=args.id, lines=args.lines)
|
||||
if not reply.get("ok") or not args.follow:
|
||||
return _finish(args, reply, lambda r: "\n".join(r.get("lines") or []))
|
||||
# --follow polls from the returned offset. The daemon holds no streaming
|
||||
# connection open, so a follower that dies costs the daemon nothing.
|
||||
for line in reply.get("lines") or []:
|
||||
print(line)
|
||||
offset = reply.get("offset", 0)
|
||||
try:
|
||||
while True:
|
||||
if reply.get("eof"):
|
||||
break
|
||||
time.sleep(_FOLLOW_INTERVAL_S)
|
||||
reply = _request(args, "logs", id=args.id, offset=offset)
|
||||
if not reply.get("ok"):
|
||||
return _finish(args, reply, lambda r: "")
|
||||
for line in reply.get("lines") or []:
|
||||
print(line)
|
||||
offset = reply.get("offset", offset)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
return int(ExitCode.OK)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# standalone use
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_parser(prog: str = "hhstream") -> argparse.ArgumentParser:
|
||||
"""A host CLI stand-in, so the handlers can be driven without one."""
|
||||
|
||||
parser = argparse.ArgumentParser(prog=prog)
|
||||
subparsers = parser.add_subparsers(dest="command", metavar="COMMAND")
|
||||
register(subparsers)
|
||||
return parser
|
||||
|
||||
|
||||
def dispatch(args: argparse.Namespace) -> int:
|
||||
"""Run the handler argparse selected. The host may call this or ``args.func``."""
|
||||
|
||||
handler = getattr(args, "func", None)
|
||||
if handler is None:
|
||||
return int(ExitCode.USAGE)
|
||||
return int(handler(args))
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
if getattr(args, "command", None) is None:
|
||||
parser.print_help(sys.stderr)
|
||||
return int(ExitCode.USAGE)
|
||||
return dispatch(args)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,9 @@
|
||||
"""``python -m hhstream_session`` — drive the handlers without a host CLI.
|
||||
|
||||
The real integration is :func:`hhstream_session.register`; this exists so the
|
||||
verbs can be exercised (and tested) standalone.
|
||||
"""
|
||||
|
||||
from . import main
|
||||
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,109 @@
|
||||
"""The control-socket client (DESIGN.md §1.6, §10).
|
||||
|
||||
Blocking, stdlib sockets, one request per connection. **One connect attempt,
|
||||
then exit 5** — a CLI that silently retries hides a down daemon, and the hint
|
||||
tells the operator where to look.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
from typing import Any
|
||||
|
||||
from hhstreamd import protocol
|
||||
from hhstreamd.protocol import ExitCode, exit_code_for
|
||||
|
||||
DEFAULT_SOCKET = "/run/hhstream/control.sock"
|
||||
ENV_SOCKET = "HHSTREAM_SOCKET"
|
||||
|
||||
#: How long to wait for the daemon to answer. The daemon's own per-request
|
||||
#: deadline is shorter for every verb except `kill` and `start`, which are
|
||||
#: allowed their grace and readiness budgets, so this is a backstop.
|
||||
DEFAULT_TIMEOUT_S = 60.0
|
||||
|
||||
|
||||
class ClientError(Exception):
|
||||
"""A failure that never reached the daemon's dispatch."""
|
||||
|
||||
def __init__(self, error: str, reason: str, hint: str | None = None) -> None:
|
||||
super().__init__(reason)
|
||||
self.error = error
|
||||
self.reason = reason
|
||||
self.hint = hint
|
||||
|
||||
@property
|
||||
def exit_code(self) -> ExitCode:
|
||||
return exit_code_for(self.error)
|
||||
|
||||
|
||||
def default_socket_path() -> str:
|
||||
return os.environ.get(ENV_SOCKET, DEFAULT_SOCKET)
|
||||
|
||||
|
||||
def call(
|
||||
op: str,
|
||||
*,
|
||||
socket_path: str | None = None,
|
||||
timeout: float = DEFAULT_TIMEOUT_S,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any]:
|
||||
"""Send one request, return the decoded reply.
|
||||
|
||||
Raises :class:`ClientError` with ``daemon_unreachable`` (exit 5) if the
|
||||
socket cannot be reached, and ``timeout`` (exit 7) if the daemon accepts the
|
||||
request but never answers.
|
||||
"""
|
||||
|
||||
path = socket_path or default_socket_path()
|
||||
message = protocol.request(op, **fields)
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
sock.settimeout(timeout)
|
||||
try:
|
||||
try:
|
||||
sock.connect(path)
|
||||
except (FileNotFoundError, ConnectionRefusedError, PermissionError, OSError) as exc:
|
||||
raise ClientError(
|
||||
"daemon_unreachable",
|
||||
f"cannot reach the hhstream session daemon at {path}: {exc}",
|
||||
hint="is hhstreamd running? (it is an ordinary process; check "
|
||||
"however you supervise it)",
|
||||
) from exc
|
||||
try:
|
||||
sock.sendall(protocol.encode(message))
|
||||
buffer = bytearray()
|
||||
while b"\n" not in buffer:
|
||||
chunk = sock.recv(65536)
|
||||
if not chunk:
|
||||
break
|
||||
buffer.extend(chunk)
|
||||
if len(buffer) > protocol.MAX_LINE_BYTES:
|
||||
raise ClientError("bad_request", "reply exceeds the frame limit")
|
||||
except TimeoutError as exc:
|
||||
raise ClientError(
|
||||
"timeout", f"the daemon did not answer {op!r} within {timeout:g}s"
|
||||
) from exc
|
||||
except OSError as exc:
|
||||
raise ClientError("daemon_unreachable", f"control socket error: {exc}") from exc
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
if not buffer.strip():
|
||||
raise ClientError("daemon_unreachable", "the daemon closed the connection without replying")
|
||||
try:
|
||||
return protocol.decode(bytes(buffer))
|
||||
except protocol.ProtocolError as exc:
|
||||
raise ClientError(exc.error, f"malformed reply from the daemon: {exc.reason}") from exc
|
||||
|
||||
|
||||
def reply_exit_code(reply: dict[str, Any]) -> ExitCode:
|
||||
"""The frozen exit code for a decoded reply.
|
||||
|
||||
A reply that is not ``ok`` never yields 0, even if it carries no error name
|
||||
the table knows: reporting success for an unexplained failure is the one
|
||||
mistake an agent cannot recover from.
|
||||
"""
|
||||
|
||||
if reply.get("ok"):
|
||||
return ExitCode.OK
|
||||
return exit_code_for(reply.get("error") or "bad_state")
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Rendering for the human views.
|
||||
|
||||
``--json`` output is the protocol's own objects, unmodified: a flag that
|
||||
reshapes the data would mean two contracts to keep in step. The tables below
|
||||
are for people.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from hhstreamd.clock import format_duration
|
||||
|
||||
DASH = "—"
|
||||
|
||||
|
||||
def dump_json(obj: Any, stream=None) -> None:
|
||||
print(json.dumps(obj, indent=2, sort_keys=True), file=stream or sys.stdout)
|
||||
|
||||
|
||||
def _cell(value: Any) -> str:
|
||||
return DASH if value is None else str(value)
|
||||
|
||||
|
||||
def render_table(rows: list[list[str]], headers: list[str]) -> str:
|
||||
widths = [len(h) for h in headers]
|
||||
for row in rows:
|
||||
for i, cell in enumerate(row):
|
||||
widths[i] = max(widths[i], len(cell))
|
||||
out = [" ".join(h.ljust(widths[i]) for i, h in enumerate(headers)).rstrip()]
|
||||
for row in rows:
|
||||
out.append(" ".join(cell.ljust(widths[i]) for i, cell in enumerate(row)).rstrip())
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def render_list(sessions: list[dict[str, Any]]) -> str:
|
||||
"""The default view.
|
||||
|
||||
``EXPIRES IN`` is on it rather than behind a flag because it is the number
|
||||
an agent actually needs.
|
||||
|
||||
There is **no ``FG`` column** (DESIGN.md §1.4, amended): foreground-ness
|
||||
needs compositor IPC, which this milestone does not have, so the column was
|
||||
``?`` on every row of every listing — a header and six columns of width
|
||||
paying for no information, and a reader who has to learn that ``?`` means
|
||||
*unknown* rather than *no*. ``foreground`` stays in the record and in
|
||||
``status``, where it is one line among many and can become real without
|
||||
changing the shape of this table.
|
||||
"""
|
||||
|
||||
if not sessions:
|
||||
return "no sessions"
|
||||
rows = []
|
||||
for s in sessions:
|
||||
live = s.get("live")
|
||||
if live:
|
||||
idle = format_duration(s.get("idle_s"))
|
||||
expires = format_duration(s.get("expires_in_s"))
|
||||
if s.get("expires_in_s") is None:
|
||||
expires = "never"
|
||||
else:
|
||||
idle = DASH
|
||||
left = s.get("tombstone_expires_in_s")
|
||||
expires = f"(tombstone, {format_duration(left)} left)" if left is not None else DASH
|
||||
rows.append(
|
||||
[
|
||||
_cell(s.get("id")),
|
||||
_cell(s.get("app")),
|
||||
_cell(s.get("state")),
|
||||
_cell(s.get("pid")) if live else DASH,
|
||||
idle,
|
||||
expires,
|
||||
]
|
||||
)
|
||||
return render_table(rows, ["ID", "APP", "STATE", "PID", "IDLE", "EXPIRES IN"])
|
||||
|
||||
|
||||
def render_status(session: dict[str, Any]) -> str:
|
||||
live = session.get("live")
|
||||
lines = [
|
||||
f"{session.get('id')} ({session.get('app')})",
|
||||
f" state {session.get('state')}"
|
||||
+ (f" [{session.get('last_error')}]" if session.get("last_error") else ""),
|
||||
f" pid {_cell(session.get('pid'))} pgid {_cell(session.get('pgid'))}"
|
||||
f" sid {_cell(session.get('sid'))}",
|
||||
f" argv {' '.join(session.get('argv') or []) or DASH}",
|
||||
f" home {_cell(session.get('home'))}",
|
||||
f" created {_cell(session.get('created_at'))}",
|
||||
f" last activity {_cell(session.get('last_activity_at'))}"
|
||||
f" ({session.get('command_count', 0)} command(s) counted)",
|
||||
f" idle {format_duration(session.get('idle_s'))}"
|
||||
f" of {format_duration(session.get('idle_timeout_s'))}",
|
||||
f" expires in {format_duration(session.get('expires_in_s'))}"
|
||||
if session.get("expires_in_s") is not None
|
||||
else " expires in never (idle timeout disabled)",
|
||||
f" ready signal {_cell(session.get('ready_signal'))}",
|
||||
f" adopted {session.get('adopted')}",
|
||||
]
|
||||
if not live:
|
||||
lines.append(
|
||||
f" exit code {_cell(session.get('exit_code'))}"
|
||||
f" detected by {_cell(session.get('exit_detected_by'))}"
|
||||
)
|
||||
if session.get("exit_detected_by") in ("pidfd", "proc_poll", "proc_check"):
|
||||
lines.append(" exit code unavailable (adopted process)")
|
||||
compositor = session.get("compositor") or {}
|
||||
lines.append(
|
||||
f" window {_cell(compositor.get('window'))}"
|
||||
f" workspace {_cell(compositor.get('workspace'))}"
|
||||
f" foreground {_cell(compositor.get('foreground'))}"
|
||||
)
|
||||
if compositor.get("note"):
|
||||
lines.append(f" {compositor['note']}")
|
||||
events = session.get("events") or []
|
||||
if events:
|
||||
lines.append(" recent events")
|
||||
for event in events[-20:]:
|
||||
lines.append(f" {event.get('ts')} {event.get('event')}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_gc(result: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
f"pruned {result.get('pruned', 0)} tombstone(s), "
|
||||
f"reconciled {len(result.get('reconciled') or [])} session(s)",
|
||||
f"{result.get('live', 0)} live, {result.get('tombstones', 0)} tombstone(s) remain",
|
||||
]
|
||||
for item in result.get("reconciled") or []:
|
||||
lines.append(f" {item['id']}: {item['why']}")
|
||||
unresolved = result.get("unresolved") or []
|
||||
if unresolved:
|
||||
# Not a routine exit: the daemon could not establish whether these pids
|
||||
# are still their sessions', so it signalled nothing. Said loudly
|
||||
# because the application may well still be running (BUG-02 item 5).
|
||||
lines.append(
|
||||
"UNRESOLVED — identity could not be established, nothing was signalled:"
|
||||
)
|
||||
for item in unresolved:
|
||||
mismatched = ", ".join(item.get("mismatched") or []) or "no usable evidence"
|
||||
lines.append(f" {item['id']}: {item['why']} ({mismatched})")
|
||||
orphans = result.get("orphans") or []
|
||||
if orphans:
|
||||
lines.append("orphaned processes (escaped the process group AND the session id):")
|
||||
for item in orphans:
|
||||
lines.append(f" {item['id']} sid {item['sid']}: pids {item['pids']}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_error(reply: dict[str, Any]) -> str:
|
||||
parts = [f"error: {reply.get('reason') or reply.get('error')}"]
|
||||
if reply.get("hint"):
|
||||
parts.append(f"hint: {reply['hint']}")
|
||||
return "\n".join(parts)
|
||||
@@ -0,0 +1,8 @@
|
||||
"""hhstreamd — the hhstream session daemon (M0).
|
||||
|
||||
Deliberately import-light: the CLI handler package imports
|
||||
:mod:`hhstreamd.protocol` for the wire format and the exit-code table, and
|
||||
must not pay for the daemon's dependencies to do it.
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,253 @@
|
||||
"""``python -m hhstreamd`` — the session daemon.
|
||||
|
||||
An ordinary long-running process. It is deliberately **agnostic about how it
|
||||
is supervised**: no unit file, no ``sd_notify``, no ``systemctl``. It creates
|
||||
the directories it needs if they are not already there, so it runs the same
|
||||
under systemd, under a shell, or under a test harness.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from . import __version__
|
||||
from .audit import AuditLog
|
||||
from .clock import Clock, parse_duration
|
||||
from .compositor import CompositorError, CompositorTarget
|
||||
from .launcher import Launcher
|
||||
from .manager import DEFAULT_SETTLE_S, ManagerConfig, SessionManager
|
||||
from .policy import Policy, PolicyError, load_policy
|
||||
from .registry import Registry
|
||||
from .server import ControlServer
|
||||
|
||||
log = logging.getLogger("hhstreamd")
|
||||
|
||||
DEFAULT_SOCKET = "/run/hhstream/control.sock"
|
||||
DEFAULT_RUNTIME_DIR = "/run/hhstream"
|
||||
DEFAULT_STATE_DIR = "/var/lib/hhstream"
|
||||
DEFAULT_CONFIG = "/etc/hhstream/config.toml"
|
||||
|
||||
ENV_SOCKET = "HHSTREAM_SOCKET"
|
||||
ENV_RUNTIME_DIR = "HHSTREAM_RUNTIME_DIR"
|
||||
ENV_STATE_DIR = "HHSTREAM_STATE_DIR"
|
||||
ENV_CONFIG = "HHSTREAM_CONFIG"
|
||||
ENV_COMPOSITOR_RUNTIME_DIR = "HHSTREAM_COMPOSITOR_RUNTIME_DIR"
|
||||
|
||||
#: Printed at startup when no compositor runtime directory was given. It is a
|
||||
#: warning and not an error because the daemon is useful without a compositor —
|
||||
#: a console app, a test harness — but on a desktop machine this is BUG-01
|
||||
#: waiting to happen, and the operator should see it before the first `start`.
|
||||
NO_COMPOSITOR_WARNING = (
|
||||
"no --compositor-runtime-dir (or $%s): launched applications will inherit "
|
||||
"this daemon's environment (XDG_RUNTIME_DIR=%s, WAYLAND_DISPLAY=%s) and may "
|
||||
"not be able to reach a compositor — a GUI app will then exit immediately "
|
||||
"with 'Failed to connect to Wayland display'"
|
||||
)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="hhstreamd",
|
||||
description=(
|
||||
"The hhstream session daemon. Owns managed application sessions, "
|
||||
"their process groups and the idle clock."
|
||||
),
|
||||
epilog=(
|
||||
"Every flag below may also be set from the environment: "
|
||||
f"{ENV_SOCKET}, {ENV_RUNTIME_DIR}, {ENV_STATE_DIR}, {ENV_CONFIG}, "
|
||||
f"{ENV_COMPOSITOR_RUNTIME_DIR}. "
|
||||
"The flag wins. The daemon never speaks the Wayland protocol (it only "
|
||||
"locates the compositor's socket so a launched app can) and never "
|
||||
"integrates with systemd; supervise it however you like."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--version", action="version", version=f"hhstreamd {__version__}")
|
||||
parser.add_argument(
|
||||
"--socket",
|
||||
default=os.environ.get(ENV_SOCKET, DEFAULT_SOCKET),
|
||||
help="control socket path (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--state-dir",
|
||||
default=os.environ.get(ENV_STATE_DIR, DEFAULT_STATE_DIR),
|
||||
help="session records, logs and the audit log (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--runtime-dir",
|
||||
default=os.environ.get(ENV_RUNTIME_DIR, DEFAULT_RUNTIME_DIR),
|
||||
help="per-session XDG_RUNTIME_DIR parent (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compositor-runtime-dir",
|
||||
default=os.environ.get(ENV_COMPOSITOR_RUNTIME_DIR),
|
||||
metavar="PATH",
|
||||
help="the COMPOSITOR's runtime directory — the one holding its wayland-N "
|
||||
"socket. This is not --runtime-dir (that is where this daemon's own "
|
||||
"control socket and per-session scratch directories live). Given, the "
|
||||
"socket is discovered by name and exported to every launched "
|
||||
"application as XDG_RUNTIME_DIR/WAYLAND_DISPLAY/XDG_SESSION_TYPE, and "
|
||||
"the daemon refuses to start if the directory holds no usable socket. "
|
||||
"Omitted, applications inherit this daemon's environment and may not "
|
||||
"reach a compositor at all.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
default=os.environ.get(ENV_CONFIG, DEFAULT_CONFIG),
|
||||
help="TOML app allowlist (default: %(default)s). A missing file means "
|
||||
"an empty allowlist and every start refused.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settle",
|
||||
type=parse_duration,
|
||||
default=DEFAULT_SETTLE_S,
|
||||
metavar="DURATION",
|
||||
help="how long after spawn to wait before calling a session ready, so an "
|
||||
"app that exits instantly is reported as exited rather than ready "
|
||||
"(default: %(default)ss)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--socket-mode",
|
||||
type=lambda v: int(v, 8),
|
||||
default=0o660,
|
||||
metavar="OCTAL",
|
||||
help="mode for the control socket (default: 0660)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-level",
|
||||
default=os.environ.get("HHSTREAM_LOG_LEVEL", "info"),
|
||||
choices=["debug", "info", "warning", "error"],
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="validate the configuration and exit without listening",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def resolve_compositor(args: argparse.Namespace) -> CompositorTarget | None:
|
||||
"""Validate ``--compositor-runtime-dir``, or warn that there is none.
|
||||
|
||||
Raises :class:`CompositorError` so the caller can refuse to start: a daemon
|
||||
that accepts work and then kills every application that arrives is worse
|
||||
than one that refuses while the operator is still watching.
|
||||
"""
|
||||
|
||||
if not args.compositor_runtime_dir:
|
||||
log.warning(
|
||||
NO_COMPOSITOR_WARNING,
|
||||
ENV_COMPOSITOR_RUNTIME_DIR,
|
||||
os.environ.get("XDG_RUNTIME_DIR", "<unset>"),
|
||||
os.environ.get("WAYLAND_DISPLAY", "<unset>"),
|
||||
)
|
||||
return None
|
||||
target = CompositorTarget.from_runtime_dir(args.compositor_runtime_dir)
|
||||
log.info(
|
||||
"compositor: %s (WAYLAND_DISPLAY=%s)",
|
||||
target.session.runtime_dir,
|
||||
target.session.wayland_display,
|
||||
)
|
||||
return target
|
||||
|
||||
|
||||
async def serve(
|
||||
args: argparse.Namespace, policy: Policy, compositor: CompositorTarget | None
|
||||
) -> int:
|
||||
clock = Clock()
|
||||
# Absolute, because these end up in a session record and in a launched
|
||||
# app's environment: a relative HOME or XDG_RUNTIME_DIR would be resolved
|
||||
# against whatever directory the app happens to be started in.
|
||||
state_dir = Path(args.state_dir).resolve()
|
||||
runtime_dir = Path(args.runtime_dir).resolve()
|
||||
config = ManagerConfig(
|
||||
state_dir=state_dir,
|
||||
runtime_dir=runtime_dir,
|
||||
policy=policy,
|
||||
settle_s=args.settle,
|
||||
compositor=compositor,
|
||||
)
|
||||
manager = SessionManager(
|
||||
config,
|
||||
clock=clock,
|
||||
registry=Registry(state_dir, clock),
|
||||
launcher=Launcher(runtime_dir, state_dir, compositor=compositor),
|
||||
audit=AuditLog(state_dir / "events.jsonl", clock),
|
||||
)
|
||||
server = ControlServer(
|
||||
manager, Path(args.socket).absolute(), socket_mode=args.socket_mode
|
||||
)
|
||||
|
||||
summary = await manager.startup()
|
||||
log.info(
|
||||
"adopted %(adopted)s, tombstoned %(tombstoned)s, quarantined %(quarantined)s, "
|
||||
"pruned %(pruned)s",
|
||||
summary,
|
||||
)
|
||||
log.info(
|
||||
"allowlist: %s",
|
||||
", ".join(sorted(policy.apps)) or f"(empty — {policy.source or 'no config'})",
|
||||
)
|
||||
await server.start()
|
||||
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
with contextlib.suppress(NotImplementedError):
|
||||
loop.add_signal_handler(sig, stop.set)
|
||||
try:
|
||||
await stop.wait()
|
||||
finally:
|
||||
log.info("shutting down; sessions are left running for the next daemon to adopt")
|
||||
await server.stop()
|
||||
await manager.shutdown()
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, args.log_level.upper()),
|
||||
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
try:
|
||||
policy = load_policy(args.config)
|
||||
except (PolicyError, OSError, ValueError) as exc:
|
||||
print(f"hhstreamd: bad config {args.config}: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
compositor = resolve_compositor(args)
|
||||
except CompositorError as exc:
|
||||
# Refuse, naming the directory: every application launched from here
|
||||
# would die on startup, and "state exited" one command later is a much
|
||||
# worse place to find that out.
|
||||
print(f"hhstreamd: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
if args.check:
|
||||
print(f"config {args.config}: {len(policy.apps)} app(s) allowlisted")
|
||||
for name, rule in sorted(policy.apps.items()):
|
||||
print(f" {name} -> {rule.exec}")
|
||||
if compositor is None:
|
||||
print("compositor: none configured; apps inherit the daemon's environment")
|
||||
else:
|
||||
print(
|
||||
f"compositor: {compositor.session.runtime_dir} "
|
||||
f"(WAYLAND_DISPLAY={compositor.session.wayland_display})"
|
||||
)
|
||||
return 0
|
||||
try:
|
||||
return asyncio.run(serve(args, policy, compositor))
|
||||
except OSError as exc:
|
||||
print(f"hhstreamd: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,116 @@
|
||||
"""The append-only event log (DESIGN.md §2.3, §8 item 5).
|
||||
|
||||
One JSON object per line in ``<state>/events.jsonl``, size-rotated. Every
|
||||
lifecycle event and every accepted command, with **both** timestamps: wall time
|
||||
for a human reading it at 09:00 to find out why Blender died at 03:00, and
|
||||
boottime so the entry can be lined up with the deadline arithmetic that
|
||||
actually drove the daemon.
|
||||
|
||||
A write failure is logged and dropped. Refusing a command because a log line
|
||||
could not be written is the wrong trade (DESIGN.md §7 #15).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .clock import Clock, iso
|
||||
from .textio import tail_lines
|
||||
|
||||
log = logging.getLogger("hhstreamd.audit")
|
||||
|
||||
DEFAULT_MAX_BYTES = 16 * 1024 * 1024
|
||||
DEFAULT_KEEP = 3
|
||||
|
||||
#: Arguments are bounded in the audit record so one pathological request cannot
|
||||
#: dominate the log.
|
||||
MAX_FIELD_CHARS = 2048
|
||||
|
||||
|
||||
class AuditLog:
|
||||
def __init__(
|
||||
self,
|
||||
path: Path | str,
|
||||
clock: Clock | None = None,
|
||||
*,
|
||||
max_bytes: int = DEFAULT_MAX_BYTES,
|
||||
keep: int = DEFAULT_KEEP,
|
||||
) -> None:
|
||||
self.path = Path(path)
|
||||
self.clock = clock or Clock()
|
||||
self.max_bytes = max_bytes
|
||||
self.keep = keep
|
||||
|
||||
def emit(self, event: str, **fields: Any) -> None:
|
||||
record = {
|
||||
"ts": iso(self.clock.wall()),
|
||||
"boottime": round(self.clock.boottime(), 3),
|
||||
"event": event,
|
||||
}
|
||||
record.update({k: _bound(v) for k, v in fields.items() if v is not None})
|
||||
try:
|
||||
self._rotate_if_needed()
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
line = json.dumps(record, ensure_ascii=True, sort_keys=True) + "\n"
|
||||
with open(self.path, "a", encoding="utf-8") as fh:
|
||||
fh.write(line)
|
||||
except OSError as exc:
|
||||
log.error("audit write failed (%s): dropping event %s", exc, event)
|
||||
|
||||
def _rotate_if_needed(self) -> None:
|
||||
try:
|
||||
size = self.path.stat().st_size
|
||||
except FileNotFoundError:
|
||||
return
|
||||
if size < self.max_bytes:
|
||||
return
|
||||
for index in range(self.keep - 1, 0, -1):
|
||||
src = self.path.with_suffix(self.path.suffix + f".{index}")
|
||||
dst = self.path.with_suffix(self.path.suffix + f".{index + 1}")
|
||||
if src.exists():
|
||||
os.replace(src, dst)
|
||||
os.replace(self.path, self.path.with_suffix(self.path.suffix + ".1"))
|
||||
|
||||
#: How far back `tail` will look. Filtering by session means the window
|
||||
#: may hold far fewer than `limit` matching events; a wider window costs
|
||||
#: one read of a file that is already on the page cache.
|
||||
TAIL_BYTES = 1 << 20
|
||||
|
||||
def tail(self, limit: int = 20, *, session: str | None = None) -> list[dict[str, Any]]:
|
||||
"""The most recent events, newest last. Used by ``status``.
|
||||
|
||||
Bounded in bytes, not just in lines: the audit log rotates at 16 MiB and
|
||||
reading all of it to show twenty entries would be absurd.
|
||||
"""
|
||||
|
||||
if limit <= 0:
|
||||
return []
|
||||
lines, _, _ = tail_lines(self.path, limit=10_000, max_bytes=self.TAIL_BYTES)
|
||||
out: list[dict[str, Any]] = []
|
||||
for line in reversed(lines):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if session is not None and record.get("session") != session:
|
||||
continue
|
||||
out.append(record)
|
||||
if len(out) >= limit:
|
||||
break
|
||||
out.reverse()
|
||||
return out
|
||||
|
||||
|
||||
def _bound(value: Any) -> Any:
|
||||
if isinstance(value, str) and len(value) > MAX_FIELD_CHARS:
|
||||
return value[:MAX_FIELD_CHARS] + "…"
|
||||
if isinstance(value, list):
|
||||
return [_bound(v) for v in value[:64]]
|
||||
return value
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Clocks (DESIGN.md §6.1).
|
||||
|
||||
Every deadline in the daemon is computed on ``CLOCK_BOOTTIME``:
|
||||
|
||||
* wall time is wrong — an NTP step or a DST change would silently extend or
|
||||
collapse an idle window;
|
||||
* ``CLOCK_MONOTONIC`` is wrong — it stops across system suspend, so a host
|
||||
suspended for four hours would wake believing a session had been idle for
|
||||
seconds;
|
||||
* ``CLOCK_BOOTTIME`` includes suspend, which is what an operator means by
|
||||
"no commands for more than three hours".
|
||||
|
||||
A wall-clock ISO timestamp is recorded alongside, for humans and logs only.
|
||||
Nothing computes with it.
|
||||
|
||||
The clock is an object so it can be injected: :class:`ManualClock` is what the
|
||||
reaper's unit tests advance by three hours without sleeping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import time
|
||||
|
||||
|
||||
class Clock:
|
||||
"""The real clock: ``CLOCK_BOOTTIME`` for arithmetic, wall time for display."""
|
||||
|
||||
def boottime(self) -> float:
|
||||
return time.clock_gettime(time.CLOCK_BOOTTIME)
|
||||
|
||||
def wall(self) -> float:
|
||||
return time.time()
|
||||
|
||||
def wall_iso(self) -> str:
|
||||
return iso(self.wall())
|
||||
|
||||
|
||||
class ManualClock(Clock):
|
||||
"""A clock under the test's control. Never used by the shipped daemon."""
|
||||
|
||||
def __init__(self, boottime: float = 1000.0, wall: float | None = None) -> None:
|
||||
self._boottime = float(boottime)
|
||||
self._wall = float(wall) if wall is not None else 1_700_000_000.0
|
||||
|
||||
def boottime(self) -> float:
|
||||
return self._boottime
|
||||
|
||||
def wall(self) -> float:
|
||||
return self._wall
|
||||
|
||||
def advance(self, seconds: float) -> None:
|
||||
"""Move both clocks forward. Suspend is indistinguishable from this."""
|
||||
|
||||
self._boottime += float(seconds)
|
||||
self._wall += float(seconds)
|
||||
|
||||
|
||||
def iso(epoch: float) -> str:
|
||||
"""Format a wall-clock epoch as a UTC ISO-8601 timestamp, second precision."""
|
||||
|
||||
return (
|
||||
_dt.datetime.fromtimestamp(epoch, _dt.UTC)
|
||||
.replace(microsecond=0)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
|
||||
|
||||
def parse_duration(text: str) -> float:
|
||||
"""Parse ``30s`` / ``5m`` / ``3h`` / ``2d`` / a bare number of seconds.
|
||||
|
||||
Raises :class:`ValueError` with a message fit for an argparse error.
|
||||
"""
|
||||
|
||||
s = text.strip().lower()
|
||||
if not s:
|
||||
raise ValueError("empty duration")
|
||||
units = {"s": 1.0, "m": 60.0, "h": 3600.0, "d": 86400.0}
|
||||
multiplier = 1.0
|
||||
if s[-1] in units:
|
||||
multiplier = units[s[-1]]
|
||||
s = s[:-1]
|
||||
try:
|
||||
value = float(s)
|
||||
except ValueError:
|
||||
raise ValueError(f"not a duration: {text!r} (expected e.g. 30s, 5m, 3h)") from None
|
||||
if value < 0:
|
||||
raise ValueError(f"negative duration: {text!r}")
|
||||
return value * multiplier
|
||||
|
||||
|
||||
def format_duration(seconds: float | None) -> str:
|
||||
"""Render a duration as ``H:MM:SS`` for the human table. ``None`` is an em dash."""
|
||||
|
||||
if seconds is None:
|
||||
return "—"
|
||||
seconds = int(max(0.0, seconds))
|
||||
hours, rest = divmod(seconds, 3600)
|
||||
minutes, secs = divmod(rest, 60)
|
||||
return f"{hours}:{minutes:02d}:{secs:02d}"
|
||||
@@ -0,0 +1,240 @@
|
||||
"""Finding the compositor's Wayland socket, and the environment that reaches it.
|
||||
|
||||
This module exists because of BUG-01: the daemon spawned children with its own
|
||||
inherited environment, and on the target machine that environment names the
|
||||
*login* session (``XDG_RUNTIME_DIR=/run/user/1000``, no ``WAYLAND_DISPLAY``),
|
||||
where no compositor is listening. Every GUI application therefore died inside a
|
||||
second with ``Failed to connect to Wayland display``, and the only symptom one
|
||||
command later was ``state exited``.
|
||||
|
||||
Three facts shape what is here.
|
||||
|
||||
**The compositor's runtime directory is not the daemon's.** ``--runtime-dir``
|
||||
is where the daemon's control socket and the per-session scratch directories
|
||||
live; ``--compositor-runtime-dir`` is where the compositor's ``wayland-N``
|
||||
lives, owned by a *different* user (DESIGN.md §3.1, user A). Conflating them is
|
||||
the bug, so they are separate options and this module only ever reads the
|
||||
latter.
|
||||
|
||||
**The socket's name is not knowable in advance.** It is ``wayland-1`` on the
|
||||
target machine, not ``wayland-0``, because a crashed compositor had already
|
||||
taken the first name; libwayland itself picks the first free one. So the socket
|
||||
is *discovered*, never assumed, and a stale leftover must not win over the live
|
||||
one — hence the connect probe in :func:`discover`.
|
||||
|
||||
**A daemon that cannot reach a compositor should refuse to start.** Accepting
|
||||
work and then killing every application that arrives is worse than refusing at
|
||||
the point where the operator is still looking at the terminal. So discovery
|
||||
raises :class:`CompositorError` and the caller exits; it never degrades quietly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import stat
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("hhstreamd.compositor")
|
||||
|
||||
#: Wayland sockets are ``wayland-<n>``; libwayland also writes a ``wayland-<n>.lock``
|
||||
#: beside each one, which is an ordinary file and never a candidate.
|
||||
SOCKET_PREFIX = "wayland-"
|
||||
|
||||
#: How long to wait for ``connect()`` on a candidate socket. The compositor is
|
||||
#: local and accepting, or it is not; this only bounds a pathological case.
|
||||
PROBE_TIMEOUT_S = 0.5
|
||||
|
||||
|
||||
class CompositorError(Exception):
|
||||
"""The configured compositor runtime directory holds no usable socket."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompositorSession:
|
||||
"""Where the compositor is, and the environment that attaches to it."""
|
||||
|
||||
runtime_dir: Path
|
||||
wayland_display: str
|
||||
|
||||
@property
|
||||
def socket_path(self) -> Path:
|
||||
return self.runtime_dir / self.wayland_display
|
||||
|
||||
def env(self) -> dict[str, str]:
|
||||
"""The variables a child needs, laid *on top of* its inherited ones.
|
||||
|
||||
``XDG_RUNTIME_DIR`` is the compositor's, not the daemon's and not the
|
||||
session's private scratch directory: a toolkit resolves
|
||||
``WAYLAND_DISPLAY`` relative to it, and the daemon's own value names a
|
||||
login session with nothing listening in it. ``WAYLAND_DISPLAY`` is the
|
||||
bare socket name rather than an absolute path, which is what every
|
||||
toolkit expects to see. ``XDG_SESSION_TYPE`` is what GTK, Qt and
|
||||
Chromium read to choose their Wayland backend over X11.
|
||||
"""
|
||||
|
||||
return {
|
||||
"XDG_RUNTIME_DIR": str(self.runtime_dir),
|
||||
"WAYLAND_DISPLAY": self.wayland_display,
|
||||
"XDG_SESSION_TYPE": "wayland",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _Candidate:
|
||||
name: str
|
||||
connectable: bool
|
||||
refused: bool # nobody is listening: a leftover from a crashed compositor
|
||||
detail: str
|
||||
|
||||
@property
|
||||
def rank(self) -> tuple[int, int, str]:
|
||||
# Live sockets first; among equals, the highest ``wayland-N`` wins,
|
||||
# because a compositor takes the first free name and so the survivor of
|
||||
# a crash-and-restart is the *later* one.
|
||||
return (0 if self.connectable else 1, -_suffix_order(self.name), self.name)
|
||||
|
||||
|
||||
def _suffix_order(name: str) -> int:
|
||||
suffix = name[len(SOCKET_PREFIX) :]
|
||||
return int(suffix) if suffix.isdigit() else -1
|
||||
|
||||
|
||||
def _probe(path: Path) -> _Candidate:
|
||||
"""Try to connect, to tell a live socket from a crashed compositor's leftover."""
|
||||
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
try:
|
||||
sock.settimeout(PROBE_TIMEOUT_S)
|
||||
sock.connect(str(path))
|
||||
except OSError as exc:
|
||||
# ECONNREFUSED is definitive: the inode is there and no process is
|
||||
# behind it. Anything else (EACCES, EPERM, a timeout) is a reason to
|
||||
# warn, not to discard the candidate — it may still be the live socket,
|
||||
# and refusing to start over an ambiguous probe is its own outage.
|
||||
refused = exc.errno in (errno.ECONNREFUSED, errno.ENOENT)
|
||||
return _Candidate(path.name, False, refused, f"{type(exc).__name__}: {exc}")
|
||||
finally:
|
||||
sock.close()
|
||||
return _Candidate(path.name, True, False, "connectable")
|
||||
|
||||
|
||||
def candidates(runtime_dir: Path) -> list[str]:
|
||||
"""Names in ``runtime_dir`` that are Wayland sockets, unordered.
|
||||
|
||||
Only ``S_ISSOCK`` entries count, so the ``.lock`` file beside each socket
|
||||
and any directory named like one are never picked.
|
||||
"""
|
||||
|
||||
found = []
|
||||
with os.scandir(runtime_dir) as entries:
|
||||
for entry in sorted(entries, key=lambda e: e.name):
|
||||
if not entry.name.startswith(SOCKET_PREFIX):
|
||||
continue
|
||||
try:
|
||||
mode = entry.stat(follow_symlinks=True).st_mode
|
||||
except OSError: # vanished, or a dangling symlink
|
||||
continue
|
||||
if stat.S_ISSOCK(mode):
|
||||
found.append(entry.name)
|
||||
return found
|
||||
|
||||
|
||||
def discover(runtime_dir: Path | str) -> CompositorSession:
|
||||
"""Resolve the compositor's session from its runtime directory.
|
||||
|
||||
Raises :class:`CompositorError`, naming the directory, if it cannot be read,
|
||||
holds no Wayland socket, or holds only sockets that nothing is listening on.
|
||||
"""
|
||||
|
||||
path = Path(runtime_dir).expanduser()
|
||||
try:
|
||||
resolved = path.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise CompositorError(f"compositor runtime dir {path}: {exc.strerror or exc}") from exc
|
||||
if not resolved.is_dir():
|
||||
raise CompositorError(f"compositor runtime dir {resolved} is not a directory")
|
||||
try:
|
||||
names = candidates(resolved)
|
||||
except OSError as exc:
|
||||
raise CompositorError(
|
||||
f"compositor runtime dir {resolved}: cannot be listed ({exc.strerror or exc})"
|
||||
) from exc
|
||||
if not names:
|
||||
raise CompositorError(
|
||||
f"compositor runtime dir {resolved} holds no Wayland socket "
|
||||
f"({SOCKET_PREFIX}N); is the compositor running, and is this the "
|
||||
"right directory? It is the compositor's XDG_RUNTIME_DIR, not the "
|
||||
"daemon's --runtime-dir."
|
||||
)
|
||||
|
||||
probed = sorted((_probe(resolved / name) for name in names), key=lambda c: c.rank)
|
||||
best = probed[0]
|
||||
if not best.connectable and all(c.refused for c in probed):
|
||||
detail = ", ".join(f"{c.name} ({c.detail})" for c in probed)
|
||||
raise CompositorError(
|
||||
f"compositor runtime dir {resolved} holds no usable Wayland socket: "
|
||||
f"{detail}. These look like leftovers from a compositor that is no "
|
||||
"longer running."
|
||||
)
|
||||
if not best.connectable:
|
||||
log.warning(
|
||||
"compositor socket %s did not accept a probe connection (%s); using it anyway",
|
||||
best.name,
|
||||
best.detail,
|
||||
)
|
||||
if len(probed) > 1:
|
||||
log.info(
|
||||
"compositor runtime dir %s holds %d Wayland sockets (%s); chose %s",
|
||||
resolved,
|
||||
len(probed),
|
||||
", ".join(f"{c.name} [{c.detail}]" for c in probed),
|
||||
best.name,
|
||||
)
|
||||
return CompositorSession(runtime_dir=resolved, wayland_display=best.name)
|
||||
|
||||
|
||||
class CompositorTarget:
|
||||
"""The configured compositor, re-resolved at every spawn.
|
||||
|
||||
Re-resolving matters because the compositor can restart under a running
|
||||
daemon and come back on a different socket name (that is exactly how the
|
||||
target machine ended up on ``wayland-1``). If re-resolution fails, the last
|
||||
known-good session is reused and the failure is logged: a transient listing
|
||||
error must not turn into every subsequent application being launched with no
|
||||
compositor at all.
|
||||
"""
|
||||
|
||||
def __init__(self, session: CompositorSession) -> None:
|
||||
self.session = session
|
||||
|
||||
@classmethod
|
||||
def from_runtime_dir(cls, runtime_dir: Path | str) -> "CompositorTarget":
|
||||
"""Validate at startup. Raises :class:`CompositorError` if unusable."""
|
||||
|
||||
return cls(discover(runtime_dir))
|
||||
|
||||
def resolve(self) -> CompositorSession:
|
||||
try:
|
||||
session = discover(self.session.runtime_dir)
|
||||
except CompositorError as exc:
|
||||
log.warning(
|
||||
"re-resolving the compositor socket failed (%s); keeping %s",
|
||||
exc,
|
||||
self.session.wayland_display,
|
||||
)
|
||||
return self.session
|
||||
if session != self.session:
|
||||
log.info(
|
||||
"compositor socket changed: %s -> %s",
|
||||
self.session.wayland_display,
|
||||
session.wayland_display,
|
||||
)
|
||||
self.session = session
|
||||
return session
|
||||
|
||||
def env(self) -> dict[str, str]:
|
||||
return self.resolve().env()
|
||||
@@ -0,0 +1,237 @@
|
||||
"""Spawning session applications (DESIGN.md §3.1, §3.3; brief boundaries).
|
||||
|
||||
Every app is spawned with ``start_new_session=True``, which gives it a fresh
|
||||
session id *and* process group with ``sid == pgid == pid``. That pair is the
|
||||
kill handle: there is no cgroup here, so the process group is what a teardown
|
||||
signals and the session id is what the sweep searches for.
|
||||
|
||||
**Never a shell.** ``argv`` is a list, built by :mod:`hhstreamd.policy` from an
|
||||
absolute path in the config plus caller arguments that matched an allowlist
|
||||
pattern. There is no ``shell=True`` and no string interpolation into a command
|
||||
line anywhere in this package.
|
||||
|
||||
**Why :class:`subprocess.Popen` and not ``asyncio.create_subprocess_exec``.**
|
||||
asyncio's subprocess transport kills the child when it is closed — which
|
||||
happens when the daemon's event loop shuts down, and again via the transport's
|
||||
``__del__``. Using it here would mean every session application was SIGKILLed
|
||||
the moment the daemon stopped, so no session would ever survive a daemon
|
||||
restart and ``adopt`` would have nothing to adopt. That is the opposite of
|
||||
what DESIGN.md §2.4 requires, and no unit file setting can fix it because the
|
||||
kill comes from inside this process. A plain ``Popen`` has no such behaviour:
|
||||
it never signals the child, and the daemon watches for exit through a pidfd
|
||||
instead (see :mod:`hhstreamd.manager`). The fork/exec briefly blocks the event
|
||||
loop, which is a few milliseconds and the right trade.
|
||||
|
||||
**The compositor environment (BUG-01).** A child does *not* get a usable
|
||||
Wayland connection from the daemon's inherited environment: on the target machine
|
||||
the daemon runs in the login session (``XDG_RUNTIME_DIR=/run/user/1000``, no
|
||||
``WAYLAND_DISPLAY``) and the compositor listens somewhere else entirely. So when
|
||||
``--compositor-runtime-dir`` names the compositor's runtime directory, the
|
||||
discovered session (:mod:`hhstreamd.compositor`) is laid over the inherited
|
||||
environment at spawn. Without that option the old inherited-environment
|
||||
behaviour stands, and the daemon warns at startup that applications may not
|
||||
reach a compositor.
|
||||
|
||||
The app runs as **the same unprivileged user as the daemon** and inherits the
|
||||
real home directory. M0 invents no service account and forces no throwaway
|
||||
``HOME``; ``--home`` pins a different one and ``--ephemeral-home`` opts in to a
|
||||
per-session one that is deleted on teardown.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .compositor import CompositorTarget
|
||||
from .registry import Session
|
||||
|
||||
log = logging.getLogger("hhstreamd.launcher")
|
||||
|
||||
#: Environment variables the daemon's own plumbing must not leak into an app.
|
||||
_STRIPPED = (
|
||||
"HHSTREAM_SOCKET",
|
||||
"HHSTREAM_STATE_DIR",
|
||||
"HHSTREAM_RUNTIME_DIR",
|
||||
"HHSTREAM_CONFIG",
|
||||
"HHSTREAM_COMPOSITOR_RUNTIME_DIR",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Spawned:
|
||||
process: subprocess.Popen
|
||||
pid: int
|
||||
pgid: int
|
||||
sid: int
|
||||
home: str
|
||||
#: The **daemon-owned** per-session runtime directory, which is what goes in
|
||||
#: the record and what ``discard_runtime_dir`` may delete. It is not
|
||||
#: necessarily the child's ``XDG_RUNTIME_DIR``: with a compositor
|
||||
#: configured, that is the compositor's directory, owned by another user.
|
||||
runtime_dir: str
|
||||
log_path: str
|
||||
|
||||
|
||||
class Launcher:
|
||||
def __init__(
|
||||
self,
|
||||
runtime_dir: Path | str,
|
||||
state_dir: Path | str,
|
||||
*,
|
||||
compositor: CompositorTarget | None = None,
|
||||
) -> None:
|
||||
self.runtime_dir = Path(runtime_dir)
|
||||
self.state_dir = Path(state_dir)
|
||||
#: The compositor to attach children to, or ``None`` to leave the
|
||||
#: inherited environment alone (``--compositor-runtime-dir`` unset).
|
||||
self.compositor = compositor
|
||||
|
||||
# ---- per-session directories -------------------------------------
|
||||
|
||||
def session_runtime_dir(self, session_id: str) -> Path:
|
||||
return self.runtime_dir / "sessions" / session_id
|
||||
|
||||
def session_home(self, session_id: str) -> Path:
|
||||
return self.state_dir / "homes" / session_id
|
||||
|
||||
def log_path(self, session_id: str) -> Path:
|
||||
return self.state_dir / "logs" / f"{session_id}.log"
|
||||
|
||||
# ---- environment -------------------------------------------------
|
||||
|
||||
def build_env(self, session_id: str, home: Path) -> dict[str, str]:
|
||||
"""The app's environment.
|
||||
|
||||
It inherits the daemon's, minus the daemon's own configuration, plus a
|
||||
private ``XDG_RUNTIME_DIR`` for the app's own locks and sockets.
|
||||
|
||||
When a compositor is configured, its session is laid **on top** of that:
|
||||
``XDG_RUNTIME_DIR`` becomes the compositor's (a toolkit resolves
|
||||
``WAYLAND_DISPLAY`` against it), ``WAYLAND_DISPLAY`` the discovered
|
||||
socket name, ``XDG_SESSION_TYPE`` ``wayland``. This is BUG-01: without
|
||||
it a child inherits the daemon's login-session environment, finds no
|
||||
compositor there, and dies within a second — and the only symptom is
|
||||
``state exited`` one command later. The private directory is still
|
||||
created and still recorded, because the daemon owns and cleans it up;
|
||||
it is simply not what the app sees as ``XDG_RUNTIME_DIR`` any more.
|
||||
|
||||
With no compositor configured, ``WAYLAND_DISPLAY`` is passed through
|
||||
untouched when the daemon has one: the daemon still never speaks the
|
||||
Wayland protocol itself, but it must not *prevent* an app from doing so.
|
||||
"""
|
||||
|
||||
env = {k: v for k, v in os.environ.items() if k not in _STRIPPED}
|
||||
runtime = self.session_runtime_dir(session_id)
|
||||
runtime.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
env["HOME"] = str(home)
|
||||
env["XDG_RUNTIME_DIR"] = str(runtime)
|
||||
env["HHSTREAM_SESSION_ID"] = session_id
|
||||
if self.compositor is not None:
|
||||
env.update(self.compositor.env())
|
||||
return env
|
||||
|
||||
def resolve_home(self, session_id: str, home: str | None, ephemeral: bool) -> tuple[Path, bool]:
|
||||
"""Decide the app's ``HOME``. Returns ``(path, is_ephemeral)``."""
|
||||
|
||||
if home is not None and ephemeral:
|
||||
raise ValueError("--home and --ephemeral-home are mutually exclusive")
|
||||
if ephemeral:
|
||||
path = self.session_home(session_id)
|
||||
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
return path, True
|
||||
if home is not None:
|
||||
return Path(home), False
|
||||
inherited = os.environ.get("HOME") or str(Path.home())
|
||||
return Path(inherited), False
|
||||
|
||||
def discard_home(self, session: Session) -> None:
|
||||
"""Delete an ephemeral home on teardown. Best effort, never fatal."""
|
||||
|
||||
if not session.ephemeral_home or not session.home:
|
||||
return
|
||||
expected = self.session_home(session.id)
|
||||
if Path(session.home) != expected: # never remove a directory we did not create
|
||||
log.warning("refusing to delete %s: not the session's ephemeral home", session.home)
|
||||
return
|
||||
shutil.rmtree(session.home, ignore_errors=True)
|
||||
|
||||
def discard_runtime_dir(self, session: Session) -> None:
|
||||
"""Remove the session's private ``XDG_RUNTIME_DIR`` once it is pruned.
|
||||
|
||||
Kept until the tombstone is pruned rather than removed at teardown, so
|
||||
a post-mortem can still look at what the app left behind.
|
||||
"""
|
||||
|
||||
if not session.runtime_dir:
|
||||
return
|
||||
expected = self.session_runtime_dir(session.id)
|
||||
if Path(session.runtime_dir) != expected:
|
||||
log.warning("refusing to delete %s: not the session's runtime dir", session.runtime_dir)
|
||||
return
|
||||
shutil.rmtree(session.runtime_dir, ignore_errors=True)
|
||||
|
||||
# ---- spawn -------------------------------------------------------
|
||||
|
||||
def spawn(
|
||||
self,
|
||||
session_id: str,
|
||||
argv: list[str],
|
||||
*,
|
||||
home: str | None = None,
|
||||
ephemeral_home: bool = False,
|
||||
) -> Spawned:
|
||||
"""Start the application. Raises ``OSError`` if it cannot be executed."""
|
||||
|
||||
if not argv:
|
||||
raise ValueError("argv is empty")
|
||||
if not os.path.isabs(argv[0]):
|
||||
# Policy already guarantees this; asserting it here means a future
|
||||
# caller cannot bypass the allowlist by handing in a bare name that
|
||||
# would be resolved against PATH.
|
||||
raise ValueError(f"argv[0] must be an absolute path, got {argv[0]!r}")
|
||||
|
||||
home_path, is_ephemeral = self.resolve_home(session_id, home, ephemeral_home)
|
||||
env = self.build_env(session_id, home_path)
|
||||
log_file = self.log_path(session_id)
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True, mode=0o750)
|
||||
|
||||
# stdout and stderr are captured to a file the daemon owns, because
|
||||
# `session logs` is served from here rather than from the journal.
|
||||
fh = open(log_file, "ab", buffering=0)
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
argv,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=fh,
|
||||
stderr=subprocess.STDOUT,
|
||||
cwd=str(home_path) if home_path.is_dir() else "/",
|
||||
env=env,
|
||||
start_new_session=True, # sid == pgid == pid; the kill handle
|
||||
close_fds=True,
|
||||
)
|
||||
finally:
|
||||
fh.close()
|
||||
|
||||
pid = process.pid
|
||||
# A process spawned with start_new_session has sid == pgid == pid by
|
||||
# construction. Read them back anyway: if the app has already exited,
|
||||
# /proc is gone and the recorded values must still be the right ones to
|
||||
# signal, which they are.
|
||||
pgid = sid = pid
|
||||
return Spawned(
|
||||
process=process,
|
||||
pid=pid,
|
||||
pgid=pgid,
|
||||
sid=sid,
|
||||
home=str(home_path),
|
||||
# The daemon-owned per-session directory, which is what
|
||||
# `discard_runtime_dir` is allowed to delete — *not* the compositor's
|
||||
# XDG_RUNTIME_DIR, which belongs to another user entirely.
|
||||
runtime_dir=str(self.session_runtime_dir(session_id)),
|
||||
log_path=str(log_file),
|
||||
)
|
||||
@@ -0,0 +1,1000 @@
|
||||
"""The session manager: every verb's implementation, and the one kill path.
|
||||
|
||||
This module owns the state machine. :mod:`hhstreamd.server` is only framing
|
||||
and dispatch; everything that decides anything lives here.
|
||||
|
||||
Two things are worth finding quickly:
|
||||
|
||||
* :meth:`SessionManager.teardown` is **the** teardown — ``killpg(SIGTERM)``,
|
||||
grace, ``killpg(SIGKILL)``, then the ``/proc`` session-id sweep (DESIGN.md
|
||||
§3.3). ``kill`` and the reaper both call it with a different ``reason``, so
|
||||
there is one kill path and not two.
|
||||
* :meth:`SessionManager.touch` is the **only** mark of activity in M0. There is
|
||||
no input verb and no session-scoped frame verb, so an agent observing a
|
||||
long-running app has no other way to say "still working". ``list``,
|
||||
``status``, ``logs`` and ``gc`` deliberately do not stamp it: if monitoring
|
||||
refreshed the clock, a supervisor polling every 60 s would keep every
|
||||
abandoned session alive forever and the reaper would never fire.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import procmon
|
||||
from .audit import AuditLog
|
||||
from .clock import Clock, iso
|
||||
from .compositor import CompositorTarget
|
||||
from .launcher import Launcher
|
||||
from .policy import Policy, PolicyError, check_app_name
|
||||
from .reaper import Reaper
|
||||
from .registry import (
|
||||
TERMINAL_ERROR,
|
||||
Registry,
|
||||
Session,
|
||||
State,
|
||||
)
|
||||
from .textio import tail_lines
|
||||
|
||||
log = logging.getLogger("hhstreamd.manager")
|
||||
|
||||
#: How long after spawn the daemon waits before calling a session ready.
|
||||
#: It exists for one real failure mode: an app that hands off to an existing
|
||||
#: instance and exits immediately (DESIGN.md §7 #3). Without the settle,
|
||||
#: `start` would return a healthy-looking id for a process that is already
|
||||
#: gone. It is deliberately short.
|
||||
DEFAULT_SETTLE_S = 0.1
|
||||
|
||||
#: How long a teardown waits for SIGKILL to take effect before it calls a pid a
|
||||
#: survivor. SIGKILL is not synchronous: the task is woken and torn down by the
|
||||
#: kernel, which takes microseconds normally but can block on uninterruptible
|
||||
#: I/O. Short enough not to stall a verb, long enough that a healthy process is
|
||||
#: never libelled as unkillable.
|
||||
SURVIVOR_GRACE_S = 0.5
|
||||
|
||||
#: A single `logs` reply never carries more than this, so one request cannot
|
||||
#: exceed the protocol's frame limit however much the app has written.
|
||||
MAX_LOG_CHUNK_BYTES = 256 * 1024
|
||||
|
||||
SIGNALS_BY_NAME = {
|
||||
"TERM": signal.SIGTERM,
|
||||
"SIGTERM": signal.SIGTERM,
|
||||
"KILL": signal.SIGKILL,
|
||||
"SIGKILL": signal.SIGKILL,
|
||||
"INT": signal.SIGINT,
|
||||
"SIGINT": signal.SIGINT,
|
||||
"HUP": signal.SIGHUP,
|
||||
"SIGHUP": signal.SIGHUP,
|
||||
"QUIT": signal.SIGQUIT,
|
||||
"SIGQUIT": signal.SIGQUIT,
|
||||
}
|
||||
|
||||
|
||||
class ManagerError(Exception):
|
||||
"""A verb that failed for a reason the protocol has a name for."""
|
||||
|
||||
def __init__(self, error: str, reason: str, hint: str | None = None, **fields: Any) -> None:
|
||||
super().__init__(reason)
|
||||
self.error = error
|
||||
self.reason = reason
|
||||
self.hint = hint
|
||||
self.fields = fields
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ManagerConfig:
|
||||
state_dir: Path
|
||||
runtime_dir: Path
|
||||
policy: Policy
|
||||
settle_s: float = DEFAULT_SETTLE_S
|
||||
#: The compositor launched applications attach to, or ``None`` for the
|
||||
#: inherited environment (``--compositor-runtime-dir`` unset).
|
||||
compositor: CompositorTarget | None = None
|
||||
|
||||
|
||||
class SessionManager:
|
||||
def __init__(
|
||||
self,
|
||||
config: ManagerConfig,
|
||||
*,
|
||||
clock: Clock | None = None,
|
||||
registry: Registry | None = None,
|
||||
launcher: Launcher | None = None,
|
||||
audit: AuditLog | None = None,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.clock = clock or Clock()
|
||||
self.registry = registry or Registry(config.state_dir, self.clock)
|
||||
self.launcher = launcher or Launcher(
|
||||
config.runtime_dir, config.state_dir, compositor=config.compositor
|
||||
)
|
||||
self.audit = audit or AuditLog(Path(config.state_dir) / "events.jsonl", self.clock)
|
||||
self.reaper = Reaper(
|
||||
now=self.clock.boottime,
|
||||
sessions=self.registry.live,
|
||||
on_expire=self._reap,
|
||||
)
|
||||
# Per-session asyncio plumbing, never persisted.
|
||||
self._exited: dict[str, asyncio.Event] = {}
|
||||
self._watchers: dict[str, asyncio.Task] = {}
|
||||
self._busy: set[str] = set()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# startup / shutdown
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def startup(self) -> dict[str, Any]:
|
||||
"""Load the mirror and adopt what is still alive (DESIGN.md §2.4)."""
|
||||
|
||||
self.registry.ensure_dirs()
|
||||
records, quarantined = self.registry.load_records()
|
||||
summary = {
|
||||
"adopted": 0,
|
||||
"tombstoned": 0,
|
||||
"quarantined": len(quarantined),
|
||||
"pruned": 0,
|
||||
# Records whose pid could be neither confirmed nor ruled out.
|
||||
# Never silent: each one is logged at ERROR as it happens.
|
||||
"unresolved": 0,
|
||||
}
|
||||
for record in records:
|
||||
if not record.live:
|
||||
self.registry.insert_loaded(record)
|
||||
continue
|
||||
self.registry.insert_loaded(record)
|
||||
if self.registry.stale_boot(record):
|
||||
# The record predates this boot. Its processes cannot exist:
|
||||
# pids restart from zero, so /proc would lie if we asked it.
|
||||
self.registry.terminate(
|
||||
record, State.EXITED, exit_detected_by="stale_boot", last_error="stale_boot"
|
||||
)
|
||||
summary["tombstoned"] += 1
|
||||
self.audit.emit("adopt_stale_boot", session=record.id, app=record.app)
|
||||
continue
|
||||
verdict = self._identity_of(record)
|
||||
if not verdict.ok:
|
||||
# A wrapper-launched app used to land here, because its argv had
|
||||
# changed since launch; it is now adopted like any other
|
||||
# (BUG-02 item 4). What reaches this branch is a process that
|
||||
# really is gone, or a pid the daemon cannot claim — and in the
|
||||
# latter case `_identity_of` has already said so at ERROR, and
|
||||
# `gc` will report any process left sharing the session id.
|
||||
self.registry.terminate(
|
||||
record,
|
||||
State.EXITED,
|
||||
exit_detected_by="proc_check",
|
||||
last_error=verdict.reason,
|
||||
)
|
||||
summary["tombstoned"] += 1
|
||||
if verdict.unresolved:
|
||||
summary["unresolved"] += 1
|
||||
self.audit.emit(
|
||||
"adopt_failed",
|
||||
session=record.id,
|
||||
app=record.app,
|
||||
why=verdict.reason,
|
||||
checked=list(verdict.checked),
|
||||
mismatched=list(verdict.mismatched),
|
||||
)
|
||||
continue
|
||||
record.adopted = True
|
||||
record.exit_detected_by = None
|
||||
self.registry.persist(record, force=True)
|
||||
self._watch(record)
|
||||
summary["adopted"] += 1
|
||||
self.audit.emit("adopted", session=record.id, app=record.app, pid=record.pid)
|
||||
summary["pruned"] = self._prune_tombstones()
|
||||
self.audit.emit("daemon_started", pid=os.getpid(), **summary)
|
||||
# Expired deadlines are reaped on the reaper's first pass, which runs
|
||||
# before its first sleep.
|
||||
self.reaper.start()
|
||||
return summary
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
"""Stop watching, flush the mirror. **Sessions are left running.**
|
||||
|
||||
The daemon is supervised by someone else and may be restarted at any
|
||||
time; killing every app on the way out would make a restart destructive.
|
||||
``startup`` adopts them back.
|
||||
"""
|
||||
|
||||
await self.reaper.stop()
|
||||
for task in list(self._watchers.values()):
|
||||
task.cancel()
|
||||
for task in list(self._watchers.values()):
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
await task
|
||||
self._watchers.clear()
|
||||
self.registry.flush_all()
|
||||
self.audit.emit("daemon_stopping", pid=os.getpid(), live=len(self.registry.live()))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def now(self) -> float:
|
||||
return self.clock.boottime()
|
||||
|
||||
def _require(self, session_id: Any) -> Session:
|
||||
if not isinstance(session_id, str) or not session_id:
|
||||
raise ManagerError("invalid_argument", "a session id is required")
|
||||
session = self.registry.get(session_id)
|
||||
if session is None:
|
||||
raise ManagerError("no_such_session", f"no session {session_id!r}")
|
||||
return session
|
||||
|
||||
def _require_live(self, session_id: Any) -> Session:
|
||||
session = self._require(session_id)
|
||||
if not session.live:
|
||||
if session.last_error in procmon.UNRESOLVED:
|
||||
# BUG-02 item 5. This session was not observed to exit — the
|
||||
# daemon gave up on it because it could not prove the pid was
|
||||
# still ours, and signalled nothing. Answering a later `kill`
|
||||
# with the same "is exited" as a routine exit would hide exactly
|
||||
# the fact the caller needs: an application may still be running.
|
||||
raise ManagerError(
|
||||
session.last_error,
|
||||
f"session {session.id} was given up on: pid {session.pid} could not "
|
||||
f"be identified as this session ({session.last_error}), so nothing "
|
||||
f"was ever signalled",
|
||||
hint="if the application is still running, `hhstream session gc` "
|
||||
"lists it under orphaned processes; kill it by hand",
|
||||
id=session.id,
|
||||
state=session.state,
|
||||
)
|
||||
raise ManagerError(
|
||||
TERMINAL_ERROR.get(session.state, "bad_state"),
|
||||
f"session {session.id} is {session.state}",
|
||||
id=session.id,
|
||||
state=session.state,
|
||||
)
|
||||
return session
|
||||
|
||||
def _event(self, session_id: str) -> asyncio.Event:
|
||||
return self._exited.setdefault(session_id, asyncio.Event())
|
||||
|
||||
def _identity_of(self, session: Session) -> procmon.Verdict:
|
||||
"""The one identity gate. **Every** caller goes through here (BUG-02).
|
||||
|
||||
``kill``, the reaper, ``gc`` and restart adoption all have to answer the
|
||||
same question — "is the pid in this record still the process we
|
||||
recorded?" — and before BUG-02 they each asked it with their own call to
|
||||
a check that was wrong in the same way. Routing them through one method
|
||||
means the evidence, the logging and the drift reporting cannot drift
|
||||
apart between callers.
|
||||
"""
|
||||
|
||||
verdict = procmon.check(
|
||||
session.pid,
|
||||
starttime=session.pid_starttime,
|
||||
sid=session.sid,
|
||||
pgid=session.pgid,
|
||||
cmdline_sha256=session.cmdline_sha256,
|
||||
)
|
||||
if verdict.ok and verdict.cmdline_drifted:
|
||||
# Entirely normal: a wrapper script `exec`ed the real binary, or the
|
||||
# app rewrote its own argv. Worth one line in the log and nothing
|
||||
# more — it is not evidence of anything (BUG-02).
|
||||
log.debug(
|
||||
"%s (pid %s): argv changed since launch; identity confirmed by %s",
|
||||
session.id,
|
||||
session.pid,
|
||||
"+".join(verdict.checked),
|
||||
)
|
||||
elif verdict.unresolved:
|
||||
log.error(
|
||||
"%s: refusing to treat pid %s as this session — %s (checked %s, mismatched %s)",
|
||||
session.id,
|
||||
session.pid,
|
||||
verdict.reason,
|
||||
"+".join(verdict.checked) or "nothing",
|
||||
"+".join(verdict.mismatched) or "nothing",
|
||||
)
|
||||
return verdict
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# exit watching
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _watch(self, session: Session, process: subprocess.Popen | None = None) -> None:
|
||||
"""Watch one session for exit. One path for own children and adopted ones.
|
||||
|
||||
``os.pidfd_open`` (Linux >= 5.3) notifies on exit for a process that is
|
||||
**not** our child, which is the adopted case, and works just as well for
|
||||
one that is. The pidfd becomes readable when the process terminates, so
|
||||
there is no polling and no latency.
|
||||
|
||||
**The documented fallback** for a kernel without ``pidfd_open``, or a
|
||||
pidfd this event loop cannot poll, is a one-second ``/proc`` liveness
|
||||
poll (``procmon.POLL_INTERVAL_S``). It costs one ``stat`` per session
|
||||
per second and detects an exit up to a second late; nothing else
|
||||
changes, because both paths converge on :meth:`_on_exit`.
|
||||
|
||||
**The exit status** comes from ``waitpid`` and is therefore available
|
||||
only for the daemon's own children. For an adopted process the record
|
||||
says ``exit_code: null`` with ``exit_detected_by`` naming the mechanism,
|
||||
rather than inventing a zero.
|
||||
"""
|
||||
|
||||
def reap() -> int | None:
|
||||
if process is None:
|
||||
return None
|
||||
# The pidfd fired, so the child is a zombie and waitpid will not
|
||||
# block. poll() is subprocess's waitpid(WNOHANG).
|
||||
for _ in range(50):
|
||||
code = process.poll()
|
||||
if code is not None:
|
||||
return code
|
||||
time.sleep(0.002)
|
||||
return None # pragma: no cover - the kernel would have to lie
|
||||
|
||||
async def watch_pidfd(fd: int) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
done = asyncio.Event()
|
||||
try:
|
||||
loop.add_reader(fd, done.set)
|
||||
except (OSError, NotImplementedError): # pragma: no cover
|
||||
os.close(fd)
|
||||
log.warning("pidfd is not pollable here; falling back to /proc polling")
|
||||
await watch_poll()
|
||||
return
|
||||
try:
|
||||
await done.wait()
|
||||
finally:
|
||||
with contextlib.suppress(OSError, ValueError):
|
||||
loop.remove_reader(fd)
|
||||
os.close(fd)
|
||||
self._on_exit(session.id, exit_code=reap(), detected_by="pidfd")
|
||||
|
||||
async def watch_poll() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(procmon.POLL_INTERVAL_S)
|
||||
if not procmon.is_alive(session.pid):
|
||||
self._on_exit(session.id, exit_code=reap(), detected_by="proc_poll")
|
||||
return
|
||||
|
||||
fd = procmon.pidfd_open(session.pid)
|
||||
if fd is None and not procmon.is_alive(session.pid):
|
||||
# Already gone between spawn/adopt and here.
|
||||
self._on_exit(session.id, exit_code=reap(), detected_by="proc_check")
|
||||
return
|
||||
coro = watch_pidfd(fd) if fd is not None else watch_poll()
|
||||
self._watchers[session.id] = asyncio.create_task(coro, name=f"hhstreamd.watch:{session.id}")
|
||||
|
||||
def _on_exit(self, session_id: str, *, exit_code: int | None, detected_by: str) -> None:
|
||||
self._event(session_id).set()
|
||||
session = self.registry.get(session_id)
|
||||
if session is None or not session.live:
|
||||
return
|
||||
self.registry.terminate(
|
||||
session, State.EXITED, exit_code=exit_code, exit_detected_by=detected_by
|
||||
)
|
||||
self.audit.emit(
|
||||
"session_exited",
|
||||
session=session.id,
|
||||
app=session.app,
|
||||
pid=session.pid,
|
||||
exit_code=exit_code,
|
||||
detected_by=detected_by,
|
||||
)
|
||||
self.reaper.wake()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# verbs
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def start(
|
||||
self,
|
||||
app: str,
|
||||
argv: list[str] | None = None,
|
||||
*,
|
||||
idle_timeout_s: float | None = None,
|
||||
wait_ready_s: float | None = None,
|
||||
home: str | None = None,
|
||||
ephemeral_home: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
policy = self.config.policy
|
||||
policy.check_capacity(len(self.registry.live()))
|
||||
rule, full_argv = policy.resolve(app, argv)
|
||||
if idle_timeout_s is None and rule.idle_timeout_s is not None:
|
||||
idle_timeout_s = rule.idle_timeout_s
|
||||
timeout = policy.check_idle_timeout(idle_timeout_s)
|
||||
wait_ready = policy.wait_ready_s if wait_ready_s is None else float(wait_ready_s)
|
||||
|
||||
session_id = self.registry.new_id(app)
|
||||
try:
|
||||
spawned = self.launcher.spawn(
|
||||
session_id, full_argv, home=home, ephemeral_home=ephemeral_home
|
||||
)
|
||||
except (OSError, ValueError) as exc:
|
||||
self.audit.emit("start_failed", app=app, argv=full_argv, reason=str(exc))
|
||||
raise ManagerError(
|
||||
"refused", f"could not execute {full_argv[0]!r}: {exc}"
|
||||
) from exc
|
||||
|
||||
identity = procmon.identity(spawned.pid)
|
||||
now = self.now()
|
||||
session = Session(
|
||||
id=session_id,
|
||||
app=app,
|
||||
argv=full_argv,
|
||||
pid=spawned.pid,
|
||||
pgid=identity.pgid if identity else spawned.pgid,
|
||||
sid=identity.sid if identity else spawned.sid,
|
||||
pid_starttime=identity.starttime if identity else 0,
|
||||
cmdline_sha256=identity.cmdline_sha256 if identity else "",
|
||||
home=spawned.home,
|
||||
runtime_dir=spawned.runtime_dir,
|
||||
log_path=spawned.log_path,
|
||||
created_at=iso(self.clock.wall()),
|
||||
created_boottime=now,
|
||||
last_activity_at=iso(self.clock.wall()),
|
||||
last_activity_boottime=now, # `start` stamps activity implicitly
|
||||
idle_timeout_s=timeout,
|
||||
state=State.STARTING,
|
||||
ephemeral_home=ephemeral_home,
|
||||
command_count=1,
|
||||
)
|
||||
self.registry.add(session)
|
||||
self._exited.pop(session_id, None)
|
||||
self._watch(session, spawned.process)
|
||||
self.audit.emit(
|
||||
"session_started",
|
||||
session=session.id,
|
||||
app=app,
|
||||
pid=session.pid,
|
||||
pgid=session.pgid,
|
||||
sid=session.sid,
|
||||
argv=full_argv,
|
||||
idle_timeout_s=timeout,
|
||||
)
|
||||
|
||||
ready = await self._settle(session, wait_ready)
|
||||
self.reaper.wake()
|
||||
record = session.public(self.now())
|
||||
if session.state == State.EXITED:
|
||||
# The app exited during the settle: surface it rather than handing
|
||||
# back a healthy-looking id (DESIGN.md §7 #3).
|
||||
raise ManagerError(
|
||||
"session_exited",
|
||||
f"{app} exited immediately (exit code {session.exit_code})",
|
||||
hint="the app may have handed off to an existing instance; "
|
||||
"try --ephemeral-home to force a fresh one",
|
||||
id=session.id,
|
||||
session=record,
|
||||
)
|
||||
if not ready:
|
||||
# DESIGN.md §7 #4: the session is left alive and the id is printed;
|
||||
# killing an app that took 20 s to start is worse than a slow
|
||||
# success. Unreachable in M0, where readiness is "still running".
|
||||
raise ManagerError(
|
||||
"readiness_timeout",
|
||||
f"{app} did not signal readiness within {wait_ready:g}s",
|
||||
id=session.id,
|
||||
session=record,
|
||||
)
|
||||
return record
|
||||
|
||||
async def _settle(self, session: Session, wait_ready: float) -> bool:
|
||||
"""Wait briefly for the process to prove it is still there.
|
||||
|
||||
M0 has no compositor, so the design's ``wayland-fd`` readiness probe
|
||||
degrades to **absent**: the only honest signal available here is that
|
||||
the process is alive and is still the process we spawned. That is
|
||||
recorded as ``ready_signal: "spawned"`` so nothing downstream mistakes
|
||||
it for evidence that a window was mapped.
|
||||
"""
|
||||
|
||||
settle = min(self.config.settle_s, wait_ready) if wait_ready > 0 else 0.0
|
||||
if settle > 0:
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(self._event(session.id).wait(), timeout=settle)
|
||||
if not session.live:
|
||||
return False
|
||||
session.state = State.READY
|
||||
session.ready_signal = "spawned"
|
||||
self.registry.persist(session, force=True)
|
||||
return True
|
||||
|
||||
def list(self, *, include_tombstones: bool = False) -> dict[str, Any]:
|
||||
"""Read-only. **Does not** count as activity."""
|
||||
|
||||
now = self.now()
|
||||
sessions = self.registry.all() if include_tombstones else self.registry.live()
|
||||
return {
|
||||
"sessions": [s.public(now) for s in sessions],
|
||||
"live": len(self.registry.live()),
|
||||
"tombstones": len(self.registry.tombstones()),
|
||||
"boottime": now,
|
||||
}
|
||||
|
||||
def status(self, session_id: str, *, events: int = 20) -> dict[str, Any]:
|
||||
"""Read-only. **Does not** count as activity."""
|
||||
|
||||
session = self._require(session_id)
|
||||
record = session.public(self.now())
|
||||
record["events"] = self.audit.tail(events, session=session.id)
|
||||
record["compositor"] = {
|
||||
# Enrichment the design calls optional, absent by construction in
|
||||
# M0: the daemon never speaks the Wayland protocol and never talks
|
||||
# to Hyprland's IPC. Locating the compositor's socket so a child
|
||||
# can attach to it (see hhstreamd.compositor) tells us nothing
|
||||
# about windows, so none of this becomes knowable.
|
||||
"ipc": "absent",
|
||||
"window": None,
|
||||
"workspace": None,
|
||||
"foreground": None,
|
||||
"note": "M0 does not talk to the compositor; these are never guessed",
|
||||
}
|
||||
return record
|
||||
|
||||
def touch(self, session_id: str) -> dict[str, Any]:
|
||||
"""**The only mark of activity in M0.** Resets the idle clock, injects nothing."""
|
||||
|
||||
session = self._require_live(session_id)
|
||||
self.registry.touch(session, force_flush=True)
|
||||
self.audit.emit(
|
||||
"touch",
|
||||
session=session.id,
|
||||
app=session.app,
|
||||
expires_in_s=round(session.expires_in(self.now()) or 0.0, 3),
|
||||
)
|
||||
self.reaper.wake()
|
||||
return session.public(self.now())
|
||||
|
||||
async def kill(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
grace_s: float | None = None,
|
||||
sig: str | int = signal.SIGTERM,
|
||||
) -> dict[str, Any]:
|
||||
session = self._require_live(session_id)
|
||||
signum = _signal_number(sig)
|
||||
grace = self.config.policy.grace_s if grace_s is None else float(grace_s)
|
||||
if grace < 0:
|
||||
raise ManagerError("invalid_argument", "grace may not be negative")
|
||||
result = await self.teardown(session, reason="killed", grace_s=grace, sig=signum)
|
||||
record = {**session.public(self.now()), **result}
|
||||
if result["refused"]:
|
||||
# BUG-02 item 5: the caller asked for a kill that did not happen, so
|
||||
# they get an error and exit 4, not a success line that reads like
|
||||
# the process was already gone.
|
||||
raise ManagerError(
|
||||
result["refused"],
|
||||
f"refused to signal session {session.id}: pid {session.pid} can no longer "
|
||||
f"be identified as this session "
|
||||
f"({', '.join(result['identity_mismatched']) or 'no usable evidence'} "
|
||||
f"of {', '.join(result['identity_checked']) or 'nothing'})",
|
||||
hint="nothing was signalled; if an application is still running, "
|
||||
"find it with `hhstream session gc` and kill it by hand",
|
||||
id=session.id,
|
||||
session=record,
|
||||
)
|
||||
return record
|
||||
|
||||
async def _survivors(self, session: Session) -> list[int]:
|
||||
"""Pids of the session still alive after the teardown has finished.
|
||||
|
||||
The leader plus anything still carrying the session id. This is the
|
||||
honest answer to "did the kill work", and it is what stops ``kill`` from
|
||||
reporting success over a process that is still on the screen (BUG-02).
|
||||
"""
|
||||
|
||||
# Real time, not the injected clock: this waits on the kernel, not on
|
||||
# session bookkeeping, and a test that freezes the clock must not turn
|
||||
# it into an infinite loop.
|
||||
deadline = time.monotonic() + SURVIVOR_GRACE_S
|
||||
while True:
|
||||
alive = set(procmon.session_members(session.sid, exclude={os.getpid()}))
|
||||
if procmon.is_alive(session.pid):
|
||||
alive.add(session.pid)
|
||||
if not alive or time.monotonic() >= deadline:
|
||||
return sorted(alive)
|
||||
await asyncio.sleep(0.02)
|
||||
|
||||
async def _reap(self, session_id: str) -> None:
|
||||
session = self.registry.get(session_id)
|
||||
if session is None or not session.live:
|
||||
return
|
||||
result = await self.teardown(session, reason="reaped", grace_s=self.config.policy.grace_s)
|
||||
if result["refused"] or result["survivors"]:
|
||||
# There is no caller to raise at here, so the loudness has to be in
|
||||
# the log and the audit trail. The session is tombstoned either
|
||||
# way, which is what keeps the reaper from spinning on it.
|
||||
log.error(
|
||||
"reaping %s did not complete: refused=%s survivors=%s",
|
||||
session_id,
|
||||
result["refused"],
|
||||
result["survivors"],
|
||||
)
|
||||
|
||||
async def teardown(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
reason: str,
|
||||
grace_s: float,
|
||||
sig: int = signal.SIGTERM,
|
||||
) -> dict[str, Any]:
|
||||
"""The one kill path (DESIGN.md §3.3, §6.3).
|
||||
|
||||
``killpg(TERM)`` → grace → ``killpg(KILL)`` → sweep ``/proc`` for the
|
||||
session id. The sweep is the substitute for a cgroup kill: a child that
|
||||
called ``setsid()`` once has left the process group but kept the session
|
||||
id, so it is caught here. A child that called ``setsid()`` twice escapes
|
||||
even this; ``gc`` reports it.
|
||||
|
||||
**It reports what it actually did** (BUG-02 item 1): the signal that was
|
||||
sent and to which process group, whether it escalated, how many
|
||||
stragglers the sweep took, and — the one that matters — whether anything
|
||||
is *still alive* afterwards. Before BUG-02 the only thing a caller got
|
||||
when the gate refused was ``signalled: false``, which the CLI rendered as
|
||||
"the process was already gone"; an application that cannot be killed
|
||||
while the tool reports success is the failure mode this exists to
|
||||
prevent. A refusal is now carried back in ``refused`` and turned into an
|
||||
error by :meth:`kill`.
|
||||
"""
|
||||
|
||||
if session.id in self._busy:
|
||||
raise ManagerError("busy", f"session {session.id} is already being torn down")
|
||||
self._busy.add(session.id)
|
||||
state = State.REAPED if reason == "reaped" else State.KILLED
|
||||
idle_s = session.idle_for(self.now())
|
||||
self.audit.emit(
|
||||
f"{reason}_started",
|
||||
session=session.id,
|
||||
app=session.app,
|
||||
pid=session.pid,
|
||||
pgid=session.pgid,
|
||||
signal=signal.Signals(sig).name,
|
||||
grace_s=grace_s,
|
||||
idle_s=None if idle_s is None else round(idle_s, 3),
|
||||
)
|
||||
try:
|
||||
exited = self._event(session.id)
|
||||
# Only signal a process group we can still prove is ours.
|
||||
verdict = self._identity_of(session)
|
||||
escalated = False
|
||||
sent: str | None = None
|
||||
refused: str | None = None
|
||||
if verdict.ok:
|
||||
# The wrapper case lands here now: `argv` changed when the
|
||||
# wrapper `exec`ed the real binary, and that is not evidence of
|
||||
# anything, so the application is signalled like any other.
|
||||
procmon.killpg(session.pgid, sig)
|
||||
sent = signal.Signals(sig).name
|
||||
if grace_s > 0:
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(exited.wait(), timeout=grace_s)
|
||||
if procmon.is_alive(session.pid) or procmon.session_members(
|
||||
session.sid, exclude={os.getpid()}
|
||||
):
|
||||
escalated = True
|
||||
procmon.killpg(session.pgid, signal.SIGKILL)
|
||||
# `sent` stays the signal the caller asked for; the
|
||||
# escalation is reported on its own, so the two are not
|
||||
# confused for one another.
|
||||
# Give the kernel a moment to deliver SIGKILL before the
|
||||
# sweep decides what is left.
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(exited.wait(), timeout=1.0)
|
||||
elif verdict.unresolved:
|
||||
# Nothing is signalled, and the caller is *told* so: the pid
|
||||
# cannot be claimed as this session's, so a kill aimed at it
|
||||
# would be aimed at a stranger. `kill` raises on this.
|
||||
refused = verdict.reason
|
||||
session.last_error = verdict.reason
|
||||
|
||||
if refused is not None:
|
||||
# The recorded session id may now belong to something else
|
||||
# entirely, so the sweep would be aiming at strangers. Skip it.
|
||||
stragglers: list[int] = []
|
||||
else:
|
||||
stragglers = procmon.sweep_session(session.sid, exclude={os.getpid()})
|
||||
|
||||
# What is left standing. The sweep has just sent SIGKILL, so give
|
||||
# the kernel a bounded moment to act on it before calling anything a
|
||||
# survivor: a pid that is still here after this is a real failure to
|
||||
# kill, and the caller has to hear about it rather than read
|
||||
# "killed" and assume. On a refusal nothing was signalled and the
|
||||
# pids involved may not be ours at all, so there is nothing to call
|
||||
# a survivor: `refused` is the thing the caller has to hear.
|
||||
survivors = [] if refused else await self._survivors(session)
|
||||
|
||||
if refused is None:
|
||||
state_now = state
|
||||
else:
|
||||
# Not killed — the daemon never signalled anything. Recording
|
||||
# KILLED here would be the same lie in the registry that the CLI
|
||||
# was telling on stdout.
|
||||
state_now = State.EXITED
|
||||
self.registry.terminate(
|
||||
session,
|
||||
state_now,
|
||||
exit_detected_by=session.exit_detected_by
|
||||
or ("identity_refused" if refused else "killpg"),
|
||||
last_error=session.last_error,
|
||||
)
|
||||
self.launcher.discard_home(session)
|
||||
self.audit.emit(
|
||||
f"{reason}_completed",
|
||||
session=session.id,
|
||||
app=session.app,
|
||||
escalated_to_sigkill=escalated,
|
||||
stragglers=len(stragglers),
|
||||
straggler_pids=stragglers,
|
||||
signalled=verdict.ok,
|
||||
signal_sent=sent,
|
||||
pgid=session.pgid,
|
||||
refused=refused,
|
||||
survivors=survivors,
|
||||
identity=verdict.reason or "verified",
|
||||
cmdline_drifted=verdict.cmdline_drifted,
|
||||
idle_s=None if idle_s is None else round(idle_s, 3),
|
||||
)
|
||||
if survivors:
|
||||
log.error(
|
||||
"%s: %d process(es) survived the teardown: %s",
|
||||
session.id,
|
||||
len(survivors),
|
||||
survivors,
|
||||
)
|
||||
self.reaper.wake()
|
||||
return {
|
||||
"reason": reason,
|
||||
"signalled": verdict.ok,
|
||||
#: The signal actually delivered, or ``None`` if none was.
|
||||
"signal_sent": sent,
|
||||
"pgid": session.pgid,
|
||||
"escalated_to_sigkill": escalated,
|
||||
"stragglers": len(stragglers),
|
||||
"straggler_pids": stragglers,
|
||||
#: Still alive after everything. Non-empty means the teardown
|
||||
#: did not do what it was asked to do.
|
||||
"survivors": survivors,
|
||||
#: ``"verified"``, or the reason the gate refused.
|
||||
"identity": verdict.reason or "verified",
|
||||
"identity_checked": list(verdict.checked),
|
||||
"identity_mismatched": list(verdict.mismatched),
|
||||
#: The recorded argv no longer matches the live one. Normal for
|
||||
#: an app behind a wrapper script; never a reason to refuse.
|
||||
"cmdline_drifted": verdict.cmdline_drifted,
|
||||
#: Set when the daemon refused to signal at all (BUG-02 item 5).
|
||||
"refused": refused,
|
||||
}
|
||||
finally:
|
||||
self._busy.discard(session.id)
|
||||
|
||||
async def adopt(
|
||||
self,
|
||||
pid: int,
|
||||
*,
|
||||
app: str | None = None,
|
||||
idle_timeout_s: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Bring an already-running process under management.
|
||||
|
||||
The same machinery the daemon uses to re-adopt its own sessions after a
|
||||
restart, exposed as a verb — and the bridge from an unmanaged launch
|
||||
into management.
|
||||
"""
|
||||
|
||||
try:
|
||||
pid = int(pid)
|
||||
except (TypeError, ValueError):
|
||||
raise ManagerError("invalid_argument", f"not a pid: {pid!r}") from None
|
||||
if pid <= 1:
|
||||
raise ManagerError("refused", f"refusing to adopt pid {pid}")
|
||||
if pid == os.getpid():
|
||||
raise ManagerError("refused", "refusing to adopt the daemon itself")
|
||||
for existing in self.registry.live():
|
||||
if existing.pid == pid:
|
||||
raise ManagerError(
|
||||
"already_managed",
|
||||
f"pid {pid} is already session {existing.id}",
|
||||
id=existing.id,
|
||||
)
|
||||
identity = procmon.identity(pid)
|
||||
if identity is None:
|
||||
raise ManagerError("no_such_process", f"no process with pid {pid}")
|
||||
argv = procmon.cmdline_argv(pid) or []
|
||||
if app is None:
|
||||
app = Path(argv[0]).name if argv else f"pid{pid}"
|
||||
try:
|
||||
check_app_name(app)
|
||||
except PolicyError as exc:
|
||||
raise ManagerError(exc.error, exc.reason, exc.hint) from exc
|
||||
self.config.policy.check_capacity(len(self.registry.live()))
|
||||
timeout = self.config.policy.check_idle_timeout(idle_timeout_s)
|
||||
|
||||
now = self.now()
|
||||
session_id = self.registry.new_id(app)
|
||||
session = Session(
|
||||
id=session_id,
|
||||
app=app,
|
||||
argv=argv,
|
||||
pid=pid,
|
||||
pgid=identity.pgid,
|
||||
sid=identity.sid,
|
||||
pid_starttime=identity.starttime,
|
||||
cmdline_sha256=identity.cmdline_sha256,
|
||||
home=os.environ.get("HOME", "/"),
|
||||
runtime_dir=str(self.launcher.session_runtime_dir(session_id)),
|
||||
log_path=None, # the daemon did not spawn it, so it captured no output
|
||||
created_at=iso(self.clock.wall()),
|
||||
created_boottime=now,
|
||||
last_activity_at=iso(self.clock.wall()),
|
||||
last_activity_boottime=now,
|
||||
idle_timeout_s=timeout,
|
||||
state=State.READY,
|
||||
ready_signal="adopted",
|
||||
adopted=True,
|
||||
command_count=1,
|
||||
)
|
||||
self.registry.add(session)
|
||||
self._exited.pop(session_id, None)
|
||||
self._watch(session)
|
||||
self.audit.emit(
|
||||
"session_adopted", session=session.id, app=app, pid=pid, idle_timeout_s=timeout
|
||||
)
|
||||
self.reaper.wake()
|
||||
return session.public(self.now())
|
||||
|
||||
def gc(self) -> dict[str, Any]:
|
||||
"""Prune tombstones and reconcile the table with ``/proc``.
|
||||
|
||||
Read-only with respect to the idle clock: **``gc`` does not count as
|
||||
activity.**
|
||||
"""
|
||||
|
||||
now = self.now()
|
||||
reconciled: list[dict[str, Any]] = []
|
||||
#: Reconciled sessions whose pid could not be resolved either way.
|
||||
#: Reported separately so a caller can tell "it exited" (routine)
|
||||
#: from "the daemon has lost track of a running application".
|
||||
unresolved: list[dict[str, Any]] = []
|
||||
for session in self.registry.live():
|
||||
verdict = self._identity_of(session)
|
||||
if verdict.ok:
|
||||
# Including the wrapper case: a session whose argv changed after
|
||||
# launch is healthy, and `gc` used to tombstone it while its
|
||||
# application kept running (BUG-02 item 4).
|
||||
continue
|
||||
why = verdict.reason
|
||||
self.registry.terminate(
|
||||
session,
|
||||
State.EXITED,
|
||||
exit_detected_by="gc_proc_scan",
|
||||
last_error=None if why == procmon.GONE else why,
|
||||
)
|
||||
entry = {"id": session.id, "why": why}
|
||||
if verdict.unresolved:
|
||||
entry["mismatched"] = list(verdict.mismatched)
|
||||
unresolved.append(entry)
|
||||
reconciled.append(entry)
|
||||
self.audit.emit(
|
||||
"gc_reconciled",
|
||||
session=session.id,
|
||||
why=why,
|
||||
mismatched=list(verdict.mismatched),
|
||||
)
|
||||
|
||||
orphans: list[dict[str, Any]] = []
|
||||
for session in self.registry.tombstones():
|
||||
survivors = procmon.session_members(session.sid, exclude={os.getpid()})
|
||||
if survivors:
|
||||
# Processes still sharing a dead session's session id: the
|
||||
# double-`setsid()` leak the design admits to (§7 #7).
|
||||
orphans.append({"id": session.id, "sid": session.sid, "pids": survivors})
|
||||
pruned = self._prune_tombstones()
|
||||
self.reaper.wake()
|
||||
self.audit.emit(
|
||||
"gc",
|
||||
pruned=pruned,
|
||||
reconciled=len(reconciled),
|
||||
unresolved=len(unresolved),
|
||||
orphans=len(orphans),
|
||||
)
|
||||
return {
|
||||
"pruned": pruned,
|
||||
"reconciled": reconciled,
|
||||
"unresolved": unresolved,
|
||||
"orphans": orphans,
|
||||
"live": len(self.registry.live()),
|
||||
"tombstones": len(self.registry.tombstones()),
|
||||
"boottime": now,
|
||||
}
|
||||
|
||||
def _prune_tombstones(self) -> int:
|
||||
now = self.now()
|
||||
pruned = 0
|
||||
for session in self.registry.tombstones():
|
||||
expires = session.tombstone_expires()
|
||||
if expires is None or expires > now:
|
||||
continue
|
||||
self.launcher.discard_home(session)
|
||||
self.launcher.discard_runtime_dir(session)
|
||||
self.registry.drop(session.id)
|
||||
self._exited.pop(session.id, None)
|
||||
self._watchers.pop(session.id, None)
|
||||
pruned += 1
|
||||
return pruned
|
||||
|
||||
def logs(self, session_id: str, *, lines: int = 100, offset: int | None = None) -> dict[str, Any]:
|
||||
"""The daemon's capture of the app's stdout/stderr.
|
||||
|
||||
Served from ``<state>/logs/<id>.log`` rather than the journal, because
|
||||
apps are ordinary child processes here. Read-only: **does not** count
|
||||
as activity. ``offset`` supports ``--follow`` without the daemon
|
||||
holding a streaming connection open.
|
||||
"""
|
||||
|
||||
session = self._require(session_id)
|
||||
if not session.log_path:
|
||||
return {
|
||||
"id": session.id,
|
||||
"path": None,
|
||||
"lines": [],
|
||||
"offset": 0,
|
||||
"eof": True,
|
||||
"note": "adopted sessions have no captured output",
|
||||
}
|
||||
path = Path(session.log_path)
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
except OSError:
|
||||
return {"id": session.id, "path": str(path), "lines": [], "offset": 0, "eof": True}
|
||||
if offset is None:
|
||||
tail, truncated, size = tail_lines(path, max(0, int(lines)))
|
||||
return {
|
||||
"id": session.id,
|
||||
"path": str(path),
|
||||
"lines": tail,
|
||||
"offset": size,
|
||||
"truncated": truncated,
|
||||
# `eof` means "no more output will ever arrive", which is true
|
||||
# exactly when the session has ended. A follower polls until it.
|
||||
"eof": not session.live,
|
||||
}
|
||||
offset = max(0, min(int(offset), size))
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
fh.seek(offset)
|
||||
chunk = fh.read(MAX_LOG_CHUNK_BYTES)
|
||||
new_offset = fh.tell()
|
||||
except OSError as exc:
|
||||
raise ManagerError("refused", f"could not read logs: {exc}") from exc
|
||||
return {
|
||||
"id": session.id,
|
||||
"path": str(path),
|
||||
"lines": chunk.decode("utf-8", "replace").splitlines(),
|
||||
"offset": new_offset,
|
||||
"truncated": new_offset < size,
|
||||
"eof": not session.live and new_offset >= size,
|
||||
}
|
||||
|
||||
|
||||
def _signal_number(sig: str | int) -> int:
|
||||
if isinstance(sig, int):
|
||||
try:
|
||||
return signal.Signals(sig).value
|
||||
except ValueError:
|
||||
raise ManagerError("invalid_argument", f"unknown signal: {sig!r}") from None
|
||||
name = str(sig).upper()
|
||||
if name in SIGNALS_BY_NAME:
|
||||
return SIGNALS_BY_NAME[name]
|
||||
raise ManagerError(
|
||||
"invalid_argument",
|
||||
f"unknown signal: {sig!r}",
|
||||
hint=f"one of {', '.join(sorted(set(SIGNALS_BY_NAME)))}",
|
||||
)
|
||||
@@ -0,0 +1,237 @@
|
||||
"""The app allowlist and the input bounds on ``start`` (DESIGN.md §8 item 1).
|
||||
|
||||
This is the main control, and the only part of §8 that is in M0:
|
||||
|
||||
* ``start`` resolves an app *name* against the config file to an **absolute
|
||||
path**. An arbitrary binary is refused with exit 6. Without this,
|
||||
``hhstream session start /bin/sh`` makes every other control decorative.
|
||||
* ``argv`` is a **list**. There is never a shell: no ``shell=True``, no string
|
||||
interpolation into a command line, anywhere in this package.
|
||||
* Caller-supplied arguments are matched against ``args_allow`` patterns. An
|
||||
app with no ``args_allow`` accepts no caller arguments at all.
|
||||
|
||||
Deliberately **not** in M0 (brief: "nothing else from §8"): the ``SO_PEERCRED``
|
||||
uid/gid authorisation, per-session ``HOME`` by default, ``nice``,
|
||||
``oom_score_adj``, rlimits, the RSS watchdog and the keybinding denylist.
|
||||
``max_sessions`` is kept, because exit code 6 in the frozen table names it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import logging
|
||||
import os
|
||||
import tomllib
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
log = logging.getLogger("hhstreamd.policy")
|
||||
|
||||
DEFAULT_IDLE_TIMEOUT_S = 3 * 3600.0
|
||||
DEFAULT_MAX_IDLE_TIMEOUT_S = 24 * 3600.0
|
||||
DEFAULT_MAX_SESSIONS = 4
|
||||
DEFAULT_GRACE_S = 10.0
|
||||
DEFAULT_WAIT_READY_S = 15.0
|
||||
|
||||
# Input bounds that apply to `start`. They exist so a malformed or hostile
|
||||
# request is refused cheaply, before anything is spawned.
|
||||
MAX_APP_NAME = 64
|
||||
MAX_ARGV_ITEMS = 64
|
||||
MAX_ARG_BYTES = 4096
|
||||
MAX_ARGV_BYTES = 64 * 1024
|
||||
APP_NAME_CHARS = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-+")
|
||||
|
||||
|
||||
class PolicyError(Exception):
|
||||
"""A request refused by policy. ``error`` maps to an exit code."""
|
||||
|
||||
def __init__(self, error: str, reason: str, hint: str | None = None) -> None:
|
||||
super().__init__(reason)
|
||||
self.error = error
|
||||
self.reason = reason
|
||||
self.hint = hint
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AppRule:
|
||||
name: str
|
||||
exec: str
|
||||
args: list[str] = field(default_factory=list)
|
||||
args_allow: list[str] = field(default_factory=list)
|
||||
idle_timeout_s: float | None = None
|
||||
|
||||
def check_args(self, argv: list[str]) -> None:
|
||||
for arg in argv:
|
||||
if not any(fnmatch.fnmatchcase(arg, pattern) for pattern in self.args_allow):
|
||||
raise PolicyError(
|
||||
"argument_refused",
|
||||
f"argument {arg!r} is not permitted for app {self.name!r}",
|
||||
hint="args_allow in the hhstream config controls this",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Policy:
|
||||
apps: dict[str, AppRule] = field(default_factory=dict)
|
||||
max_sessions: int = DEFAULT_MAX_SESSIONS
|
||||
idle_timeout_s: float = DEFAULT_IDLE_TIMEOUT_S
|
||||
max_idle_timeout_s: float = DEFAULT_MAX_IDLE_TIMEOUT_S
|
||||
grace_s: float = DEFAULT_GRACE_S
|
||||
wait_ready_s: float = DEFAULT_WAIT_READY_S
|
||||
source: str | None = None
|
||||
|
||||
# ---- resolution --------------------------------------------------
|
||||
|
||||
def resolve(self, app: str, argv: list[str] | None) -> tuple[AppRule, list[str]]:
|
||||
"""Resolve an app name and caller argv to the full argv to execute.
|
||||
|
||||
Returns ``(rule, argv)`` where ``argv[0]`` is the rule's absolute exec
|
||||
path. Raises :class:`PolicyError` for anything not allowlisted.
|
||||
"""
|
||||
|
||||
check_app_name(app)
|
||||
argv = list(argv or [])
|
||||
check_argv(argv)
|
||||
rule = self.apps.get(app)
|
||||
if rule is None:
|
||||
known = ", ".join(sorted(self.apps)) or "(the allowlist is empty)"
|
||||
raise PolicyError(
|
||||
"not_allowlisted",
|
||||
f"app {app!r} is not in the allowlist",
|
||||
hint=f"allowlisted apps: {known}; see {self.source or 'the hhstream config'}",
|
||||
)
|
||||
rule.check_args(argv)
|
||||
return rule, [rule.exec, *rule.args, *argv]
|
||||
|
||||
def check_idle_timeout(self, seconds: float | None) -> float:
|
||||
if seconds is None:
|
||||
return self.idle_timeout_s
|
||||
try:
|
||||
value = float(seconds)
|
||||
except (TypeError, ValueError):
|
||||
raise PolicyError("invalid_argument", f"idle timeout is not a number: {seconds!r}") from None
|
||||
if value < 0:
|
||||
raise PolicyError("invalid_argument", "idle timeout may not be negative")
|
||||
if value > self.max_idle_timeout_s:
|
||||
raise PolicyError(
|
||||
"refused",
|
||||
f"idle timeout {value:g}s exceeds the configured maximum "
|
||||
f"{self.max_idle_timeout_s:g}s",
|
||||
)
|
||||
return value
|
||||
|
||||
def check_capacity(self, live_count: int) -> None:
|
||||
if self.max_sessions and live_count >= self.max_sessions:
|
||||
raise PolicyError(
|
||||
"session_cap_reached",
|
||||
f"{live_count} live sessions, max_sessions is {self.max_sessions}",
|
||||
hint="kill or gc a session, or raise max_sessions in the config",
|
||||
)
|
||||
|
||||
|
||||
def check_app_name(app: Any) -> None:
|
||||
if not isinstance(app, str) or not app:
|
||||
raise PolicyError("invalid_argument", "app name must be a non-empty string")
|
||||
if len(app) > MAX_APP_NAME:
|
||||
raise PolicyError("invalid_argument", f"app name longer than {MAX_APP_NAME} characters")
|
||||
bad = set(app) - APP_NAME_CHARS
|
||||
if bad:
|
||||
raise PolicyError(
|
||||
"invalid_argument",
|
||||
f"app name contains disallowed characters: {''.join(sorted(bad))!r}",
|
||||
)
|
||||
|
||||
|
||||
def check_argv(argv: Any) -> None:
|
||||
"""Bounds on the caller-supplied argv. A list of strings, and not a large one."""
|
||||
|
||||
if not isinstance(argv, list):
|
||||
raise PolicyError("invalid_argument", "argv must be a list of strings")
|
||||
if len(argv) > MAX_ARGV_ITEMS:
|
||||
raise PolicyError("invalid_argument", f"argv has more than {MAX_ARGV_ITEMS} items")
|
||||
total = 0
|
||||
for arg in argv:
|
||||
if not isinstance(arg, str):
|
||||
raise PolicyError("invalid_argument", f"argv item is not a string: {arg!r}")
|
||||
if "\0" in arg:
|
||||
raise PolicyError("invalid_argument", "argv item contains a NUL byte")
|
||||
encoded = len(arg.encode("utf-8"))
|
||||
if encoded > MAX_ARG_BYTES:
|
||||
raise PolicyError("invalid_argument", f"argv item longer than {MAX_ARG_BYTES} bytes")
|
||||
total += encoded
|
||||
if total > MAX_ARGV_BYTES:
|
||||
raise PolicyError("invalid_argument", f"argv longer than {MAX_ARGV_BYTES} bytes in total")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Config loading
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_policy(path: Path | str | None) -> Policy:
|
||||
"""Read the allowlist from a TOML file.
|
||||
|
||||
A missing file is not an error: it yields an empty allowlist, so the daemon
|
||||
starts and every ``start`` is refused with exit 6 and a hint naming the file
|
||||
it looked for. Refusing to start would make a misconfigured deployment look
|
||||
like a crash.
|
||||
"""
|
||||
|
||||
if path is None:
|
||||
return Policy(source=None)
|
||||
path = Path(path)
|
||||
if not path.exists():
|
||||
log.warning("no config at %s: the app allowlist is empty, every start will be refused", path)
|
||||
return Policy(source=str(path))
|
||||
with open(path, "rb") as fh:
|
||||
data = tomllib.load(fh)
|
||||
return policy_from_dict(data, source=str(path))
|
||||
|
||||
|
||||
def policy_from_dict(data: dict[str, Any], *, source: str | None = None) -> Policy:
|
||||
defaults = data.get("defaults", {})
|
||||
if not isinstance(defaults, dict):
|
||||
raise PolicyError("invalid_argument", "[defaults] must be a table")
|
||||
apps: dict[str, AppRule] = {}
|
||||
raw_apps = data.get("apps", {})
|
||||
if not isinstance(raw_apps, dict):
|
||||
raise PolicyError("invalid_argument", "[apps] must be a table")
|
||||
for name, spec in raw_apps.items():
|
||||
apps[name] = _app_rule(name, spec)
|
||||
return Policy(
|
||||
apps=apps,
|
||||
max_sessions=int(defaults.get("max_sessions", DEFAULT_MAX_SESSIONS)),
|
||||
idle_timeout_s=float(defaults.get("idle_timeout_s", DEFAULT_IDLE_TIMEOUT_S)),
|
||||
max_idle_timeout_s=float(defaults.get("max_idle_timeout_s", DEFAULT_MAX_IDLE_TIMEOUT_S)),
|
||||
grace_s=float(defaults.get("grace_s", DEFAULT_GRACE_S)),
|
||||
wait_ready_s=float(defaults.get("wait_ready_s", DEFAULT_WAIT_READY_S)),
|
||||
source=source,
|
||||
)
|
||||
|
||||
|
||||
def _app_rule(name: str, spec: Any) -> AppRule:
|
||||
if not isinstance(spec, dict):
|
||||
raise PolicyError("invalid_argument", f"[apps.{name}] must be a table")
|
||||
check_app_name(name)
|
||||
exec_path = spec.get("exec")
|
||||
if not isinstance(exec_path, str) or not exec_path:
|
||||
raise PolicyError("invalid_argument", f"[apps.{name}] needs an 'exec' string")
|
||||
if not os.path.isabs(exec_path):
|
||||
raise PolicyError(
|
||||
"invalid_argument",
|
||||
f"[apps.{name}].exec must be an absolute path, got {exec_path!r}",
|
||||
)
|
||||
args = spec.get("args", [])
|
||||
args_allow = spec.get("args_allow", [])
|
||||
for key, value in (("args", args), ("args_allow", args_allow)):
|
||||
if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
|
||||
raise PolicyError("invalid_argument", f"[apps.{name}].{key} must be a list of strings")
|
||||
idle = spec.get("idle_timeout_s")
|
||||
return AppRule(
|
||||
name=name,
|
||||
exec=exec_path,
|
||||
args=list(args),
|
||||
args_allow=list(args_allow),
|
||||
idle_timeout_s=None if idle is None else float(idle),
|
||||
)
|
||||
@@ -0,0 +1,406 @@
|
||||
"""``/proc`` inspection: liveness, identity, and the session-id sweep.
|
||||
|
||||
This is the substitute for the cgroup membership the design lost when the
|
||||
daemon stopped being a systemd *user* service (DESIGN.md §2.4, §3.3):
|
||||
|
||||
* **Liveness** is ``/proc/<pid>`` existing.
|
||||
* **Identity** is the evidence that makes it safe to signal a pid the daemon
|
||||
did not spawn — see :func:`check` and the note below.
|
||||
* **The sweep** finds processes whose session id matches a recorded session,
|
||||
which catches a child that called ``setsid()`` once and so escaped the
|
||||
process group. A child that calls ``setsid()`` twice escapes even this; that
|
||||
is the honest cost of having no cgroup, and ``gc`` reports it.
|
||||
|
||||
**Why the command line is not identity evidence (BUG-02).** The original
|
||||
triple-check hashed ``/proc/<pid>/cmdline`` and refused to signal when the hash
|
||||
moved. ``/proc/<pid>/cmdline`` is not a property of the *process*; it is a
|
||||
window onto the current image's argument vector, and it is replaced wholesale by
|
||||
every ``execve()``. On the target platform ``/run/current-system/sw/bin/<app>``
|
||||
is almost always a wrapper script that ``exec``s the real binary, which arrives
|
||||
with an argv of its own; Chromium then rewrites argv in place for its zygote and
|
||||
renderer processes on top of that. So a perfectly healthy session was being
|
||||
classified as a recycled pid, and ``kill``, ``gc`` and restart adoption all
|
||||
quietly did the wrong thing. The hash is still recorded and still reported —
|
||||
as *drift*, which is useful in a post-mortem — but it never gates a signal.
|
||||
|
||||
**What is left, and why it is sound.**
|
||||
|
||||
* ``starttime`` (``stat`` field 22) is written once by the kernel when the task
|
||||
is created and is never writable afterwards, by the process or by anyone
|
||||
else. ``execve()`` replaces the address space, not the task, so it survives
|
||||
an arbitrarily long chain of wrappers. Two processes holding the same pid are
|
||||
necessarily created at different times, so a difference here is proof of
|
||||
reuse. The one residual window is the clock-tick granularity of the field
|
||||
(``USER_HZ``, 10 ms on this kernel): a pid could in principle be recycled
|
||||
within the same tick, which would require the kernel to wrap the whole pid
|
||||
space — ``/proc/sys/kernel/pid_max``, four million by default — in under 10 ms.
|
||||
``boot_id`` (see :mod:`hhstreamd.registry`) closes the across-reboot case
|
||||
separately, so this is the entire remaining exposure and it is accepted
|
||||
knowingly. **This field is mandatory**: without it there is no identity, and
|
||||
the caller is refused rather than silently obliged.
|
||||
* ``sid`` and ``pgid`` are inherited across ``execve()`` and survive a wrapper
|
||||
just as well, but they are only *immutable* for a session/group leader: a
|
||||
leader's ``setsid()`` and ``setpgid()`` both fail with ``EPERM``, whereas an
|
||||
ordinary process may lawfully leave at any moment. Every session the daemon
|
||||
spawns is a leader by construction (``start_new_session=True`` gives
|
||||
``sid == pgid == pid``), so the check is applied exactly when the recorded
|
||||
value equals the recorded pid and skipped otherwise — an *adopted* mid-session
|
||||
process that later calls ``setsid()`` must not be mistaken for a stranger.
|
||||
They corroborate ``starttime``; they are never trusted alone, because a
|
||||
recycled pid that calls ``setsid()`` acquires ``sid == pgid == pid`` too.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import os
|
||||
import signal
|
||||
from dataclasses import dataclass
|
||||
|
||||
PROC = "/proc"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProcStat:
|
||||
"""The fields of ``/proc/<pid>/stat`` this daemon cares about."""
|
||||
|
||||
pid: int
|
||||
comm: str
|
||||
state: str
|
||||
ppid: int
|
||||
pgid: int
|
||||
sid: int
|
||||
starttime: int
|
||||
|
||||
|
||||
def read_stat(pid: int, *, proc: str = PROC) -> ProcStat | None:
|
||||
"""Parse ``/proc/<pid>/stat``. ``None`` if the process is gone.
|
||||
|
||||
``comm`` may contain spaces and parentheses, so the fields after it are
|
||||
located from the *last* ``)`` rather than by splitting the whole line.
|
||||
"""
|
||||
|
||||
try:
|
||||
with open(f"{proc}/{pid}/stat", "rb") as fh:
|
||||
raw = fh.read()
|
||||
except (FileNotFoundError, ProcessLookupError, NotADirectoryError):
|
||||
return None
|
||||
except OSError as exc: # ESRCH racing with exit, EACCES on a hardened /proc
|
||||
if exc.errno in (errno.ESRCH, errno.EACCES, errno.EPERM):
|
||||
return None
|
||||
raise
|
||||
open_paren = raw.find(b"(")
|
||||
close = raw.rfind(b")")
|
||||
if open_paren == -1 or close < open_paren:
|
||||
return None
|
||||
comm = raw[open_paren + 1 : close].decode("utf-8", "replace")
|
||||
rest = raw[close + 2 :].split()
|
||||
# `rest` begins at field 3 (state), so field N lives at index N - 3.
|
||||
if len(rest) < 20:
|
||||
return None
|
||||
try:
|
||||
return ProcStat(
|
||||
pid=int(raw[:open_paren].strip()),
|
||||
comm=comm,
|
||||
state=rest[0].decode(),
|
||||
ppid=int(rest[1]),
|
||||
pgid=int(rest[2]),
|
||||
sid=int(rest[3]),
|
||||
starttime=int(rest[19]),
|
||||
)
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def read_cmdline(pid: int, *, proc: str = PROC) -> bytes | None:
|
||||
"""Raw NUL-separated ``/proc/<pid>/cmdline``. ``None`` if the process is gone."""
|
||||
|
||||
try:
|
||||
with open(f"{proc}/{pid}/cmdline", "rb") as fh:
|
||||
return fh.read()
|
||||
except (FileNotFoundError, ProcessLookupError, NotADirectoryError):
|
||||
return None
|
||||
except OSError as exc:
|
||||
if exc.errno in (errno.ESRCH, errno.EACCES, errno.EPERM):
|
||||
return None
|
||||
raise
|
||||
|
||||
|
||||
def cmdline_argv(pid: int, *, proc: str = PROC) -> list[str] | None:
|
||||
"""``/proc/<pid>/cmdline`` split into an argv list."""
|
||||
|
||||
raw = read_cmdline(pid, proc=proc)
|
||||
if raw is None:
|
||||
return None
|
||||
parts = raw.split(b"\0")
|
||||
if parts and parts[-1] == b"":
|
||||
parts.pop()
|
||||
return [p.decode("utf-8", "replace") for p in parts]
|
||||
|
||||
|
||||
def hash_cmdline(raw: bytes) -> str:
|
||||
"""The ``cmdline_sha256`` of DESIGN.md §2.1."""
|
||||
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def cmdline_hash(pid: int, *, proc: str = PROC) -> str | None:
|
||||
raw = read_cmdline(pid, proc=proc)
|
||||
return None if raw is None else hash_cmdline(raw)
|
||||
|
||||
|
||||
#: `/proc/<pid>/stat` states for a process that has already terminated. A
|
||||
#: zombie still has a `/proc` entry — it holds its pid until its real parent
|
||||
#: reaps it — but it will never run again and cannot be signalled into doing
|
||||
#: anything. Treating it as alive would make the daemon escalate to SIGKILL
|
||||
#: against a corpse and report sessions as running after they ended, which is
|
||||
#: the common case for an *adopted* process whose real parent is slow to wait().
|
||||
DEAD_STATES = frozenset("ZX")
|
||||
|
||||
|
||||
def is_alive(pid: int, *, proc: str = PROC) -> bool:
|
||||
"""True if the process exists and has not already terminated."""
|
||||
|
||||
stat = read_stat(pid, proc=proc)
|
||||
return stat is not None and stat.state not in DEAD_STATES
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Identity:
|
||||
"""The evidence recorded at spawn and re-checked after a daemon restart."""
|
||||
|
||||
pid: int
|
||||
pgid: int
|
||||
sid: int
|
||||
starttime: int
|
||||
cmdline_sha256: str
|
||||
|
||||
|
||||
def identity(pid: int, *, proc: str = PROC) -> Identity | None:
|
||||
"""Capture the anti-pid-reuse evidence for a live process."""
|
||||
|
||||
stat = read_stat(pid, proc=proc)
|
||||
if stat is None:
|
||||
return None
|
||||
raw = read_cmdline(pid, proc=proc)
|
||||
if raw is None:
|
||||
return None
|
||||
return Identity(
|
||||
pid=pid,
|
||||
pgid=stat.pgid,
|
||||
sid=stat.sid,
|
||||
starttime=stat.starttime,
|
||||
cmdline_sha256=hash_cmdline(raw),
|
||||
)
|
||||
|
||||
|
||||
#: Reasons :func:`check` can refuse. ``gone`` is routine — the process simply
|
||||
#: exited. The other two are not: ``pid_reuse_detected`` means a *different*
|
||||
#: process now holds the pid, and ``identity_unknown`` means the record carries
|
||||
#: no usable evidence either way. A caller may never signal on any of them, and
|
||||
#: must never report the latter two as a quiet success (BUG-02).
|
||||
GONE = "gone"
|
||||
PID_REUSE = "pid_reuse_detected"
|
||||
IDENTITY_UNKNOWN = "identity_unknown"
|
||||
|
||||
#: Refusals the caller has to be *loud* about: both mean the daemon could not
|
||||
#: establish that the pid is still the session's, so an operation the caller
|
||||
#: asked for did not happen.
|
||||
UNRESOLVED = frozenset({PID_REUSE, IDENTITY_UNKNOWN})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Verdict:
|
||||
"""The outcome of an identity check, with the evidence that produced it.
|
||||
|
||||
``checked`` and ``mismatched`` are what makes this falsifiable in a log or a
|
||||
test: they name the fields that actually gated the decision, so weakening
|
||||
the gate cannot be done silently.
|
||||
"""
|
||||
|
||||
ok: bool
|
||||
reason: str | None
|
||||
#: Fields that were in a position to refuse, in the order they were tried.
|
||||
checked: tuple[str, ...] = ()
|
||||
#: The subset of ``checked`` that did not match.
|
||||
mismatched: tuple[str, ...] = ()
|
||||
#: The recorded ``cmdline_sha256`` no longer matches the live one. This is
|
||||
#: *reported*, never acted on: see the module docstring.
|
||||
cmdline_drifted: bool = False
|
||||
|
||||
@property
|
||||
def unresolved(self) -> bool:
|
||||
"""True when the identity could be neither confirmed nor ruled out."""
|
||||
|
||||
return self.reason in UNRESOLVED
|
||||
|
||||
|
||||
def check(
|
||||
pid: int,
|
||||
*,
|
||||
starttime: int | None,
|
||||
sid: int | None = None,
|
||||
pgid: int | None = None,
|
||||
cmdline_sha256: str | None = None,
|
||||
proc: str = PROC,
|
||||
) -> Verdict:
|
||||
"""Decide whether ``pid`` is still the process whose identity was recorded.
|
||||
|
||||
The gate is ``starttime``, corroborated by ``sid`` and ``pgid`` when the
|
||||
recorded process was its own session/group leader — which every session the
|
||||
daemon spawns is. ``cmdline_sha256`` is accepted and compared, but a
|
||||
difference only sets :attr:`Verdict.cmdline_drifted`; it never refuses,
|
||||
because an ``exec``ing wrapper script changes it as a matter of routine
|
||||
(BUG-02). The reasoning for each field is in the module docstring.
|
||||
"""
|
||||
|
||||
stat = read_stat(pid, proc=proc)
|
||||
if stat is None or stat.state in DEAD_STATES:
|
||||
return Verdict(False, GONE)
|
||||
|
||||
drifted = False
|
||||
if cmdline_sha256:
|
||||
live_hash = cmdline_hash(pid, proc=proc)
|
||||
drifted = live_hash is not None and live_hash != cmdline_sha256
|
||||
|
||||
if not starttime or starttime <= 0:
|
||||
# No evidence at all. Refusing is the only safe answer, and the caller
|
||||
# is required to say so rather than skip whatever it was asked to do.
|
||||
return Verdict(False, IDENTITY_UNKNOWN, cmdline_drifted=drifted)
|
||||
|
||||
checked = ["starttime"]
|
||||
mismatched = [] if stat.starttime == starttime else ["starttime"]
|
||||
# `sid`/`pgid` are immutable only for a leader; see the module docstring.
|
||||
if sid is not None and sid == pid:
|
||||
checked.append("sid")
|
||||
if stat.sid != sid:
|
||||
mismatched.append("sid")
|
||||
if pgid is not None and pgid == pid:
|
||||
checked.append("pgid")
|
||||
if stat.pgid != pgid:
|
||||
mismatched.append("pgid")
|
||||
|
||||
if mismatched:
|
||||
return Verdict(
|
||||
False, PID_REUSE, tuple(checked), tuple(mismatched), cmdline_drifted=drifted
|
||||
)
|
||||
return Verdict(True, None, tuple(checked), (), cmdline_drifted=drifted)
|
||||
|
||||
|
||||
def verify(
|
||||
pid: int,
|
||||
*,
|
||||
starttime: int | None,
|
||||
sid: int | None = None,
|
||||
pgid: int | None = None,
|
||||
cmdline_sha256: str | None = None,
|
||||
proc: str = PROC,
|
||||
) -> tuple[bool, str | None]:
|
||||
""":func:`check` reduced to ``(ok, reason)`` for callers that want no detail."""
|
||||
|
||||
verdict = check(
|
||||
pid,
|
||||
starttime=starttime,
|
||||
sid=sid,
|
||||
pgid=pgid,
|
||||
cmdline_sha256=cmdline_sha256,
|
||||
proc=proc,
|
||||
)
|
||||
return verdict.ok, verdict.reason
|
||||
|
||||
|
||||
def iter_pids(*, proc: str = PROC) -> list[int]:
|
||||
try:
|
||||
entries = os.listdir(proc)
|
||||
except OSError:
|
||||
return []
|
||||
return sorted(int(e) for e in entries if e.isdigit())
|
||||
|
||||
|
||||
def session_members(sid: int, *, proc: str = PROC, exclude: set[int] | None = None) -> list[int]:
|
||||
"""Every live pid whose ``/proc/<pid>/stat`` session id equals ``sid``.
|
||||
|
||||
Refuses ``sid <= 1``: sweeping session 0 or 1 would select init and most of
|
||||
the machine.
|
||||
"""
|
||||
|
||||
if sid is None or sid <= 1:
|
||||
return []
|
||||
exclude = exclude or set()
|
||||
found = []
|
||||
for pid in iter_pids(proc=proc):
|
||||
if pid in exclude:
|
||||
continue
|
||||
stat = read_stat(pid, proc=proc)
|
||||
if stat is not None and stat.sid == sid and stat.state not in DEAD_STATES:
|
||||
found.append(pid)
|
||||
return found
|
||||
|
||||
|
||||
def kill_pid(pid: int, sig: int = signal.SIGKILL) -> bool:
|
||||
"""Signal one pid, tolerating the race where it has already exited."""
|
||||
|
||||
try:
|
||||
os.kill(pid, sig)
|
||||
return True
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return False
|
||||
|
||||
|
||||
def killpg(pgid: int, sig: int) -> bool:
|
||||
"""Signal a whole process group, refusing the dangerous group ids.
|
||||
|
||||
``pgid <= 1`` would mean "every process we may signal" or init's group;
|
||||
neither is ever a session's group, because every session is spawned with
|
||||
``start_new_session=True`` and so has ``pgid == pid``.
|
||||
"""
|
||||
|
||||
if pgid is None or pgid <= 1:
|
||||
return False
|
||||
try:
|
||||
os.killpg(pgid, sig)
|
||||
return True
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return False
|
||||
|
||||
|
||||
def sweep_session(
|
||||
sid: int, *, proc: str = PROC, exclude: set[int] | None = None, sig: int = signal.SIGKILL
|
||||
) -> list[int]:
|
||||
"""Kill every straggler sharing the session id. Returns the pids signalled."""
|
||||
|
||||
victims = session_members(sid, proc=proc, exclude=exclude)
|
||||
killed = [pid for pid in victims if kill_pid(pid, sig)]
|
||||
return killed
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Exit notification
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: ``os.pidfd_open`` needs Linux >= 5.3 and gives exit notification for
|
||||
#: processes that are *not* our children — which is exactly the adopted case.
|
||||
#: Where it is unavailable the daemon falls back to a 1 s ``/proc`` liveness
|
||||
#: poll, which costs one ``stat`` per adopted session per second and detects an
|
||||
#: exit up to a second late. Own children never need either path: asyncio's
|
||||
#: child watcher gives both notification and the exit status.
|
||||
HAVE_PIDFD = hasattr(os, "pidfd_open")
|
||||
|
||||
#: How often the fallback polls, in seconds.
|
||||
POLL_INTERVAL_S = 1.0
|
||||
|
||||
|
||||
def pidfd_open(pid: int) -> int | None:
|
||||
"""Open a pidfd, or ``None`` if unsupported or the process is already gone."""
|
||||
|
||||
if not HAVE_PIDFD:
|
||||
return None
|
||||
try:
|
||||
return os.pidfd_open(pid)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
return None
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Wire format and the exit-code table (DESIGN.md §1.6, §3.4).
|
||||
|
||||
Newline-delimited JSON over a unix ``SOCK_STREAM`` socket: one request object,
|
||||
one response object, connection closed. This module is shared by the daemon
|
||||
and by the CLI handlers, so the framing and the exit-code mapping exist exactly
|
||||
once. It imports nothing outside the standard library and nothing from the
|
||||
rest of ``hhstreamd``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
#: A single request or response line may not exceed this many bytes, including
|
||||
#: the terminating newline. A peer that sends more is answered with
|
||||
#: ``message_too_large`` and disconnected rather than being allowed to grow the
|
||||
#: daemon's read buffer without bound.
|
||||
MAX_LINE_BYTES = 1 << 20 # 1 MiB
|
||||
|
||||
|
||||
class ProtocolError(Exception):
|
||||
"""A frame could not be read or decoded.
|
||||
|
||||
``error`` is the machine-readable name reported to the peer.
|
||||
"""
|
||||
|
||||
def __init__(self, error: str, reason: str) -> None:
|
||||
super().__init__(reason)
|
||||
self.error = error
|
||||
self.reason = reason
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Exit codes (DESIGN.md §1.6). Frozen: agents branch on these.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ExitCode(enum.IntEnum):
|
||||
OK = 0
|
||||
USAGE = 2
|
||||
NO_SUCH_SESSION = 3
|
||||
BAD_STATE = 4
|
||||
DAEMON_UNREACHABLE = 5
|
||||
POLICY = 6
|
||||
TIMEOUT = 7
|
||||
BUSY = 8
|
||||
|
||||
|
||||
#: Machine-readable error names, grouped by the exit code they map to. Every
|
||||
#: ``{"ok": false, "error": ...}`` the daemon can emit appears here; anything
|
||||
#: unknown falls back to :data:`ExitCode.BAD_STATE` (see :func:`exit_code_for`).
|
||||
EXIT_CODE_FOR_ERROR: dict[str, ExitCode] = {
|
||||
# 2 — usage
|
||||
"bad_request": ExitCode.USAGE,
|
||||
"unknown_op": ExitCode.USAGE,
|
||||
"unsupported_version": ExitCode.USAGE,
|
||||
"message_too_large": ExitCode.USAGE,
|
||||
"invalid_argument": ExitCode.USAGE,
|
||||
# 3 — no such session
|
||||
"no_such_session": ExitCode.NO_SUCH_SESSION,
|
||||
"no_such_process": ExitCode.NO_SUCH_SESSION,
|
||||
# 4 — not in a usable state
|
||||
"session_exited": ExitCode.BAD_STATE,
|
||||
"session_killed": ExitCode.BAD_STATE,
|
||||
"session_reaped": ExitCode.BAD_STATE,
|
||||
"session_starting": ExitCode.BAD_STATE,
|
||||
"compositor_lost": ExitCode.BAD_STATE,
|
||||
"bad_state": ExitCode.BAD_STATE,
|
||||
"pid_reuse_detected": ExitCode.BAD_STATE,
|
||||
# The session's identity could not be established either way, so the
|
||||
# daemon refused rather than signalling blind (BUG-02 item 5).
|
||||
"identity_unknown": ExitCode.BAD_STATE,
|
||||
# 5 — daemon unreachable (produced by the client, never by the daemon)
|
||||
"daemon_unreachable": ExitCode.DAEMON_UNREACHABLE,
|
||||
# 6 — refused by policy
|
||||
"not_allowlisted": ExitCode.POLICY,
|
||||
"argument_refused": ExitCode.POLICY,
|
||||
"session_cap_reached": ExitCode.POLICY,
|
||||
"refused": ExitCode.POLICY,
|
||||
# 7 — timed out
|
||||
"timeout": ExitCode.TIMEOUT,
|
||||
"readiness_timeout": ExitCode.TIMEOUT,
|
||||
# 8 — busy
|
||||
"busy": ExitCode.BUSY,
|
||||
"already_managed": ExitCode.BUSY,
|
||||
}
|
||||
|
||||
|
||||
def exit_code_for(error: str | None) -> ExitCode:
|
||||
"""Map a machine-readable error name to its frozen exit code.
|
||||
|
||||
An unrecognised name maps to :data:`ExitCode.BAD_STATE`: a caller that does
|
||||
not know what went wrong should still be told the session is not usable,
|
||||
which is strictly safer than reporting success.
|
||||
"""
|
||||
|
||||
if not error:
|
||||
return ExitCode.OK
|
||||
return EXIT_CODE_FOR_ERROR.get(error, ExitCode.BAD_STATE)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Message construction
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def request(op: str, **fields: Any) -> dict[str, Any]:
|
||||
"""Build a request object with the protocol version attached."""
|
||||
|
||||
msg: dict[str, Any] = {"v": PROTOCOL_VERSION, "op": op}
|
||||
msg.update({k: v for k, v in fields.items() if v is not None})
|
||||
return msg
|
||||
|
||||
|
||||
def ok(**fields: Any) -> dict[str, Any]:
|
||||
return {"ok": True, **fields}
|
||||
|
||||
|
||||
def error(err: str, reason: str, *, hint: str | None = None, **fields: Any) -> dict[str, Any]:
|
||||
"""Build an error reply.
|
||||
|
||||
``error`` is the machine field agents branch on, ``reason`` the human one
|
||||
(DESIGN.md §1.6).
|
||||
"""
|
||||
|
||||
msg: dict[str, Any] = {"ok": False, "error": err, "reason": reason}
|
||||
if hint is not None:
|
||||
msg["hint"] = hint
|
||||
msg.update(fields)
|
||||
return msg
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Framing
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def encode(message: dict[str, Any]) -> bytes:
|
||||
"""Serialise one message to a single newline-terminated line.
|
||||
|
||||
``ensure_ascii`` keeps the line free of raw newlines inside strings by
|
||||
construction, so the framing cannot be broken by a session id, an argv
|
||||
element or an error message.
|
||||
"""
|
||||
|
||||
line = json.dumps(message, ensure_ascii=True, separators=(",", ":")).encode("ascii")
|
||||
if len(line) + 1 > MAX_LINE_BYTES:
|
||||
raise ProtocolError("message_too_large", f"message is {len(line)} bytes")
|
||||
return line + b"\n"
|
||||
|
||||
|
||||
def decode(line: bytes) -> dict[str, Any]:
|
||||
"""Parse one frame (with or without its trailing newline) into a dict."""
|
||||
|
||||
if len(line) > MAX_LINE_BYTES:
|
||||
raise ProtocolError("message_too_large", f"frame is {len(line)} bytes")
|
||||
text = line.rstrip(b"\n")
|
||||
if not text.strip():
|
||||
raise ProtocolError("bad_request", "empty frame")
|
||||
try:
|
||||
obj = json.loads(text)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ProtocolError("bad_request", f"malformed JSON: {exc}") from exc
|
||||
if not isinstance(obj, dict):
|
||||
raise ProtocolError("bad_request", f"expected a JSON object, got {type(obj).__name__}")
|
||||
return obj
|
||||
|
||||
|
||||
def decode_request(line: bytes) -> dict[str, Any]:
|
||||
"""Parse and validate a request frame: an object with a known ``v`` and ``op``."""
|
||||
|
||||
obj = decode(line)
|
||||
version = obj.get("v", PROTOCOL_VERSION)
|
||||
if version != PROTOCOL_VERSION:
|
||||
raise ProtocolError(
|
||||
"unsupported_version",
|
||||
f"protocol version {version!r} is not supported (daemon speaks {PROTOCOL_VERSION})",
|
||||
)
|
||||
op = obj.get("op")
|
||||
if not isinstance(op, str) or not op:
|
||||
raise ProtocolError("bad_request", "request has no 'op' field")
|
||||
return obj
|
||||
@@ -0,0 +1,187 @@
|
||||
"""The idle reaper (DESIGN.md §6).
|
||||
|
||||
Deadline-scheduled, not polled: the reaper computes
|
||||
``min(last_activity_boottime + idle_timeout_s)`` across live sessions and
|
||||
sleeps until that instant or until a state change wakes it. Zero wakeups while
|
||||
idle; no one-second tick burning a core on an otherwise quiet server.
|
||||
|
||||
**The clock is injected at construction** — ``now()`` returns a
|
||||
``CLOCK_BOOTTIME`` reading in production and whatever a test says otherwise.
|
||||
The selection arithmetic below is therefore pure and covered by unit tests that
|
||||
never sleep and never spawn anything.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Awaitable, Callable, Iterable, Protocol
|
||||
|
||||
log = logging.getLogger("hhstreamd.reaper")
|
||||
|
||||
|
||||
class Expirable(Protocol):
|
||||
"""What the reaper needs of a session. ``Session`` satisfies it."""
|
||||
|
||||
id: str
|
||||
|
||||
@property
|
||||
def live(self) -> bool: ...
|
||||
|
||||
def deadline(self) -> float | None: ...
|
||||
|
||||
|
||||
def deadlines(sessions: Iterable[Expirable]) -> dict[str, float]:
|
||||
"""Every live session's deadline, keyed by id.
|
||||
|
||||
Sessions in a terminal state and sessions with the idle timeout disabled
|
||||
(``idle_timeout_s == 0``) have no deadline and do not appear.
|
||||
"""
|
||||
|
||||
out: dict[str, float] = {}
|
||||
for session in sessions:
|
||||
deadline = session.deadline()
|
||||
if deadline is not None:
|
||||
out[session.id] = deadline
|
||||
return out
|
||||
|
||||
|
||||
def next_deadline(sessions: Iterable[Expirable]) -> float | None:
|
||||
"""The earliest deadline, or ``None`` when nothing can expire.
|
||||
|
||||
``None`` is the empty-set case and means "sleep until woken".
|
||||
"""
|
||||
|
||||
found = deadlines(sessions)
|
||||
return min(found.values()) if found else None
|
||||
|
||||
|
||||
def select_expired(sessions: Iterable[Expirable], now: float) -> list[str]:
|
||||
"""Ids whose deadline has arrived, oldest deadline first.
|
||||
|
||||
The comparison is ``deadline <= now`` so a session adopted after a daemon
|
||||
restart with an already-expired deadline is selected on the first pass
|
||||
rather than waiting another full timeout (DESIGN.md §2.4 item 6).
|
||||
"""
|
||||
|
||||
expired = [(deadline, sid) for sid, deadline in deadlines(sessions).items() if deadline <= now]
|
||||
expired.sort()
|
||||
return [sid for _, sid in expired]
|
||||
|
||||
|
||||
def sleep_for(sessions: Iterable[Expirable], now: float) -> float | None:
|
||||
"""How long to sleep before the next deadline.
|
||||
|
||||
``None`` means "indefinitely"; ``0.0`` means something is already expired.
|
||||
"""
|
||||
|
||||
deadline = next_deadline(sessions)
|
||||
if deadline is None:
|
||||
return None
|
||||
return max(0.0, deadline - now)
|
||||
|
||||
|
||||
class Reaper:
|
||||
"""The deadline-scheduled task.
|
||||
|
||||
``sessions`` is a callable returning the current live set — the registry is
|
||||
authoritative, and the reaper never holds its own copy of it.
|
||||
``on_expire(session_id)`` performs the teardown; it is exactly ``kill`` with
|
||||
a different reason, so there is one kill path, not two (DESIGN.md §6.3).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
now: Callable[[], float],
|
||||
sessions: Callable[[], Iterable[Expirable]],
|
||||
on_expire: Callable[[str], Awaitable[None]],
|
||||
*,
|
||||
max_sleep: float = 3600.0,
|
||||
retry_delay: float = 1.0,
|
||||
) -> None:
|
||||
self._now = now
|
||||
self._sessions = sessions
|
||||
self._on_expire = on_expire
|
||||
self._max_sleep = max_sleep
|
||||
self._retry_delay = retry_delay
|
||||
self._wake = asyncio.Event()
|
||||
self._task: asyncio.Task | None = None
|
||||
self.reaped: list[str] = []
|
||||
|
||||
# ---- pure helpers, exposed for tests and for `status` -------------
|
||||
|
||||
def next_deadline(self) -> float | None:
|
||||
return next_deadline(self._sessions())
|
||||
|
||||
def expired(self) -> list[str]:
|
||||
return select_expired(self._sessions(), self._now())
|
||||
|
||||
def sleep_for(self) -> float | None:
|
||||
return sleep_for(self._sessions(), self._now())
|
||||
|
||||
# ---- lifecycle ---------------------------------------------------
|
||||
|
||||
def wake(self) -> None:
|
||||
"""Re-evaluate the schedule now. Called on every state change."""
|
||||
|
||||
self._wake.set()
|
||||
|
||||
def start(self) -> asyncio.Task:
|
||||
if self._task is None:
|
||||
self._task = asyncio.create_task(self.run(), name="hhstreamd.reaper")
|
||||
return self._task
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._task = None
|
||||
|
||||
async def tick(self) -> list[str]:
|
||||
"""Reap everything currently expired. Returns the ids reaped."""
|
||||
|
||||
reaped: list[str] = []
|
||||
for session_id in self.expired():
|
||||
try:
|
||||
await self._on_expire(session_id)
|
||||
except Exception: # pragma: no cover - teardown must not kill the task
|
||||
log.exception("reaping %s failed", session_id)
|
||||
else:
|
||||
reaped.append(session_id)
|
||||
self.reaped.extend(reaped)
|
||||
return reaped
|
||||
|
||||
async def run(self) -> None:
|
||||
while True:
|
||||
# Cleared *before* the tick, so a state change that lands while we
|
||||
# are tearing a session down still forces a re-evaluation instead
|
||||
# of being swallowed.
|
||||
self._wake.clear()
|
||||
reaped = await self.tick()
|
||||
delay = self.sleep_for()
|
||||
# Nothing can expire: sleep until something wakes us. Zero
|
||||
# wakeups on a quiet server, which is the point of scheduling on a
|
||||
# deadline rather than ticking.
|
||||
#
|
||||
# With a deadline, the wait is additionally capped at `max_sleep`.
|
||||
# That is not belt-and-braces: deadlines are on CLOCK_BOOTTIME but
|
||||
# asyncio's timers run on CLOCK_MONOTONIC, which *stops* across
|
||||
# system suspend. A host suspended for four hours would otherwise
|
||||
# wake and keep waiting. Re-evaluating at most an hour of
|
||||
# monotonic time later bounds that error to an hour, per daemon,
|
||||
# not per session.
|
||||
timeout: float | None = None if delay is None else min(delay, self._max_sleep)
|
||||
if timeout == 0:
|
||||
if reaped:
|
||||
continue
|
||||
# Still expired but nothing was reaped: the teardown failed.
|
||||
# Back off rather than spinning on it.
|
||||
log.warning("session(s) past their deadline could not be reaped; retrying")
|
||||
timeout = self._retry_delay
|
||||
try:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=timeout)
|
||||
except TimeoutError:
|
||||
pass
|
||||
@@ -0,0 +1,394 @@
|
||||
"""The session record and its persistence (DESIGN.md §2.1, §2.3, §2.4).
|
||||
|
||||
Authoritative state is in daemon memory; the files under ``<state>/sessions/``
|
||||
are a mirror, written ``tmp`` + :func:`os.replace` so a reader never sees a
|
||||
half-written record. **A mirror write failure never fails a command** — it is
|
||||
logged, and memory stays truthful.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
from .clock import Clock, iso
|
||||
|
||||
log = logging.getLogger("hhstreamd.registry")
|
||||
|
||||
SCHEMA = 2
|
||||
|
||||
#: Tombstones survive this long so a ``status`` after an unexplained failure
|
||||
#: still answers the question. ``gc`` prunes them.
|
||||
DEFAULT_TOMBSTONE_TTL_S = 3600.0
|
||||
|
||||
#: ``last_activity`` is hot, so the mirror is flushed at most this often for a
|
||||
#: touch-only change — losing 15 s of an idle clock across a crash is
|
||||
#: irrelevant against a three-hour timeout. State transitions always flush.
|
||||
ACTIVITY_FLUSH_INTERVAL_S = 15.0
|
||||
|
||||
|
||||
class State:
|
||||
STARTING = "starting"
|
||||
READY = "ready"
|
||||
EXITED = "exited"
|
||||
KILLED = "killed"
|
||||
REAPED = "reaped"
|
||||
COMPOSITOR_LOST = "compositor_lost"
|
||||
|
||||
|
||||
#: States a session can never leave. Reaching one writes a tombstone.
|
||||
TERMINAL = frozenset({State.EXITED, State.KILLED, State.REAPED, State.COMPOSITOR_LOST})
|
||||
|
||||
#: The error name a verb should be answered with when it arrives at a session
|
||||
#: in a terminal state (all map to exit 4).
|
||||
TERMINAL_ERROR = {
|
||||
State.EXITED: "session_exited",
|
||||
State.KILLED: "session_killed",
|
||||
State.REAPED: "session_reaped",
|
||||
State.COMPOSITOR_LOST: "compositor_lost",
|
||||
}
|
||||
|
||||
|
||||
def boot_id() -> str:
|
||||
"""The kernel's boot id, or ``""`` if it cannot be read.
|
||||
|
||||
Sessions cannot outlive a reboot — the compositor does not — so a record
|
||||
whose ``boot_id`` differs from the running kernel's describes processes
|
||||
that are certainly gone, whatever ``/proc`` says about the pid today. This
|
||||
is cheaper and far safer than hoping ``CLOCK_BOOTTIME`` values from a
|
||||
previous boot happen not to collide.
|
||||
"""
|
||||
|
||||
try:
|
||||
return Path("/proc/sys/kernel/random/boot_id").read_text().strip()
|
||||
except OSError: # pragma: no cover - /proc always has it on Linux
|
||||
return ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Session:
|
||||
"""DESIGN.md §2.1, minus the fields M0 cannot honestly fill.
|
||||
|
||||
``window``, ``workspace`` and ``foreground`` exist in the record and are
|
||||
always ``None`` in M0: the compositor is not reachable from here, and the
|
||||
design is explicit that nothing may be guessed.
|
||||
"""
|
||||
|
||||
id: str
|
||||
app: str
|
||||
argv: list[str]
|
||||
pid: int
|
||||
pgid: int
|
||||
sid: int
|
||||
pid_starttime: int
|
||||
cmdline_sha256: str
|
||||
home: str
|
||||
runtime_dir: str
|
||||
created_at: str
|
||||
created_boottime: float
|
||||
last_activity_at: str
|
||||
last_activity_boottime: float
|
||||
idle_timeout_s: float
|
||||
state: str = State.STARTING
|
||||
adopted: bool = False
|
||||
ready_signal: str | None = None
|
||||
workspace: int | None = None
|
||||
window: Any = None
|
||||
foreground: bool | None = None
|
||||
command_count: int = 0
|
||||
exit_code: int | None = None
|
||||
exit_detected_by: str | None = None
|
||||
last_error: str | None = None
|
||||
log_path: str | None = None
|
||||
ephemeral_home: bool = False
|
||||
tombstoned_boottime: float | None = None
|
||||
tombstone_ttl_s: float = DEFAULT_TOMBSTONE_TTL_S
|
||||
boot_id: str = ""
|
||||
schema: int = SCHEMA
|
||||
|
||||
# ---- derived -----------------------------------------------------
|
||||
|
||||
@property
|
||||
def live(self) -> bool:
|
||||
return self.state not in TERMINAL
|
||||
|
||||
def deadline(self) -> float | None:
|
||||
"""The boottime instant at which this session becomes idle-expired.
|
||||
|
||||
``None`` for a session that can never expire: one in a terminal state,
|
||||
or one whose idle timeout is zero (disabled).
|
||||
"""
|
||||
|
||||
if not self.live or not self.idle_timeout_s:
|
||||
return None
|
||||
return self.last_activity_boottime + self.idle_timeout_s
|
||||
|
||||
def expires_in(self, now: float) -> float | None:
|
||||
deadline = self.deadline()
|
||||
return None if deadline is None else deadline - now
|
||||
|
||||
def idle_for(self, now: float) -> float | None:
|
||||
return None if not self.live else now - self.last_activity_boottime
|
||||
|
||||
def tombstone_expires(self) -> float | None:
|
||||
if self.tombstoned_boottime is None:
|
||||
return None
|
||||
return self.tombstoned_boottime + self.tombstone_ttl_s
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "Session":
|
||||
known = {f for f in cls.__dataclass_fields__}
|
||||
unknown = set(data) - known
|
||||
if unknown:
|
||||
# Forward compatibility: a newer daemon's extra fields are dropped
|
||||
# rather than crashing the adopt path.
|
||||
log.warning("dropping unknown record fields %s", sorted(unknown))
|
||||
return cls(**{k: v for k, v in data.items() if k in known})
|
||||
|
||||
def public(self, now: float) -> dict[str, Any]:
|
||||
"""The record as the protocol reports it, with the derived numbers."""
|
||||
|
||||
data = self.to_dict()
|
||||
data["idle_s"] = self.idle_for(now)
|
||||
data["expires_in_s"] = self.expires_in(now)
|
||||
data["live"] = self.live
|
||||
data["tombstone_expires_in_s"] = (
|
||||
None if self.tombstone_expires() is None else self.tombstone_expires() - now
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def mint_id(app: str, taken: set[str], *, rng: Any = secrets) -> str:
|
||||
"""``<app-slug>-<6 base32 chars>`` (DESIGN.md §2.2).
|
||||
|
||||
Handles are never reused: an id present in the live *or* tombstone set is
|
||||
refused and another drawn. Nothing parses the slug; it is a readability
|
||||
affordance.
|
||||
"""
|
||||
|
||||
slug = "".join(c if c.isalnum() else "-" for c in app.lower()).strip("-") or "app"
|
||||
slug = slug[:24]
|
||||
for _ in range(1000):
|
||||
raw = rng.token_bytes(5)
|
||||
suffix = base64.b32encode(raw).decode("ascii").rstrip("=").lower()[:6]
|
||||
candidate = f"{slug}-{suffix}"
|
||||
if candidate not in taken:
|
||||
return candidate
|
||||
raise RuntimeError("could not mint an unused session id") # pragma: no cover
|
||||
|
||||
|
||||
def atomic_write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
"""Write ``data`` to ``path`` atomically: temp file in the same directory,
|
||||
fsync, then :func:`os.replace`.
|
||||
|
||||
Same directory matters — ``os.replace`` is only atomic within a filesystem.
|
||||
"""
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.parent / f".{path.name}.{os.getpid()}.{secrets.token_hex(4)}.tmp"
|
||||
try:
|
||||
with open(tmp, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, ensure_ascii=True, sort_keys=True)
|
||||
fh.write("\n")
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(tmp, path)
|
||||
except BaseException:
|
||||
try:
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
class Registry:
|
||||
"""The in-memory session table and its on-disk mirror."""
|
||||
|
||||
def __init__(self, state_dir: Path | str, clock: Clock | None = None) -> None:
|
||||
self.state_dir = Path(state_dir)
|
||||
self.sessions_dir = self.state_dir / "sessions"
|
||||
self.quarantine_dir = self.sessions_dir / "bad"
|
||||
self.clock = clock or Clock()
|
||||
self._sessions: dict[str, Session] = {}
|
||||
self._last_flush: dict[str, float] = {}
|
||||
self._boot_id = boot_id()
|
||||
|
||||
# ---- directories -------------------------------------------------
|
||||
|
||||
def ensure_dirs(self) -> None:
|
||||
"""Create the state directories if the supervisor has not.
|
||||
|
||||
In production these come from the unit's ``StateDirectory=``; M0 ships
|
||||
as an ordinary process, so the daemon creates what is missing rather
|
||||
than depending on how it is supervised.
|
||||
"""
|
||||
|
||||
self.sessions_dir.mkdir(parents=True, exist_ok=True, mode=0o750)
|
||||
|
||||
# ---- table -------------------------------------------------------
|
||||
|
||||
def __contains__(self, session_id: object) -> bool:
|
||||
return session_id in self._sessions
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._sessions)
|
||||
|
||||
def __iter__(self) -> Iterator[Session]:
|
||||
return iter(list(self._sessions.values()))
|
||||
|
||||
def get(self, session_id: str) -> Session | None:
|
||||
return self._sessions.get(session_id)
|
||||
|
||||
def all(self) -> list[Session]:
|
||||
return sorted(self._sessions.values(), key=lambda s: s.created_boottime)
|
||||
|
||||
def live(self) -> list[Session]:
|
||||
return [s for s in self.all() if s.live]
|
||||
|
||||
def tombstones(self) -> list[Session]:
|
||||
return [s for s in self.all() if not s.live]
|
||||
|
||||
def taken_ids(self) -> set[str]:
|
||||
"""Live *and* tombstoned ids, so handles are never reused."""
|
||||
|
||||
return set(self._sessions)
|
||||
|
||||
def new_id(self, app: str) -> str:
|
||||
return mint_id(app, self.taken_ids())
|
||||
|
||||
def add(self, session: Session) -> Session:
|
||||
if session.id in self._sessions:
|
||||
raise KeyError(f"session id already in use: {session.id}")
|
||||
if not session.boot_id:
|
||||
session.boot_id = self._boot_id
|
||||
self._sessions[session.id] = session
|
||||
self.persist(session, force=True)
|
||||
return session
|
||||
|
||||
def record_path(self, session_id: str) -> Path:
|
||||
return self.sessions_dir / f"{session_id}.json"
|
||||
|
||||
def persist(self, session: Session, *, force: bool = False) -> None:
|
||||
"""Mirror one record. Never raises: a mirror failure is not a command failure."""
|
||||
|
||||
now = self.clock.boottime()
|
||||
if not force and now - self._last_flush.get(session.id, 0.0) < ACTIVITY_FLUSH_INTERVAL_S:
|
||||
return
|
||||
try:
|
||||
self.ensure_dirs()
|
||||
atomic_write_json(self.record_path(session.id), session.to_dict())
|
||||
self._last_flush[session.id] = now
|
||||
except OSError as exc:
|
||||
log.error("mirror write failed for %s: %s (memory is authoritative)", session.id, exc)
|
||||
|
||||
def flush_all(self) -> None:
|
||||
for session in self.all():
|
||||
self.persist(session, force=True)
|
||||
|
||||
def drop(self, session_id: str) -> None:
|
||||
"""Forget a record entirely and remove its mirror (used by ``gc``)."""
|
||||
|
||||
self._sessions.pop(session_id, None)
|
||||
self._last_flush.pop(session_id, None)
|
||||
try:
|
||||
self.record_path(session_id).unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError as exc:
|
||||
log.error("could not remove mirror for %s: %s", session_id, exc)
|
||||
|
||||
# ---- transitions -------------------------------------------------
|
||||
|
||||
def touch(self, session: Session, *, force_flush: bool = False) -> None:
|
||||
"""Stamp activity. Callers stamp *after* a command is accepted."""
|
||||
|
||||
session.last_activity_boottime = self.clock.boottime()
|
||||
session.last_activity_at = iso(self.clock.wall())
|
||||
session.command_count += 1
|
||||
self.persist(session, force=force_flush)
|
||||
|
||||
def terminate(
|
||||
self,
|
||||
session: Session,
|
||||
state: str,
|
||||
*,
|
||||
exit_code: int | None = None,
|
||||
exit_detected_by: str | None = None,
|
||||
last_error: str | None = None,
|
||||
) -> None:
|
||||
"""Move a session to a terminal state and write its tombstone."""
|
||||
|
||||
session.state = state
|
||||
if exit_code is not None:
|
||||
session.exit_code = exit_code
|
||||
if exit_detected_by is not None:
|
||||
session.exit_detected_by = exit_detected_by
|
||||
if last_error is not None:
|
||||
session.last_error = last_error
|
||||
session.tombstoned_boottime = self.clock.boottime()
|
||||
self.persist(session, force=True)
|
||||
|
||||
# ---- startup adoption -------------------------------------------
|
||||
|
||||
def load_records(self) -> tuple[list[Session], list[tuple[Path, str]]]:
|
||||
"""Read the mirror. Returns ``(records, quarantined)``.
|
||||
|
||||
A record that will not parse is moved to ``sessions/bad/`` and logged
|
||||
loudly; its processes are left running and reported by ``gc`` rather
|
||||
than killed on the basis of unreadable state (DESIGN.md §7 #13).
|
||||
"""
|
||||
|
||||
records: list[Session] = []
|
||||
quarantined: list[tuple[Path, str]] = []
|
||||
if not self.sessions_dir.is_dir():
|
||||
return records, quarantined
|
||||
for path in sorted(self.sessions_dir.glob("*.json")):
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("record is not an object")
|
||||
session = Session.from_dict(data)
|
||||
except (OSError, ValueError, TypeError) as exc:
|
||||
reason = f"{type(exc).__name__}: {exc}"
|
||||
log.error("quarantining unreadable record %s: %s", path, reason)
|
||||
quarantined.append((path, reason))
|
||||
self._quarantine(path)
|
||||
continue
|
||||
records.append(session)
|
||||
return records, quarantined
|
||||
|
||||
def _quarantine(self, path: Path) -> None:
|
||||
try:
|
||||
self.quarantine_dir.mkdir(parents=True, exist_ok=True, mode=0o750)
|
||||
os.replace(path, self.quarantine_dir / f"{path.name}.{int(time.time())}")
|
||||
except OSError as exc: # pragma: no cover - best effort
|
||||
log.error("could not quarantine %s: %s", path, exc)
|
||||
|
||||
def insert_loaded(self, session: Session) -> Session:
|
||||
"""Put a record read from disk back into the table without re-persisting."""
|
||||
|
||||
self._sessions[session.id] = session
|
||||
return session
|
||||
|
||||
def stale_boot(self, session: Session) -> bool:
|
||||
"""True if the record was written before the current boot.
|
||||
|
||||
Its processes are gone whatever ``/proc`` says, because pids are
|
||||
recycled from zero at boot.
|
||||
"""
|
||||
|
||||
return bool(session.boot_id) and bool(self._boot_id) and session.boot_id != self._boot_id
|
||||
|
||||
|
||||
def clone(session: Session, **changes: Any) -> Session:
|
||||
return replace(session, **changes)
|
||||
@@ -0,0 +1,335 @@
|
||||
"""The control socket (DESIGN.md §3.4).
|
||||
|
||||
An asyncio unix ``SOCK_STREAM`` server speaking newline-delimited JSON: one
|
||||
request object, one response object, connection closed. Chosen over HTTP
|
||||
(framing and a server for no benefit on a local socket), D-Bus (no stdlib
|
||||
client) and a bespoke binary protocol (unreadable at 3am). It is debuggable
|
||||
with ``socat - UNIX-CONNECT:/run/hhstream/control.sock`` and costs nothing.
|
||||
|
||||
This module is framing and dispatch only. Every decision lives in
|
||||
:mod:`hhstreamd.manager`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import collections
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
from . import protocol
|
||||
from .manager import ManagerError, SessionManager
|
||||
from .policy import PolicyError
|
||||
|
||||
log = logging.getLogger("hhstreamd.server")
|
||||
|
||||
#: A request that has not produced a reply in this long is abandoned and the
|
||||
#: caller told so, rather than holding the connection open indefinitely.
|
||||
DEFAULT_REQUEST_TIMEOUT_S = 5.0
|
||||
|
||||
#: A `kill` may legitimately take `grace` seconds, so it gets its own budget.
|
||||
KILL_TIMEOUT_MARGIN_S = 5.0
|
||||
|
||||
#: Replies to mutating verbs are remembered so a CLI that died before reading
|
||||
#: the reply can retry without launching a second application.
|
||||
DEDUP_ENTRIES = 64
|
||||
|
||||
#: Verbs whose replies are cached by `request_id`. The read verbs are absent
|
||||
#: deliberately: they are idempotent, so a retry costs nothing and a cached
|
||||
#: `list` would be a stale one.
|
||||
MUTATING_VERBS = frozenset({"start", "touch", "kill", "adopt", "gc"})
|
||||
|
||||
|
||||
class ControlServer:
|
||||
def __init__(
|
||||
self,
|
||||
manager: SessionManager,
|
||||
socket_path: Path | str,
|
||||
*,
|
||||
request_timeout_s: float = DEFAULT_REQUEST_TIMEOUT_S,
|
||||
socket_mode: int = 0o660,
|
||||
) -> None:
|
||||
self.manager = manager
|
||||
self.socket_path = Path(socket_path)
|
||||
self.request_timeout_s = request_timeout_s
|
||||
self.socket_mode = socket_mode
|
||||
self._server: asyncio.AbstractServer | None = None
|
||||
self._dedup: collections.OrderedDict[str, dict[str, Any]] = collections.OrderedDict()
|
||||
self._ops: dict[str, Callable[[dict[str, Any], dict[str, Any]], Awaitable[dict[str, Any]]]] = {
|
||||
"ping": self._op_ping,
|
||||
"start": self._op_start,
|
||||
"list": self._op_list,
|
||||
"status": self._op_status,
|
||||
"touch": self._op_touch,
|
||||
"kill": self._op_kill,
|
||||
"gc": self._op_gc,
|
||||
"adopt": self._op_adopt,
|
||||
"logs": self._op_logs,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def start(self) -> None:
|
||||
self._prepare_socket_path()
|
||||
self._server = await asyncio.start_unix_server(
|
||||
self._handle,
|
||||
path=str(self.socket_path),
|
||||
# The stream limit is the frame limit: a peer cannot grow the
|
||||
# daemon's read buffer past one message.
|
||||
limit=protocol.MAX_LINE_BYTES,
|
||||
)
|
||||
with contextlib.suppress(OSError):
|
||||
os.chmod(self.socket_path, self.socket_mode)
|
||||
log.info("listening on %s", self.socket_path)
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._server is not None:
|
||||
self._server.close()
|
||||
with contextlib.suppress(Exception):
|
||||
await self._server.wait_closed()
|
||||
self._server = None
|
||||
with contextlib.suppress(FileNotFoundError, OSError):
|
||||
self.socket_path.unlink()
|
||||
|
||||
def _prepare_socket_path(self) -> None:
|
||||
"""Create the parent directory and clear a stale socket.
|
||||
|
||||
A socket file left by a crashed daemon is unlinked only after proving
|
||||
nobody is listening on it: connecting and being refused is the proof.
|
||||
Unlinking a *live* socket would silently steal another daemon's clients.
|
||||
"""
|
||||
|
||||
self.socket_path.parent.mkdir(parents=True, exist_ok=True, mode=0o750)
|
||||
if not self.socket_path.exists():
|
||||
return
|
||||
probe = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
try:
|
||||
probe.settimeout(0.5)
|
||||
probe.connect(str(self.socket_path))
|
||||
except (ConnectionRefusedError, FileNotFoundError):
|
||||
log.warning("removing stale socket %s", self.socket_path)
|
||||
with contextlib.suppress(OSError):
|
||||
self.socket_path.unlink()
|
||||
except OSError as exc:
|
||||
raise OSError(
|
||||
f"{self.socket_path} exists and could not be probed: {exc}"
|
||||
) from exc
|
||||
else:
|
||||
raise OSError(f"another daemon is already listening on {self.socket_path}")
|
||||
finally:
|
||||
probe.close()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# connection handling
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
||||
caller = _peer_credentials(writer)
|
||||
try:
|
||||
try:
|
||||
line = await reader.readuntil(b"\n")
|
||||
except asyncio.IncompleteReadError as exc:
|
||||
if not exc.partial:
|
||||
return # a peer that connected and went away; not an error
|
||||
line = exc.partial
|
||||
except asyncio.LimitOverrunError:
|
||||
await self._send(
|
||||
writer,
|
||||
protocol.error("message_too_large", "request exceeds the frame limit"),
|
||||
)
|
||||
return
|
||||
if len(line) > protocol.MAX_LINE_BYTES:
|
||||
await self._send(
|
||||
writer,
|
||||
protocol.error("message_too_large", "request exceeds the frame limit"),
|
||||
)
|
||||
return
|
||||
response = await self._dispatch(line, caller)
|
||||
await self._send(writer, response)
|
||||
except (ConnectionResetError, BrokenPipeError):
|
||||
pass
|
||||
except Exception:
|
||||
log.exception("unhandled error serving a request")
|
||||
with contextlib.suppress(Exception):
|
||||
await self._send(
|
||||
writer, protocol.error("refused", "internal error; see the daemon log")
|
||||
)
|
||||
finally:
|
||||
with contextlib.suppress(Exception):
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
|
||||
async def _send(self, writer: asyncio.StreamWriter, message: dict[str, Any]) -> None:
|
||||
writer.write(protocol.encode(message))
|
||||
await writer.drain()
|
||||
|
||||
async def _dispatch(self, line: bytes, caller: dict[str, Any]) -> dict[str, Any]:
|
||||
started = self.manager.clock.boottime()
|
||||
try:
|
||||
request = protocol.decode_request(line)
|
||||
except protocol.ProtocolError as exc:
|
||||
self.manager.audit.emit(
|
||||
"request_rejected", error=exc.error, reason=exc.reason, caller=caller
|
||||
)
|
||||
return protocol.error(exc.error, exc.reason)
|
||||
|
||||
op = request["op"]
|
||||
request_id = request.get("request_id")
|
||||
handler = self._ops.get(op)
|
||||
if handler is None:
|
||||
return protocol.error(
|
||||
"unknown_op", f"unknown op {op!r}", hint=f"known: {', '.join(sorted(self._ops))}"
|
||||
)
|
||||
|
||||
if isinstance(request_id, str) and op in MUTATING_VERBS and request_id in self._dedup:
|
||||
cached = dict(self._dedup[request_id])
|
||||
cached["deduplicated"] = True
|
||||
return cached
|
||||
|
||||
timeout = self.request_timeout_s
|
||||
if op == "kill":
|
||||
# A kill is allowed to take its grace period plus the SIGKILL step.
|
||||
grace = request.get("grace_s")
|
||||
grace = self.manager.config.policy.grace_s if grace is None else float(grace or 0)
|
||||
timeout = max(timeout, grace + KILL_TIMEOUT_MARGIN_S)
|
||||
elif op == "start":
|
||||
wait_ready = request.get("wait_ready_s")
|
||||
wait_ready = (
|
||||
self.manager.config.policy.wait_ready_s if wait_ready is None else float(wait_ready or 0)
|
||||
)
|
||||
timeout = max(timeout, wait_ready + 1.0)
|
||||
|
||||
try:
|
||||
response = await asyncio.wait_for(handler(request, caller), timeout=timeout)
|
||||
except TimeoutError:
|
||||
response = protocol.error("timeout", f"{op} did not complete within {timeout:g}s")
|
||||
except ManagerError as exc:
|
||||
response = protocol.error(exc.error, exc.reason, hint=exc.hint, **exc.fields)
|
||||
except PolicyError as exc:
|
||||
response = protocol.error(exc.error, exc.reason, hint=exc.hint)
|
||||
except (TypeError, ValueError) as exc:
|
||||
response = protocol.error("invalid_argument", str(exc))
|
||||
except Exception as exc:
|
||||
log.exception("op %s failed", op)
|
||||
response = protocol.error("refused", f"internal error: {type(exc).__name__}")
|
||||
|
||||
elapsed_ms = round((self.manager.clock.boottime() - started) * 1000, 3)
|
||||
response.setdefault("elapsed_ms", elapsed_ms)
|
||||
if isinstance(request_id, str):
|
||||
response["request_id"] = request_id
|
||||
if op in MUTATING_VERBS:
|
||||
self._remember(request_id, response)
|
||||
|
||||
self.manager.audit.emit(
|
||||
"request",
|
||||
verb=op,
|
||||
session=request.get("id") or response.get("id"),
|
||||
caller=caller,
|
||||
ok=bool(response.get("ok")),
|
||||
error=response.get("error"),
|
||||
duration_ms=elapsed_ms,
|
||||
request_id=request_id if isinstance(request_id, str) else None,
|
||||
)
|
||||
return response
|
||||
|
||||
def _remember(self, request_id: str, response: dict[str, Any]) -> None:
|
||||
self._dedup[request_id] = dict(response)
|
||||
self._dedup.move_to_end(request_id)
|
||||
while len(self._dedup) > DEDUP_ENTRIES:
|
||||
self._dedup.popitem(last=False)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# ops
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _op_ping(self, request: dict[str, Any], caller: dict[str, Any]) -> dict[str, Any]:
|
||||
from . import __version__
|
||||
|
||||
return protocol.ok(
|
||||
version=__version__,
|
||||
pid=os.getpid(),
|
||||
boottime=self.manager.now(),
|
||||
live=len(self.manager.registry.live()),
|
||||
)
|
||||
|
||||
async def _op_start(self, request: dict[str, Any], caller: dict[str, Any]) -> dict[str, Any]:
|
||||
record = await self.manager.start(
|
||||
request.get("app"),
|
||||
request.get("argv"),
|
||||
idle_timeout_s=request.get("idle_timeout_s"),
|
||||
wait_ready_s=request.get("wait_ready_s"),
|
||||
home=request.get("home"),
|
||||
ephemeral_home=bool(request.get("ephemeral_home", False)),
|
||||
)
|
||||
return protocol.ok(id=record["id"], state=record["state"], session=record)
|
||||
|
||||
async def _op_list(self, request: dict[str, Any], caller: dict[str, Any]) -> dict[str, Any]:
|
||||
return protocol.ok(**self.manager.list(include_tombstones=bool(request.get("all", False))))
|
||||
|
||||
async def _op_status(self, request: dict[str, Any], caller: dict[str, Any]) -> dict[str, Any]:
|
||||
record = self.manager.status(request.get("id"), events=int(request.get("events", 20)))
|
||||
return protocol.ok(id=record["id"], state=record["state"], session=record)
|
||||
|
||||
async def _op_touch(self, request: dict[str, Any], caller: dict[str, Any]) -> dict[str, Any]:
|
||||
record = self.manager.touch(request.get("id"))
|
||||
return protocol.ok(id=record["id"], state=record["state"], session=record)
|
||||
|
||||
async def _op_kill(self, request: dict[str, Any], caller: dict[str, Any]) -> dict[str, Any]:
|
||||
record = await self.manager.kill(
|
||||
request.get("id"),
|
||||
grace_s=request.get("grace_s"),
|
||||
sig=request.get("signal", "TERM"),
|
||||
)
|
||||
return protocol.ok(id=record["id"], state=record["state"], session=record)
|
||||
|
||||
async def _op_gc(self, request: dict[str, Any], caller: dict[str, Any]) -> dict[str, Any]:
|
||||
return protocol.ok(**self.manager.gc())
|
||||
|
||||
async def _op_adopt(self, request: dict[str, Any], caller: dict[str, Any]) -> dict[str, Any]:
|
||||
record = await self.manager.adopt(
|
||||
request.get("pid"),
|
||||
app=request.get("app"),
|
||||
idle_timeout_s=request.get("idle_timeout_s"),
|
||||
)
|
||||
return protocol.ok(id=record["id"], state=record["state"], session=record)
|
||||
|
||||
async def _op_logs(self, request: dict[str, Any], caller: dict[str, Any]) -> dict[str, Any]:
|
||||
return protocol.ok(
|
||||
**self.manager.logs(
|
||||
request.get("id"),
|
||||
lines=int(request.get("lines", 100)),
|
||||
offset=request.get("offset"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _peer_credentials(writer: asyncio.StreamWriter) -> dict[str, Any]:
|
||||
"""``SO_PEERCRED`` of the connecting process, for the audit log.
|
||||
|
||||
M0 records who asked but does **not** authorise on it: the uid/gid
|
||||
allowlist is §8 item 2, which the brief holds out of this milestone. The
|
||||
socket's own mode and its directory are the access control here.
|
||||
"""
|
||||
|
||||
sock = writer.get_extra_info("socket")
|
||||
if sock is None:
|
||||
return {}
|
||||
try:
|
||||
raw = sock.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))
|
||||
pid, uid, gid = struct.unpack("3i", raw)
|
||||
except OSError:
|
||||
return {}
|
||||
caller: dict[str, Any] = {"pid": pid, "uid": uid, "gid": gid}
|
||||
try:
|
||||
with open(f"/proc/{pid}/comm", encoding="utf-8") as fh:
|
||||
caller["comm"] = fh.read().strip()
|
||||
except OSError:
|
||||
pass
|
||||
return caller
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Reading the tail of a file that may be arbitrarily large.
|
||||
|
||||
Both the audit log and an application's captured output are append-only files
|
||||
with no upper bound in M0, and both are read by verbs that must stay cheap.
|
||||
Reading either whole — ``readlines()`` on a gigabyte of Chromium stderr — would
|
||||
turn a ``status`` into an out-of-memory event on a machine that already has no
|
||||
resource caps (DESIGN.md §3.3). So every tail read is bounded in bytes, and
|
||||
says when it truncated.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
#: The most any single tail read will pull into memory.
|
||||
DEFAULT_TAIL_BYTES = 1 << 20 # 1 MiB
|
||||
|
||||
|
||||
def tail_lines(
|
||||
path: Path | str, limit: int, *, max_bytes: int = DEFAULT_TAIL_BYTES
|
||||
) -> tuple[list[str], bool, int]:
|
||||
"""The last ``limit`` lines of ``path``.
|
||||
|
||||
Returns ``(lines, truncated, size)``. ``truncated`` is true when the window
|
||||
did not reach the start of the file, so a caller can say so rather than
|
||||
implying it showed everything.
|
||||
"""
|
||||
|
||||
if limit <= 0:
|
||||
try:
|
||||
return [], False, os.stat(path).st_size
|
||||
except OSError:
|
||||
return [], False, 0
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
size = fh.seek(0, os.SEEK_END)
|
||||
window = min(size, max_bytes)
|
||||
fh.seek(size - window)
|
||||
data = fh.read(window)
|
||||
except OSError:
|
||||
return [], False, 0
|
||||
partial = window < size
|
||||
if partial:
|
||||
# The window almost certainly starts mid-line; drop that fragment
|
||||
# rather than reporting a half line as if it were whole.
|
||||
newline = data.find(b"\n")
|
||||
data = data[newline + 1 :] if newline != -1 else b""
|
||||
lines = data.decode("utf-8", "replace").splitlines()
|
||||
truncated = partial or len(lines) > limit
|
||||
return lines[-limit:], truncated, size
|
||||
Reference in new issue
Block a user