diff --git a/src/hhstream_live/publisher.py b/src/hhstream_live/publisher.py index 081c2e2..ed56809 100644 --- a/src/hhstream_live/publisher.py +++ b/src/hhstream_live/publisher.py @@ -31,9 +31,12 @@ log = logging.getLogger("hhstream.live") #: the box is also running the compositor and the main publisher. DEFAULT_FPS = 5 FPS_CAP = 10 -#: After this many capture failures with no successful frame, a stream gives up -#: and closes its viewers. `grim` against a removed output ("unknown output") -#: fails every time, and retrying forever would be a busy loop on a dead session. +#: After this many *consecutive* capture failures a stream gives up and closes +#: its viewers. `grim` against a removed output ("unknown output") fails every +#: time, and retrying forever would be a busy loop on a dead session -- which is +#: as true after a thousand good frames as it is from a cold start, so the count +#: is of failures in a row, not of failures-before-the-first-frame. A successful +#: grab resets it; giving up does not reset, because a dead output stays dead. FAILURE_LIMIT = 10 @@ -95,8 +98,11 @@ class Stream: # ---- lifecycle --------------------------------------------------- def start(self) -> None: + # Named for the output, never the phrase: faulthandler, the default + # `threading.excepthook` and any profiler that samples the process all + # print thread names, and the phrase is this stream's only credential. self._thread = threading.Thread( - target=self._run, name=f"hhstream-live-{self.spec.phrase}", daemon=True + target=self._run, name=f"hhstream-live-{self.spec.output}", daemon=True ) self._thread.start() @@ -129,15 +135,22 @@ class Stream: with self._cond: self._failures += 1 self._consecutive_failures += 1 - give_up = self._consecutive_failures >= FAILURE_LIMIT and self._captured == 0 + give_up = self._consecutive_failures >= FAILURE_LIMIT if give_up: + # Sticky on purpose: a stream that reached the cap is + # not re-armed by `add_viewer`, because the case this + # guards -- an output that was removed -- does not come + # back. Re-enabling the session is how it comes back. self._stopped = True self._cond.notify_all() if give_up: - log.warning("stream %s: giving up after %d capture failures: %s", - self.spec.phrase, FAILURE_LIMIT, exc) + # By output, not by phrase: a log line is the easiest place + # for a credential to escape to. + log.warning("stream on %s: giving up after %d consecutive " + "capture failures: %s", + self.spec.output, FAILURE_LIMIT, exc) return - log.debug("stream %s: capture failed: %s", self.spec.phrase, exc) + log.debug("stream on %s: capture failed: %s", self.spec.output, exc) if self._sleep(0.5): return continue diff --git a/tests/live/live_helpers.py b/tests/live/live_helpers.py index 6cb45cf..2992690 100644 --- a/tests/live/live_helpers.py +++ b/tests/live/live_helpers.py @@ -25,11 +25,19 @@ PHRASE = "amber-otter-lantern-harbor-7k2qp" class StubCapture: - """A `Capture` under the test's control: counts grabs, can be told to fail.""" + """A `Capture` under the test's control: counts grabs, can be told to fail. - def __init__(self, frames=None, fail_first: int = 0, delay: float = 0.0): + `fail_first` is a capture that is broken from the start (a cold start that + never works); `fail_after` is one that works and then stops -- the output + was removed while someone was watching, which is grim's "unknown output" on + every grab from then on. + """ + + def __init__(self, frames=None, fail_first: int = 0, delay: float = 0.0, + fail_after: int | None = None): self.frames = list(frames) if frames else [jpeg_bytes(i) for i in range(1, 50)] self.fail_first = fail_first + self.fail_after = fail_after self.delay = delay self.calls = 0 self._lock = threading.Lock() @@ -42,6 +50,8 @@ class StubCapture: time.sleep(self.delay) if call <= self.fail_first: raise RuntimeError(f"stub capture failing on purpose (call {call})") + if self.fail_after is not None and call > self.fail_after: + raise RuntimeError(f"stub capture: unknown output (call {call})") return self.frames[(call - 1 - self.fail_first) % len(self.frames)] diff --git a/tests/live/test_live_publisher.py b/tests/live/test_live_publisher.py index 24150d5..9807d9b 100644 --- a/tests/live/test_live_publisher.py +++ b/tests/live/test_live_publisher.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import time import pytest @@ -94,6 +95,64 @@ def test_a_dead_capture_gives_up_and_closes_viewers(): stream.stop() +def test_a_capture_that_dies_after_a_good_frame_also_gives_up(): + """The cap counts failures *in a row*, not failures before the first frame. + + An output removed while someone is watching fails every grab from then on. + A give-up rule that only applied to a cold start would leave this stream + retrying twice a second for as long as the viewer held the page open -- + exactly the busy loop on a dead session the cap exists to stop. + """ + capture = StubCapture(fail_after=1) # one good frame, then the output is gone + stream = make_stream(capture, fps=10) + stream.start() + try: + stream.add_viewer() # and the viewer stays attached for the whole test + frame, seq = stream.wait_frame(0, timeout=5) + assert frame is not None and seq >= 1 + + # Poll through the viewer's own wait, so a `wait_frame` that hangs is a + # failure here rather than a test that quietly waits for the deadline. + deadline = time.monotonic() + 15 + while not stream.stopped and time.monotonic() < deadline: + assert stream.wait_frame(seq, timeout=1.0) == (None, seq) + assert stream.stopped, "a dead output must stop the stream, frame or no frame" + + # And the viewer in the loop is released at once rather than on timeout. + started = time.monotonic() + assert stream.wait_frame(seq, timeout=10.0) == (None, seq) + assert time.monotonic() - started < 1.0 + assert capture.calls >= FAILURE_LIMIT + finally: + stream.stop() + + +def test_giving_up_never_writes_the_phrase_to_the_log(caplog): + """A log line is the easiest place for the stream's one credential to leak. + + faulthandler and `threading.excepthook` print thread names, so the capture + thread is checked here too. + """ + capture = StubCapture(fail_first=10_000) + stream = make_stream(capture, fps=10) + with caplog.at_level(logging.DEBUG, logger="hhstream.live"): + stream.start() + thread_name = stream._thread.name + try: + stream.add_viewer() + assert stream.wait_frame(0, timeout=15) == (None, 0) + assert stream.stopped + finally: + stream.stop() + + messages = [record.getMessage() for record in caplog.records] + assert messages, "the give-up must be logged at all" + assert any("hh-demo" in message for message in messages) + for message in messages: + assert PHRASE not in message, message + assert PHRASE not in thread_name and "hh-demo" in thread_name + + # ---- registry -------------------------------------------------------------- def test_registry_adds_and_finds_a_phrase_case_insensitively():