fix(live): a dead output stops the stream even after a good frame

The failure cap read `consecutive_failures >= FAILURE_LIMIT and captured ==
0`, so it could only ever fire on a cold start. An output removed while
someone was watching -- grim answering "unknown output" to every grab from
then on -- kept the loop retrying twice a second for as long as the viewer
held the page open. That is the busy loop the cap exists to stop, so the
count is now of failures in a row and nothing else. Giving up stays sticky:
a removed output does not come back, and re-enabling the session is how it
returns.

While here, the capture thread is named for its output and the give-up is
logged by output. The phrase is the stream's only credential, and
faulthandler, `threading.excepthook` and any sampling profiler all print
thread names.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01URhyHGQnmKH9zBXxoznfQY
This commit is contained in:
HermenixandClaude Opus 5 committed 2026-10-06 16:19:32 +00:00
1 parent 9744d35945
commit 52e3786bfb
3 files changed
+92 -10

No files matched your search

+21 -8
View File
@@ -31,9 +31,12 @@ log = logging.getLogger("hhstream.live")
#: the box is also running the compositor and the main publisher.
DEFAULT_FPS = 5
FPS_CAP = 10
#: After this many capture failures with no successful frame, a stream gives up
#: and closes its viewers. `grim` against a removed output ("unknown output")
#: fails every time, and retrying forever would be a busy loop on a dead session.
#: After this many *consecutive* capture failures a stream gives up and closes
#: its viewers. `grim` against a removed output ("unknown output") fails every
#: time, and retrying forever would be a busy loop on a dead session -- which is
#: as true after a thousand good frames as it is from a cold start, so the count
#: is of failures in a row, not of failures-before-the-first-frame. A successful
#: grab resets it; giving up does not reset, because a dead output stays dead.
FAILURE_LIMIT = 10
@@ -95,8 +98,11 @@ class Stream:
# ---- lifecycle ---------------------------------------------------
def start(self) -> None:
# Named for the output, never the phrase: faulthandler, the default
# `threading.excepthook` and any profiler that samples the process all
# print thread names, and the phrase is this stream's only credential.
self._thread = threading.Thread(
target=self._run, name=f"hhstream-live-{self.spec.phrase}", daemon=True
target=self._run, name=f"hhstream-live-{self.spec.output}", daemon=True
)
self._thread.start()
@@ -129,15 +135,22 @@ class Stream:
with self._cond:
self._failures += 1
self._consecutive_failures += 1
give_up = self._consecutive_failures >= FAILURE_LIMIT and self._captured == 0
give_up = self._consecutive_failures >= FAILURE_LIMIT
if give_up:
# Sticky on purpose: a stream that reached the cap is
# not re-armed by `add_viewer`, because the case this
# guards -- an output that was removed -- does not come
# back. Re-enabling the session is how it comes back.
self._stopped = True
self._cond.notify_all()
if give_up:
log.warning("stream %s: giving up after %d capture failures: %s",
self.spec.phrase, FAILURE_LIMIT, exc)
# By output, not by phrase: a log line is the easiest place
# for a credential to escape to.
log.warning("stream on %s: giving up after %d consecutive "
"capture failures: %s",
self.spec.output, FAILURE_LIMIT, exc)
return
log.debug("stream %s: capture failed: %s", self.spec.phrase, exc)
log.debug("stream on %s: capture failed: %s", self.spec.output, exc)
if self._sleep(0.5):
return
continue
+12 -2
View File
@@ -25,11 +25,19 @@ PHRASE = "amber-otter-lantern-harbor-7k2qp"
class StubCapture:
"""A `Capture` under the test's control: counts grabs, can be told to fail."""
"""A `Capture` under the test's control: counts grabs, can be told to fail.
def __init__(self, frames=None, fail_first: int = 0, delay: float = 0.0):
`fail_first` is a capture that is broken from the start (a cold start that
never works); `fail_after` is one that works and then stops -- the output
was removed while someone was watching, which is grim's "unknown output" on
every grab from then on.
"""
def __init__(self, frames=None, fail_first: int = 0, delay: float = 0.0,
fail_after: int | None = None):
self.frames = list(frames) if frames else [jpeg_bytes(i) for i in range(1, 50)]
self.fail_first = fail_first
self.fail_after = fail_after
self.delay = delay
self.calls = 0
self._lock = threading.Lock()
@@ -42,6 +50,8 @@ class StubCapture:
time.sleep(self.delay)
if call <= self.fail_first:
raise RuntimeError(f"stub capture failing on purpose (call {call})")
if self.fail_after is not None and call > self.fail_after:
raise RuntimeError(f"stub capture: unknown output (call {call})")
return self.frames[(call - 1 - self.fail_first) % len(self.frames)]
+59
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import logging
import time
import pytest
@@ -94,6 +95,64 @@ def test_a_dead_capture_gives_up_and_closes_viewers():
stream.stop()
def test_a_capture_that_dies_after_a_good_frame_also_gives_up():
"""The cap counts failures *in a row*, not failures before the first frame.
An output removed while someone is watching fails every grab from then on.
A give-up rule that only applied to a cold start would leave this stream
retrying twice a second for as long as the viewer held the page open --
exactly the busy loop on a dead session the cap exists to stop.
"""
capture = StubCapture(fail_after=1) # one good frame, then the output is gone
stream = make_stream(capture, fps=10)
stream.start()
try:
stream.add_viewer() # and the viewer stays attached for the whole test
frame, seq = stream.wait_frame(0, timeout=5)
assert frame is not None and seq >= 1
# Poll through the viewer's own wait, so a `wait_frame` that hangs is a
# failure here rather than a test that quietly waits for the deadline.
deadline = time.monotonic() + 15
while not stream.stopped and time.monotonic() < deadline:
assert stream.wait_frame(seq, timeout=1.0) == (None, seq)
assert stream.stopped, "a dead output must stop the stream, frame or no frame"
# And the viewer in the loop is released at once rather than on timeout.
started = time.monotonic()
assert stream.wait_frame(seq, timeout=10.0) == (None, seq)
assert time.monotonic() - started < 1.0
assert capture.calls >= FAILURE_LIMIT
finally:
stream.stop()
def test_giving_up_never_writes_the_phrase_to_the_log(caplog):
"""A log line is the easiest place for the stream's one credential to leak.
faulthandler and `threading.excepthook` print thread names, so the capture
thread is checked here too.
"""
capture = StubCapture(fail_first=10_000)
stream = make_stream(capture, fps=10)
with caplog.at_level(logging.DEBUG, logger="hhstream.live"):
stream.start()
thread_name = stream._thread.name
try:
stream.add_viewer()
assert stream.wait_frame(0, timeout=15) == (None, 0)
assert stream.stopped
finally:
stream.stop()
messages = [record.getMessage() for record in caplog.records]
assert messages, "the give-up must be logged at all"
assert any("hh-demo" in message for message in messages)
for message in messages:
assert PHRASE not in message, message
assert PHRASE not in thread_name and "hh-demo" in thread_name
# ---- registry --------------------------------------------------------------
def test_registry_adds_and_finds_a_phrase_case_insensitively():