fix(live): a dead output stops the stream even after a good frame
The failure cap read `consecutive_failures >= FAILURE_LIMIT and captured == 0`, so it could only ever fire on a cold start. An output removed while someone was watching -- grim answering "unknown output" to every grab from then on -- kept the loop retrying twice a second for as long as the viewer held the page open. That is the busy loop the cap exists to stop, so the count is now of failures in a row and nothing else. Giving up stays sticky: a removed output does not come back, and re-enabling the session is how it returns. While here, the capture thread is named for its output and the give-up is logged by output. The phrase is the stream's only credential, and faulthandler, `threading.excepthook` and any sampling profiler all print thread names. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01URhyHGQnmKH9zBXxoznfQY
This commit is contained in:
1 parent
9744d35945
commit
52e3786bfb
3 files changed
+92
-10
No files matched your search
@@ -31,9 +31,12 @@ log = logging.getLogger("hhstream.live")
|
||||
#: the box is also running the compositor and the main publisher.
|
||||
DEFAULT_FPS = 5
|
||||
FPS_CAP = 10
|
||||
#: After this many capture failures with no successful frame, a stream gives up
|
||||
#: and closes its viewers. `grim` against a removed output ("unknown output")
|
||||
#: fails every time, and retrying forever would be a busy loop on a dead session.
|
||||
#: After this many *consecutive* capture failures a stream gives up and closes
|
||||
#: its viewers. `grim` against a removed output ("unknown output") fails every
|
||||
#: time, and retrying forever would be a busy loop on a dead session -- which is
|
||||
#: as true after a thousand good frames as it is from a cold start, so the count
|
||||
#: is of failures in a row, not of failures-before-the-first-frame. A successful
|
||||
#: grab resets it; giving up does not reset, because a dead output stays dead.
|
||||
FAILURE_LIMIT = 10
|
||||
|
||||
|
||||
@@ -95,8 +98,11 @@ class Stream:
|
||||
# ---- lifecycle ---------------------------------------------------
|
||||
|
||||
def start(self) -> None:
|
||||
# Named for the output, never the phrase: faulthandler, the default
|
||||
# `threading.excepthook` and any profiler that samples the process all
|
||||
# print thread names, and the phrase is this stream's only credential.
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name=f"hhstream-live-{self.spec.phrase}", daemon=True
|
||||
target=self._run, name=f"hhstream-live-{self.spec.output}", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
@@ -129,15 +135,22 @@ class Stream:
|
||||
with self._cond:
|
||||
self._failures += 1
|
||||
self._consecutive_failures += 1
|
||||
give_up = self._consecutive_failures >= FAILURE_LIMIT and self._captured == 0
|
||||
give_up = self._consecutive_failures >= FAILURE_LIMIT
|
||||
if give_up:
|
||||
# Sticky on purpose: a stream that reached the cap is
|
||||
# not re-armed by `add_viewer`, because the case this
|
||||
# guards -- an output that was removed -- does not come
|
||||
# back. Re-enabling the session is how it comes back.
|
||||
self._stopped = True
|
||||
self._cond.notify_all()
|
||||
if give_up:
|
||||
log.warning("stream %s: giving up after %d capture failures: %s",
|
||||
self.spec.phrase, FAILURE_LIMIT, exc)
|
||||
# By output, not by phrase: a log line is the easiest place
|
||||
# for a credential to escape to.
|
||||
log.warning("stream on %s: giving up after %d consecutive "
|
||||
"capture failures: %s",
|
||||
self.spec.output, FAILURE_LIMIT, exc)
|
||||
return
|
||||
log.debug("stream %s: capture failed: %s", self.spec.phrase, exc)
|
||||
log.debug("stream on %s: capture failed: %s", self.spec.output, exc)
|
||||
if self._sleep(0.5):
|
||||
return
|
||||
continue
|
||||
|
||||
@@ -25,11 +25,19 @@ PHRASE = "amber-otter-lantern-harbor-7k2qp"
|
||||
|
||||
|
||||
class StubCapture:
|
||||
"""A `Capture` under the test's control: counts grabs, can be told to fail."""
|
||||
"""A `Capture` under the test's control: counts grabs, can be told to fail.
|
||||
|
||||
def __init__(self, frames=None, fail_first: int = 0, delay: float = 0.0):
|
||||
`fail_first` is a capture that is broken from the start (a cold start that
|
||||
never works); `fail_after` is one that works and then stops -- the output
|
||||
was removed while someone was watching, which is grim's "unknown output" on
|
||||
every grab from then on.
|
||||
"""
|
||||
|
||||
def __init__(self, frames=None, fail_first: int = 0, delay: float = 0.0,
|
||||
fail_after: int | None = None):
|
||||
self.frames = list(frames) if frames else [jpeg_bytes(i) for i in range(1, 50)]
|
||||
self.fail_first = fail_first
|
||||
self.fail_after = fail_after
|
||||
self.delay = delay
|
||||
self.calls = 0
|
||||
self._lock = threading.Lock()
|
||||
@@ -42,6 +50,8 @@ class StubCapture:
|
||||
time.sleep(self.delay)
|
||||
if call <= self.fail_first:
|
||||
raise RuntimeError(f"stub capture failing on purpose (call {call})")
|
||||
if self.fail_after is not None and call > self.fail_after:
|
||||
raise RuntimeError(f"stub capture: unknown output (call {call})")
|
||||
return self.frames[(call - 1 - self.fail_first) % len(self.frames)]
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
import pytest
|
||||
@@ -94,6 +95,64 @@ def test_a_dead_capture_gives_up_and_closes_viewers():
|
||||
stream.stop()
|
||||
|
||||
|
||||
def test_a_capture_that_dies_after_a_good_frame_also_gives_up():
|
||||
"""The cap counts failures *in a row*, not failures before the first frame.
|
||||
|
||||
An output removed while someone is watching fails every grab from then on.
|
||||
A give-up rule that only applied to a cold start would leave this stream
|
||||
retrying twice a second for as long as the viewer held the page open --
|
||||
exactly the busy loop on a dead session the cap exists to stop.
|
||||
"""
|
||||
capture = StubCapture(fail_after=1) # one good frame, then the output is gone
|
||||
stream = make_stream(capture, fps=10)
|
||||
stream.start()
|
||||
try:
|
||||
stream.add_viewer() # and the viewer stays attached for the whole test
|
||||
frame, seq = stream.wait_frame(0, timeout=5)
|
||||
assert frame is not None and seq >= 1
|
||||
|
||||
# Poll through the viewer's own wait, so a `wait_frame` that hangs is a
|
||||
# failure here rather than a test that quietly waits for the deadline.
|
||||
deadline = time.monotonic() + 15
|
||||
while not stream.stopped and time.monotonic() < deadline:
|
||||
assert stream.wait_frame(seq, timeout=1.0) == (None, seq)
|
||||
assert stream.stopped, "a dead output must stop the stream, frame or no frame"
|
||||
|
||||
# And the viewer in the loop is released at once rather than on timeout.
|
||||
started = time.monotonic()
|
||||
assert stream.wait_frame(seq, timeout=10.0) == (None, seq)
|
||||
assert time.monotonic() - started < 1.0
|
||||
assert capture.calls >= FAILURE_LIMIT
|
||||
finally:
|
||||
stream.stop()
|
||||
|
||||
|
||||
def test_giving_up_never_writes_the_phrase_to_the_log(caplog):
|
||||
"""A log line is the easiest place for the stream's one credential to leak.
|
||||
|
||||
faulthandler and `threading.excepthook` print thread names, so the capture
|
||||
thread is checked here too.
|
||||
"""
|
||||
capture = StubCapture(fail_first=10_000)
|
||||
stream = make_stream(capture, fps=10)
|
||||
with caplog.at_level(logging.DEBUG, logger="hhstream.live"):
|
||||
stream.start()
|
||||
thread_name = stream._thread.name
|
||||
try:
|
||||
stream.add_viewer()
|
||||
assert stream.wait_frame(0, timeout=15) == (None, 0)
|
||||
assert stream.stopped
|
||||
finally:
|
||||
stream.stop()
|
||||
|
||||
messages = [record.getMessage() for record in caplog.records]
|
||||
assert messages, "the give-up must be logged at all"
|
||||
assert any("hh-demo" in message for message in messages)
|
||||
for message in messages:
|
||||
assert PHRASE not in message, message
|
||||
assert PHRASE not in thread_name and "hh-demo" in thread_name
|
||||
|
||||
|
||||
# ---- registry --------------------------------------------------------------
|
||||
|
||||
def test_registry_adds_and_finds_a_phrase_case_insensitively():
|
||||
|
||||
Reference in new issue
Block a user