fix(live): a dead output stops the stream even after a good frame
The failure cap read `consecutive_failures >= FAILURE_LIMIT and captured == 0`, so it could only ever fire on a cold start. An output removed while someone was watching -- grim answering "unknown output" to every grab from then on -- kept the loop retrying twice a second for as long as the viewer held the page open. That is the busy loop the cap exists to stop, so the count is now of failures in a row and nothing else. Giving up stays sticky: a removed output does not come back, and re-enabling the session is how it returns. While here, the capture thread is named for its output and the give-up is logged by output. The phrase is the stream's only credential, and faulthandler, `threading.excepthook` and any sampling profiler all print thread names. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01URhyHGQnmKH9zBXxoznfQY
This commit is contained in:
1 parent
9744d35945
commit
52e3786bfb
3 files changed
+92
-10
No files matched your search
@@ -25,11 +25,19 @@ PHRASE = "amber-otter-lantern-harbor-7k2qp"
|
||||
|
||||
|
||||
class StubCapture:
|
||||
"""A `Capture` under the test's control: counts grabs, can be told to fail."""
|
||||
"""A `Capture` under the test's control: counts grabs, can be told to fail.
|
||||
|
||||
def __init__(self, frames=None, fail_first: int = 0, delay: float = 0.0):
|
||||
`fail_first` is a capture that is broken from the start (a cold start that
|
||||
never works); `fail_after` is one that works and then stops -- the output
|
||||
was removed while someone was watching, which is grim's "unknown output" on
|
||||
every grab from then on.
|
||||
"""
|
||||
|
||||
def __init__(self, frames=None, fail_first: int = 0, delay: float = 0.0,
|
||||
fail_after: int | None = None):
|
||||
self.frames = list(frames) if frames else [jpeg_bytes(i) for i in range(1, 50)]
|
||||
self.fail_first = fail_first
|
||||
self.fail_after = fail_after
|
||||
self.delay = delay
|
||||
self.calls = 0
|
||||
self._lock = threading.Lock()
|
||||
@@ -42,6 +50,8 @@ class StubCapture:
|
||||
time.sleep(self.delay)
|
||||
if call <= self.fail_first:
|
||||
raise RuntimeError(f"stub capture failing on purpose (call {call})")
|
||||
if self.fail_after is not None and call > self.fail_after:
|
||||
raise RuntimeError(f"stub capture: unknown output (call {call})")
|
||||
return self.frames[(call - 1 - self.fail_first) % len(self.frames)]
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
import pytest
|
||||
@@ -94,6 +95,64 @@ def test_a_dead_capture_gives_up_and_closes_viewers():
|
||||
stream.stop()
|
||||
|
||||
|
||||
def test_a_capture_that_dies_after_a_good_frame_also_gives_up():
|
||||
"""The cap counts failures *in a row*, not failures before the first frame.
|
||||
|
||||
An output removed while someone is watching fails every grab from then on.
|
||||
A give-up rule that only applied to a cold start would leave this stream
|
||||
retrying twice a second for as long as the viewer held the page open --
|
||||
exactly the busy loop on a dead session the cap exists to stop.
|
||||
"""
|
||||
capture = StubCapture(fail_after=1) # one good frame, then the output is gone
|
||||
stream = make_stream(capture, fps=10)
|
||||
stream.start()
|
||||
try:
|
||||
stream.add_viewer() # and the viewer stays attached for the whole test
|
||||
frame, seq = stream.wait_frame(0, timeout=5)
|
||||
assert frame is not None and seq >= 1
|
||||
|
||||
# Poll through the viewer's own wait, so a `wait_frame` that hangs is a
|
||||
# failure here rather than a test that quietly waits for the deadline.
|
||||
deadline = time.monotonic() + 15
|
||||
while not stream.stopped and time.monotonic() < deadline:
|
||||
assert stream.wait_frame(seq, timeout=1.0) == (None, seq)
|
||||
assert stream.stopped, "a dead output must stop the stream, frame or no frame"
|
||||
|
||||
# And the viewer in the loop is released at once rather than on timeout.
|
||||
started = time.monotonic()
|
||||
assert stream.wait_frame(seq, timeout=10.0) == (None, seq)
|
||||
assert time.monotonic() - started < 1.0
|
||||
assert capture.calls >= FAILURE_LIMIT
|
||||
finally:
|
||||
stream.stop()
|
||||
|
||||
|
||||
def test_giving_up_never_writes_the_phrase_to_the_log(caplog):
|
||||
"""A log line is the easiest place for the stream's one credential to leak.
|
||||
|
||||
faulthandler and `threading.excepthook` print thread names, so the capture
|
||||
thread is checked here too.
|
||||
"""
|
||||
capture = StubCapture(fail_first=10_000)
|
||||
stream = make_stream(capture, fps=10)
|
||||
with caplog.at_level(logging.DEBUG, logger="hhstream.live"):
|
||||
stream.start()
|
||||
thread_name = stream._thread.name
|
||||
try:
|
||||
stream.add_viewer()
|
||||
assert stream.wait_frame(0, timeout=15) == (None, 0)
|
||||
assert stream.stopped
|
||||
finally:
|
||||
stream.stop()
|
||||
|
||||
messages = [record.getMessage() for record in caplog.records]
|
||||
assert messages, "the give-up must be logged at all"
|
||||
assert any("hh-demo" in message for message in messages)
|
||||
for message in messages:
|
||||
assert PHRASE not in message, message
|
||||
assert PHRASE not in thread_name and "hh-demo" in thread_name
|
||||
|
||||
|
||||
# ---- registry --------------------------------------------------------------
|
||||
|
||||
def test_registry_adds_and_finds_a_phrase_case_insensitively():
|
||||
|
||||
Reference in new issue
Block a user