fix(live): bound connections, a delimited stream, and a quiet access log
Four things the HTTP side owed. A connection had no read timeout, so a client that connected and sent nothing held a handler thread in `readline` for ever -- and on the MJPEG route it also held a viewer slot, which keeps grim running for nobody. Handlers now carry a 15 s header timeout, cleared once the MJPEG headers are out because from there the wait is on frames, not on the client. TCP keepalive covers the other half: a black-holed viewer that never closes. The MJPEG response had no `Content-Length`, no chunked encoding and no `Connection: close` -- `close_connection = True` was set after `end_headers()`, so it never became a header. An intermediary was entitled to buffer the whole thing waiting for an EOF that never comes. `log_message` formatted the request line, which is the phrase, at debug level; so did the client-error line. Both now log the method, the status and the peer, never the path. `RateLimiter` pruned only the key it was asked about, so every source address that ever missed stayed in the table for the life of the process -- a leak any LAN host could drive. The sweep now covers the table. Finally, `LiveServer` says where it is really bound: `address` still returns something dialable, and `describe_bind` is what names `0.0.0.0` as all interfaces. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01URhyHGQnmKH9zBXxoznfQY
This commit is contained in:
1 parent
19338af17c
commit
a9299682d2
2 files changed
+196
-15
No files matched your search
+91
-14
@@ -22,6 +22,7 @@ nothing a dependency would make easier.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
@@ -43,6 +44,11 @@ DEFAULT_HOST = "127.0.0.1"
|
||||
#: Wrong-phrase lookups allowed per client per window before 429.
|
||||
RATE_LIMIT = 30
|
||||
RATE_WINDOW_S = 60.0
|
||||
#: How long a connection may stay silent before it is dropped. It covers the
|
||||
#: request line and the headers only: a client that connects and says nothing
|
||||
#: would otherwise hold a handler thread in `readline` for ever, and on the
|
||||
#: MJPEG route it would hold a viewer slot -- which keeps `grim` running.
|
||||
HEADER_TIMEOUT_S = 15.0
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
@@ -51,6 +57,10 @@ class RateLimiter:
|
||||
A phrase carries ~77 bits, so guessing is already hopeless; this is here so
|
||||
that a LAN attacker cannot even *try* at line speed, and so the attempt is
|
||||
visible in one place. Only misses are counted.
|
||||
|
||||
The table holds only clients that missed inside the current window: the key
|
||||
is a client address, which on a LAN is the attacker's to choose, so a table
|
||||
that only ever grew would be a slow memory leak driven from the network.
|
||||
"""
|
||||
|
||||
def __init__(self, limit: int = RATE_LIMIT, window: float = RATE_WINDOW_S,
|
||||
@@ -63,16 +73,29 @@ class RateLimiter:
|
||||
|
||||
def allow(self, key: str) -> bool:
|
||||
now = self._clock()
|
||||
cutoff = now - self.window
|
||||
with self._lock:
|
||||
# Prune every key, not just this one. Pruning only `key` would leave
|
||||
# a client that missed once and never came back in the table for the
|
||||
# life of the process; the sweep is over the clients seen in the
|
||||
# last window, which is what the limit already bounds.
|
||||
for seen, hits in list(self._hits.items()):
|
||||
while hits and hits[0] < cutoff:
|
||||
hits.popleft()
|
||||
if not hits:
|
||||
del self._hits[seen]
|
||||
hits = self._hits.setdefault(key, deque())
|
||||
cutoff = now - self.window
|
||||
while hits and hits[0] < cutoff:
|
||||
hits.popleft()
|
||||
if len(hits) >= self.limit:
|
||||
return False
|
||||
hits.append(now)
|
||||
return True
|
||||
|
||||
@property
|
||||
def tracked(self) -> int:
|
||||
"""How many clients the table is holding. For the leak test and status."""
|
||||
with self._lock:
|
||||
return len(self._hits)
|
||||
|
||||
def reset(self) -> None:
|
||||
with self._lock:
|
||||
self._hits.clear()
|
||||
@@ -93,6 +116,9 @@ class _Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
server_version = "hhstream-live"
|
||||
sys_version = ""
|
||||
#: `StreamRequestHandler.setup` applies this to the socket. Without it a
|
||||
#: client that opens a connection and sends nothing pins a handler thread.
|
||||
timeout = HEADER_TIMEOUT_S
|
||||
|
||||
# ---- plumbing ----------------------------------------------------
|
||||
|
||||
@@ -100,8 +126,31 @@ class _Handler(BaseHTTPRequestHandler):
|
||||
def registry(self) -> StreamRegistry:
|
||||
return self.server.registry # type: ignore[attr-defined]
|
||||
|
||||
def setup(self) -> None:
|
||||
super().setup()
|
||||
# Keepalive is for the other half of the same problem: a viewer whose
|
||||
# network black-holes never closes, so `sendall` blocks and the viewer
|
||||
# count never drops -- which keeps `grim` capturing for nobody. The
|
||||
# kernel's probes end that connection instead.
|
||||
try:
|
||||
self.connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
|
||||
except OSError as exc: # a platform without it is not worth failing over
|
||||
log.debug("could not enable TCP keepalive: %s", exc)
|
||||
|
||||
def log_message(self, format: str, *args) -> None: # noqa: A002 - base signature
|
||||
log.debug("%s %s", self.address_string(), format % args)
|
||||
"""Log that something happened, never *what was asked for*.
|
||||
|
||||
The base class formats `self.requestline` into almost every message it
|
||||
emits, and the request line contains the phrase -- the stream's only
|
||||
credential. So the format string and its arguments are dropped here and
|
||||
the method is logged on its own; `log_request` adds the status.
|
||||
"""
|
||||
log.debug("%s %s", self.address_string(), self.command or "-")
|
||||
|
||||
def log_request(self, code="-", size="-") -> None:
|
||||
if isinstance(code, HTTPStatus):
|
||||
code = code.value
|
||||
log.debug("%s %s -> %s", self.address_string(), self.command or "-", code)
|
||||
|
||||
def _send(self, status: HTTPStatus, body: bytes, content_type: str,
|
||||
headers: dict[str, str] | None = None) -> None:
|
||||
@@ -128,7 +177,8 @@ class _Handler(BaseHTTPRequestHandler):
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
pass # the viewer went away mid-write; that is not an error
|
||||
except OSError as exc:
|
||||
log.debug("client error on %s: %s", path, exc)
|
||||
# Not the path: it carries the phrase.
|
||||
log.debug("client error from %s: %s", self.address_string(), exc)
|
||||
|
||||
def _route(self, path: str) -> None:
|
||||
if path in ("", "/"):
|
||||
@@ -191,12 +241,19 @@ class _Handler(BaseHTTPRequestHandler):
|
||||
self.send_header("Content-Type", f"multipart/x-mixed-replace; boundary={BOUNDARY}")
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
# The connection *is* the stream: it has no `Content-Length` and is not
|
||||
# chunked, so "the response ended" can only mean "the connection ended".
|
||||
# HTTP/1.1 keeps a connection alive by default, and an intermediary that
|
||||
# believed that would be entitled to buffer this response until an EOF
|
||||
# that never comes. Saying so has to happen before `end_headers`, or it
|
||||
# is not a header at all -- `send_header` also sets `close_connection`.
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
|
||||
# The header timeout has done its job; from here the handler waits on
|
||||
# frames, not on the client, and a quiet minute is normal.
|
||||
self.connection.settimeout(None)
|
||||
stream.add_viewer()
|
||||
# The connection is the stream: after this handler returns there is no
|
||||
# next request to read, so it must not be kept alive.
|
||||
self.close_connection = True
|
||||
last_seq = 0
|
||||
try:
|
||||
while True:
|
||||
@@ -225,6 +282,9 @@ class _Handler(BaseHTTPRequestHandler):
|
||||
class LiveServer:
|
||||
"""Owns the listener and its lifetime. One instance, many streams."""
|
||||
|
||||
#: Binds that mean "every interface" rather than one address of its own.
|
||||
WILDCARDS = ("", "0.0.0.0", "::")
|
||||
|
||||
def __init__(self, registry: StreamRegistry, *, host: str = DEFAULT_HOST,
|
||||
port: int = DEFAULT_PORT, limiter: RateLimiter | None = None) -> None:
|
||||
self.registry = registry
|
||||
@@ -242,12 +302,28 @@ class LiveServer:
|
||||
return self.requested_port
|
||||
return self._httpd.server_address[1]
|
||||
|
||||
@property
|
||||
def binds_everywhere(self) -> bool:
|
||||
return self.host in self.WILDCARDS
|
||||
|
||||
@property
|
||||
def address(self) -> str:
|
||||
advertised = self.host
|
||||
if advertised in ("0.0.0.0", "::"):
|
||||
advertised = "127.0.0.1"
|
||||
return advertised
|
||||
"""A host a browser can dial. A wildcard bind has none of its own, so
|
||||
the loopback name stands in; `describe_bind` is what tells the truth
|
||||
about the reach."""
|
||||
return "127.0.0.1" if self.binds_everywhere else self.host
|
||||
|
||||
def describe_bind(self) -> str:
|
||||
"""One line naming the real bind, and saying when it is every interface.
|
||||
|
||||
`url_for` has to print something dialable, which for `0.0.0.0` is a lie
|
||||
of omission -- the stream is on the LAN, not on loopback. This is where
|
||||
that is said out loud.
|
||||
"""
|
||||
where = f"{self.host or '0.0.0.0'}:{self.port}"
|
||||
if self.binds_everywhere:
|
||||
return f"bound to {where} -- all interfaces, reachable from the LAN"
|
||||
return f"bound to {where}"
|
||||
|
||||
def url_for(self, phrase: str) -> str:
|
||||
return f"http://{self.address}:{self.port}/{phrase}/"
|
||||
@@ -264,7 +340,7 @@ class LiveServer:
|
||||
name="hhstream-live-http", daemon=True)
|
||||
self._thread.start()
|
||||
self.ready.set()
|
||||
log.info("listening on http://%s:%d", self.host, self.port)
|
||||
log.info("listening, %s", self.describe_bind())
|
||||
|
||||
def stop(self) -> None:
|
||||
if self._httpd is not None:
|
||||
@@ -277,4 +353,5 @@ class LiveServer:
|
||||
self.registry.stop_all()
|
||||
|
||||
|
||||
__all__ = ["LiveServer", "RateLimiter", "DEFAULT_PORT", "DEFAULT_HOST", "BOUNDARY"]
|
||||
__all__ = ["LiveServer", "RateLimiter", "DEFAULT_PORT", "DEFAULT_HOST", "BOUNDARY",
|
||||
"HEADER_TIMEOUT_S"]
|
||||
@@ -2,11 +2,13 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
|
||||
from live_helpers import PHRASE, StubCapture, read_parts
|
||||
from hhstream_live.server import BOUNDARY
|
||||
from hhstream_live.server import BOUNDARY, HEADER_TIMEOUT_S, LiveServer, RateLimiter
|
||||
|
||||
WRONG = "no-such-phrase-here-00000"
|
||||
|
||||
@@ -94,6 +96,10 @@ def test_stream_is_mjpeg_and_delivers_valid_jpeg_parts(serve, http_conn):
|
||||
assert resp.getheader("Content-Type") == (
|
||||
f"multipart/x-mixed-replace; boundary={BOUNDARY}"
|
||||
)
|
||||
# The stream has no length and is not chunked, so the only thing that
|
||||
# can delimit it is the close -- which an intermediary is only entitled
|
||||
# to expect if the response says so.
|
||||
assert resp.getheader("Connection") == "close"
|
||||
parts = read_parts(resp, 2)
|
||||
assert len(parts) == 2
|
||||
for part in parts:
|
||||
@@ -150,6 +156,104 @@ def test_no_capture_until_a_request_arrives(serve):
|
||||
assert capture.calls >= 1
|
||||
|
||||
|
||||
def test_a_silent_client_does_not_pin_a_handler_thread(serve):
|
||||
"""A connection that never sends a request line must not hold a thread.
|
||||
|
||||
The socket timeout is the only thing that ends it; without one the handler
|
||||
sits in `readline` for the life of the process.
|
||||
"""
|
||||
assert HEADER_TIMEOUT_S <= 30 # a cap that never fires is not a cap
|
||||
|
||||
with serve() as (server, _):
|
||||
before = threading.active_count()
|
||||
sock = socket.create_connection(("127.0.0.1", server.port), timeout=5)
|
||||
try:
|
||||
time.sleep(0.2)
|
||||
assert threading.active_count() > before # a handler is waiting on us
|
||||
sock.settimeout(HEADER_TIMEOUT_S + 10)
|
||||
started = time.monotonic()
|
||||
# The server closes on timeout, which arrives here as EOF.
|
||||
assert sock.recv(4096) == b""
|
||||
waited = time.monotonic() - started
|
||||
assert waited <= HEADER_TIMEOUT_S + 5, waited
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
def test_a_viewer_that_drops_mid_stream_releases_its_slot(serve, http_conn):
|
||||
"""A dropped viewer must bring `viewers` back to 0, or grim keeps running."""
|
||||
import json
|
||||
|
||||
with serve(fps=10) as (server, _):
|
||||
sock = socket.create_connection(("127.0.0.1", server.port), timeout=10)
|
||||
sock.sendall(f"GET /{PHRASE}/stream.mjpg HTTP/1.1\r\n"
|
||||
f"Host: 127.0.0.1\r\n\r\n".encode("ascii"))
|
||||
assert sock.recv(4096) # headers, and at least the start of a part
|
||||
|
||||
def viewers() -> int:
|
||||
return json.loads(get(http_conn(server), f"/{PHRASE}/status.json")[2])["viewers"]
|
||||
|
||||
# The headers can reach us a moment before the handler counts itself in.
|
||||
deadline = time.monotonic() + 10
|
||||
while viewers() == 0 and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
assert viewers() == 1
|
||||
|
||||
sock.close() # the phone walked out of range
|
||||
|
||||
deadline = time.monotonic() + 10
|
||||
while viewers() and time.monotonic() < deadline:
|
||||
time.sleep(0.1)
|
||||
assert viewers() == 0
|
||||
|
||||
|
||||
def test_the_access_log_never_carries_the_phrase(serve, http_conn, caplog):
|
||||
"""Debug logging is on in development, and the request line is the phrase."""
|
||||
with caplog.at_level(logging.DEBUG, logger="hhstream.live"):
|
||||
with serve() as (server, _):
|
||||
assert get(http_conn(server), f"/{PHRASE}/status.json")[0] == 200
|
||||
assert get(http_conn(server), f"/{WRONG}/")[0] == 404
|
||||
|
||||
messages = [record.getMessage() for record in caplog.records]
|
||||
assert messages, "something must be logged, or this proves nothing"
|
||||
for message in messages:
|
||||
assert PHRASE not in message, message
|
||||
assert WRONG not in message, message
|
||||
|
||||
|
||||
def test_the_server_says_when_it_is_bound_to_every_interface(registry):
|
||||
"""`url_for` must print something dialable; the bind line tells the truth."""
|
||||
loopback = LiveServer(registry, host="127.0.0.1", port=9988)
|
||||
assert loopback.describe_bind() == "bound to 127.0.0.1:9988"
|
||||
assert not loopback.binds_everywhere
|
||||
|
||||
everywhere = LiveServer(registry, host="0.0.0.0", port=9988)
|
||||
assert everywhere.binds_everywhere
|
||||
assert everywhere.address == "127.0.0.1" # still dialable
|
||||
described = everywhere.describe_bind()
|
||||
assert "0.0.0.0:9988" in described and "all interfaces" in described
|
||||
|
||||
|
||||
def test_the_rate_limiter_counts_each_client_separately_and_forgets_them():
|
||||
"""One noisy client must not spend another's budget, and must not be kept.
|
||||
|
||||
`_hits` is keyed by client address, so a table that only grew would be a
|
||||
memory leak any LAN host could drive.
|
||||
"""
|
||||
now = [0.0]
|
||||
limiter = RateLimiter(limit=2, window=60.0, clock=lambda: now[0])
|
||||
|
||||
assert limiter.allow("10.0.0.1")
|
||||
assert limiter.allow("10.0.0.1")
|
||||
assert not limiter.allow("10.0.0.1") # that client is out of budget
|
||||
assert limiter.allow("10.0.0.2") # this one is not
|
||||
assert limiter.tracked == 2
|
||||
|
||||
now[0] += 61.0 # both windows have passed
|
||||
assert limiter.allow("10.0.0.1")
|
||||
assert limiter.tracked == 1, "a client that went quiet is not kept"
|
||||
|
||||
|
||||
def test_repeated_wrong_phrases_are_rate_limited(serve, http_conn):
|
||||
with serve(limit=3) as (server, _):
|
||||
conn = http_conn(server)
|
||||
|
||||
Reference in new issue
Block a user