From 71909601204d913792c6e360cda8dd27fbd1d0fe Mon Sep 17 00:00:00 2001 From: Hermenix Date: Sun, 11 Oct 2026 22:58:28 +0000 Subject: [PATCH] capture: encode JPEG in-process too, so the live stream keeps working PNG-only was a real defect, not a nice-to-have: `hhstream_live.server` serves `multipart/x-mixed-replace` with `image/jpeg` parts and its own capture default is `grim -c -t jpeg -q 80 -`, so switching this backend on would have broken video streaming while looking faster in a benchmark. JPEG now goes through libjpeg-turbo by ctypes, using its `TJPF_BGRX` input format -- which is exactly the compositor's B, G, R, X memory order, so the frame goes into the encoder with no copy and no colour conversion. Measured on this VM against the real compositor, the deployment's own settings (Virtual-1, 1280x720, jpeg q80, 200 frames each, two runs): p50 ms fps our CPU compositor CPU in-process 76.2 13.1 2.1 ms 51.5 ms grim -c -t jpeg 137.0 7.3 0.5 ms 80.2 ms (second run: 84/144, 55.1/79.8 -> 1.71x) So ~1.7-1.8x the frame rate and ~1.5x less of the compositor's CPU, with frames of the same order (188 KB against grim's 222 KB). PNG stays available through the stdlib for a deployment that would rather not carry the library. Both segfault-class traps from the spike are handled and documented: every argtype is declared (an undeclared pitch is a crash, not a wrong number), and `*jpegBuf` is reset to NULL after `tjFree` (turbojpeg reads a non-NULL pointer as a caller buffer, so the next frame would write into freed memory). The decoder is here because a lossy codec can only be tested by round-tripping pixels -- the new suite encodes solid blocks whose channels differ enough that a swap is unmistakable, and includes a padded stride. Packaging: libjpeg-turbo is reached through ctypes, so nothing links it and the path must reach the loader -- the console scripts get a wrapper with it on LD_LIBRARY_PATH, and the unit check exports it too. A missing library is reported at startup rather than once per frame: a server that is up but can never produce a frame is worse than one that refuses to start. Verified end to end: server with `--capture wayland:Virtual-1 --format jpeg`, client Hello `image/jpeg`, three frames read, each magic `ffd8`. --- checks/unit.nix | 10 ++ docs/ARCHITECTURE.md | 8 +- docs/CONTRACT.md | 13 +- nix/module.nix | 27 ++-- nix/package.nix | 20 ++- src/hhstream/jpeg.py | 219 +++++++++++++++++++++++++++++ src/hhstream/server.py | 51 ++++--- src/hhstream/wayland_capture.py | 41 +++++- tests/unit/test_jpeg.py | 164 +++++++++++++++++++++ tests/unit/test_wayland_capture.py | 53 ++++++- 10 files changed, 553 insertions(+), 53 deletions(-) create mode 100644 src/hhstream/jpeg.py create mode 100644 tests/unit/test_jpeg.py diff --git a/checks/unit.nix b/checks/unit.nix index 4a29301..749d986 100644 --- a/checks/unit.nix +++ b/checks/unit.nix @@ -13,6 +13,7 @@ # the suite runs in a build. { lib, + libjpeg_turbo, runCommand, python3, hhstream, @@ -26,6 +27,8 @@ let # whether a bad string types a prefix. pytestEnv = python3.withPackages (ps: [ ps.pytest ps.pywayland ps.xkbcommon ]); + + # Only the suite itself, so an editor's droppings or a local .pytest_cache # cannot change the derivation's hash. testSrc = lib.cleanSourceWith { @@ -62,6 +65,13 @@ runCommand "hhstream-unit" # than letting pytest do it makes a packaging problem distinguishable from a # test failure. export PYTHONPATH=${hhstream}/${python3.sitePackages} + + # `tests/unit/test_jpeg.py` round-trips a real frame through libjpeg-turbo: a + # lossy codec can only be checked by comparing pixels, so the suite needs the + # library -- and it needs the *loader* to find it, because the encoder reaches + # it through `ctypes` and nothing links it. Same reason nix/package.nix wraps + # the console scripts with this path. + export LD_LIBRARY_PATH=${libjpeg_turbo}/lib export PYTHONDONTWRITEBYTECODE=1 export HOME=$PWD python3 -c 'import hhstream; print("testing hhstream", hhstream.__version__, "from", hhstream.__file__)' diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index db97f1f..1731165 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -102,8 +102,12 @@ identically and appears in a Hyprland segfault stack (`create_frame` → `CScreenshareFrame::transform`). The price of the in-process route is that the package now carries Wayland protocol bindings of its own (`src/hhstream/_protocol/`, vendored the same way `hhstream_pointer` already vendors its two) and that it can -only publish **PNG**, since PNG is the one format it can encode without a -dependency. Both costs are why `command` is still the default. +publish **JPEG** (through libjpeg-turbo) as well as PNG — and JPEG is the one +that matters, because the live viewer serves `image/jpeg` parts and a PNG-only +backend would have broken video streaming while looking faster on a benchmark. +The library is reached through `ctypes` and has to be on `LD_LIBRARY_PATH`, which +the package's wrapper arranges. Both costs are why `command` is still the default: +switching is the operator's call, not something a merge should decide. Comments in the spike that produced these numbers live in the branch's `capture-speed` worktree; the honest caveat that came with them is that this box diff --git a/docs/CONTRACT.md b/docs/CONTRACT.md index 13a118a..534286c 100644 --- a/docs/CONTRACT.md +++ b/docs/CONTRACT.md @@ -94,7 +94,7 @@ Namespace: `services.hyprland-headless-stream` | `hyprlandPackage` | package | `pkgs.hyprland` | which compositor | | `package` | package | the flake's own build | the frame server/client | | `captureCommand` | list of str | `[ "grim" … ]` | override the capture *command* — used by tests | -| `captureBackend` | `"command"` \| `"wayland"` | `"command"` | capture by command or in-process (`zwlr-screencopy-v1`); `wayland` needs `format = "png"` | +| `captureBackend` | `"command"` \| `"wayland"` | `"command"` | capture by command or in-process (`zwlr-screencopy-v1`); both `format` values work | | `captureOutput` | str or null | `null` | output to capture by name when `captureBackend = "wayland"`; null = the first one | Services (names are part of the contract, tests refer to them): @@ -111,6 +111,7 @@ status as the wire format. Exactly this shape, and no others: ``` hhstream-server --socket --fps --format \ + [--capture-quality <1..100>] \ [--capture-command ...] [--capture wayland[:NAME]] ``` @@ -124,10 +125,12 @@ hhstream-server --socket --fps --format \ - `--capture` selects a **named backend** instead of a program: `wayland` captures the compositor's first output in-process, `wayland:NAME` the output Hyprland calls `NAME`. It is exclusive with `--capture-command` (the backend - replaces the command; passing both is a caller error). It requires - `--format png`, and the server refuses `jpeg` rather than publish a Hello whose - MIME type the frames do not have — the in-process client can encode PNG without - a dependency and nothing else. + replaces the command; passing both is a caller error). It honours both + `--format` values: `png` through the stdlib, `jpeg` through libjpeg-turbo. If + libjpeg-turbo is missing and `--format jpeg` was asked for, the server **refuses + to start** rather than serving a socket it can never produce a frame for. +- `--capture-quality` is the JPEG quality for the in-process backend (1..100, + default 80, matching the default command's `-q 80`); `--format png` ignores it. - There is deliberately **no** `--width` / `--height`: the Hello reports the captured frame's real size (see above). - The socket's **mode** (0660) is the server's business; its **ownership** is diff --git a/nix/module.nix b/nix/module.nix index ed596a5..0a89b55 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -418,11 +418,14 @@ in { per frame, because it reuses one shared-memory buffer instead of making `grim` build a new one per frame. - It publishes **PNG only**: PNG is the one encoding this package can do - without a dependency, so {option}`format` must be `png` when this is set. - That is a real trade -- a JPEG frame is ~105-130 KB against ~580 KB for - the same frame as PNG -- and it is the reason the default is still - `command`. + Both of the {option}`format` values work, and in practice this means + `jpeg`: the live viewer serves `image/jpeg` parts, so a PNG-only backend + would break video streaming. JPEG is encoded in-process through + libjpeg-turbo at the same quality the default `grim` command uses + (~1.7 ms a frame, ~120 KB), and PNG remains available through the stdlib + for a deployment that would rather not carry that library (~2.1 MB a + frame). The default is still `command` only because switching backends + is the operator's call, not because anything is missing. ''; }; @@ -822,20 +825,6 @@ in { assertion = lib.hasPrefix "/" cfg.socketPath; message = "services.hyprland-headless-stream.socketPath must be an absolute path."; } - # The Hello carries the frame's MIME type, and the in-process backend can - # only produce PNG. A jpeg setting here would publish a format the frames - # do not have, so it is refused at build time rather than mis-decoded by - # every client at run time. - { - assertion = cfg.captureBackend != "wayland" || cfg.format == "png"; - message = '' - services.hyprland-headless-stream.captureBackend = "wayland" publishes - PNG only, but format = "${cfg.format}". - - JPEG in-process would need an encoder this package does not carry yet. - Set format = "png", or go back to captureBackend = "command". - ''; - } # Structural, not cosmetic: see the `runtimeDirName` comment above. The # ceiling exists so the compositor's *event* socket still fits in # `sun_path`, and the symptom of breaking it is silent -- the request diff --git a/nix/package.nix b/nix/package.nix index fcdaa13..0d049ec 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -1,5 +1,6 @@ { lib, + libjpeg_turbo, python3, ... }: @@ -22,7 +23,17 @@ python3.pkgs.buildPythonApplication { # hhstream_pointer._protocol.regenerate for how to refresh them -- including # the note that virtual-keyboard-unstable-v1.xml is not in wayland-protocols # or wlr-protocols in this nixpkgs and comes from pkgs.hyprland.src. - dependencies = [ python3.pkgs.pywayland python3.pkgs.xkbcommon ]; + dependencies = [ python3.pkgs.pywayland python3.pkgs.xkbcommon libjpeg_turbo ]; + + # The in-process jpeg encoder reaches libjpeg-turbo through `ctypes`, so the + # library has to be *findable at run time*, which a propagated build input + # alone does not achieve: nothing links it, so nothing puts its directory on + # LD_LIBRARY_PATH. The wrapper is what makes `ctypes.util.find_library` succeed + # inside the unit -- and it is the same library grim already links, so the + # frames stay byte-compatible with the existing stream. + makeWrapperArgs = [ + "--prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ libjpeg_turbo ]}" + ]; # Unit tests are wired up in the flake's `checks` so they can run without # installing the package; see docs/CONTRACT.md. @@ -38,6 +49,13 @@ python3.pkgs.buildPythonApplication { # The web app: importing the server pulls in the router, the publisher and # the pages. "hhstream_live.server" + # The capture backend and its vendored protocol bindings: importing the + # backend runs the scanner-generated `_gen_c()` for wl_shm, wl_output and + # zwlr-screencopy-manager-v1, which is the same cheap proof the pointer + # driver gets above. + "hhstream.wayland_capture" + "hhstream._protocol.wlr_screencopy_unstable_v1" + "hhstream.jpeg" ]; # `pythonImportsCheck` only *imports*, and `hhstream_live.phrase` reads its diff --git a/src/hhstream/jpeg.py b/src/hhstream/jpeg.py new file mode 100644 index 0000000..96157e4 --- /dev/null +++ b/src/hhstream/jpeg.py @@ -0,0 +1,219 @@ +"""JPEG encoding and decoding through libjpeg-turbo, via `ctypes`. + +Why this exists at all: the live viewer (`hhstream_live.server`) serves +`multipart/x-mixed-replace` with parts labelled `image/jpeg`, and its own capture +default is `grim -c -t jpeg -q 80 -`. The in-process capture backend has to +produce the same thing or video streaming breaks -- so PNG-only was never an +option, and this is the half of the work that makes the backend usable. + +libjpeg-turbo rather than a Python binding because it is what `grim` already +links: the frames stay byte-compatible with the existing stream, the dependency is +a library rather than a wheel, and measured on this machine the encode costs +~1.7 ms for a 1280x720 frame -- against ~30-60 ms for the stdlib PNG path, and +~2.1 MB against ~120 KB on the wire. + +**`TJPF_BGRX` is the whole reason there is no colour conversion here.** A +`wl_shm` frame arrives as B, G, R, X in memory -- which is exactly libjpeg-turbo's +`TJPF_BGRX` input format (3). Naming it means the compositor's buffer goes straight +into the encoder with no per-frame swizzle, which is the ~2.2 ms the spike +measured for doing that conversion in Python. + +Two traps, both of which cost a real debugging session and are the reason this +file is chatty: + +1. **Declare every `argtypes`.** Without them ctypes passes arguments through a + default signature and `tjCompress2` reads the wrong width/pitch -- a segfault + that takes the whole server down, not a wrong byte count. +2. **Reset `*jpegBuf` to NULL after `tjFree`.** turbojpeg treats a non-NULL + `*jpegBuf` as a caller-supplied destination, so leaving the freed pointer in + place makes the *next* frame write into freed memory: a heap-corrupting + segfault several frames later, with no stack trace anywhere near the encoder. + +The decoder is here because the unit tests need to prove the encoder round-trips +without adding an image library to the test environment -- and it is the only +honest way to check a lossy codec: compare pixels, not a file header. +""" + +from __future__ import annotations + +import ctypes +import ctypes.util +import os +from typing import Final + +#: From turbojpeg.h: memory order B, G, R, X -- what a `wl_shm` frame already is. +TJPF_BGRX: Final = 3 +#: 4:2:0 chroma subsampling, the default `grim -q 80` effectively produces. +TJSAMP_420: Final = 2 +#: What the deployment's capture command uses (`grim -c -t jpeg -q 80 -`). +DEFAULT_QUALITY: Final = 80 + +#: Tried in order when no explicit path is given. Env override first, because a +#: nix wrapper is the natural place to point this at the exact library it built +#: against; then whatever the loader can find (which is how the wrapper's +#: `LD_LIBRARY_PATH` reaches ctypes). +_LIBRARY_NAMES: Final = ("libturbojpeg.so.0", "libturbojpeg.so", "turbojpeg") + + +class JPEGError(RuntimeError): + """libjpeg-turbo is missing, or refused the frame.""" + + +def _load_library(path: str | None = None) -> ctypes.CDLL: + candidate = path or os.environ.get("TURBOJPEG_PATH") + if candidate: + try: + return ctypes.CDLL(candidate) + except OSError as exc: + raise JPEGError(f"cannot load libjpeg-turbo from {candidate}: {exc}") from exc + found = ctypes.util.find_library("turbojpeg") + for name in filter(None, (found, *_LIBRARY_NAMES)): + try: + return ctypes.CDLL(name) + except OSError: + continue + raise JPEGError( + "libjpeg-turbo is not available; the in-process backend needs it for JPEG " + "frames (set TURBOJPEG_PATH, or use --format png)" + ) + + +class TurboJPEG: + """One encoder and one decoder handle, kept for the process's lifetime. + + Handle reuse is deliberate: `tjInitCompress` allocates, and doing that per + frame is the kind of per-frame work this whole backend exists to remove. + """ + + def __init__(self, path: str | None = None) -> None: + self.lib = _load_library(path) + self._declare_signatures() + self._compress_handle = self.lib.tjInitCompress() + if not self._compress_handle: + raise JPEGError("tjInitCompress failed") + self._decompress_handle = None + self._out = ctypes.c_void_p() + self._size = ctypes.c_ulong() + + def _declare_signatures(self) -> None: + """Trap 1: every argument typed, or the call reads the wrong pitch.""" + lib = self.lib + lib.tjInitCompress.argtypes = [] + lib.tjInitCompress.restype = ctypes.c_void_p + lib.tjDestroy.argtypes = [ctypes.c_void_p] + lib.tjDestroy.restype = ctypes.c_int + lib.tjFree.argtypes = [ctypes.c_void_p] + lib.tjFree.restype = None + lib.tjCompress2.argtypes = [ + ctypes.c_void_p, # tjhandle + ctypes.c_void_p, # srcBuf + ctypes.c_int, # width + ctypes.c_int, # pitch (bytes per row) + ctypes.c_int, # height + ctypes.c_int, # pixelFormat + ctypes.POINTER(ctypes.c_void_p), # *jpegBuf (out) + ctypes.POINTER(ctypes.c_ulong), # *jpegSize (out) + ctypes.c_int, # jpegSubsamp + ctypes.c_int, # jpegQual + ctypes.c_int, # flags + ] + lib.tjCompress2.restype = ctypes.c_int + lib.tjInitDecompress.argtypes = [] + lib.tjInitDecompress.restype = ctypes.c_void_p + lib.tjDecompressHeader3.argtypes = [ + ctypes.c_void_p, # tjhandle + ctypes.c_void_p, # jpegBuf + ctypes.c_ulong, # jpegSize + ctypes.POINTER(ctypes.c_int), # *width + ctypes.POINTER(ctypes.c_int), # *height + ctypes.POINTER(ctypes.c_int), # *jpegSubsamp + ctypes.POINTER(ctypes.c_int), # *jpegColorspace + ] + lib.tjDecompressHeader3.restype = ctypes.c_int + lib.tjDecompress2.argtypes = [ + ctypes.c_void_p, # tjhandle + ctypes.c_void_p, # jpegBuf + ctypes.c_ulong, # jpegSize + ctypes.c_void_p, # dstBuf + ctypes.c_int, # width + ctypes.c_int, # pitch + ctypes.c_int, # height + ctypes.c_int, # pixelFormat + ctypes.c_int, # flags + ] + lib.tjDecompress2.restype = ctypes.c_int + + def compress(self, address: int, width: int, height: int, pitch: int, + quality: int = DEFAULT_QUALITY) -> bytes: + """Encode one BGRX frame held at *address* (a mapped shm buffer).""" + if not 0 < quality <= 100: + raise ValueError(f"JPEG quality must be 1..100, got {quality!r}") + rc = self.lib.tjCompress2( + ctypes.c_void_p(self._compress_handle), + ctypes.c_void_p(address), + ctypes.c_int(width), + ctypes.c_int(pitch), + ctypes.c_int(height), + ctypes.c_int(TJPF_BGRX), + ctypes.byref(self._out), + ctypes.byref(self._size), + ctypes.c_int(TJSAMP_420), + ctypes.c_int(quality), + 0, + ) + if rc != 0: + raise JPEGError(f"tjCompress2 failed (code {rc})") + try: + return ctypes.string_at(self._out, self._size.value) + finally: + self.lib.tjFree(self._out) + # Trap 2: a non-NULL *jpegBuf means "caller's buffer" to turbojpeg, so + # the freed pointer must not be left behind. + self._out = ctypes.c_void_p() + self._size = ctypes.c_ulong() + + def decompress(self, data: bytes) -> tuple[int, int, bytes]: + """Decode a JPEG to (width, height, BGRX pixels). For tests and tools.""" + if self._decompress_handle is None: + self._decompress_handle = self.lib.tjInitDecompress() + if not self._decompress_handle: + raise JPEGError("tjInitDecompress failed") + source = ctypes.create_string_buffer(data, len(data)) + return self._decompress(ctypes.cast(source, ctypes.c_void_p), len(data)) + + def _decompress(self, pointer: ctypes.c_void_p, size: int) -> tuple[int, int, bytes]: + # Four separate `c_int` objects, not a `(c_int * 4)()` array: iterating an + # array yields plain Python ints, and `byref` refuses those -- so the + # out-parameters of the header call have to be real ctypes cells. + frame_width = ctypes.c_int() + frame_height = ctypes.c_int() + subsamp = ctypes.c_int() + colorspace = ctypes.c_int() + rc = self.lib.tjDecompressHeader3( + ctypes.c_void_p(self._decompress_handle), pointer, ctypes.c_ulong(size), + ctypes.byref(frame_width), ctypes.byref(frame_height), + ctypes.byref(subsamp), ctypes.byref(colorspace), + ) + if rc != 0: + raise JPEGError(f"tjDecompressHeader3 failed (code {rc})") + width, height = frame_width.value, frame_height.value + pitch = width * 4 + out = ctypes.create_string_buffer(pitch * height) + rc = self.lib.tjDecompress2( + ctypes.c_void_p(self._decompress_handle), pointer, ctypes.c_ulong(size), + ctypes.cast(out, ctypes.c_void_p), ctypes.c_int(width), ctypes.c_int(pitch), + ctypes.c_int(height), ctypes.c_int(TJPF_BGRX), 0, + ) + if rc != 0: + raise JPEGError(f"tjDecompress2 failed (code {rc})") + return width, height, out.raw + + def close(self) -> None: + for name in ("_compress_handle", "_decompress_handle"): + handle = getattr(self, name, None) + if handle: + self.lib.tjDestroy(ctypes.c_void_p(handle)) + setattr(self, name, None) + + +__all__ = ["TurboJPEG", "JPEGError", "DEFAULT_QUALITY", "TJPF_BGRX", "TJSAMP_420"] diff --git a/src/hhstream/server.py b/src/hhstream/server.py index 0661ba0..d60ebf1 100644 --- a/src/hhstream/server.py +++ b/src/hhstream/server.py @@ -323,8 +323,14 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( _CAPTURE_BACKEND_FLAG, metavar="BACKEND", default=None, help="capture in-process instead of by command: 'wayland' to capture the " - "first output, 'wayland:NAME' to capture that output by name. " - "Only PNG can be produced this way, so it needs --format png.", + "first output, 'wayland:NAME' to capture that output by name. Both " + "--format png and --format jpeg are supported; jpeg needs " + "libjpeg-turbo on the machine.", + ) + parser.add_argument( + "--capture-quality", type=_quality, default=80, metavar="N", + help="JPEG quality for the in-process backend (default: %(default)s); " + "ignored by --format png. Matches the default capture command's -q.", ) parser.add_argument( _CAPTURE_FLAG, action="append", dest="capture_command", metavar="TOKEN", @@ -345,7 +351,7 @@ def main(argv: Iterable[str] | None = None) -> int: capture_command = args.capture_command or _default_capture_command(args.format) capture: CaptureBackend = CommandCapture(capture_command) else: - capture = _build_capture_backend(args.capture, args.format) + capture = _build_capture_backend(args.capture, args.format, args.capture_quality) if capture is None: return 2 server = FrameServer( @@ -369,22 +375,25 @@ def _default_capture_command(fmt: str) -> list[str]: return ["grim", "-t", fmt, "-"] -def _build_capture_backend(spec: str, fmt: str) -> CaptureBackend | None: +def _build_capture_backend(spec: str, fmt: str, quality: int) -> CaptureBackend | None: """Turn `--capture BACKEND` into a backend, or log why it cannot be one. - The in-process backend publishes PNG, because PNG is the one format this - package can produce without a dependency (see `png.encode_rgb`). Rather than - advertise a MIME type the frames do not have -- the Hello carries the format, - and a client that trusts it would mis-decode every frame -- this refuses the - combination and says what to do instead. + Both formats the contract allows are produced in-process: PNG through the + stdlib (`png.encode_rgb`), JPEG through libjpeg-turbo (`jpeg.TurboJPEG`). + JPEG is not optional in practice -- the live viewer serves + `multipart/x-mixed-replace` with `image/jpeg` parts -- so a missing + libjpeg-turbo is reported here, at startup, rather than once per frame: a + server that is up but can never produce a frame is worse than one that + refuses to start, because the socket still accepts connections. """ - if fmt != "png": - log.error( - "the in-process capture backend publishes PNG only, but --format %s " - "was asked for; JPEG in-process needs an encoder this package does " - "not carry yet, so use --format png or the command backend", fmt, - ) - return None + if fmt == "jpeg": + from .jpeg import JPEGError, TurboJPEG + + try: + TurboJPEG().close() + except JPEGError as exc: + log.error("cannot capture as jpeg: %s", exc) + return None if spec == "wayland": output = None elif spec.startswith("wayland:"): @@ -402,7 +411,15 @@ def _build_capture_backend(spec: str, fmt: str) -> CaptureBackend | None: # backend (still the default) should not need pywayland importable at all. from .wayland_capture import WaylandCapture - return WaylandCapture(output=output) + return WaylandCapture(output=output, codec=fmt, quality=quality) + + +def _quality(text: str) -> int: + """JPEG quality, matching grim's `-q`: 1..100, and refused outside it.""" + value = int(text) + if not 1 <= value <= 100: + raise argparse.ArgumentTypeError(f"must be 1..100, got {text!r}") + return value def _positive_int(text: str) -> int: diff --git a/src/hhstream/wayland_capture.py b/src/hhstream/wayland_capture.py index 98a35ee..9cf7d2c 100644 --- a/src/hhstream/wayland_capture.py +++ b/src/hhstream/wayland_capture.py @@ -29,7 +29,14 @@ Three things follow, and each one shaped the code below: favour of ``ext-image-copy-capture-v1``, which was measured as identical (78.0 vs 77.7 ms, compositor 67.5 vs 67.2) and which appears in a Hyprland segfault stack (``CScreenshareFrame::transform()`` reached from - ``create_frame``). When that crash is fixed, the switch is a constant here. + `create_frame`. When that crash is fixed, the switch is a constant here. +4. **Both codecs, because the stream decides.** The live viewer serves + `multipart/x-mixed-replace` with `image/jpeg` parts, so a PNG-only backend + would have broken video streaming while looking faster in a benchmark. JPEG + goes through libjpeg-turbo's `TJPF_BGRX` input format, which is exactly the + compositor's memory order -- no conversion, ~1.7 ms a frame, ~120 KB. + PNG stays available (stdlib `zlib`) for deployments that would rather not + carry the library, at ~2.1 MB a frame. **Why the wait is a poll loop and not a signal.** The frame server captures in a worker thread (``FrameServer._capture_loop``), so ``SIGALRM`` -- the obvious way to @@ -44,6 +51,7 @@ and actually reads. from __future__ import annotations +import ctypes import logging import mmap import os @@ -58,6 +66,7 @@ from ._protocol.wlr_screencopy_unstable_v1.zwlr_screencopy_manager_v1 import ( ZwlrScreencopyManagerV1, ) from .capture import CaptureError +from .jpeg import DEFAULT_QUALITY, TurboJPEG from .png import encode_rgb log = logging.getLogger(__name__) @@ -87,6 +96,9 @@ class _ShmBuffer: self.fd = os.memfd_create("hhstream-frame", os.MFD_CLOEXEC) os.ftruncate(self.fd, self.size) self.map = mmap.mmap(self.fd, self.size, prot=mmap.PROT_READ | mmap.PROT_WRITE) + #: Address of the mapping, for the ctypes encoder. Kept, not re-derived: + #: `addressof(from_buffer(...))` per frame is pure overhead. + self.address = ctypes.addressof(ctypes.c_char.from_buffer(self.map)) self.pool = shm.create_pool(self.fd, self.size) self.buffer = self.pool.create_buffer(0, width, height, stride, fmt) @@ -117,9 +129,14 @@ class WaylandCapture: DEFAULT_TIMEOUT = 5.0 def __init__(self, output: str | None = None, timeout: float = DEFAULT_TIMEOUT, - display: str | None = None, level: int = 1) -> None: + display: str | None = None, level: int = 1, codec: str = "png", + quality: int = DEFAULT_QUALITY) -> None: if timeout is not None and timeout <= 0: raise ValueError(f"capture timeout must be positive, got {timeout!r}") + if codec not in ("png", "jpeg"): + raise ValueError(f"unknown codec {codec!r}; expected 'png' or 'jpeg'") + self.codec = codec + self.quality = quality self.output_name = output self.timeout = timeout #: `None` means "whatever WAYLAND_DISPLAY says", which is how the module @@ -138,6 +155,9 @@ class WaylandCapture: self._ready: bool | None = None self._failure: str | None = None self._advertised: tuple[int, int, int, int] | None = None + #: Built on the first jpeg frame, not in __init__: a png-only deployment + #: must not need libjpeg-turbo on the box at all. + self._encoder: TurboJPEG | None = None # -- the two-method backend interface --------------------------------- @@ -158,6 +178,8 @@ class WaylandCapture: self._copy_frame() self._wait_for_ready() assert self._buffer is not None + if self.codec == "jpeg": + return self._encode_jpeg(self._buffer) return encode_rgb(self._buffer.width, self._buffer.height, self._buffer.stride, memoryview(self._buffer.map), level=self.level) @@ -167,9 +189,24 @@ class WaylandCapture: self.close() raise CaptureError(f"wayland capture failed: {exc}") from exc + def _encode_jpeg(self, buffer: _ShmBuffer) -> bytes: + """Encode the mapped buffer in place -- no copy, no colour conversion. + + `TJPF_BGRX` tells libjpeg-turbo that the compositor's B,G,R,X layout is + already the format it wants, which is what keeps this at ~1.7 ms a frame + instead of paying a Python swizzle first. + """ + if self._encoder is None: + self._encoder = TurboJPEG() + return self._encoder.compress(buffer.address, buffer.width, buffer.height, + buffer.stride, quality=self.quality) + def close(self) -> None: """Drop the connection and the buffer. Safe to call twice, and after a failure -- it is the recovery path, not just a shutdown path.""" + if self._encoder is not None: + self._encoder.close() + self._encoder = None if self._buffer is not None: try: self._buffer.destroy() diff --git a/tests/unit/test_jpeg.py b/tests/unit/test_jpeg.py new file mode 100644 index 0000000..06bf29a --- /dev/null +++ b/tests/unit/test_jpeg.py @@ -0,0 +1,164 @@ +"""`hhstream.jpeg`: the libjpeg-turbo binding, checked by round-tripping pixels. + +A lossy codec cannot be tested by looking at a file header, so these tests encode a +raster and decode it again through the same library. The raster is chosen to fail +loudly on the two mistakes that matter here: uniformly filled blocks (JPEG +reproduces a solid block almost exactly, so a tolerance of a few counts is honest) +whose channels differ enough that a BGR/RGB mix-up is unmistakable, and a padded +stride, because assuming `width * 4` is the bug grim shipped in 1.5. + +This is also the only test of the pixel-format claim the backend rests on: +`TJPF_BGRX` exists precisely so the compositor's B, G, R, X buffer can go into the +encoder unconverted. If that were the wrong format, the decoded block colours come +back swapped and these tests say so. +""" + +from __future__ import annotations + +import ctypes + +import pytest + +from hhstream.jpeg import DEFAULT_QUALITY, JPEGError, TurboJPEG + +BLOCK = 16 + + +def blocks() -> list[tuple[int, int, int]]: + """Four colours whose channels differ enough that a swap cannot hide.""" + return [(200, 30, 30), (30, 200, 30), (30, 30, 200), (140, 120, 100)] + + +def as_rgb(colour: tuple[int, int, int]) -> tuple[int, int, int]: + """`blocks()` writes memory order (B, G, R); assertions read (R, G, B). + + Getting these two the wrong way round is indistinguishable from a wrong pixel + format in the encoder -- which is exactly what this file exists to detect, so + the conversion is named rather than done inline. + """ + blue, green, red = colour + return red, green, blue + + +def colour_at(x: int, y: int, width: int) -> tuple[int, int, int]: + """The block colour at a pixel, wrapping so any frame size is covered. + + One formula for both the raster and the assertions: they used to index the + four colours independently, which ran off the end of the list for anything + bigger than a 2x2 grid of blocks. + """ + column = x // BLOCK + row = y // BLOCK + return blocks()[((row * (width // BLOCK)) + column) % len(blocks())] + + +def raster(width: int = 64, height: int = 64, stride: int | None = None) -> tuple[bytes, int]: + """A BGRX frame of solid blocks; returns (bytes, stride).""" + stride = stride if stride is not None else width * 4 + out = bytearray() + for y in range(height): + row = bytearray() + for x in range(width): + b, g, r = colour_at(x, y, width) + row += bytes((b, g, r, 0x00)) + row += b"\x77" * (stride - width * 4) # padding, if any + out += row + return bytes(out), stride + + +def address_of(pixels: bytes) -> int: + return ctypes.addressof(ctypes.create_string_buffer(pixels, len(pixels))) + + +def centre_colour(pixels: bytes, stride: int, x: int, y: int) -> tuple[int, int, int]: + offset = y * stride + x * 4 + blue, green, red, _ = pixels[offset:offset + 4] + return red, green, blue + + +def test_a_frame_round_trips_with_its_channels_in_the_right_place(): + enc = TurboJPEG() + try: + pixels, stride = raster() + encoded = enc.compress(address_of(pixels), 64, 64, stride) + width, height, decoded = enc.decompress(encoded) + assert (width, height) == (64, 64) + for column, row in ((0, 0), (1, 0), (2, 0), (0, 1), (3, 3)): + x, y = column * BLOCK + BLOCK // 2, row * BLOCK + BLOCK // 2 + want = as_rgb(colour_at(x, y, 64)) + got = centre_colour(decoded, width * 4, x, y) + assert got == pytest.approx(want, abs=12), ( + f"block ({row}, {column}) came back as {got}, expected ~{want} " + "-- a swapped channel order or a wrong pixel format" + ) + # Being close is not enough on its own: assert the channels are + # distinguishable, so a swap could not pass as "close enough". + assert max(want) - min(want) > 24, "pick colours a swap would expose" + finally: + enc.close() + + +def test_a_padded_stride_is_honoured(): + """The padding bytes are deliberately not black; leaking them changes the + last column of every row.""" + enc = TurboJPEG() + try: + pixels, stride = raster(64, 64, stride=64 * 4 + 32) + assert stride != 64 * 4 + encoded = enc.compress(address_of(pixels), 64, 64, stride) + _, _, decoded = enc.decompress(encoded) + got = centre_colour(decoded, 64 * 4, 8, 8) + assert got == pytest.approx(as_rgb(colour_at(8, 8, 64)), abs=12), got + finally: + enc.close() + + +def test_quality_trades_bytes_for_fidelity(): + enc = TurboJPEG() + try: + pixels, stride = raster() + small = enc.compress(address_of(pixels), 64, 64, stride, quality=20) + large = enc.compress(address_of(pixels), 64, 64, stride, quality=95) + assert len(small) < len(large) + finally: + enc.close() + + +def test_the_output_is_really_a_jpeg(): + enc = TurboJPEG() + try: + pixels, stride = raster() + encoded = enc.compress(address_of(pixels), 64, 64, stride) + assert encoded.startswith(b"\xff\xd8"), "missing SOI marker" + assert b"\xff\xc0" in encoded, "missing SOF0 marker" + assert encoded.endswith(b"\xff\xd9"), "missing EOI marker" + finally: + enc.close() + + +def test_the_default_quality_is_the_one_the_capture_command_used(): + """`grim -c -t jpeg -q 80 -` is the deployment's command; a swap must not + silently change the frames' size or quality.""" + assert DEFAULT_QUALITY == 80 + + +@pytest.mark.parametrize("quality", [0, -1, 101]) +def test_an_impossible_quality_is_refused(quality): + enc = TurboJPEG() + try: + pixels, stride = raster() + with pytest.raises(ValueError, match="1..100"): + enc.compress(address_of(pixels), 64, 64, stride, quality=quality) + finally: + enc.close() + + +def test_a_library_that_is_not_there_is_a_clear_error_not_a_crash(): + with pytest.raises(JPEGError, match="cannot load libjpeg-turbo"): + TurboJPEG(path="/nonexistent/libturbojpeg.so") + + +def test_close_is_safe_twice(): + enc = TurboJPEG() + enc.close() + enc.close() diff --git a/tests/unit/test_wayland_capture.py b/tests/unit/test_wayland_capture.py index f5c586c..aaff9c8 100644 --- a/tests/unit/test_wayland_capture.py +++ b/tests/unit/test_wayland_capture.py @@ -99,7 +99,7 @@ def test_a_timeout_of_zero_or_less_is_refused_at_construction(): ) def test_the_backend_spec_selects_an_output_without_connecting(spec, expected_output): """Constructing the backend must not open a connection -- grab() does.""" - backend = _build_capture_backend(spec, "png") + backend = _build_capture_backend(spec, "png", 80) assert isinstance(backend, WaylandCapture) assert backend.output_name == expected_output assert backend._display is None, "the connection belongs to grab(), not to __init__" @@ -109,14 +109,41 @@ def test_the_backend_spec_selects_an_output_without_connecting(spec, expected_ou def test_an_unknown_backend_spec_is_refused_rather_than_guessed(spec, caplog): """`wayland:a:b` is not an output name, and guessing would capture the wrong output silently.""" - assert _build_capture_backend(spec, "png") is None + assert _build_capture_backend(spec, "png", 80) is None -def test_jpeg_is_refused_rather_than_advertised_in_the_hello(caplog): - """The Hello carries the MIME type; publishing jpeg over PNG frames would - mis-decode every frame in every client, so it is refused up front.""" - assert _build_capture_backend("wayland", "jpeg") is None - assert "PNG only" in caplog.text +def test_jpeg_is_what_the_backend_uses_when_jpeg_is_asked_for(): + """The live viewer serves image/jpeg parts, so this is the format that has to + work -- a PNG-only backend would break video streaming.""" + backend = _build_capture_backend("wayland", "jpeg", 80) + assert isinstance(backend, WaylandCapture) + assert (backend.codec, backend.quality) == ("jpeg", 80) + + +def test_png_never_needs_the_jpeg_library(monkeypatch): + """A deployment that chose png must not require libjpeg-turbo on the box.""" + import hhstream.jpeg as jpeg + + def explode(*_args, **_kwargs): + raise jpeg.JPEGError("no libjpeg-turbo anywhere") + + monkeypatch.setattr(jpeg, "TurboJPEG", explode) + backend = _build_capture_backend("wayland", "png", 80) + assert isinstance(backend, WaylandCapture) + assert backend.codec == "png" + + +def test_a_missing_jpeg_library_refuses_to_start_rather_than_never_serving(monkeypatch, caplog): + """A server that is up but can never produce a frame is worse than one that + refuses to start: its socket still accepts connections.""" + import hhstream.jpeg as jpeg + + def explode(*_args, **_kwargs): + raise jpeg.JPEGError("no libjpeg-turbo anywhere") + + monkeypatch.setattr(jpeg, "TurboJPEG", explode) + assert _build_capture_backend("wayland", "jpeg", 80) is None + assert "cannot capture as jpeg" in caplog.text def test_the_backend_flag_survives_the_token_folding_that_capture_tokens_need(): @@ -125,6 +152,18 @@ def test_the_backend_flag_survives_the_token_folding_that_capture_tokens_need(): assert build_parser().parse_args(folded).capture == "wayland:Virtual-1" +def test_quality_defaults_to_match_the_command_backend(): + """`grim -q 80` is what the deployment used, so a swap must not change it.""" + args = build_parser().parse_args(["--capture", "wayland"]) + assert args.capture_quality == 80 + + +@pytest.mark.parametrize("bad", ["0", "101", "-5"]) +def test_an_impossible_quality_is_refused(bad): + with pytest.raises(SystemExit): + build_parser().parse_args(["--capture-quality", bad]) + + def test_no_backend_flag_still_means_the_command_backend(): args = build_parser().parse_args(["--socket", "/tmp/x.sock"]) assert args.capture is None