From 1ce39ba2106455c5ebf8fa871a4d5f9cbfe70b93 Mon Sep 17 00:00:00 2001 From: Hermenix Date: Wed, 30 Sep 2026 18:47:24 +0000 Subject: [PATCH] examples: agent-eyes-demo.py -- drive a browser on the headless screen and film it --- examples/agent-eyes-demo.py | 423 ++++++++++++++++++++++++++++++++++++ flake.nix | 3 + 2 files changed, 426 insertions(+) create mode 100644 examples/agent-eyes-demo.py diff --git a/examples/agent-eyes-demo.py b/examples/agent-eyes-demo.py new file mode 100644 index 0000000..76a98e3 --- /dev/null +++ b/examples/agent-eyes-demo.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +"""Agent Eyes demo: put a browser on the headless screen, drive it, film it. + +This is the *consumer* side of `docs/CONTRACT.md`: it reads frames off the +published socket (not from a screenshot API of its own) while a real browser, +running inside the headless Hyprland session, does something visible. The output +is an MP4 you can post anywhere. + +Why it exists: "Agent Eyes" is this machine's ability to look at a screen it can +drive. Everything else in this repo is the plumbing that makes the screen exist; +this is the smallest honest demonstration that a human can watch. + + nix develop + sg hhstream -c 'python3 examples/agent-eyes-demo.py --query "Ari Sosnovsky"' + +Preconditions, all of them checked below rather than assumed: + + * the headless session is up and `stream.sock` exists (services + `hyprland-headless` + `hyprland-headless-stream`); + * you can read it — that means being in the stream's group, hence `sg + hhstream` for an already-logged-in shell (`id` will not show it until you + log in again); + * nothing else is on that screen. Hyprland tiles, so a leftover window would + take half the frame. `--clear-screen` kills leftovers (needs sudo); + * `chromium` and `ffmpeg` on PATH — both come from the dev shell. + +It talks to the browser over the DevTools protocol rather than synthesising +input events, so the typing is real text entry into the real page and the script +can *read back* what happened. That read-back is the point: the demo asserts the +result page loaded instead of just hoping the frames look right. +""" + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import shutil +import socket +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from typing import NoReturn + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "src")) + +from hhstream import protocol # noqa: E402 (path set above) + +DEFAULT_SOCKET = "/run/hyprland-headless-stream/stream.sock" +DEFAULT_RUNTIME_DIR = "/run/hyprland-headless-stream" + + +def log(message: str) -> None: + print(f"[eyes] {message}", flush=True) + + +def fail(message: str) -> NoReturn: + print(f"[eyes] {message}", file=sys.stderr, flush=True) + raise SystemExit(1) + + +# -------------------------------------------------------------------------- +# The session: where the screen is, and how to reach it +# -------------------------------------------------------------------------- + +def wayland_display(runtime_dir: str) -> str: + """The compositor names its own socket (`wayland-N`), so ask the filesystem. + + Hardcoding `wayland-0` is the trap the module documents at length: a + preserved runtime directory can leave `wayland-0` behind from a crashed run + and push the live session to `wayland-1`. + """ + for entry in sorted(pathlib.Path(runtime_dir).glob("wayland-*")): + if entry.is_socket() and not entry.name.endswith(".lock"): + return entry.name + fail(f"no Wayland socket in {runtime_dir} — is hyprland-headless running?") + + +def session_env(runtime_dir: str) -> dict: + display = wayland_display(runtime_dir) + log(f"session: XDG_RUNTIME_DIR={runtime_dir} WAYLAND_DISPLAY={display}") + return { + **os.environ, + "XDG_RUNTIME_DIR": runtime_dir, + "WAYLAND_DISPLAY": display, + "XDG_SESSION_TYPE": "wayland", + } + + +def check_readable(socket_path: str) -> None: + if not pathlib.Path(socket_path).exists(): + fail(f"{socket_path} does not exist — is hyprland-headless-stream running?") + if not os.access(socket_path, os.R_OK | os.W_OK): + fail( + f"cannot use {socket_path} as {os.environ.get('USER', '?')}. The socket " + "is `0660 hhstream:hhstream` in a `0750` directory, so your user must " + "be in the `hhstream` group — run this under `sg hhstream -c '...'` if " + "the group was added since you logged in." + ) + + +# -------------------------------------------------------------------------- +# The browser +# -------------------------------------------------------------------------- + +def clear_screen(runtime_dir: str) -> None: + """Kill leftover browsers on the shared screen (needs sudo).""" + log("--clear-screen: killing leftover chromium processes") + result = subprocess.run( + ["sudo", "-n", "pkill", "-f", "chromium"], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + ) + if result.returncode not in (0, 1): # 1 == "nothing matched", which is fine + fail(f"could not clear the screen: {result.stderr.decode().strip()}") + time.sleep(1.0) + + +def start_browser(env: dict, *, url: str, port: int, home: pathlib.Path) -> subprocess.Popen: + home.mkdir(parents=True, exist_ok=True) + profile = home / "profile" + env = {**env, "HOME": str(home)} + argv = [ + "chromium", + "--ozone-platform=wayland", + # The compositor is a system service with no user manager and the kernel + # here has no working user-namespace sandbox set up; this is the same + # invocation the module's own manual testing used. + "--no-sandbox", + "--disable-dev-shm-usage", + "--kiosk", + "--window-size=1280,720", + f"--user-data-dir={profile}", + "--no-first-run", + "--no-default-browser-check", + f"--remote-debugging-port={port}", + # Keep the debug port on loopback: nothing outside this machine has any + # business driving the browser. + "--remote-debugging-address=127.0.0.1", + url, + ] + log(f"browser: starting chromium (debug port {port}) at {url}") + return subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + +def cdp_target(port: int, timeout: float = 30.0) -> str: + """Wait for the DevTools endpoint and return the page target's ws URL.""" + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list", + timeout=2) as response: + targets = json.load(response) + pages = [t for t in targets if t.get("type") == "page"] + if pages: + return pages[0]["webSocketDebuggerUrl"] + last = "endpoint up, no page target yet" + except (urllib.error.URLError, OSError, ValueError) as exc: + last = str(exc) + time.sleep(0.5) + fail(f"DevTools endpoint on port {port} never came up ({last})") + + +class DevTools: + """A very small CDP client: send a method, wait for its reply.""" + + def __init__(self, ws_url: str) -> None: + import websocket # type: ignore[import-not-found] # from the dev shell + + self._ws = websocket.create_connection(ws_url, timeout=20, + suppress_origin=True) + self._next_id = 0 + + def call(self, method: str, **params): + self._next_id += 1 + ident = self._next_id + self._ws.send(json.dumps({"id": ident, "method": method, "params": params})) + while True: + message = json.loads(self._ws.recv()) + if message.get("id") == ident: + if "error" in message: + raise RuntimeError(f"{method}: {message['error']}") + return message.get("result", {}) + + def evaluate(self, expression: str): + result = self.call("Runtime.evaluate", expression=expression, + returnByValue=True, awaitPromise=True) + return result.get("result", {}).get("value") + + def close(self) -> None: + self._ws.close() + + +def type_query(devtools: DevTools, query: str, per_char: float = 0.11) -> None: + """Focus the search box and type into it, one character at a time. + + `Input.insertText` goes through the renderer's normal text-entry path, so the + page sees exactly what a keyboard would produce (its autocomplete included) + and the characters appear on screen as they are sent — which is the whole + reason this is worth filming. + """ + focused = devtools.evaluate( + "(() => { const el = document.querySelector('textarea[name=q], input[name=q]');" + " if (!el) return false; el.focus(); return document.activeElement === el; })()" + ) + if not focused: + fail("could not focus the search box — did the page load?") + log(f"typing {query!r}") + for char in query: + devtools.call("Input.insertText", text=char) + time.sleep(per_char) + + +def submit_and_wait(devtools: DevTools, timeout: float = 15.0) -> str: + """Press Enter, then confirm the results page actually loaded.""" + before = devtools.evaluate("location.href") + devtools.call("Input.dispatchKeyEvent", type="rawKeyDown", key="Enter", + code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13) + devtools.call("Input.dispatchKeyEvent", type="keyUp", key="Enter", + code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13) + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + time.sleep(0.25) + now = devtools.evaluate("location.href") + if now and now != before and "search" in now: + return now + # Enter did not navigate: submit the form directly rather than leaving a + # video of a page that never went anywhere. + log("Enter did not navigate; submitting the form directly") + devtools.evaluate( + "(() => { const f = document.querySelector('form');" + " if (f) { f.submit(); return true; } return false; })()" + ) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + time.sleep(0.25) + now = devtools.evaluate("location.href") + if now and now != before: + return now + fail("the search never navigated — nothing to film") + + +# -------------------------------------------------------------------------- +# Recording: read the real frames off the socket +# -------------------------------------------------------------------------- + +class Recorder(threading.Thread): + """Pull frames off the published socket and write them as a numbered series. + + Deliberately not `hhstream-client`: that reads a fixed count and exits, and + here the frames have to arrive *while* something else happens. + """ + + def __init__(self, socket_path: str, out_dir: pathlib.Path, target_fps: float) -> None: + super().__init__(daemon=True) + self.socket_path = socket_path + self.out_dir = out_dir + self.target_fps = target_fps + self.hello: dict = {} + self.count = 0 + self.error: str | None = None + self._stop = threading.Event() + self._started = threading.Event() + + def run(self) -> None: + try: + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + conn.settimeout(10.0) + conn.connect(self.socket_path) + except OSError as exc: + self.error = str(exc) + self._started.set() + return + + try: + with conn.makefile("rb") as stream: + # CONTRACT: the Hello can take a moment (it is written on the + # first successful capture), so give it the socket timeout. + self.hello = protocol.read_hello(stream) + server_fps = float(self.hello.get("fps") or 30) + step = max(1, round(server_fps / self.target_fps)) + self._started.set() + index = 0 + while not self._stop.is_set(): + payload = protocol.read_frame(stream) + if payload is None: + break + index += 1 + if (index - 1) % step: + continue # decimate to the target rate + self.count += 1 + (self.out_dir / f"frame-{self.count:05d}.png").write_bytes(payload) + except (OSError, EOFError, ValueError) as exc: + self.error = str(exc) + finally: + conn.close() + self._started.set() + + def stop(self) -> None: + self._stop.set() + self.join(timeout=10) + + +def encode(frames: pathlib.Path, fps: float, out: pathlib.Path) -> None: + argv = [ + "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-framerate", f"{fps:.4f}", + "-i", str(frames / "frame-%05d.png"), + "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", + # yuv420p + faststart: what every player and every chat client wants. + "-pix_fmt", "yuv420p", "-movflags", "+faststart", + str(out), + ] + log(f"encoding {out.name}") + result = subprocess.run(argv, stderr=subprocess.PIPE) + if result.returncode != 0: + fail(f"ffmpeg failed: {result.stderr.decode().strip()}") + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="agent-eyes-demo.py", + description="Drive a browser on the headless screen and film it.", + ) + parser.add_argument("--query", default="Ari Sosnovsky", + help="what to type into the search box") + parser.add_argument("--output", default="agent-eyes.mp4", metavar="PATH") + parser.add_argument("--socket", default=DEFAULT_SOCKET) + parser.add_argument("--runtime-dir", default=DEFAULT_RUNTIME_DIR) + parser.add_argument("--seconds", type=float, default=14.0, + help="how long to record (default: %(default)s)") + parser.add_argument("--capture-fps", type=float, default=15.0, + help="frames per second to keep (default: %(default)s)") + parser.add_argument("--debug-port", type=int, default=9222) + parser.add_argument("--work-dir", type=pathlib.Path, default=None, + help="where frames land (default: a temp dir)") + parser.add_argument("--clear-screen", action="store_true", + help="kill leftover browsers first (needs passwordless sudo)") + parser.add_argument("--keep-work", action="store_true", + help="keep the frames and the browser profile") + args = parser.parse_args() + + for tool in ("chromium", "ffmpeg"): + if not shutil.which(tool): + fail(f"{tool} is not on PATH — run this inside `nix develop`") + + check_readable(args.socket) + env = session_env(args.runtime_dir) + if args.clear_screen: + clear_screen(args.runtime_dir) + + work = args.work_dir or pathlib.Path( + os.environ.get("TMPDIR", "/tmp")) / "agent-eyes-demo" + frames = work / "frames" + if frames.exists(): + shutil.rmtree(frames) + frames.mkdir(parents=True) + + out = pathlib.Path(args.output).resolve() + out.parent.mkdir(parents=True, exist_ok=True) + + browser = start_browser(env, url="https://www.google.com", + port=args.debug_port, home=work / "home") + devtools = None + recorder = None + try: + devtools = DevTools(cdp_target(args.debug_port)) + + recorder = Recorder(args.socket, frames, args.capture_fps) + recorder.start() + if not recorder._started.wait(timeout=15): + fail("the frame socket never answered") + if recorder.error: + fail(f"recording failed: {recorder.error}") + log(f"recording {args.seconds:.0f}s at {args.capture_fps:.0f}fps " + f"(server says {recorder.hello.get('fps')}fps, " + f"{recorder.hello.get('width')}x{recorder.hello.get('height')})") + + began = time.monotonic() + time.sleep(1.5) # let the page settle on screen + type_query(devtools, args.query) + landed = submit_and_wait(devtools) + log(f"landed on {landed}") + devtools.evaluate("window.scrollBy(0, 320)") # show actual results + + # Record for the requested wall-clock duration, so the video's length is + # what was asked for regardless of how long the page took to load. + remaining = args.seconds - (time.monotonic() - began) + if remaining > 0: + time.sleep(remaining) + finally: + if recorder is not None: + recorder.stop() + if devtools is not None: + devtools.close() + browser.terminate() + try: + browser.wait(timeout=10) + except subprocess.TimeoutExpired: + browser.kill() + + if recorder is None or recorder.count < 2: + fail("fewer than two frames arrived — nothing to encode") + + fps = recorder.count / args.seconds + log(f"captured {recorder.count} frames ({fps:.2f} fps effective)") + encode(frames, fps, out) + size = out.stat().st_size + log(f"wrote {out} ({size / 1024:.0f} KiB, {recorder.count} frames)") + + if not args.keep_work: + shutil.rmtree(work, ignore_errors=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/flake.nix b/flake.nix index b95600f..17138a9 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,9 @@ # The client side of the contract (hhstream-client), so a shell can # read the stream without a `nix build` first. (callPackage ./nix/package.nix { }) + # `websocket-client` drives Chromium over the DevTools protocol, which + # is how the demo types into the page and reads the result back. + python3.pkgs.websocket-client # Something to *put on* the headless screen. Chromium is the only # browser here that both runs on Wayland and can be driven over the # DevTools protocol, which is how examples/agent-eyes-demo.py types