#!/usr/bin/env python3 """Agent Eyes demo: put a browser on the headless screen, drive it, film it. This is the *consumer* side of `docs/CONTRACT.md`: it reads frames off the published socket (not from a screenshot API of its own) while a real browser, running inside the headless Hyprland session, does something visible. The output is an MP4 you can post anywhere. Why it exists: "Agent Eyes" is this machine's ability to look at a screen it can drive. Everything else in this repo is the plumbing that makes the screen exist; this is the smallest honest demonstration that a human can watch. nix develop sg hhstream -c 'python3 examples/agent-eyes-demo.py --query "Ari Sosnovsky"' Preconditions, all of them checked below rather than assumed: * the headless session is up and `stream.sock` exists (services `hyprland-headless` + `hyprland-headless-stream`); * you can read it — that means being in the stream's group, hence `sg hhstream` for an already-logged-in shell (`id` will not show it until you log in again); * nothing else is on that screen. Hyprland tiles, so a leftover window would take half the frame. `--clear-screen` kills leftovers (needs sudo); * `chromium` and `ffmpeg` on PATH — both come from the dev shell. It talks to the browser over the DevTools protocol rather than synthesising input events, so the typing is real text entry into the real page and the script can *read back* what happened. That read-back is the point: the demo asserts the result page loaded instead of just hoping the frames look right. """ from __future__ import annotations import argparse import json import os import pathlib import shutil import socket import subprocess import sys import threading import time import urllib.error import urllib.request from typing import NoReturn sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "src")) from hhstream import protocol # noqa: E402 (path set above) DEFAULT_SOCKET = "/run/hyprland-headless-stream/stream.sock" DEFAULT_RUNTIME_DIR = "/run/hyprland-headless-stream" def log(message: str) -> None: print(f"[eyes] {message}", flush=True) def fail(message: str) -> NoReturn: print(f"[eyes] {message}", file=sys.stderr, flush=True) raise SystemExit(1) # -------------------------------------------------------------------------- # The session: where the screen is, and how to reach it # -------------------------------------------------------------------------- def wayland_display(runtime_dir: str) -> str: """The compositor names its own socket (`wayland-N`), so ask the filesystem. Hardcoding `wayland-0` is the trap the module documents at length: a preserved runtime directory can leave `wayland-0` behind from a crashed run and push the live session to `wayland-1`. """ for entry in sorted(pathlib.Path(runtime_dir).glob("wayland-*")): if entry.is_socket() and not entry.name.endswith(".lock"): return entry.name fail(f"no Wayland socket in {runtime_dir} — is hyprland-headless running?") def session_env(runtime_dir: str) -> dict: display = wayland_display(runtime_dir) log(f"session: XDG_RUNTIME_DIR={runtime_dir} WAYLAND_DISPLAY={display}") return { **os.environ, "XDG_RUNTIME_DIR": runtime_dir, "WAYLAND_DISPLAY": display, "XDG_SESSION_TYPE": "wayland", } def check_readable(socket_path: str) -> None: if not pathlib.Path(socket_path).exists(): fail(f"{socket_path} does not exist — is hyprland-headless-stream running?") if not os.access(socket_path, os.R_OK | os.W_OK): fail( f"cannot use {socket_path} as {os.environ.get('USER', '?')}. The socket " "is `0660 hhstream:hhstream` in a `0750` directory, so your user must " "be in the `hhstream` group — run this under `sg hhstream -c '...'` if " "the group was added since you logged in." ) # -------------------------------------------------------------------------- # The browser # -------------------------------------------------------------------------- def clear_screen(runtime_dir: str) -> None: """Kill leftover browsers on the shared screen (needs sudo).""" log("--clear-screen: killing leftover chromium processes") # The bracket keeps the pattern from matching `pkill`'s own command line — # `pkill -f chromium` kills the shell that ran it, which is a confusing way # to lose a session. result = subprocess.run( ["sudo", "-n", "pkill", "-f", "[c]hromium"], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, ) if result.returncode not in (0, 1): # 1 == "nothing matched", which is fine fail(f"could not clear the screen: {result.stderr.decode().strip()}") time.sleep(1.0) def _browser_argv(env: dict, *, url: str, port: int, home: pathlib.Path, fullscreen: bool, unsandboxed: bool) -> list[str]: profile = home / "profile" return [ "chromium", "--ozone-platform=wayland", # The sandbox works on this machine, so it is on by default. When it # does not, the fallback needs `--test-type` as well: without it Chromium # paints "you are using an unsupported command-line flag: --no-sandbox" # across the top of the page, and that banner would be in every frame of # the recording. *(["--no-sandbox", "--test-type"] if unsandboxed else []), "--disable-dev-shm-usage", # Windowed by default: the session's wallpaper, gaps and window borders # are the point of filming a *desktop*, and a kiosk window hides all of # them. `--fullscreen` brings back the full-bleed browser. *(["--kiosk"] if fullscreen else []), "--window-size=1040,560", f"--user-data-dir={profile}", "--no-first-run", "--no-default-browser-check", f"--remote-debugging-port={port}", # Keep the debug port on loopback: nothing outside this machine has any # business driving the browser. "--remote-debugging-address=127.0.0.1", url, ] def start_browser(env: dict, *, url: str, port: int, home: pathlib.Path, fullscreen: bool = False) -> subprocess.Popen: home.mkdir(parents=True, exist_ok=True) env = {**env, "HOME": str(home)} argv = _browser_argv(env, url=url, port=port, home=home, fullscreen=fullscreen, unsandboxed=False) log(f"browser: starting chromium (debug port {port}, " f"{'kiosk' if fullscreen else 'windowed'}) at {url}") proc = subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # A sandboxed Chromium can die on launch (no user namespaces, no setuid # helper). Give it a moment, and fall back rather than filming an empty # desktop and calling it a demo. time.sleep(2.5) if proc.poll() is not None: log("browser: sandboxed launch exited; retrying with --no-sandbox") argv = _browser_argv(env, url=url, port=port, home=home, fullscreen=fullscreen, unsandboxed=True) proc = subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) return proc def cdp_target(port: int, timeout: float = 30.0) -> str: """Wait for the DevTools endpoint and return the page target's ws URL.""" deadline = time.monotonic() + timeout last = "" while time.monotonic() < deadline: try: with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list", timeout=2) as response: targets = json.load(response) pages = [t for t in targets if t.get("type") == "page"] if pages: return pages[0]["webSocketDebuggerUrl"] last = "endpoint up, no page target yet" except (urllib.error.URLError, OSError, ValueError) as exc: last = str(exc) time.sleep(0.5) fail(f"DevTools endpoint on port {port} never came up ({last})") class DevTools: """A very small CDP client: send a method, wait for its reply.""" def __init__(self, ws_url: str) -> None: import websocket # type: ignore[import-not-found] # from the dev shell self._ws = websocket.create_connection(ws_url, timeout=20, suppress_origin=True) self._next_id = 0 def call(self, method: str, **params): self._next_id += 1 ident = self._next_id self._ws.send(json.dumps({"id": ident, "method": method, "params": params})) while True: message = json.loads(self._ws.recv()) if message.get("id") == ident: if "error" in message: raise RuntimeError(f"{method}: {message['error']}") return message.get("result", {}) def evaluate(self, expression: str): result = self.call("Runtime.evaluate", expression=expression, returnByValue=True, awaitPromise=True) return result.get("result", {}).get("value") def close(self) -> None: self._ws.close() def is_walled(devtools: DevTools) -> bool: """Google's bot wall, however it presents itself. Both checks are needed. The URL is the obvious tell (`/sorry/index?...`), but the redirect can arrive *after* the results page has already painted — so a guard that only runs right after the search passes and then films a CAPTCHA for sixteen seconds. Hence: check the URL, check the page text, and call this again once the recording has finished. """ url = devtools.evaluate("location.href") or "" if "/sorry/" in url: return True text = devtools.evaluate( "(() => { const b = document.body; return b ? b.innerText.slice(0, 600) : ''; })()") or "" return "unusual traffic" in text.lower() def type_query(devtools: DevTools, query: str, per_char: float = 0.11) -> None: """Focus the search box and type into it, one character at a time. `Input.insertText` goes through the renderer's normal text-entry path, so the page sees exactly what a keyboard would produce (its autocomplete included) and the characters appear on screen as they are sent — which is the whole reason this is worth filming. """ focused = devtools.evaluate( "(() => { const el = document.querySelector('textarea[name=q], input[name=q]');" " if (!el) return false; el.focus(); return document.activeElement === el; })()" ) if not focused: fail("could not focus the search box — did the page load?") log(f"typing {query!r}") for char in query: devtools.call("Input.insertText", text=char) time.sleep(per_char) def submit_and_wait(devtools: DevTools, timeout: float = 15.0) -> str: """Press Enter, then confirm the results page actually loaded.""" before = devtools.evaluate("location.href") devtools.call("Input.dispatchKeyEvent", type="rawKeyDown", key="Enter", code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13) devtools.call("Input.dispatchKeyEvent", type="keyUp", key="Enter", code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13) deadline = time.monotonic() + timeout while time.monotonic() < deadline: time.sleep(0.25) now = devtools.evaluate("location.href") if now and now != before and "search" in now: return now # Enter did not navigate: submit the form directly rather than leaving a # video of a page that never went anywhere. log("Enter did not navigate; submitting the form directly") devtools.evaluate( "(() => { const f = document.querySelector('form');" " if (f) { f.submit(); return true; } return false; })()" ) deadline = time.monotonic() + timeout while time.monotonic() < deadline: time.sleep(0.25) now = devtools.evaluate("location.href") if now and now != before: return now fail("the search never navigated — nothing to film") # -------------------------------------------------------------------------- # Recording: read the real frames off the socket # -------------------------------------------------------------------------- class Recorder(threading.Thread): """Pull frames off the published socket and write them as a numbered series. Deliberately not `hhstream-client`: that reads a fixed count and exits, and here the frames have to arrive *while* something else happens. """ def __init__(self, socket_path: str, out_dir: pathlib.Path, target_fps: float) -> None: super().__init__(daemon=True) self.socket_path = socket_path self.out_dir = out_dir self.target_fps = target_fps self.hello: dict = {} # Frame extension follows the Hello, not an assumption: the stream can # legitimately be JPEG, and the caller must not have to know which. self.ext = "png" self.count = 0 self.error: str | None = None # NOT `_stop` / `_started`: `threading.Thread` owns both names. It sets # `_started` itself as the thread boots — *before* `run()` — so a # `wait()` on it returns instantly and every attribute `run()` is # supposed to fill in is still empty. It also calls `self._stop()` on # the way out, which an Event cannot answer to. self._stop_event = threading.Event() self._ready = threading.Event() def run(self) -> None: try: conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) conn.settimeout(10.0) conn.connect(self.socket_path) except OSError as exc: self.error = str(exc) self._ready.set() return try: with conn.makefile("rb") as stream: # CONTRACT: the Hello can take a moment (it is written on the # first successful capture), so give it the socket timeout. self.hello = protocol.read_hello(stream) self.ext = {"image/png": "png", "image/jpeg": "jpg"}.get( str(self.hello.get("format", "image/png")), "png") server_fps = float(self.hello.get("fps") or 30) step = max(1, round(server_fps / self.target_fps)) self._ready.set() index = 0 while not self._stop_event.is_set(): payload = protocol.read_frame(stream) if payload is None: break index += 1 if (index - 1) % step: continue # decimate to the target rate self.count += 1 (self.out_dir / f"frame-{self.count:05d}.{self.ext}").write_bytes(payload) except (OSError, EOFError, ValueError) as exc: self.error = str(exc) finally: conn.close() self._ready.set() def stop(self) -> None: self._stop_event.set() self.join(timeout=10) def encode(frames: pathlib.Path, fps: float, out: pathlib.Path, ext: str = "png") -> None: argv = [ "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-framerate", f"{fps:.4f}", "-i", str(frames / f"frame-%05d.{ext}"), "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", # yuv420p + faststart: what every player and every chat client wants. "-pix_fmt", "yuv420p", "-movflags", "+faststart", # Strip every metadata tag. Xiph/Blossom media stores reject files that # carry them — Buzz's does, with `422 media contains metadata or a # non-canonical metadata channel`, and the only thing to fix is ffmpeg's # own `encoder`/`handler_name` tags. `bitexact` keeps them from being # written back in. "-map_metadata", "-1", "-fflags", "+bitexact", "-flags:v", "+bitexact", "-an", str(out), ] log(f"encoding {out.name}") result = subprocess.run(argv, stderr=subprocess.PIPE) if result.returncode != 0: fail(f"ffmpeg failed: {result.stderr.decode().strip()}") def main() -> int: parser = argparse.ArgumentParser( prog="agent-eyes-demo.py", description="Drive a browser on the headless screen and film it.", ) parser.add_argument("--query", default="Ari Sosnovsky", help="what to type into the search box") parser.add_argument("--output", default="agent-eyes.mp4", metavar="PATH") parser.add_argument("--socket", default=DEFAULT_SOCKET) parser.add_argument("--runtime-dir", default=DEFAULT_RUNTIME_DIR) parser.add_argument("--seconds", type=float, default=14.0, help="how long to record (default: %(default)s)") parser.add_argument("--capture-fps", type=float, default=15.0, help="frames per second to keep (default: %(default)s)") parser.add_argument("--debug-port", type=int, default=9222) parser.add_argument("--fullscreen", action="store_true", help="kiosk/full-bleed browser instead of a windowed one " "(hides the session's wallpaper and window borders)") parser.add_argument("--work-dir", type=pathlib.Path, default=None, help="where frames land (default: a temp dir)") parser.add_argument("--clear-screen", action="store_true", help="kill leftover browsers first (needs passwordless sudo)") parser.add_argument("--keep-work", action="store_true", help="keep the frames and the browser profile") args = parser.parse_args() for tool in ("chromium", "ffmpeg"): if not shutil.which(tool): fail(f"{tool} is not on PATH — run this inside `nix develop`") check_readable(args.socket) env = session_env(args.runtime_dir) if args.clear_screen: clear_screen(args.runtime_dir) work = args.work_dir or pathlib.Path( os.environ.get("TMPDIR", "/tmp")) / "agent-eyes-demo" frames = work / "frames" if frames.exists(): shutil.rmtree(frames) frames.mkdir(parents=True) out = pathlib.Path(args.output).resolve() out.parent.mkdir(parents=True, exist_ok=True) browser = start_browser(env, url="https://www.google.com", port=args.debug_port, home=work / "home", fullscreen=args.fullscreen) devtools = None recorder = None try: devtools = DevTools(cdp_target(args.debug_port)) recorder = Recorder(args.socket, frames, args.capture_fps) recorder.start() if not recorder._ready.wait(timeout=15): fail("the frame socket never answered") if recorder.error: fail(f"recording failed: {recorder.error}") log(f"recording {args.seconds:.0f}s at {args.capture_fps:.0f}fps " f"(server says {recorder.hello.get('fps')}fps, " f"{recorder.hello.get('width')}x{recorder.hello.get('height')})") began = time.monotonic() time.sleep(1.5) # let the page settle on screen type_query(devtools, args.query) landed = submit_and_wait(devtools) log(f"landed on {landed}") # The wall can arrive as a redirect *after* the results paint, so give # it a beat and then look again rather than trusting the first URL. time.sleep(1.5) if is_walled(devtools): # A video that looks like a working demo and is not one is worse # than no video. fail( "Google served its 'unusual traffic' wall instead of results " "(rate-limited IP, not a broken browser). Wait a few minutes and " "run again; the first search from a fresh profile usually passes." ) devtools.evaluate("window.scrollBy(0, 320)") # show actual results # Record for the requested wall-clock duration, so the video's length is # what was asked for regardless of how long the page took to load. remaining = args.seconds - (time.monotonic() - began) if remaining > 0: time.sleep(remaining) # Final gate, after the whole recording: if the page ended up on the # wall at any point, nothing is encoded. if is_walled(devtools): fail("Google walled the search during recording — no video written") finally: if recorder is not None: recorder.stop() if devtools is not None: devtools.close() browser.terminate() try: browser.wait(timeout=10) except subprocess.TimeoutExpired: browser.kill() if recorder is None or recorder.count < 2: fail("fewer than two frames arrived — nothing to encode") fps = recorder.count / args.seconds log(f"captured {recorder.count} frames ({fps:.2f} fps effective)") encode(frames, fps, out, recorder.ext) size = out.stat().st_size log(f"wrote {out} ({size / 1024:.0f} KiB, {recorder.count} frames)") if not args.keep_work: shutil.rmtree(work, ignore_errors=True) return 0 if __name__ == "__main__": raise SystemExit(main())