#!/usr/bin/env python3 """Agent Eyes demo: put a browser on the headless screen, drive it, film it. This is the *consumer* side of `docs/CONTRACT.md`: it reads frames off the published socket (not from a screenshot API of its own) while a real browser, running inside the headless Hyprland session, does something visible. The output is an MP4 you can post anywhere. Why it exists: "Agent Eyes" is this machine's ability to look at a screen it can drive. Everything else in this repo is the plumbing that makes the screen exist; this is the smallest honest demonstration that a human can watch. nix develop sg hhstream -c 'python3 examples/agent-eyes-demo.py --query "Ari Sosnovsky"' Preconditions, all of them checked below rather than assumed: * the headless session is up and `stream.sock` exists (services `hyprland-headless` + `hyprland-headless-stream`); * you can read it — that means being in the stream's group, hence `sg hhstream` for an already-logged-in shell (`id` will not show it until you log in again); * nothing else is on that screen. Hyprland tiles, so a leftover window would take half the frame. `--clear-screen` kills leftovers (needs sudo); * `chromium` and `ffmpeg` on PATH — both come from the dev shell. It talks to the browser over the DevTools protocol rather than synthesising input events, so the typing is real text entry into the real page and the script can *read back* what happened. That read-back is the point: the demo asserts the result page loaded instead of just hoping the frames look right. """ from __future__ import annotations import argparse import json import os import pathlib import shutil import socket import subprocess import sys import threading import time import urllib.error import urllib.request from typing import NoReturn sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "src")) from hhstream import protocol # noqa: E402 (path set above) DEFAULT_SOCKET = "/run/hyprland-headless-stream/stream.sock" DEFAULT_RUNTIME_DIR = "/run/hyprland-headless-stream" def log(message: str) -> None: print(f"[eyes] {message}", flush=True) def fail(message: str) -> NoReturn: print(f"[eyes] {message}", file=sys.stderr, flush=True) raise SystemExit(1) # -------------------------------------------------------------------------- # The session: where the screen is, and how to reach it # -------------------------------------------------------------------------- def wayland_display(runtime_dir: str) -> str: """The compositor names its own socket (`wayland-N`), so ask the filesystem. Hardcoding `wayland-0` is the trap the module documents at length: a preserved runtime directory can leave `wayland-0` behind from a crashed run and push the live session to `wayland-1`. """ for entry in sorted(pathlib.Path(runtime_dir).glob("wayland-*")): if entry.is_socket() and not entry.name.endswith(".lock"): return entry.name fail(f"no Wayland socket in {runtime_dir} — is hyprland-headless running?") def session_env(runtime_dir: str) -> dict: display = wayland_display(runtime_dir) log(f"session: XDG_RUNTIME_DIR={runtime_dir} WAYLAND_DISPLAY={display}") return { **os.environ, "XDG_RUNTIME_DIR": runtime_dir, "WAYLAND_DISPLAY": display, "XDG_SESSION_TYPE": "wayland", } def check_readable(socket_path: str) -> None: if not pathlib.Path(socket_path).exists(): fail(f"{socket_path} does not exist — is hyprland-headless-stream running?") if not os.access(socket_path, os.R_OK | os.W_OK): fail( f"cannot use {socket_path} as {os.environ.get('USER', '?')}. The socket " "is `0660 hhstream:hhstream` in a `0750` directory, so your user must " "be in the `hhstream` group — run this under `sg hhstream -c '...'` if " "the group was added since you logged in." ) # -------------------------------------------------------------------------- # The browser # -------------------------------------------------------------------------- def clear_screen(runtime_dir: str) -> None: """Kill leftover browsers on the shared screen (needs sudo).""" log("--clear-screen: killing leftover chromium processes") result = subprocess.run( ["sudo", "-n", "pkill", "-f", "chromium"], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, ) if result.returncode not in (0, 1): # 1 == "nothing matched", which is fine fail(f"could not clear the screen: {result.stderr.decode().strip()}") time.sleep(1.0) def start_browser(env: dict, *, url: str, port: int, home: pathlib.Path) -> subprocess.Popen: home.mkdir(parents=True, exist_ok=True) profile = home / "profile" env = {**env, "HOME": str(home)} argv = [ "chromium", "--ozone-platform=wayland", # The compositor is a system service with no user manager and the kernel # here has no working user-namespace sandbox set up; this is the same # invocation the module's own manual testing used. "--no-sandbox", "--disable-dev-shm-usage", "--kiosk", "--window-size=1280,720", f"--user-data-dir={profile}", "--no-first-run", "--no-default-browser-check", f"--remote-debugging-port={port}", # Keep the debug port on loopback: nothing outside this machine has any # business driving the browser. "--remote-debugging-address=127.0.0.1", url, ] log(f"browser: starting chromium (debug port {port}) at {url}") return subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def cdp_target(port: int, timeout: float = 30.0) -> str: """Wait for the DevTools endpoint and return the page target's ws URL.""" deadline = time.monotonic() + timeout last = "" while time.monotonic() < deadline: try: with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list", timeout=2) as response: targets = json.load(response) pages = [t for t in targets if t.get("type") == "page"] if pages: return pages[0]["webSocketDebuggerUrl"] last = "endpoint up, no page target yet" except (urllib.error.URLError, OSError, ValueError) as exc: last = str(exc) time.sleep(0.5) fail(f"DevTools endpoint on port {port} never came up ({last})") class DevTools: """A very small CDP client: send a method, wait for its reply.""" def __init__(self, ws_url: str) -> None: import websocket # type: ignore[import-not-found] # from the dev shell self._ws = websocket.create_connection(ws_url, timeout=20, suppress_origin=True) self._next_id = 0 def call(self, method: str, **params): self._next_id += 1 ident = self._next_id self._ws.send(json.dumps({"id": ident, "method": method, "params": params})) while True: message = json.loads(self._ws.recv()) if message.get("id") == ident: if "error" in message: raise RuntimeError(f"{method}: {message['error']}") return message.get("result", {}) def evaluate(self, expression: str): result = self.call("Runtime.evaluate", expression=expression, returnByValue=True, awaitPromise=True) return result.get("result", {}).get("value") def close(self) -> None: self._ws.close() def type_query(devtools: DevTools, query: str, per_char: float = 0.11) -> None: """Focus the search box and type into it, one character at a time. `Input.insertText` goes through the renderer's normal text-entry path, so the page sees exactly what a keyboard would produce (its autocomplete included) and the characters appear on screen as they are sent — which is the whole reason this is worth filming. """ focused = devtools.evaluate( "(() => { const el = document.querySelector('textarea[name=q], input[name=q]');" " if (!el) return false; el.focus(); return document.activeElement === el; })()" ) if not focused: fail("could not focus the search box — did the page load?") log(f"typing {query!r}") for char in query: devtools.call("Input.insertText", text=char) time.sleep(per_char) def submit_and_wait(devtools: DevTools, timeout: float = 15.0) -> str: """Press Enter, then confirm the results page actually loaded.""" before = devtools.evaluate("location.href") devtools.call("Input.dispatchKeyEvent", type="rawKeyDown", key="Enter", code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13) devtools.call("Input.dispatchKeyEvent", type="keyUp", key="Enter", code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13) deadline = time.monotonic() + timeout while time.monotonic() < deadline: time.sleep(0.25) now = devtools.evaluate("location.href") if now and now != before and "search" in now: return now # Enter did not navigate: submit the form directly rather than leaving a # video of a page that never went anywhere. log("Enter did not navigate; submitting the form directly") devtools.evaluate( "(() => { const f = document.querySelector('form');" " if (f) { f.submit(); return true; } return false; })()" ) deadline = time.monotonic() + timeout while time.monotonic() < deadline: time.sleep(0.25) now = devtools.evaluate("location.href") if now and now != before: return now fail("the search never navigated — nothing to film") # -------------------------------------------------------------------------- # Recording: read the real frames off the socket # -------------------------------------------------------------------------- class Recorder(threading.Thread): """Pull frames off the published socket and write them as a numbered series. Deliberately not `hhstream-client`: that reads a fixed count and exits, and here the frames have to arrive *while* something else happens. """ def __init__(self, socket_path: str, out_dir: pathlib.Path, target_fps: float) -> None: super().__init__(daemon=True) self.socket_path = socket_path self.out_dir = out_dir self.target_fps = target_fps self.hello: dict = {} self.count = 0 self.error: str | None = None self._stop = threading.Event() self._started = threading.Event() def run(self) -> None: try: conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) conn.settimeout(10.0) conn.connect(self.socket_path) except OSError as exc: self.error = str(exc) self._started.set() return try: with conn.makefile("rb") as stream: # CONTRACT: the Hello can take a moment (it is written on the # first successful capture), so give it the socket timeout. self.hello = protocol.read_hello(stream) server_fps = float(self.hello.get("fps") or 30) step = max(1, round(server_fps / self.target_fps)) self._started.set() index = 0 while not self._stop.is_set(): payload = protocol.read_frame(stream) if payload is None: break index += 1 if (index - 1) % step: continue # decimate to the target rate self.count += 1 (self.out_dir / f"frame-{self.count:05d}.png").write_bytes(payload) except (OSError, EOFError, ValueError) as exc: self.error = str(exc) finally: conn.close() self._started.set() def stop(self) -> None: self._stop.set() self.join(timeout=10) def encode(frames: pathlib.Path, fps: float, out: pathlib.Path) -> None: argv = [ "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-framerate", f"{fps:.4f}", "-i", str(frames / "frame-%05d.png"), "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", # yuv420p + faststart: what every player and every chat client wants. "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(out), ] log(f"encoding {out.name}") result = subprocess.run(argv, stderr=subprocess.PIPE) if result.returncode != 0: fail(f"ffmpeg failed: {result.stderr.decode().strip()}") def main() -> int: parser = argparse.ArgumentParser( prog="agent-eyes-demo.py", description="Drive a browser on the headless screen and film it.", ) parser.add_argument("--query", default="Ari Sosnovsky", help="what to type into the search box") parser.add_argument("--output", default="agent-eyes.mp4", metavar="PATH") parser.add_argument("--socket", default=DEFAULT_SOCKET) parser.add_argument("--runtime-dir", default=DEFAULT_RUNTIME_DIR) parser.add_argument("--seconds", type=float, default=14.0, help="how long to record (default: %(default)s)") parser.add_argument("--capture-fps", type=float, default=15.0, help="frames per second to keep (default: %(default)s)") parser.add_argument("--debug-port", type=int, default=9222) parser.add_argument("--work-dir", type=pathlib.Path, default=None, help="where frames land (default: a temp dir)") parser.add_argument("--clear-screen", action="store_true", help="kill leftover browsers first (needs passwordless sudo)") parser.add_argument("--keep-work", action="store_true", help="keep the frames and the browser profile") args = parser.parse_args() for tool in ("chromium", "ffmpeg"): if not shutil.which(tool): fail(f"{tool} is not on PATH — run this inside `nix develop`") check_readable(args.socket) env = session_env(args.runtime_dir) if args.clear_screen: clear_screen(args.runtime_dir) work = args.work_dir or pathlib.Path( os.environ.get("TMPDIR", "/tmp")) / "agent-eyes-demo" frames = work / "frames" if frames.exists(): shutil.rmtree(frames) frames.mkdir(parents=True) out = pathlib.Path(args.output).resolve() out.parent.mkdir(parents=True, exist_ok=True) browser = start_browser(env, url="https://www.google.com", port=args.debug_port, home=work / "home") devtools = None recorder = None try: devtools = DevTools(cdp_target(args.debug_port)) recorder = Recorder(args.socket, frames, args.capture_fps) recorder.start() if not recorder._started.wait(timeout=15): fail("the frame socket never answered") if recorder.error: fail(f"recording failed: {recorder.error}") log(f"recording {args.seconds:.0f}s at {args.capture_fps:.0f}fps " f"(server says {recorder.hello.get('fps')}fps, " f"{recorder.hello.get('width')}x{recorder.hello.get('height')})") began = time.monotonic() time.sleep(1.5) # let the page settle on screen type_query(devtools, args.query) landed = submit_and_wait(devtools) log(f"landed on {landed}") devtools.evaluate("window.scrollBy(0, 320)") # show actual results # Record for the requested wall-clock duration, so the video's length is # what was asked for regardless of how long the page took to load. remaining = args.seconds - (time.monotonic() - began) if remaining > 0: time.sleep(remaining) finally: if recorder is not None: recorder.stop() if devtools is not None: devtools.close() browser.terminate() try: browser.wait(timeout=10) except subprocess.TimeoutExpired: browser.kill() if recorder is None or recorder.count < 2: fail("fewer than two frames arrived — nothing to encode") fps = recorder.count / args.seconds log(f"captured {recorder.count} frames ({fps:.2f} fps effective)") encode(frames, fps, out) size = out.stat().st_size log(f"wrote {out} ({size / 1024:.0f} KiB, {recorder.count} frames)") if not args.keep_work: shutil.rmtree(work, ignore_errors=True) return 0 if __name__ == "__main__": raise SystemExit(main())