examples: agent-eyes-demo.py -- drive a browser on the headless screen and film it

This commit is contained in:
Hermenix committed 2026-09-30 18:47:24 +00:00
1 parent a77eaa96d3
commit 1ce39ba210
2 files changed
+426

No files matched your search

+423
View File
@@ -0,0 +1,423 @@
#!/usr/bin/env python3
"""Agent Eyes demo: put a browser on the headless screen, drive it, film it.
This is the *consumer* side of `docs/CONTRACT.md`: it reads frames off the
published socket (not from a screenshot API of its own) while a real browser,
running inside the headless Hyprland session, does something visible. The output
is an MP4 you can post anywhere.
Why it exists: "Agent Eyes" is this machine's ability to look at a screen it can
drive. Everything else in this repo is the plumbing that makes the screen exist;
this is the smallest honest demonstration that a human can watch.
nix develop
sg hhstream -c 'python3 examples/agent-eyes-demo.py --query "Ari Sosnovsky"'
Preconditions, all of them checked below rather than assumed:
* the headless session is up and `stream.sock` exists (services
`hyprland-headless` + `hyprland-headless-stream`);
* you can read it — that means being in the stream's group, hence `sg
hhstream` for an already-logged-in shell (`id` will not show it until you
log in again);
* nothing else is on that screen. Hyprland tiles, so a leftover window would
take half the frame. `--clear-screen` kills leftovers (needs sudo);
* `chromium` and `ffmpeg` on PATH — both come from the dev shell.
It talks to the browser over the DevTools protocol rather than synthesising
input events, so the typing is real text entry into the real page and the script
can *read back* what happened. That read-back is the point: the demo asserts the
result page loaded instead of just hoping the frames look right.
"""
from __future__ import annotations
import argparse
import json
import os
import pathlib
import shutil
import socket
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from typing import NoReturn
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "src"))
from hhstream import protocol # noqa: E402 (path set above)
DEFAULT_SOCKET = "/run/hyprland-headless-stream/stream.sock"
DEFAULT_RUNTIME_DIR = "/run/hyprland-headless-stream"
def log(message: str) -> None:
print(f"[eyes] {message}", flush=True)
def fail(message: str) -> NoReturn:
print(f"[eyes] {message}", file=sys.stderr, flush=True)
raise SystemExit(1)
# --------------------------------------------------------------------------
# The session: where the screen is, and how to reach it
# --------------------------------------------------------------------------
def wayland_display(runtime_dir: str) -> str:
"""The compositor names its own socket (`wayland-N`), so ask the filesystem.
Hardcoding `wayland-0` is the trap the module documents at length: a
preserved runtime directory can leave `wayland-0` behind from a crashed run
and push the live session to `wayland-1`.
"""
for entry in sorted(pathlib.Path(runtime_dir).glob("wayland-*")):
if entry.is_socket() and not entry.name.endswith(".lock"):
return entry.name
fail(f"no Wayland socket in {runtime_dir} — is hyprland-headless running?")
def session_env(runtime_dir: str) -> dict:
display = wayland_display(runtime_dir)
log(f"session: XDG_RUNTIME_DIR={runtime_dir} WAYLAND_DISPLAY={display}")
return {
**os.environ,
"XDG_RUNTIME_DIR": runtime_dir,
"WAYLAND_DISPLAY": display,
"XDG_SESSION_TYPE": "wayland",
}
def check_readable(socket_path: str) -> None:
if not pathlib.Path(socket_path).exists():
fail(f"{socket_path} does not exist — is hyprland-headless-stream running?")
if not os.access(socket_path, os.R_OK | os.W_OK):
fail(
f"cannot use {socket_path} as {os.environ.get('USER', '?')}. The socket "
"is `0660 hhstream:hhstream` in a `0750` directory, so your user must "
"be in the `hhstream` group — run this under `sg hhstream -c '...'` if "
"the group was added since you logged in."
)
# --------------------------------------------------------------------------
# The browser
# --------------------------------------------------------------------------
def clear_screen(runtime_dir: str) -> None:
"""Kill leftover browsers on the shared screen (needs sudo)."""
log("--clear-screen: killing leftover chromium processes")
result = subprocess.run(
["sudo", "-n", "pkill", "-f", "chromium"],
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
)
if result.returncode not in (0, 1): # 1 == "nothing matched", which is fine
fail(f"could not clear the screen: {result.stderr.decode().strip()}")
time.sleep(1.0)
def start_browser(env: dict, *, url: str, port: int, home: pathlib.Path) -> subprocess.Popen:
home.mkdir(parents=True, exist_ok=True)
profile = home / "profile"
env = {**env, "HOME": str(home)}
argv = [
"chromium",
"--ozone-platform=wayland",
# The compositor is a system service with no user manager and the kernel
# here has no working user-namespace sandbox set up; this is the same
# invocation the module's own manual testing used.
"--no-sandbox",
"--disable-dev-shm-usage",
"--kiosk",
"--window-size=1280,720",
f"--user-data-dir={profile}",
"--no-first-run",
"--no-default-browser-check",
f"--remote-debugging-port={port}",
# Keep the debug port on loopback: nothing outside this machine has any
# business driving the browser.
"--remote-debugging-address=127.0.0.1",
url,
]
log(f"browser: starting chromium (debug port {port}) at {url}")
return subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
def cdp_target(port: int, timeout: float = 30.0) -> str:
"""Wait for the DevTools endpoint and return the page target's ws URL."""
deadline = time.monotonic() + timeout
last = ""
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list",
timeout=2) as response:
targets = json.load(response)
pages = [t for t in targets if t.get("type") == "page"]
if pages:
return pages[0]["webSocketDebuggerUrl"]
last = "endpoint up, no page target yet"
except (urllib.error.URLError, OSError, ValueError) as exc:
last = str(exc)
time.sleep(0.5)
fail(f"DevTools endpoint on port {port} never came up ({last})")
class DevTools:
"""A very small CDP client: send a method, wait for its reply."""
def __init__(self, ws_url: str) -> None:
import websocket # type: ignore[import-not-found] # from the dev shell
self._ws = websocket.create_connection(ws_url, timeout=20,
suppress_origin=True)
self._next_id = 0
def call(self, method: str, **params):
self._next_id += 1
ident = self._next_id
self._ws.send(json.dumps({"id": ident, "method": method, "params": params}))
while True:
message = json.loads(self._ws.recv())
if message.get("id") == ident:
if "error" in message:
raise RuntimeError(f"{method}: {message['error']}")
return message.get("result", {})
def evaluate(self, expression: str):
result = self.call("Runtime.evaluate", expression=expression,
returnByValue=True, awaitPromise=True)
return result.get("result", {}).get("value")
def close(self) -> None:
self._ws.close()
def type_query(devtools: DevTools, query: str, per_char: float = 0.11) -> None:
"""Focus the search box and type into it, one character at a time.
`Input.insertText` goes through the renderer's normal text-entry path, so the
page sees exactly what a keyboard would produce (its autocomplete included)
and the characters appear on screen as they are sent — which is the whole
reason this is worth filming.
"""
focused = devtools.evaluate(
"(() => { const el = document.querySelector('textarea[name=q], input[name=q]');"
" if (!el) return false; el.focus(); return document.activeElement === el; })()"
)
if not focused:
fail("could not focus the search box — did the page load?")
log(f"typing {query!r}")
for char in query:
devtools.call("Input.insertText", text=char)
time.sleep(per_char)
def submit_and_wait(devtools: DevTools, timeout: float = 15.0) -> str:
"""Press Enter, then confirm the results page actually loaded."""
before = devtools.evaluate("location.href")
devtools.call("Input.dispatchKeyEvent", type="rawKeyDown", key="Enter",
code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13)
devtools.call("Input.dispatchKeyEvent", type="keyUp", key="Enter",
code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13)
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
time.sleep(0.25)
now = devtools.evaluate("location.href")
if now and now != before and "search" in now:
return now
# Enter did not navigate: submit the form directly rather than leaving a
# video of a page that never went anywhere.
log("Enter did not navigate; submitting the form directly")
devtools.evaluate(
"(() => { const f = document.querySelector('form');"
" if (f) { f.submit(); return true; } return false; })()"
)
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
time.sleep(0.25)
now = devtools.evaluate("location.href")
if now and now != before:
return now
fail("the search never navigated — nothing to film")
# --------------------------------------------------------------------------
# Recording: read the real frames off the socket
# --------------------------------------------------------------------------
class Recorder(threading.Thread):
"""Pull frames off the published socket and write them as a numbered series.
Deliberately not `hhstream-client`: that reads a fixed count and exits, and
here the frames have to arrive *while* something else happens.
"""
def __init__(self, socket_path: str, out_dir: pathlib.Path, target_fps: float) -> None:
super().__init__(daemon=True)
self.socket_path = socket_path
self.out_dir = out_dir
self.target_fps = target_fps
self.hello: dict = {}
self.count = 0
self.error: str | None = None
self._stop = threading.Event()
self._started = threading.Event()
def run(self) -> None:
try:
conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
conn.settimeout(10.0)
conn.connect(self.socket_path)
except OSError as exc:
self.error = str(exc)
self._started.set()
return
try:
with conn.makefile("rb") as stream:
# CONTRACT: the Hello can take a moment (it is written on the
# first successful capture), so give it the socket timeout.
self.hello = protocol.read_hello(stream)
server_fps = float(self.hello.get("fps") or 30)
step = max(1, round(server_fps / self.target_fps))
self._started.set()
index = 0
while not self._stop.is_set():
payload = protocol.read_frame(stream)
if payload is None:
break
index += 1
if (index - 1) % step:
continue # decimate to the target rate
self.count += 1
(self.out_dir / f"frame-{self.count:05d}.png").write_bytes(payload)
except (OSError, EOFError, ValueError) as exc:
self.error = str(exc)
finally:
conn.close()
self._started.set()
def stop(self) -> None:
self._stop.set()
self.join(timeout=10)
def encode(frames: pathlib.Path, fps: float, out: pathlib.Path) -> None:
argv = [
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
"-framerate", f"{fps:.4f}",
"-i", str(frames / "frame-%05d.png"),
"-c:v", "libx264", "-preset", "veryfast", "-crf", "23",
# yuv420p + faststart: what every player and every chat client wants.
"-pix_fmt", "yuv420p", "-movflags", "+faststart",
str(out),
]
log(f"encoding {out.name}")
result = subprocess.run(argv, stderr=subprocess.PIPE)
if result.returncode != 0:
fail(f"ffmpeg failed: {result.stderr.decode().strip()}")
def main() -> int:
parser = argparse.ArgumentParser(
prog="agent-eyes-demo.py",
description="Drive a browser on the headless screen and film it.",
)
parser.add_argument("--query", default="Ari Sosnovsky",
help="what to type into the search box")
parser.add_argument("--output", default="agent-eyes.mp4", metavar="PATH")
parser.add_argument("--socket", default=DEFAULT_SOCKET)
parser.add_argument("--runtime-dir", default=DEFAULT_RUNTIME_DIR)
parser.add_argument("--seconds", type=float, default=14.0,
help="how long to record (default: %(default)s)")
parser.add_argument("--capture-fps", type=float, default=15.0,
help="frames per second to keep (default: %(default)s)")
parser.add_argument("--debug-port", type=int, default=9222)
parser.add_argument("--work-dir", type=pathlib.Path, default=None,
help="where frames land (default: a temp dir)")
parser.add_argument("--clear-screen", action="store_true",
help="kill leftover browsers first (needs passwordless sudo)")
parser.add_argument("--keep-work", action="store_true",
help="keep the frames and the browser profile")
args = parser.parse_args()
for tool in ("chromium", "ffmpeg"):
if not shutil.which(tool):
fail(f"{tool} is not on PATH — run this inside `nix develop`")
check_readable(args.socket)
env = session_env(args.runtime_dir)
if args.clear_screen:
clear_screen(args.runtime_dir)
work = args.work_dir or pathlib.Path(
os.environ.get("TMPDIR", "/tmp")) / "agent-eyes-demo"
frames = work / "frames"
if frames.exists():
shutil.rmtree(frames)
frames.mkdir(parents=True)
out = pathlib.Path(args.output).resolve()
out.parent.mkdir(parents=True, exist_ok=True)
browser = start_browser(env, url="https://www.google.com",
port=args.debug_port, home=work / "home")
devtools = None
recorder = None
try:
devtools = DevTools(cdp_target(args.debug_port))
recorder = Recorder(args.socket, frames, args.capture_fps)
recorder.start()
if not recorder._started.wait(timeout=15):
fail("the frame socket never answered")
if recorder.error:
fail(f"recording failed: {recorder.error}")
log(f"recording {args.seconds:.0f}s at {args.capture_fps:.0f}fps "
f"(server says {recorder.hello.get('fps')}fps, "
f"{recorder.hello.get('width')}x{recorder.hello.get('height')})")
began = time.monotonic()
time.sleep(1.5) # let the page settle on screen
type_query(devtools, args.query)
landed = submit_and_wait(devtools)
log(f"landed on {landed}")
devtools.evaluate("window.scrollBy(0, 320)") # show actual results
# Record for the requested wall-clock duration, so the video's length is
# what was asked for regardless of how long the page took to load.
remaining = args.seconds - (time.monotonic() - began)
if remaining > 0:
time.sleep(remaining)
finally:
if recorder is not None:
recorder.stop()
if devtools is not None:
devtools.close()
browser.terminate()
try:
browser.wait(timeout=10)
except subprocess.TimeoutExpired:
browser.kill()
if recorder is None or recorder.count < 2:
fail("fewer than two frames arrived — nothing to encode")
fps = recorder.count / args.seconds
log(f"captured {recorder.count} frames ({fps:.2f} fps effective)")
encode(frames, fps, out)
size = out.stat().st_size
log(f"wrote {out} ({size / 1024:.0f} KiB, {recorder.count} frames)")
if not args.keep_work:
shutil.rmtree(work, ignore_errors=True)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+3
View File
@@ -42,6 +42,9 @@
# The client side of the contract (hhstream-client), so a shell can
# read the stream without a `nix build` first.
(callPackage ./nix/package.nix { })
# `websocket-client` drives Chromium over the DevTools protocol, which
# is how the demo types into the page and reads the result back.
python3.pkgs.websocket-client
# Something to *put on* the headless screen. Chromium is the only
# browser here that both runs on Wayland and can be driven over the
# DevTools protocol, which is how examples/agent-eyes-demo.py types