examples: agent-eyes-demo.py -- drive a browser on the headless screen and film it
This commit is contained in:
1 parent
a77eaa96d3
commit
1ce39ba210
2 files changed
+426
No files matched your search
@@ -0,0 +1,423 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Agent Eyes demo: put a browser on the headless screen, drive it, film it.
|
||||
|
||||
This is the *consumer* side of `docs/CONTRACT.md`: it reads frames off the
|
||||
published socket (not from a screenshot API of its own) while a real browser,
|
||||
running inside the headless Hyprland session, does something visible. The output
|
||||
is an MP4 you can post anywhere.
|
||||
|
||||
Why it exists: "Agent Eyes" is this machine's ability to look at a screen it can
|
||||
drive. Everything else in this repo is the plumbing that makes the screen exist;
|
||||
this is the smallest honest demonstration that a human can watch.
|
||||
|
||||
nix develop
|
||||
sg hhstream -c 'python3 examples/agent-eyes-demo.py --query "Ari Sosnovsky"'
|
||||
|
||||
Preconditions, all of them checked below rather than assumed:
|
||||
|
||||
* the headless session is up and `stream.sock` exists (services
|
||||
`hyprland-headless` + `hyprland-headless-stream`);
|
||||
* you can read it — that means being in the stream's group, hence `sg
|
||||
hhstream` for an already-logged-in shell (`id` will not show it until you
|
||||
log in again);
|
||||
* nothing else is on that screen. Hyprland tiles, so a leftover window would
|
||||
take half the frame. `--clear-screen` kills leftovers (needs sudo);
|
||||
* `chromium` and `ffmpeg` on PATH — both come from the dev shell.
|
||||
|
||||
It talks to the browser over the DevTools protocol rather than synthesising
|
||||
input events, so the typing is real text entry into the real page and the script
|
||||
can *read back* what happened. That read-back is the point: the demo asserts the
|
||||
result page loaded instead of just hoping the frames look right.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import NoReturn
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "src"))
|
||||
|
||||
from hhstream import protocol # noqa: E402 (path set above)
|
||||
|
||||
DEFAULT_SOCKET = "/run/hyprland-headless-stream/stream.sock"
|
||||
DEFAULT_RUNTIME_DIR = "/run/hyprland-headless-stream"
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[eyes] {message}", flush=True)
|
||||
|
||||
|
||||
def fail(message: str) -> NoReturn:
|
||||
print(f"[eyes] {message}", file=sys.stderr, flush=True)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The session: where the screen is, and how to reach it
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def wayland_display(runtime_dir: str) -> str:
|
||||
"""The compositor names its own socket (`wayland-N`), so ask the filesystem.
|
||||
|
||||
Hardcoding `wayland-0` is the trap the module documents at length: a
|
||||
preserved runtime directory can leave `wayland-0` behind from a crashed run
|
||||
and push the live session to `wayland-1`.
|
||||
"""
|
||||
for entry in sorted(pathlib.Path(runtime_dir).glob("wayland-*")):
|
||||
if entry.is_socket() and not entry.name.endswith(".lock"):
|
||||
return entry.name
|
||||
fail(f"no Wayland socket in {runtime_dir} — is hyprland-headless running?")
|
||||
|
||||
|
||||
def session_env(runtime_dir: str) -> dict:
|
||||
display = wayland_display(runtime_dir)
|
||||
log(f"session: XDG_RUNTIME_DIR={runtime_dir} WAYLAND_DISPLAY={display}")
|
||||
return {
|
||||
**os.environ,
|
||||
"XDG_RUNTIME_DIR": runtime_dir,
|
||||
"WAYLAND_DISPLAY": display,
|
||||
"XDG_SESSION_TYPE": "wayland",
|
||||
}
|
||||
|
||||
|
||||
def check_readable(socket_path: str) -> None:
|
||||
if not pathlib.Path(socket_path).exists():
|
||||
fail(f"{socket_path} does not exist — is hyprland-headless-stream running?")
|
||||
if not os.access(socket_path, os.R_OK | os.W_OK):
|
||||
fail(
|
||||
f"cannot use {socket_path} as {os.environ.get('USER', '?')}. The socket "
|
||||
"is `0660 hhstream:hhstream` in a `0750` directory, so your user must "
|
||||
"be in the `hhstream` group — run this under `sg hhstream -c '...'` if "
|
||||
"the group was added since you logged in."
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The browser
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def clear_screen(runtime_dir: str) -> None:
|
||||
"""Kill leftover browsers on the shared screen (needs sudo)."""
|
||||
log("--clear-screen: killing leftover chromium processes")
|
||||
result = subprocess.run(
|
||||
["sudo", "-n", "pkill", "-f", "chromium"],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
|
||||
)
|
||||
if result.returncode not in (0, 1): # 1 == "nothing matched", which is fine
|
||||
fail(f"could not clear the screen: {result.stderr.decode().strip()}")
|
||||
time.sleep(1.0)
|
||||
|
||||
|
||||
def start_browser(env: dict, *, url: str, port: int, home: pathlib.Path) -> subprocess.Popen:
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
profile = home / "profile"
|
||||
env = {**env, "HOME": str(home)}
|
||||
argv = [
|
||||
"chromium",
|
||||
"--ozone-platform=wayland",
|
||||
# The compositor is a system service with no user manager and the kernel
|
||||
# here has no working user-namespace sandbox set up; this is the same
|
||||
# invocation the module's own manual testing used.
|
||||
"--no-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--kiosk",
|
||||
"--window-size=1280,720",
|
||||
f"--user-data-dir={profile}",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
f"--remote-debugging-port={port}",
|
||||
# Keep the debug port on loopback: nothing outside this machine has any
|
||||
# business driving the browser.
|
||||
"--remote-debugging-address=127.0.0.1",
|
||||
url,
|
||||
]
|
||||
log(f"browser: starting chromium (debug port {port}) at {url}")
|
||||
return subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def cdp_target(port: int, timeout: float = 30.0) -> str:
|
||||
"""Wait for the DevTools endpoint and return the page target's ws URL."""
|
||||
deadline = time.monotonic() + timeout
|
||||
last = ""
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list",
|
||||
timeout=2) as response:
|
||||
targets = json.load(response)
|
||||
pages = [t for t in targets if t.get("type") == "page"]
|
||||
if pages:
|
||||
return pages[0]["webSocketDebuggerUrl"]
|
||||
last = "endpoint up, no page target yet"
|
||||
except (urllib.error.URLError, OSError, ValueError) as exc:
|
||||
last = str(exc)
|
||||
time.sleep(0.5)
|
||||
fail(f"DevTools endpoint on port {port} never came up ({last})")
|
||||
|
||||
|
||||
class DevTools:
|
||||
"""A very small CDP client: send a method, wait for its reply."""
|
||||
|
||||
def __init__(self, ws_url: str) -> None:
|
||||
import websocket # type: ignore[import-not-found] # from the dev shell
|
||||
|
||||
self._ws = websocket.create_connection(ws_url, timeout=20,
|
||||
suppress_origin=True)
|
||||
self._next_id = 0
|
||||
|
||||
def call(self, method: str, **params):
|
||||
self._next_id += 1
|
||||
ident = self._next_id
|
||||
self._ws.send(json.dumps({"id": ident, "method": method, "params": params}))
|
||||
while True:
|
||||
message = json.loads(self._ws.recv())
|
||||
if message.get("id") == ident:
|
||||
if "error" in message:
|
||||
raise RuntimeError(f"{method}: {message['error']}")
|
||||
return message.get("result", {})
|
||||
|
||||
def evaluate(self, expression: str):
|
||||
result = self.call("Runtime.evaluate", expression=expression,
|
||||
returnByValue=True, awaitPromise=True)
|
||||
return result.get("result", {}).get("value")
|
||||
|
||||
def close(self) -> None:
|
||||
self._ws.close()
|
||||
|
||||
|
||||
def type_query(devtools: DevTools, query: str, per_char: float = 0.11) -> None:
|
||||
"""Focus the search box and type into it, one character at a time.
|
||||
|
||||
`Input.insertText` goes through the renderer's normal text-entry path, so the
|
||||
page sees exactly what a keyboard would produce (its autocomplete included)
|
||||
and the characters appear on screen as they are sent — which is the whole
|
||||
reason this is worth filming.
|
||||
"""
|
||||
focused = devtools.evaluate(
|
||||
"(() => { const el = document.querySelector('textarea[name=q], input[name=q]');"
|
||||
" if (!el) return false; el.focus(); return document.activeElement === el; })()"
|
||||
)
|
||||
if not focused:
|
||||
fail("could not focus the search box — did the page load?")
|
||||
log(f"typing {query!r}")
|
||||
for char in query:
|
||||
devtools.call("Input.insertText", text=char)
|
||||
time.sleep(per_char)
|
||||
|
||||
|
||||
def submit_and_wait(devtools: DevTools, timeout: float = 15.0) -> str:
|
||||
"""Press Enter, then confirm the results page actually loaded."""
|
||||
before = devtools.evaluate("location.href")
|
||||
devtools.call("Input.dispatchKeyEvent", type="rawKeyDown", key="Enter",
|
||||
code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13)
|
||||
devtools.call("Input.dispatchKeyEvent", type="keyUp", key="Enter",
|
||||
code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13)
|
||||
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
time.sleep(0.25)
|
||||
now = devtools.evaluate("location.href")
|
||||
if now and now != before and "search" in now:
|
||||
return now
|
||||
# Enter did not navigate: submit the form directly rather than leaving a
|
||||
# video of a page that never went anywhere.
|
||||
log("Enter did not navigate; submitting the form directly")
|
||||
devtools.evaluate(
|
||||
"(() => { const f = document.querySelector('form');"
|
||||
" if (f) { f.submit(); return true; } return false; })()"
|
||||
)
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
time.sleep(0.25)
|
||||
now = devtools.evaluate("location.href")
|
||||
if now and now != before:
|
||||
return now
|
||||
fail("the search never navigated — nothing to film")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Recording: read the real frames off the socket
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class Recorder(threading.Thread):
|
||||
"""Pull frames off the published socket and write them as a numbered series.
|
||||
|
||||
Deliberately not `hhstream-client`: that reads a fixed count and exits, and
|
||||
here the frames have to arrive *while* something else happens.
|
||||
"""
|
||||
|
||||
def __init__(self, socket_path: str, out_dir: pathlib.Path, target_fps: float) -> None:
|
||||
super().__init__(daemon=True)
|
||||
self.socket_path = socket_path
|
||||
self.out_dir = out_dir
|
||||
self.target_fps = target_fps
|
||||
self.hello: dict = {}
|
||||
self.count = 0
|
||||
self.error: str | None = None
|
||||
self._stop = threading.Event()
|
||||
self._started = threading.Event()
|
||||
|
||||
def run(self) -> None:
|
||||
try:
|
||||
conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
conn.settimeout(10.0)
|
||||
conn.connect(self.socket_path)
|
||||
except OSError as exc:
|
||||
self.error = str(exc)
|
||||
self._started.set()
|
||||
return
|
||||
|
||||
try:
|
||||
with conn.makefile("rb") as stream:
|
||||
# CONTRACT: the Hello can take a moment (it is written on the
|
||||
# first successful capture), so give it the socket timeout.
|
||||
self.hello = protocol.read_hello(stream)
|
||||
server_fps = float(self.hello.get("fps") or 30)
|
||||
step = max(1, round(server_fps / self.target_fps))
|
||||
self._started.set()
|
||||
index = 0
|
||||
while not self._stop.is_set():
|
||||
payload = protocol.read_frame(stream)
|
||||
if payload is None:
|
||||
break
|
||||
index += 1
|
||||
if (index - 1) % step:
|
||||
continue # decimate to the target rate
|
||||
self.count += 1
|
||||
(self.out_dir / f"frame-{self.count:05d}.png").write_bytes(payload)
|
||||
except (OSError, EOFError, ValueError) as exc:
|
||||
self.error = str(exc)
|
||||
finally:
|
||||
conn.close()
|
||||
self._started.set()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self.join(timeout=10)
|
||||
|
||||
|
||||
def encode(frames: pathlib.Path, fps: float, out: pathlib.Path) -> None:
|
||||
argv = [
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-framerate", f"{fps:.4f}",
|
||||
"-i", str(frames / "frame-%05d.png"),
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "23",
|
||||
# yuv420p + faststart: what every player and every chat client wants.
|
||||
"-pix_fmt", "yuv420p", "-movflags", "+faststart",
|
||||
str(out),
|
||||
]
|
||||
log(f"encoding {out.name}")
|
||||
result = subprocess.run(argv, stderr=subprocess.PIPE)
|
||||
if result.returncode != 0:
|
||||
fail(f"ffmpeg failed: {result.stderr.decode().strip()}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="agent-eyes-demo.py",
|
||||
description="Drive a browser on the headless screen and film it.",
|
||||
)
|
||||
parser.add_argument("--query", default="Ari Sosnovsky",
|
||||
help="what to type into the search box")
|
||||
parser.add_argument("--output", default="agent-eyes.mp4", metavar="PATH")
|
||||
parser.add_argument("--socket", default=DEFAULT_SOCKET)
|
||||
parser.add_argument("--runtime-dir", default=DEFAULT_RUNTIME_DIR)
|
||||
parser.add_argument("--seconds", type=float, default=14.0,
|
||||
help="how long to record (default: %(default)s)")
|
||||
parser.add_argument("--capture-fps", type=float, default=15.0,
|
||||
help="frames per second to keep (default: %(default)s)")
|
||||
parser.add_argument("--debug-port", type=int, default=9222)
|
||||
parser.add_argument("--work-dir", type=pathlib.Path, default=None,
|
||||
help="where frames land (default: a temp dir)")
|
||||
parser.add_argument("--clear-screen", action="store_true",
|
||||
help="kill leftover browsers first (needs passwordless sudo)")
|
||||
parser.add_argument("--keep-work", action="store_true",
|
||||
help="keep the frames and the browser profile")
|
||||
args = parser.parse_args()
|
||||
|
||||
for tool in ("chromium", "ffmpeg"):
|
||||
if not shutil.which(tool):
|
||||
fail(f"{tool} is not on PATH — run this inside `nix develop`")
|
||||
|
||||
check_readable(args.socket)
|
||||
env = session_env(args.runtime_dir)
|
||||
if args.clear_screen:
|
||||
clear_screen(args.runtime_dir)
|
||||
|
||||
work = args.work_dir or pathlib.Path(
|
||||
os.environ.get("TMPDIR", "/tmp")) / "agent-eyes-demo"
|
||||
frames = work / "frames"
|
||||
if frames.exists():
|
||||
shutil.rmtree(frames)
|
||||
frames.mkdir(parents=True)
|
||||
|
||||
out = pathlib.Path(args.output).resolve()
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
browser = start_browser(env, url="https://www.google.com",
|
||||
port=args.debug_port, home=work / "home")
|
||||
devtools = None
|
||||
recorder = None
|
||||
try:
|
||||
devtools = DevTools(cdp_target(args.debug_port))
|
||||
|
||||
recorder = Recorder(args.socket, frames, args.capture_fps)
|
||||
recorder.start()
|
||||
if not recorder._started.wait(timeout=15):
|
||||
fail("the frame socket never answered")
|
||||
if recorder.error:
|
||||
fail(f"recording failed: {recorder.error}")
|
||||
log(f"recording {args.seconds:.0f}s at {args.capture_fps:.0f}fps "
|
||||
f"(server says {recorder.hello.get('fps')}fps, "
|
||||
f"{recorder.hello.get('width')}x{recorder.hello.get('height')})")
|
||||
|
||||
began = time.monotonic()
|
||||
time.sleep(1.5) # let the page settle on screen
|
||||
type_query(devtools, args.query)
|
||||
landed = submit_and_wait(devtools)
|
||||
log(f"landed on {landed}")
|
||||
devtools.evaluate("window.scrollBy(0, 320)") # show actual results
|
||||
|
||||
# Record for the requested wall-clock duration, so the video's length is
|
||||
# what was asked for regardless of how long the page took to load.
|
||||
remaining = args.seconds - (time.monotonic() - began)
|
||||
if remaining > 0:
|
||||
time.sleep(remaining)
|
||||
finally:
|
||||
if recorder is not None:
|
||||
recorder.stop()
|
||||
if devtools is not None:
|
||||
devtools.close()
|
||||
browser.terminate()
|
||||
try:
|
||||
browser.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
browser.kill()
|
||||
|
||||
if recorder is None or recorder.count < 2:
|
||||
fail("fewer than two frames arrived — nothing to encode")
|
||||
|
||||
fps = recorder.count / args.seconds
|
||||
log(f"captured {recorder.count} frames ({fps:.2f} fps effective)")
|
||||
encode(frames, fps, out)
|
||||
size = out.stat().st_size
|
||||
log(f"wrote {out} ({size / 1024:.0f} KiB, {recorder.count} frames)")
|
||||
|
||||
if not args.keep_work:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -42,6 +42,9 @@
|
||||
# The client side of the contract (hhstream-client), so a shell can
|
||||
# read the stream without a `nix build` first.
|
||||
(callPackage ./nix/package.nix { })
|
||||
# `websocket-client` drives Chromium over the DevTools protocol, which
|
||||
# is how the demo types into the page and reads the result back.
|
||||
python3.pkgs.websocket-client
|
||||
# Something to *put on* the headless screen. Chromium is the only
|
||||
# browser here that both runs on Wayland and can be driven over the
|
||||
# DevTools protocol, which is how examples/agent-eyes-demo.py types
|
||||
|
||||
Reference in new issue
Block a user