456 lines
19 KiB
Python
456 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Agent Eyes demo: put a browser on the headless screen, drive it, film it.
|
|
|
|
This is the *consumer* side of `docs/CONTRACT.md`: it reads frames off the
|
|
published socket (not from a screenshot API of its own) while a real browser,
|
|
running inside the headless Hyprland session, does something visible. The output
|
|
is an MP4 you can post anywhere.
|
|
|
|
Why it exists: "Agent Eyes" is this machine's ability to look at a screen it can
|
|
drive. Everything else in this repo is the plumbing that makes the screen exist;
|
|
this is the smallest honest demonstration that a human can watch.
|
|
|
|
nix develop
|
|
sg hhstream -c 'python3 examples/agent-eyes-demo.py --query "Ari Sosnovsky"'
|
|
|
|
Preconditions, all of them checked below rather than assumed:
|
|
|
|
* the headless session is up and `stream.sock` exists (services
|
|
`hyprland-headless` + `hyprland-headless-stream`);
|
|
* you can read it — that means being in the stream's group, hence `sg
|
|
hhstream` for an already-logged-in shell (`id` will not show it until you
|
|
log in again);
|
|
* nothing else is on that screen. Hyprland tiles, so a leftover window would
|
|
take half the frame. `--clear-screen` kills leftovers (needs sudo);
|
|
* `chromium` and `ffmpeg` on PATH — both come from the dev shell.
|
|
|
|
It talks to the browser over the DevTools protocol rather than synthesising
|
|
input events, so the typing is real text entry into the real page and the script
|
|
can *read back* what happened. That read-back is the point: the demo asserts the
|
|
result page loaded instead of just hoping the frames look right.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from typing import NoReturn
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "src"))
|
|
|
|
from hhstream import protocol # noqa: E402 (path set above)
|
|
|
|
DEFAULT_SOCKET = "/run/hyprland-headless-stream/stream.sock"
|
|
DEFAULT_RUNTIME_DIR = "/run/hyprland-headless-stream"
|
|
|
|
|
|
def log(message: str) -> None:
|
|
print(f"[eyes] {message}", flush=True)
|
|
|
|
|
|
def fail(message: str) -> NoReturn:
|
|
print(f"[eyes] {message}", file=sys.stderr, flush=True)
|
|
raise SystemExit(1)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The session: where the screen is, and how to reach it
|
|
# --------------------------------------------------------------------------
|
|
|
|
def wayland_display(runtime_dir: str) -> str:
|
|
"""The compositor names its own socket (`wayland-N`), so ask the filesystem.
|
|
|
|
Hardcoding `wayland-0` is the trap the module documents at length: a
|
|
preserved runtime directory can leave `wayland-0` behind from a crashed run
|
|
and push the live session to `wayland-1`.
|
|
"""
|
|
for entry in sorted(pathlib.Path(runtime_dir).glob("wayland-*")):
|
|
if entry.is_socket() and not entry.name.endswith(".lock"):
|
|
return entry.name
|
|
fail(f"no Wayland socket in {runtime_dir} — is hyprland-headless running?")
|
|
|
|
|
|
def session_env(runtime_dir: str) -> dict:
|
|
display = wayland_display(runtime_dir)
|
|
log(f"session: XDG_RUNTIME_DIR={runtime_dir} WAYLAND_DISPLAY={display}")
|
|
return {
|
|
**os.environ,
|
|
"XDG_RUNTIME_DIR": runtime_dir,
|
|
"WAYLAND_DISPLAY": display,
|
|
"XDG_SESSION_TYPE": "wayland",
|
|
}
|
|
|
|
|
|
def check_readable(socket_path: str) -> None:
|
|
if not pathlib.Path(socket_path).exists():
|
|
fail(f"{socket_path} does not exist — is hyprland-headless-stream running?")
|
|
if not os.access(socket_path, os.R_OK | os.W_OK):
|
|
fail(
|
|
f"cannot use {socket_path} as {os.environ.get('USER', '?')}. The socket "
|
|
"is `0660 hhstream:hhstream` in a `0750` directory, so your user must "
|
|
"be in the `hhstream` group — run this under `sg hhstream -c '...'` if "
|
|
"the group was added since you logged in."
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The browser
|
|
# --------------------------------------------------------------------------
|
|
|
|
def clear_screen(runtime_dir: str) -> None:
|
|
"""Kill leftover browsers on the shared screen (needs sudo)."""
|
|
log("--clear-screen: killing leftover chromium processes")
|
|
# The bracket keeps the pattern from matching `pkill`'s own command line —
|
|
# `pkill -f chromium` kills the shell that ran it, which is a confusing way
|
|
# to lose a session.
|
|
result = subprocess.run(
|
|
["sudo", "-n", "pkill", "-f", "[c]hromium"],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
|
|
)
|
|
if result.returncode not in (0, 1): # 1 == "nothing matched", which is fine
|
|
fail(f"could not clear the screen: {result.stderr.decode().strip()}")
|
|
time.sleep(1.0)
|
|
|
|
|
|
def start_browser(env: dict, *, url: str, port: int, home: pathlib.Path,
|
|
fullscreen: bool = False) -> subprocess.Popen:
|
|
home.mkdir(parents=True, exist_ok=True)
|
|
profile = home / "profile"
|
|
env = {**env, "HOME": str(home)}
|
|
argv = [
|
|
"chromium",
|
|
"--ozone-platform=wayland",
|
|
# The compositor is a system service with no user manager and the kernel
|
|
# here has no working user-namespace sandbox set up; this is the same
|
|
# invocation the module's own manual testing used.
|
|
"--no-sandbox",
|
|
"--disable-dev-shm-usage",
|
|
# Windowed by default: the session's wallpaper, gaps and window borders
|
|
# are the point of filming a *desktop*, and a kiosk window hides all of
|
|
# them. `--fullscreen` brings back the full-bleed browser.
|
|
*(["--kiosk"] if fullscreen else []),
|
|
"--window-size=1040,560",
|
|
f"--user-data-dir={profile}",
|
|
"--no-first-run",
|
|
"--no-default-browser-check",
|
|
f"--remote-debugging-port={port}",
|
|
# Keep the debug port on loopback: nothing outside this machine has any
|
|
# business driving the browser.
|
|
"--remote-debugging-address=127.0.0.1",
|
|
url,
|
|
]
|
|
log(f"browser: starting chromium (debug port {port}, "
|
|
f"{'kiosk' if fullscreen else 'windowed'}) at {url}")
|
|
return subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL)
|
|
|
|
|
|
def cdp_target(port: int, timeout: float = 30.0) -> str:
|
|
"""Wait for the DevTools endpoint and return the page target's ws URL."""
|
|
deadline = time.monotonic() + timeout
|
|
last = ""
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list",
|
|
timeout=2) as response:
|
|
targets = json.load(response)
|
|
pages = [t for t in targets if t.get("type") == "page"]
|
|
if pages:
|
|
return pages[0]["webSocketDebuggerUrl"]
|
|
last = "endpoint up, no page target yet"
|
|
except (urllib.error.URLError, OSError, ValueError) as exc:
|
|
last = str(exc)
|
|
time.sleep(0.5)
|
|
fail(f"DevTools endpoint on port {port} never came up ({last})")
|
|
|
|
|
|
class DevTools:
|
|
"""A very small CDP client: send a method, wait for its reply."""
|
|
|
|
def __init__(self, ws_url: str) -> None:
|
|
import websocket # type: ignore[import-not-found] # from the dev shell
|
|
|
|
self._ws = websocket.create_connection(ws_url, timeout=20,
|
|
suppress_origin=True)
|
|
self._next_id = 0
|
|
|
|
def call(self, method: str, **params):
|
|
self._next_id += 1
|
|
ident = self._next_id
|
|
self._ws.send(json.dumps({"id": ident, "method": method, "params": params}))
|
|
while True:
|
|
message = json.loads(self._ws.recv())
|
|
if message.get("id") == ident:
|
|
if "error" in message:
|
|
raise RuntimeError(f"{method}: {message['error']}")
|
|
return message.get("result", {})
|
|
|
|
def evaluate(self, expression: str):
|
|
result = self.call("Runtime.evaluate", expression=expression,
|
|
returnByValue=True, awaitPromise=True)
|
|
return result.get("result", {}).get("value")
|
|
|
|
def close(self) -> None:
|
|
self._ws.close()
|
|
|
|
|
|
def type_query(devtools: DevTools, query: str, per_char: float = 0.11) -> None:
|
|
"""Focus the search box and type into it, one character at a time.
|
|
|
|
`Input.insertText` goes through the renderer's normal text-entry path, so the
|
|
page sees exactly what a keyboard would produce (its autocomplete included)
|
|
and the characters appear on screen as they are sent — which is the whole
|
|
reason this is worth filming.
|
|
"""
|
|
focused = devtools.evaluate(
|
|
"(() => { const el = document.querySelector('textarea[name=q], input[name=q]');"
|
|
" if (!el) return false; el.focus(); return document.activeElement === el; })()"
|
|
)
|
|
if not focused:
|
|
fail("could not focus the search box — did the page load?")
|
|
log(f"typing {query!r}")
|
|
for char in query:
|
|
devtools.call("Input.insertText", text=char)
|
|
time.sleep(per_char)
|
|
|
|
|
|
def submit_and_wait(devtools: DevTools, timeout: float = 15.0) -> str:
|
|
"""Press Enter, then confirm the results page actually loaded."""
|
|
before = devtools.evaluate("location.href")
|
|
devtools.call("Input.dispatchKeyEvent", type="rawKeyDown", key="Enter",
|
|
code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13)
|
|
devtools.call("Input.dispatchKeyEvent", type="keyUp", key="Enter",
|
|
code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13)
|
|
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
time.sleep(0.25)
|
|
now = devtools.evaluate("location.href")
|
|
if now and now != before and "search" in now:
|
|
return now
|
|
# Enter did not navigate: submit the form directly rather than leaving a
|
|
# video of a page that never went anywhere.
|
|
log("Enter did not navigate; submitting the form directly")
|
|
devtools.evaluate(
|
|
"(() => { const f = document.querySelector('form');"
|
|
" if (f) { f.submit(); return true; } return false; })()"
|
|
)
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
time.sleep(0.25)
|
|
now = devtools.evaluate("location.href")
|
|
if now and now != before:
|
|
return now
|
|
fail("the search never navigated — nothing to film")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Recording: read the real frames off the socket
|
|
# --------------------------------------------------------------------------
|
|
|
|
class Recorder(threading.Thread):
|
|
"""Pull frames off the published socket and write them as a numbered series.
|
|
|
|
Deliberately not `hhstream-client`: that reads a fixed count and exits, and
|
|
here the frames have to arrive *while* something else happens.
|
|
"""
|
|
|
|
def __init__(self, socket_path: str, out_dir: pathlib.Path, target_fps: float) -> None:
|
|
super().__init__(daemon=True)
|
|
self.socket_path = socket_path
|
|
self.out_dir = out_dir
|
|
self.target_fps = target_fps
|
|
self.hello: dict = {}
|
|
self.count = 0
|
|
self.error: str | None = None
|
|
# NOT `_stop` / `_started`: `threading.Thread` owns both names. It sets
|
|
# `_started` itself as the thread boots — *before* `run()` — so a
|
|
# `wait()` on it returns instantly and every attribute `run()` is
|
|
# supposed to fill in is still empty. It also calls `self._stop()` on
|
|
# the way out, which an Event cannot answer to.
|
|
self._stop_event = threading.Event()
|
|
self._ready = threading.Event()
|
|
|
|
def run(self) -> None:
|
|
try:
|
|
conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
conn.settimeout(10.0)
|
|
conn.connect(self.socket_path)
|
|
except OSError as exc:
|
|
self.error = str(exc)
|
|
self._ready.set()
|
|
return
|
|
|
|
try:
|
|
with conn.makefile("rb") as stream:
|
|
# CONTRACT: the Hello can take a moment (it is written on the
|
|
# first successful capture), so give it the socket timeout.
|
|
self.hello = protocol.read_hello(stream)
|
|
server_fps = float(self.hello.get("fps") or 30)
|
|
step = max(1, round(server_fps / self.target_fps))
|
|
self._ready.set()
|
|
index = 0
|
|
while not self._stop_event.is_set():
|
|
payload = protocol.read_frame(stream)
|
|
if payload is None:
|
|
break
|
|
index += 1
|
|
if (index - 1) % step:
|
|
continue # decimate to the target rate
|
|
self.count += 1
|
|
(self.out_dir / f"frame-{self.count:05d}.png").write_bytes(payload)
|
|
except (OSError, EOFError, ValueError) as exc:
|
|
self.error = str(exc)
|
|
finally:
|
|
conn.close()
|
|
self._ready.set()
|
|
|
|
def stop(self) -> None:
|
|
self._stop_event.set()
|
|
self.join(timeout=10)
|
|
|
|
|
|
def encode(frames: pathlib.Path, fps: float, out: pathlib.Path) -> None:
|
|
argv = [
|
|
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
|
"-framerate", f"{fps:.4f}",
|
|
"-i", str(frames / "frame-%05d.png"),
|
|
"-c:v", "libx264", "-preset", "veryfast", "-crf", "23",
|
|
# yuv420p + faststart: what every player and every chat client wants.
|
|
"-pix_fmt", "yuv420p", "-movflags", "+faststart",
|
|
# Strip every metadata tag. Xiph/Blossom media stores reject files that
|
|
# carry them — Buzz's does, with `422 media contains metadata or a
|
|
# non-canonical metadata channel`, and the only thing to fix is ffmpeg's
|
|
# own `encoder`/`handler_name` tags. `bitexact` keeps them from being
|
|
# written back in.
|
|
"-map_metadata", "-1", "-fflags", "+bitexact", "-flags:v", "+bitexact",
|
|
"-an",
|
|
str(out),
|
|
]
|
|
log(f"encoding {out.name}")
|
|
result = subprocess.run(argv, stderr=subprocess.PIPE)
|
|
if result.returncode != 0:
|
|
fail(f"ffmpeg failed: {result.stderr.decode().strip()}")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
prog="agent-eyes-demo.py",
|
|
description="Drive a browser on the headless screen and film it.",
|
|
)
|
|
parser.add_argument("--query", default="Ari Sosnovsky",
|
|
help="what to type into the search box")
|
|
parser.add_argument("--output", default="agent-eyes.mp4", metavar="PATH")
|
|
parser.add_argument("--socket", default=DEFAULT_SOCKET)
|
|
parser.add_argument("--runtime-dir", default=DEFAULT_RUNTIME_DIR)
|
|
parser.add_argument("--seconds", type=float, default=14.0,
|
|
help="how long to record (default: %(default)s)")
|
|
parser.add_argument("--capture-fps", type=float, default=15.0,
|
|
help="frames per second to keep (default: %(default)s)")
|
|
parser.add_argument("--debug-port", type=int, default=9222)
|
|
parser.add_argument("--fullscreen", action="store_true",
|
|
help="kiosk/full-bleed browser instead of a windowed one "
|
|
"(hides the session's wallpaper and window borders)")
|
|
parser.add_argument("--work-dir", type=pathlib.Path, default=None,
|
|
help="where frames land (default: a temp dir)")
|
|
parser.add_argument("--clear-screen", action="store_true",
|
|
help="kill leftover browsers first (needs passwordless sudo)")
|
|
parser.add_argument("--keep-work", action="store_true",
|
|
help="keep the frames and the browser profile")
|
|
args = parser.parse_args()
|
|
|
|
for tool in ("chromium", "ffmpeg"):
|
|
if not shutil.which(tool):
|
|
fail(f"{tool} is not on PATH — run this inside `nix develop`")
|
|
|
|
check_readable(args.socket)
|
|
env = session_env(args.runtime_dir)
|
|
if args.clear_screen:
|
|
clear_screen(args.runtime_dir)
|
|
|
|
work = args.work_dir or pathlib.Path(
|
|
os.environ.get("TMPDIR", "/tmp")) / "agent-eyes-demo"
|
|
frames = work / "frames"
|
|
if frames.exists():
|
|
shutil.rmtree(frames)
|
|
frames.mkdir(parents=True)
|
|
|
|
out = pathlib.Path(args.output).resolve()
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
browser = start_browser(env, url="https://www.google.com",
|
|
port=args.debug_port, home=work / "home",
|
|
fullscreen=args.fullscreen)
|
|
devtools = None
|
|
recorder = None
|
|
try:
|
|
devtools = DevTools(cdp_target(args.debug_port))
|
|
|
|
recorder = Recorder(args.socket, frames, args.capture_fps)
|
|
recorder.start()
|
|
if not recorder._ready.wait(timeout=15):
|
|
fail("the frame socket never answered")
|
|
if recorder.error:
|
|
fail(f"recording failed: {recorder.error}")
|
|
log(f"recording {args.seconds:.0f}s at {args.capture_fps:.0f}fps "
|
|
f"(server says {recorder.hello.get('fps')}fps, "
|
|
f"{recorder.hello.get('width')}x{recorder.hello.get('height')})")
|
|
|
|
began = time.monotonic()
|
|
time.sleep(1.5) # let the page settle on screen
|
|
type_query(devtools, args.query)
|
|
landed = submit_and_wait(devtools)
|
|
log(f"landed on {landed}")
|
|
if "/sorry/" in landed:
|
|
# Google's bot wall. Filming it would produce a video that looks
|
|
# like a working demo and is not one, so stop instead.
|
|
fail(
|
|
"Google served its 'unusual traffic' wall instead of results "
|
|
"(rate-limited IP, not a broken browser). Wait a few minutes and "
|
|
"run again; the first search from a fresh profile usually passes."
|
|
)
|
|
devtools.evaluate("window.scrollBy(0, 320)") # show actual results
|
|
|
|
# Record for the requested wall-clock duration, so the video's length is
|
|
# what was asked for regardless of how long the page took to load.
|
|
remaining = args.seconds - (time.monotonic() - began)
|
|
if remaining > 0:
|
|
time.sleep(remaining)
|
|
finally:
|
|
if recorder is not None:
|
|
recorder.stop()
|
|
if devtools is not None:
|
|
devtools.close()
|
|
browser.terminate()
|
|
try:
|
|
browser.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
browser.kill()
|
|
|
|
if recorder is None or recorder.count < 2:
|
|
fail("fewer than two frames arrived — nothing to encode")
|
|
|
|
fps = recorder.count / args.seconds
|
|
log(f"captured {recorder.count} frames ({fps:.2f} fps effective)")
|
|
encode(frames, fps, out)
|
|
size = out.stat().st_size
|
|
log(f"wrote {out} ({size / 1024:.0f} KiB, {recorder.count} frames)")
|
|
|
|
if not args.keep_work:
|
|
shutil.rmtree(work, ignore_errors=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|