Files
Hermenix e8b3855a05 move every live default to /run/hypr-stream, and resolve the slices' contract gaps
Slice A renamed the runtime directory in the module, but the old path was still
live in the code and checks it does not own, so the VM gates pointed at a
directory the module no longer creates. This moves the 21 remaining live
occurrences (client/server defaults, the unit tests that pin them, all three VM
checks, the diagnostics script, both READMEs) and deliberately leaves the three
mentions that are history rather than configuration: the plan's problem
statement, module.nix's note about the previous name, and cli.py's explanation
for spelling its default out.

Also here: docs/CONTRACT-sessions.md is now v2 with every gap the four slices
reported resolved as a decision (Monitor.id, stable_id, pid -1, Event.data,
mark(), the quarantine argv type, the corrected wlr XML path, and the state
dir/frame socket rows), and docs/CONTRACT.md is v2 because its documented socket
default moved -- wire format unchanged.

Two comments that stated the old, wrong diagnosis (a 'misnamed' IPC socket) now
state the measured one: the event socket did not fit in sun_path.
2026-10-02 18:11:46 +00:00

506 lines
21 KiB
Python

#!/usr/bin/env python3
"""Agent Eyes demo: put a browser on the headless screen, drive it, film it.
This is the *consumer* side of `docs/CONTRACT.md`: it reads frames off the
published socket (not from a screenshot API of its own) while a real browser,
running inside the headless Hyprland session, does something visible. The output
is an MP4 you can post anywhere.
Why it exists: "Agent Eyes" is this machine's ability to look at a screen it can
drive. Everything else in this repo is the plumbing that makes the screen exist;
this is the smallest honest demonstration that a human can watch.
nix develop
sg hhstream -c 'python3 examples/agent-eyes-demo.py --query "Ari Sosnovsky"'
Preconditions, all of them checked below rather than assumed:
* the headless session is up and `stream.sock` exists (services
`hyprland-headless` + `hyprland-headless-stream`);
* you can read it — that means being in the stream's group, hence `sg
hhstream` for an already-logged-in shell (`id` will not show it until you
log in again);
* nothing else is on that screen. Hyprland tiles, so a leftover window would
take half the frame. `--clear-screen` kills leftovers (needs sudo);
* `chromium` and `ffmpeg` on PATH — both come from the dev shell.
It talks to the browser over the DevTools protocol rather than synthesising
input events, so the typing is real text entry into the real page and the script
can *read back* what happened. That read-back is the point: the demo asserts the
result page loaded instead of just hoping the frames look right.
"""
from __future__ import annotations
import argparse
import json
import os
import pathlib
import shutil
import socket
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from typing import NoReturn
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "src"))
from hhstream import protocol # noqa: E402 (path set above)
DEFAULT_SOCKET = "/run/hypr-stream/stream.sock"
DEFAULT_RUNTIME_DIR = "/run/hypr-stream"
def log(message: str) -> None:
print(f"[eyes] {message}", flush=True)
def fail(message: str) -> NoReturn:
print(f"[eyes] {message}", file=sys.stderr, flush=True)
raise SystemExit(1)
# --------------------------------------------------------------------------
# The session: where the screen is, and how to reach it
# --------------------------------------------------------------------------
def wayland_display(runtime_dir: str) -> str:
"""The compositor names its own socket (`wayland-N`), so ask the filesystem.
Hardcoding `wayland-0` is the trap the module documents at length: a
preserved runtime directory can leave `wayland-0` behind from a crashed run
and push the live session to `wayland-1`.
"""
for entry in sorted(pathlib.Path(runtime_dir).glob("wayland-*")):
if entry.is_socket() and not entry.name.endswith(".lock"):
return entry.name
fail(f"no Wayland socket in {runtime_dir} — is hyprland-headless running?")
def session_env(runtime_dir: str) -> dict:
display = wayland_display(runtime_dir)
log(f"session: XDG_RUNTIME_DIR={runtime_dir} WAYLAND_DISPLAY={display}")
return {
**os.environ,
"XDG_RUNTIME_DIR": runtime_dir,
"WAYLAND_DISPLAY": display,
"XDG_SESSION_TYPE": "wayland",
}
def check_readable(socket_path: str) -> None:
if not pathlib.Path(socket_path).exists():
fail(f"{socket_path} does not exist — is hyprland-headless-stream running?")
if not os.access(socket_path, os.R_OK | os.W_OK):
fail(
f"cannot use {socket_path} as {os.environ.get('USER', '?')}. The socket "
"is `0660 hhstream:hhstream` in a `0750` directory, so your user must "
"be in the `hhstream` group — run this under `sg hhstream -c '...'` if "
"the group was added since you logged in."
)
# --------------------------------------------------------------------------
# The browser
# --------------------------------------------------------------------------
def clear_screen(runtime_dir: str) -> None:
"""Kill leftover browsers on the shared screen (needs sudo)."""
log("--clear-screen: killing leftover chromium processes")
# The bracket keeps the pattern from matching `pkill`'s own command line —
# `pkill -f chromium` kills the shell that ran it, which is a confusing way
# to lose a session.
result = subprocess.run(
["sudo", "-n", "pkill", "-f", "[c]hromium"],
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
)
if result.returncode not in (0, 1): # 1 == "nothing matched", which is fine
fail(f"could not clear the screen: {result.stderr.decode().strip()}")
time.sleep(1.0)
def _browser_argv(env: dict, *, url: str, port: int, home: pathlib.Path,
fullscreen: bool, unsandboxed: bool) -> list[str]:
profile = home / "profile"
return [
"chromium",
"--ozone-platform=wayland",
# The sandbox works on this machine, so it is on by default. When it
# does not, the fallback needs `--test-type` as well: without it Chromium
# paints "you are using an unsupported command-line flag: --no-sandbox"
# across the top of the page, and that banner would be in every frame of
# the recording.
*(["--no-sandbox", "--test-type"] if unsandboxed else []),
"--disable-dev-shm-usage",
# Windowed by default: the session's wallpaper, gaps and window borders
# are the point of filming a *desktop*, and a kiosk window hides all of
# them. `--fullscreen` brings back the full-bleed browser.
*(["--kiosk"] if fullscreen else []),
"--window-size=1040,560",
f"--user-data-dir={profile}",
"--no-first-run",
"--no-default-browser-check",
f"--remote-debugging-port={port}",
# Keep the debug port on loopback: nothing outside this machine has any
# business driving the browser.
"--remote-debugging-address=127.0.0.1",
url,
]
def start_browser(env: dict, *, url: str, port: int, home: pathlib.Path,
fullscreen: bool = False) -> subprocess.Popen:
home.mkdir(parents=True, exist_ok=True)
env = {**env, "HOME": str(home)}
argv = _browser_argv(env, url=url, port=port, home=home,
fullscreen=fullscreen, unsandboxed=False)
log(f"browser: starting chromium (debug port {port}, "
f"{'kiosk' if fullscreen else 'windowed'}) at {url}")
proc = subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
# A sandboxed Chromium can die on launch (no user namespaces, no setuid
# helper). Give it a moment, and fall back rather than filming an empty
# desktop and calling it a demo.
time.sleep(2.5)
if proc.poll() is not None:
log("browser: sandboxed launch exited; retrying with --no-sandbox")
argv = _browser_argv(env, url=url, port=port, home=home,
fullscreen=fullscreen, unsandboxed=True)
proc = subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
return proc
def cdp_target(port: int, timeout: float = 30.0) -> str:
"""Wait for the DevTools endpoint and return the page target's ws URL."""
deadline = time.monotonic() + timeout
last = ""
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list",
timeout=2) as response:
targets = json.load(response)
pages = [t for t in targets if t.get("type") == "page"]
if pages:
return pages[0]["webSocketDebuggerUrl"]
last = "endpoint up, no page target yet"
except (urllib.error.URLError, OSError, ValueError) as exc:
last = str(exc)
time.sleep(0.5)
fail(f"DevTools endpoint on port {port} never came up ({last})")
class DevTools:
"""A very small CDP client: send a method, wait for its reply."""
def __init__(self, ws_url: str) -> None:
import websocket # type: ignore[import-not-found] # from the dev shell
self._ws = websocket.create_connection(ws_url, timeout=20,
suppress_origin=True)
self._next_id = 0
def call(self, method: str, **params):
self._next_id += 1
ident = self._next_id
self._ws.send(json.dumps({"id": ident, "method": method, "params": params}))
while True:
message = json.loads(self._ws.recv())
if message.get("id") == ident:
if "error" in message:
raise RuntimeError(f"{method}: {message['error']}")
return message.get("result", {})
def evaluate(self, expression: str):
result = self.call("Runtime.evaluate", expression=expression,
returnByValue=True, awaitPromise=True)
return result.get("result", {}).get("value")
def close(self) -> None:
self._ws.close()
def is_walled(devtools: DevTools) -> bool:
"""Google's bot wall, however it presents itself.
Both checks are needed. The URL is the obvious tell (`/sorry/index?...`),
but the redirect can arrive *after* the results page has already painted —
so a guard that only runs right after the search passes and then films a
CAPTCHA for sixteen seconds. Hence: check the URL, check the page text, and
call this again once the recording has finished.
"""
url = devtools.evaluate("location.href") or ""
if "/sorry/" in url:
return True
text = devtools.evaluate(
"(() => { const b = document.body; return b ? b.innerText.slice(0, 600) : ''; })()") or ""
return "unusual traffic" in text.lower()
def type_query(devtools: DevTools, query: str, per_char: float = 0.11) -> None:
"""Focus the search box and type into it, one character at a time.
`Input.insertText` goes through the renderer's normal text-entry path, so the
page sees exactly what a keyboard would produce (its autocomplete included)
and the characters appear on screen as they are sent — which is the whole
reason this is worth filming.
"""
focused = devtools.evaluate(
"(() => { const el = document.querySelector('textarea[name=q], input[name=q]');"
" if (!el) return false; el.focus(); return document.activeElement === el; })()"
)
if not focused:
fail("could not focus the search box — did the page load?")
log(f"typing {query!r}")
for char in query:
devtools.call("Input.insertText", text=char)
time.sleep(per_char)
def submit_and_wait(devtools: DevTools, timeout: float = 15.0) -> str:
"""Press Enter, then confirm the results page actually loaded."""
before = devtools.evaluate("location.href")
devtools.call("Input.dispatchKeyEvent", type="rawKeyDown", key="Enter",
code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13)
devtools.call("Input.dispatchKeyEvent", type="keyUp", key="Enter",
code="Enter", windowsVirtualKeyCode=13, nativeVirtualKeyCode=13)
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
time.sleep(0.25)
now = devtools.evaluate("location.href")
if now and now != before and "search" in now:
return now
# Enter did not navigate: submit the form directly rather than leaving a
# video of a page that never went anywhere.
log("Enter did not navigate; submitting the form directly")
devtools.evaluate(
"(() => { const f = document.querySelector('form');"
" if (f) { f.submit(); return true; } return false; })()"
)
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
time.sleep(0.25)
now = devtools.evaluate("location.href")
if now and now != before:
return now
fail("the search never navigated — nothing to film")
# --------------------------------------------------------------------------
# Recording: read the real frames off the socket
# --------------------------------------------------------------------------
class Recorder(threading.Thread):
"""Pull frames off the published socket and write them as a numbered series.
Deliberately not `hhstream-client`: that reads a fixed count and exits, and
here the frames have to arrive *while* something else happens.
"""
def __init__(self, socket_path: str, out_dir: pathlib.Path, target_fps: float) -> None:
super().__init__(daemon=True)
self.socket_path = socket_path
self.out_dir = out_dir
self.target_fps = target_fps
self.hello: dict = {}
# Frame extension follows the Hello, not an assumption: the stream can
# legitimately be JPEG, and the caller must not have to know which.
self.ext = "png"
self.count = 0
self.error: str | None = None
# NOT `_stop` / `_started`: `threading.Thread` owns both names. It sets
# `_started` itself as the thread boots — *before* `run()` — so a
# `wait()` on it returns instantly and every attribute `run()` is
# supposed to fill in is still empty. It also calls `self._stop()` on
# the way out, which an Event cannot answer to.
self._stop_event = threading.Event()
self._ready = threading.Event()
def run(self) -> None:
try:
conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
conn.settimeout(10.0)
conn.connect(self.socket_path)
except OSError as exc:
self.error = str(exc)
self._ready.set()
return
try:
with conn.makefile("rb") as stream:
# CONTRACT: the Hello can take a moment (it is written on the
# first successful capture), so give it the socket timeout.
self.hello = protocol.read_hello(stream)
self.ext = {"image/png": "png", "image/jpeg": "jpg"}.get(
str(self.hello.get("format", "image/png")), "png")
server_fps = float(self.hello.get("fps") or 30)
step = max(1, round(server_fps / self.target_fps))
self._ready.set()
index = 0
while not self._stop_event.is_set():
payload = protocol.read_frame(stream)
if payload is None:
break
index += 1
if (index - 1) % step:
continue # decimate to the target rate
self.count += 1
(self.out_dir / f"frame-{self.count:05d}.{self.ext}").write_bytes(payload)
except (OSError, EOFError, ValueError) as exc:
self.error = str(exc)
finally:
conn.close()
self._ready.set()
def stop(self) -> None:
self._stop_event.set()
self.join(timeout=10)
def encode(frames: pathlib.Path, fps: float, out: pathlib.Path, ext: str = "png") -> None:
argv = [
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
"-framerate", f"{fps:.4f}",
"-i", str(frames / f"frame-%05d.{ext}"),
"-c:v", "libx264", "-preset", "veryfast", "-crf", "23",
# yuv420p + faststart: what every player and every chat client wants.
"-pix_fmt", "yuv420p", "-movflags", "+faststart",
# Strip every metadata tag. Xiph/Blossom media stores reject files that
# carry them — Buzz's does, with `422 media contains metadata or a
# non-canonical metadata channel`, and the only thing to fix is ffmpeg's
# own `encoder`/`handler_name` tags. `bitexact` keeps them from being
# written back in.
"-map_metadata", "-1", "-fflags", "+bitexact", "-flags:v", "+bitexact",
"-an",
str(out),
]
log(f"encoding {out.name}")
result = subprocess.run(argv, stderr=subprocess.PIPE)
if result.returncode != 0:
fail(f"ffmpeg failed: {result.stderr.decode().strip()}")
def main() -> int:
parser = argparse.ArgumentParser(
prog="agent-eyes-demo.py",
description="Drive a browser on the headless screen and film it.",
)
parser.add_argument("--query", default="Ari Sosnovsky",
help="what to type into the search box")
parser.add_argument("--output", default="agent-eyes.mp4", metavar="PATH")
parser.add_argument("--socket", default=DEFAULT_SOCKET)
parser.add_argument("--runtime-dir", default=DEFAULT_RUNTIME_DIR)
parser.add_argument("--seconds", type=float, default=14.0,
help="how long to record (default: %(default)s)")
parser.add_argument("--capture-fps", type=float, default=15.0,
help="frames per second to keep (default: %(default)s)")
parser.add_argument("--debug-port", type=int, default=9222)
parser.add_argument("--fullscreen", action="store_true",
help="kiosk/full-bleed browser instead of a windowed one "
"(hides the session's wallpaper and window borders)")
parser.add_argument("--work-dir", type=pathlib.Path, default=None,
help="where frames land (default: a temp dir)")
parser.add_argument("--clear-screen", action="store_true",
help="kill leftover browsers first (needs passwordless sudo)")
parser.add_argument("--keep-work", action="store_true",
help="keep the frames and the browser profile")
args = parser.parse_args()
for tool in ("chromium", "ffmpeg"):
if not shutil.which(tool):
fail(f"{tool} is not on PATH — run this inside `nix develop`")
check_readable(args.socket)
env = session_env(args.runtime_dir)
if args.clear_screen:
clear_screen(args.runtime_dir)
work = args.work_dir or pathlib.Path(
os.environ.get("TMPDIR", "/tmp")) / "agent-eyes-demo"
frames = work / "frames"
if frames.exists():
shutil.rmtree(frames)
frames.mkdir(parents=True)
out = pathlib.Path(args.output).resolve()
out.parent.mkdir(parents=True, exist_ok=True)
browser = start_browser(env, url="https://www.google.com",
port=args.debug_port, home=work / "home",
fullscreen=args.fullscreen)
devtools = None
recorder = None
try:
devtools = DevTools(cdp_target(args.debug_port))
recorder = Recorder(args.socket, frames, args.capture_fps)
recorder.start()
if not recorder._ready.wait(timeout=15):
fail("the frame socket never answered")
if recorder.error:
fail(f"recording failed: {recorder.error}")
log(f"recording {args.seconds:.0f}s at {args.capture_fps:.0f}fps "
f"(server says {recorder.hello.get('fps')}fps, "
f"{recorder.hello.get('width')}x{recorder.hello.get('height')})")
began = time.monotonic()
time.sleep(1.5) # let the page settle on screen
type_query(devtools, args.query)
landed = submit_and_wait(devtools)
log(f"landed on {landed}")
# The wall can arrive as a redirect *after* the results paint, so give
# it a beat and then look again rather than trusting the first URL.
time.sleep(1.5)
if is_walled(devtools):
# A video that looks like a working demo and is not one is worse
# than no video.
fail(
"Google served its 'unusual traffic' wall instead of results "
"(rate-limited IP, not a broken browser). Wait a few minutes and "
"run again; the first search from a fresh profile usually passes."
)
devtools.evaluate("window.scrollBy(0, 320)") # show actual results
# Record for the requested wall-clock duration, so the video's length is
# what was asked for regardless of how long the page took to load.
remaining = args.seconds - (time.monotonic() - began)
if remaining > 0:
time.sleep(remaining)
# Final gate, after the whole recording: if the page ended up on the
# wall at any point, nothing is encoded.
if is_walled(devtools):
fail("Google walled the search during recording — no video written")
finally:
if recorder is not None:
recorder.stop()
if devtools is not None:
devtools.close()
browser.terminate()
try:
browser.wait(timeout=10)
except subprocess.TimeoutExpired:
browser.kill()
if recorder is None or recorder.count < 2:
fail("fewer than two frames arrived — nothing to encode")
fps = recorder.count / args.seconds
log(f"captured {recorder.count} frames ({fps:.2f} fps effective)")
encode(frames, fps, out, recorder.ext)
size = out.stat().st_size
log(f"wrote {out} ({size / 1024:.0f} KiB, {recorder.count} frames)")
if not args.keep_work:
shutil.rmtree(work, ignore_errors=True)
return 0
if __name__ == "__main__":
raise SystemExit(main())