diff --git a/README.md b/README.md index 944b620..80901b1 100644 --- a/README.md +++ b/README.md @@ -40,5 +40,6 @@ connects, and keeps running when one disconnects. | `docs/CONTRACT.md` | the wire protocol and the module surface — the interface other people build against | | `docs/ARCHITECTURE.md` | how the pieces fit: compositor, capture, server, client | | `README.dev.md` | how to build, test and hack on it | +| `examples/README.md` | the demo that films the screen: `agent-eyes-demo.py` | Status: work in progress — the module and the test suite are being built. \ No newline at end of file diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..39446cf --- /dev/null +++ b/examples/README.md @@ -0,0 +1,63 @@ +# examples + +## `agent-eyes-demo.py` + +Puts a real browser on the headless screen, drives it, and films the result by +reading frames off the published socket — the consumer side of +[`docs/CONTRACT.md`](../docs/CONTRACT.md). + +```bash +nix develop +sg hhstream -c 'python3 examples/agent-eyes-demo.py --query "Ari Sosnovsky"' +``` + +Output is an MP4 (default `agent-eyes.mp4`): the Google homepage, the query +typed in a letter at a time, then the results page — captured from the +compositor, not from a screencast API. + +Everything else is a flag: `--seconds`, `--capture-fps`, `--output`, +`--work-dir`, `--keep-work`, `--debug-port`, `--clear-screen`. `--help` is +accurate. + +### What it checks rather than assumes + +- the session is up and the socket is readable (a clear message if your user is + not in the stream's group); +- the browser actually reached a results page — it presses Enter, then waits for + the URL to change, and submits the form directly if Enter did nothing; +- it *fails* on Google's `/sorry/` bot wall instead of encoding a video of a + CAPTCHA, because a video that looks like a working demo and is not one is + worse than no video. + +### Three things that will bite you + +1. **Your shell needs the group.** The socket is `0660 hhstream:hhstream` inside + a `0750` directory; a browser also has to reach the session's Wayland socket + in the same directory. Add yourself to `hhstream` (this is what + `/etc/nixos/hyprland-stream.nix` does for `hermes`) and log in again — or use + `sg hhstream -c '...'` for the shell you already have open. +2. **Google rate-limits you.** The first search from a fresh profile usually + passes; the third in a minute gets `unusual traffic`. It is the IP, not the + browser. Wait a few minutes and re-run. +3. **The frame server tops out around 12 fps** on a software-rendered VM + (`grim` plus a PNG encode per frame, competing with the browser for two + cores). Leave `--capture-fps` at its default of 30 so nothing is thrown away: + the encoder uses the *measured* rate, so the video plays at real speed either + way, and decimating only makes it choppier. + +### Posting the result to a chat + +Xiph/Blossom media stores reject files that carry metadata. `encode()` in the +script already strips it with `-map_metadata -1 -fflags +bitexact -flags:v ++bitexact`; without those flags a Buzz relay answers +`422 media contains metadata or a non-canonical metadata channel` and the upload +fails. Uploading from the command line, with the same script-encoded file: + +```bash +buzz upload file --file agent-eyes.mp4 # -> {"url": ".../media/.mp4"} +buzz media get -o check.mp4 "" # fetch it back and compare sha256 +``` + +`buzz upload file` takes `--file `, not a positional argument, and exits +**0** with a JSON error on stdout when the relay refuses the type — read the +output, never the exit status. diff --git a/examples/agent-eyes-demo.py b/examples/agent-eyes-demo.py index 76a98e3..1889be0 100644 --- a/examples/agent-eyes-demo.py +++ b/examples/agent-eyes-demo.py @@ -110,8 +110,11 @@ def check_readable(socket_path: str) -> None: def clear_screen(runtime_dir: str) -> None: """Kill leftover browsers on the shared screen (needs sudo).""" log("--clear-screen: killing leftover chromium processes") + # The bracket keeps the pattern from matching `pkill`'s own command line — + # `pkill -f chromium` kills the shell that ran it, which is a confusing way + # to lose a session. result = subprocess.run( - ["sudo", "-n", "pkill", "-f", "chromium"], + ["sudo", "-n", "pkill", "-f", "[c]hromium"], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, ) if result.returncode not in (0, 1): # 1 == "nothing matched", which is fine @@ -265,8 +268,13 @@ class Recorder(threading.Thread): self.hello: dict = {} self.count = 0 self.error: str | None = None - self._stop = threading.Event() - self._started = threading.Event() + # NOT `_stop` / `_started`: `threading.Thread` owns both names. It sets + # `_started` itself as the thread boots — *before* `run()` — so a + # `wait()` on it returns instantly and every attribute `run()` is + # supposed to fill in is still empty. It also calls `self._stop()` on + # the way out, which an Event cannot answer to. + self._stop_event = threading.Event() + self._ready = threading.Event() def run(self) -> None: try: @@ -275,7 +283,7 @@ class Recorder(threading.Thread): conn.connect(self.socket_path) except OSError as exc: self.error = str(exc) - self._started.set() + self._ready.set() return try: @@ -285,9 +293,9 @@ class Recorder(threading.Thread): self.hello = protocol.read_hello(stream) server_fps = float(self.hello.get("fps") or 30) step = max(1, round(server_fps / self.target_fps)) - self._started.set() + self._ready.set() index = 0 - while not self._stop.is_set(): + while not self._stop_event.is_set(): payload = protocol.read_frame(stream) if payload is None: break @@ -300,10 +308,10 @@ class Recorder(threading.Thread): self.error = str(exc) finally: conn.close() - self._started.set() + self._ready.set() def stop(self) -> None: - self._stop.set() + self._stop_event.set() self.join(timeout=10) @@ -315,6 +323,13 @@ def encode(frames: pathlib.Path, fps: float, out: pathlib.Path) -> None: "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", # yuv420p + faststart: what every player and every chat client wants. "-pix_fmt", "yuv420p", "-movflags", "+faststart", + # Strip every metadata tag. Xiph/Blossom media stores reject files that + # carry them — Buzz's does, with `422 media contains metadata or a + # non-canonical metadata channel`, and the only thing to fix is ffmpeg's + # own `encoder`/`handler_name` tags. `bitexact` keeps them from being + # written back in. + "-map_metadata", "-1", "-fflags", "+bitexact", "-flags:v", "+bitexact", + "-an", str(out), ] log(f"encoding {out.name}") @@ -374,7 +389,7 @@ def main() -> int: recorder = Recorder(args.socket, frames, args.capture_fps) recorder.start() - if not recorder._started.wait(timeout=15): + if not recorder._ready.wait(timeout=15): fail("the frame socket never answered") if recorder.error: fail(f"recording failed: {recorder.error}") @@ -387,6 +402,14 @@ def main() -> int: type_query(devtools, args.query) landed = submit_and_wait(devtools) log(f"landed on {landed}") + if "/sorry/" in landed: + # Google's bot wall. Filming it would produce a video that looks + # like a working demo and is not one, so stop instead. + fail( + "Google served its 'unusual traffic' wall instead of results " + "(rate-limited IP, not a broken browser). Wait a few minutes and " + "run again; the first search from a fresh profile usually passes." + ) devtools.evaluate("window.scrollBy(0, 320)") # show actual results # Record for the requested wall-clock duration, so the video's length is