diff --git a/docs/TODO/cursor-visibility-in-captures.md b/docs/TODO/cursor-visibility-in-captures.md index bdb9ded..33ce00d 100644 --- a/docs/TODO/cursor-visibility-in-captures.md +++ b/docs/TODO/cursor-visibility-in-captures.md @@ -1,5 +1,14 @@ # The compositor draws no cursor for a virtual pointer +## What is open + +**The overlay marker is not implemented.** The acceptance video was recorded +without it, so the frames show the pointer's *effects* — a page that scrolled, +a tab that opened — and never the pointer itself. Implement it by stamping +`hyprctl cursorpos` into each frame, mapped into the stitched layout, and label +it as an overlay wherever the frame is published. Nothing else about the pointer +is blocked: position, clicks and scrolling are all exact and all readable. + ## The issue — answered, and it is not a capture setting The pointer moves exactly where it is told, and **nothing is drawn**. Every diff --git a/examples/proof-video.py b/examples/proof-video.py index 60c1c82..3c07cb6 100644 --- a/examples/proof-video.py +++ b/examples/proof-video.py @@ -1,35 +1,41 @@ #!/usr/bin/env python3 -"""Drive the session's browser over CDP, then put a terminal next to it. +"""The acceptance video: an SKS search, real scrolling, tabs, and cowsay. -The acceptance proof for Agent Eyes: the agent searches Gunpost for an SKS, opens -a few listings in their own tabs, clears the browser away, and runs cowsay in a -terminal on the same screen. It is the demonstration that the agent can see and -work a real desktop, and it lives here rather than in a scratch directory because -it has been asked for more than once and a tool kept in scratch gets pruned. +One command, no arguments, and it cleans up after itself. It starts a browser +session (which gets a desk of its own — its own output and workspace), films +*that desk alone*, drives the search over the browser's own HTTP endpoints, and +scrolls the results page with the real pointer -- `hhstream session pointer +scroll`, which sends wl_pointer axis events rather than pressing a key. -Usage, with the browser session already started and its CDP port up: +Then it kills the browser, starts a terminal session on its own desk, and films +cowsay saying what it says. The two films are joined into one mp4. - hhstream session start chromium -- --ozone-platform=wayland ... \\ - --remote-debugging-port=9334 --remote-debugging-address=127.0.0.1 about:blank - hhstream record --seconds 80 --out proof.mp4 & - ./proof-video.py +Why `--output` and not the published frame socket: with desks, the socket +carries every output stitched side by side, so a viewer would watch two thirds +wallpaper. `--output` runs grim per frame against the session's own desk and +costs more; the trade is a video of the thing being demonstrated. -Only the browser's own HTTP endpoints are used (/json/new, /json/activate) -- -no websocket, no shell, no typing. The URLs are real: the search is the one the -gun watch uses (`key=`, singular, which is a bug that once cost every search its -filtering) and the listings are Gunpost pages the watch itself collected. +The cursor is **not** in the frames, and that is not a capture setting: the +virtual pointer has no cursor image for the compositor to draw. See +docs/TODO/cursor-visibility-in-captures.md. The scrolling is visible by what +moves, which is why the page is scrolled rather than merely pointed at. + + ./proof-video.py [--profile DIR] [--out FILE] """ from __future__ import annotations +import argparse import json +import pathlib import subprocess import sys import time import urllib.parse import urllib.request -CDP = "http://127.0.0.1:9334" +CDP_PORT = 9344 +CDP = f"http://127.0.0.1:{CDP_PORT}" SEARCH = "https://www.gunpost.ca/ads?key=SKS&sort=created&order=desc" LISTINGS = [ "https://www.gunpost.ca/firearms/rifles/hamilton/sks-1951-russian-excellent-rifle-762x39-100000-obo", @@ -41,6 +47,10 @@ MESSAGE = "hahaha I can see the world now" #: profile in it: a bare `cowsay` fails inside foot with "No such file or directory". COWSAY = "/nix/store/s4hlzmy3ll1bphdsy8vjdpcw447vbr93-cowsay-3.8.4/bin/cowsay" +#: How long each film runs. The beats below are timed to fit inside these. +BROWSER_SECONDS = 44 +COW_SECONDS = 12 + def call(path: str, method: str = "GET") -> dict: req = urllib.request.Request(CDP + path, method=method) @@ -50,8 +60,7 @@ def call(path: str, method: str = "GET") -> dict: def open_tab(url: str) -> str: - tab = call("/json/new?" + urllib.parse.quote(url, safe=""), method="PUT") - return tab["id"] + return call("/json/new?" + urllib.parse.quote(url, safe=""), method="PUT")["id"] def activate(tab_id: str) -> None: @@ -63,45 +72,141 @@ def activate(tab_id: str) -> None: print(f" (activate {tab_id[:8]} said: {exc})", flush=True) -def hhstream(*args: str) -> str: - out = subprocess.run(["hhstream", *args], capture_output=True, text=True, timeout=180) +def hhstream(*args: str, check: bool = False) -> str: + out = subprocess.run(["hhstream", *args], capture_output=True, text=True, timeout=300) if out.returncode != 0: - print(f" (hhstream {' '.join(args[:2])} -> {out.returncode}: {out.stderr.strip()[:120]})") + print(f" (hhstream {' '.join(args[:3])} -> {out.returncode}: {out.stderr.strip()[:160]})") + if check: + raise SystemExit(f"hhstream {' '.join(args[:3])} failed") return out.stdout.strip() -def main() -> int: - session = sys.argv[1] if len(sys.argv) > 1 else None +def desk_of(session: str) -> str: + """The output name of a session's desk, as the broker recorded it.""" - print("=== beat 1: the search ===", flush=True) + reply = json.loads(hhstream("session", "container", "--json") or "{}") + rows = reply.get("containers") or reply.get("sessions") or [] + for row in rows: + if row.get("id") == session: + desk = row.get("output") or (row.get("container") or {}).get("output") + if desk: + return desk + raise SystemExit(f"{session} has no desk output: {json.dumps(rows)[:300]}") + + +def busy_work(session: str) -> None: + """The pointer beats: scroll the results, then walk the listings. + + Scrolling is the point of these beats. The pointer is aimed into this + session's own desk, so the scroll lands on its page and nowhere else -- and + because the cursor is not drawn, what a viewer sees is the page moving. + """ + + lease = json.loads(hhstream("session", "pointer", "acquire", session, "--json") or "{}") + lease_id = lease.get("lease_id") + if not lease_id: + print(f" (no lease: {json.dumps(lease)[:200]})", flush=True) + return + # Down, down again, then back up: two directions, so it cannot be mistaken + # for a page that simply loaded a different part of itself. + hhstream("session", "pointer", "move", session, "--lease", lease_id, "--to", "0.5,0.55") + for amount, pause in ((6, 2.5), (6, 2.5), (-4, 2.0)): + print(f" scroll vertical {amount:+d}", flush=True) + hhstream( + "session", "pointer", "scroll", session, + "--lease", lease_id, "--axis", "vertical", "--amount", str(amount), + ) + time.sleep(pause) + hhstream("session", "pointer", "release", session, "--lease", lease_id) + + +def film(session: str, seconds: int, out: pathlib.Path) -> None: + """Record one desk until `seconds` have passed, then wait for the encoder.""" + + hhstream( + "record", "--output", desk_of(session), "--fps", "12", + "--seconds", str(seconds), "--out", str(out), + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", default="/var/lib/hermes/agent-eyes/gunpost-profile") + parser.add_argument("--out", default="/var/lib/hermes/agent-eyes/desks-proof.mp4") + parser.add_argument("--work-dir", default="/var/lib/hermes/agent-eyes/takes") + args = parser.parse_args() + + out = pathlib.Path(args.out) + work = pathlib.Path(args.work_dir) + work.mkdir(parents=True, exist_ok=True) + pathlib.Path(args.profile).mkdir(parents=True, exist_ok=True) + a, b = work / "desk-a.mp4", work / "desk-b.mp4" + + print("=== starting the browser on its own desk ===", flush=True) + browser = hhstream( + "session", "start", "chromium", "--", + "--ozone-platform=wayland", f"--user-data-dir={args.profile}", + "--no-first-run", "--no-default-browser-check", "--window-size=1280,720", + f"--remote-debugging-port={CDP_PORT}", "--remote-debugging-address=127.0.0.1", + "--new-window", "about:blank", + check=True, + ) + print(f" {browser} on {desk_of(browser)}", flush=True) + time.sleep(4) # let the window map and the debugging port come up + + print("=== film 1: the search, scrolling, and three listings ===", flush=True) + recording = subprocess.Popen( + ["hhstream", "record", "--output", desk_of(browser), "--fps", "12", + "--seconds", str(BROWSER_SECONDS), "--out", str(a)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + time.sleep(2) search_tab = open_tab(SEARCH) activate(search_tab) - time.sleep(13) # long enough for Cloudflare's check to clear and results to paint - - print("=== beats 2-4: three listings, each in its own tab ===", flush=True) + print(" the SKS search is loading (Cloudflare included)", flush=True) + time.sleep(13) + busy_work(browser) for n, url in enumerate(LISTINGS, 1): - tab = open_tab(url) - print(f"listing {n}: {url}", flush=True) - activate(tab) - time.sleep(7) - - print("=== beat 5: back to the results ===", flush=True) + activate(open_tab(url)) + print(f" listing {n} in its own tab", flush=True) + time.sleep(4) activate(search_tab) - time.sleep(5) + time.sleep(2) + recording.wait(timeout=300) + print(f" {a}: {a.stat().st_size} bytes", flush=True) - print("=== beat 6: clear the browser away ===", flush=True) - if session: - print(f" killed: {hhstream('session', 'kill', session)[:80]}", flush=True) - time.sleep(3) + print("=== clearing the browser away ===", flush=True) + print(f" {hhstream('session', 'kill', browser)[:90]}", flush=True) + time.sleep(2) - print("=== beat 7: a terminal, running cowsay ===", flush=True) - print( - " started: " - + hhstream("session", "start", "foot", "--", "-H", "-f", "monospace:size=20", - COWSAY, MESSAGE)[:80], - flush=True, + print("=== film 2: a terminal on its own desk, running cowsay ===", flush=True) + terminal = hhstream( + "session", "start", "foot", "--", + "-H", "-f", "monospace:size=20", COWSAY, MESSAGE, + check=True, ) - time.sleep(20) + print(f" {terminal} on {desk_of(terminal)}", flush=True) + time.sleep(2) + film(terminal, COW_SECONDS, b) + print(f" {b}: {b.stat().st_size} bytes", flush=True) + print(f" {hhstream('session', 'kill', terminal)[:90]}", flush=True) + + print("=== joining the two films ===", flush=True) + listing = work / "concat.txt" + listing.write_text(f"file '{a}'\nfile '{b}'\n") + joined = subprocess.run( + ["ffmpeg", "-v", "error", "-f", "concat", "-safe", "0", "-i", str(listing), + "-c", "copy", "-y", str(out)], + capture_output=True, text=True, + ) + if joined.returncode != 0: + raise SystemExit(f"join failed: {joined.stderr.strip()[:300]}") + probe = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "format=duration,size", + "-of", "default=noprint_wrappers=1", str(out)], + capture_output=True, text=True, + ) + print(f" {out}\n {probe.stdout.strip().replace(chr(10), ' ')}", flush=True) return 0