#!/usr/bin/env python3 """Run every route through the frozen contract and print a head-to-head. The point of a runner rather than a shell loop: the *same* cells get run for every route with identical arguments, the raw JSON of every run is kept, and the table is derived from those files instead of from what anyone remembers seeing. ./bench.py --output hh- --frames 300 --runs 2 ./bench.py --output hh- --only grim --cells output-jpeg Routes are commands, so a route that is not built yet simply fails and is reported as a failed cell rather than quietly omitted. """ from __future__ import annotations import argparse import json import os import shlex import subprocess import sys import time HERE = os.path.dirname(os.path.abspath(__file__)) PRODUCERS = os.path.join(HERE, "producers") RUST = os.path.join(HERE, "001a-grim-rs", "target", "release", "grimrs-capture") ROUTES = { # name: (description, command builder) "grim": ( "grim subprocess per frame (shipped)", lambda o, cell, frames, warmup: [ sys.executable, os.path.join(PRODUCERS, "grim_producer.py"), *(["--all"] if cell["all"] else ["--output", o]), "--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json", ], ), "py-screencopy": ( "pywayland in-process, zwlr-screencopy-v1", lambda o, cell, frames, warmup: [ sys.executable, os.path.join(PRODUCERS, "wayland_producer.py"), *(["--all"] if cell["all"] else ["--output", o]), "--protocol", "screencopy", "--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json", ], ), "py-ext": ( "pywayland in-process, ext-image-copy-capture-v1", lambda o, cell, frames, warmup: [ sys.executable, os.path.join(PRODUCERS, "wayland_producer.py"), *(["--all"] if cell["all"] else ["--output", o]), "--protocol", "ext", "--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json", ], ), "grim-rs": ( "grim-rs crate (Rust), long-lived process", lambda o, cell, frames, warmup: [ RUST, *(["--all"] if cell["all"] else ["--output", o]), "--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json", ], ), } CELLS = [ {"name": "output-raw", "all": False, "mode": "raw"}, {"name": "output-jpeg", "all": False, "mode": "jpeg"}, {"name": "all-raw", "all": True, "mode": "raw"}, {"name": "all-jpeg", "all": True, "mode": "jpeg"}, ] def idle_control(seconds: float) -> dict: """What the machine burns with nothing capturing at all. Without this the resource columns are unreadable: on a software-rendered compositor an animating page can saturate both cores on its own, and a per-frame CPU number that includes the animation would be attributed to the capture route. This is the number to subtract -- and it is also the reason the verdict has to talk about *what the machine was doing*, not just fps. """ sys.path.insert(0, PRODUCERS) from resources import MachineSample # noqa: E402 before = MachineSample(children=False) time.sleep(seconds) after = MachineSample(children=False) window_ms = seconds * 1000.0 system_ms = None compositor_ms = None if before.system_cpu is not None and after.system_cpu is not None: system_ms = (after.system_cpu - before.system_cpu) * 1000.0 if before.compositor_cpu is not None and after.compositor_cpu is not None: compositor_ms = (after.compositor_cpu - before.compositor_cpu) * 1000.0 return { "window_ms": round(window_ms, 1), "system_cpu_ms": None if system_ms is None else round(system_ms, 1), "compositor_cpu_ms": None if compositor_ms is None else round(compositor_ms, 1), "system_cpu_pct": None if system_ms is None else round(system_ms / window_ms * 100.0, 1), "compositor_cpu_pct": None if compositor_ms is None else round(compositor_ms / window_ms * 100.0, 1), } def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", required=True, help="the session container output to measure") parser.add_argument("--frames", type=int, default=300) parser.add_argument("--warmup", type=int, default=10) parser.add_argument("--runs", type=int, default=2) parser.add_argument("--only", action="append", help="restrict to these route names") parser.add_argument("--cells", action="append", help="restrict to these cell names") parser.add_argument("--results", default=os.path.join(HERE, "..", "results")) parser.add_argument("--env", action="append", default=[], help="KEY=VALUE for the producers") parser.add_argument("--control-seconds", type=float, default=0.0, help="sample machine CPU with no captures for this long, as a control") parser.add_argument("--resume", action="store_true", help="skip any cell whose result file already exists (for re-running after a kill)") args = parser.parse_args() routes = {k: v for k, v in ROUTES.items() if not args.only or k in args.only} cells = [c for c in CELLS if not args.cells or c["name"] in args.cells] os.makedirs(args.results, exist_ok=True) control = None if args.control_seconds > 0: print(f"control: sampling {args.control_seconds:.0f}s with nothing capturing...") control = idle_control(args.control_seconds) print(f" idle machine CPU: {control['system_cpu_pct']}% of one core " f"({control['system_cpu_ms'] / control['window_ms'] * 1000:.0f} ms of CPU per second), " f"compositor {control['compositor_cpu_pct']}% of one core") with open(os.path.join(args.results, "control.json"), "w") as handle: json.dump(control, handle, indent=2) env = dict(os.environ) for item in args.env: key, _, value = item.partition("=") env[key] = value records: list[dict] = [] failures: list[str] = [] for route, (description, build) in routes.items(): for cell in cells: for run in range(1, args.runs + 1): command = build(args.output, cell, args.frames, args.warmup) label = f"{route}/{cell['name']}/run{run}" stem = os.path.join(args.results, f"{route}.{cell['name']}.run{run}") if args.resume and os.path.exists(stem + ".stdout"): print(f" {label}: already measured, skipped (--resume)") try: with open(stem + ".stdout") as handle: lines = [line for line in handle.read().splitlines() if line.startswith("{")] if lines: records.append({"route": route, "description": description, "cell": cell["name"], "run": run, "rc": 0, **json.loads(lines[-1])}) except (OSError, json.JSONDecodeError): pass continue started = time.monotonic() proc = subprocess.run(command, capture_output=True, text=True, env=env) elapsed = time.monotonic() - started with open(stem + ".stdout", "w") as handle: handle.write(proc.stdout) with open(stem + ".stderr", "w") as handle: handle.write(proc.stderr) record = {"route": route, "description": description, "cell": cell["name"], "run": run, "command": " ".join(shlex.quote(part) for part in command), "rc": proc.returncode, "elapsed_s": round(elapsed, 1)} last = [line for line in proc.stdout.splitlines() if line.startswith("{")] if proc.returncode == 0 and last: record.update(json.loads(last[-1])) print(f" {label}: p50 {record['frame_ms']['p50']} ms, " f"{record['fps_sustained']} fps, cpu {record['cpu_ms_per_frame']} ms") else: record["error"] = (proc.stderr or proc.stdout).strip().splitlines()[-1:] or ["no output"] failures.append(label) print(f" {label}: FAILED rc={proc.returncode} -- {record['error']}") records.append(record) with open(os.path.join(args.results, "results.json"), "w") as handle: json.dump(records, handle, indent=2) print("\n=== head-to-head (median of runs; jpeg cells are the ones the wire carries) ===") print("speed | machine cost | footprint") print(f"{'route':<15} {'cell':<12} {'p50 ms':>8} {'fps':>6} {'own cpu':>8} {'sys cpu':>8} " f"{'comp cpu':>9} {'machine%':>9} {'rss MB':>7} {'bytes':>9}") for route in routes: for cell in cells: group = [r for r in records if r.get("route") == route and r.get("cell") == cell["name"]] good = [r for r in group if "frame_ms" in r] if not good: print(f"{route:<15} {cell['name']:<12} {'--':>8} {'--':>6} {'--':>8} {'--':>8} {'--':>9} {'--':>9} {'--':>7} (no successful run)") continue def median(key, default=0.0): values = sorted(r.get(key) or 0 for r in good) return values[len(values) // 2] if values else default p50 = sorted(r["frame_ms"]["p50"] for r in good)[len(good) // 2] print(f"{route:<15} {cell['name']:<12} {p50:>8} " f"{sorted(r['fps_sustained'] for r in good)[len(good)//2]:>6} " f"{median('cpu_ms_per_frame'):>8} {median('system_cpu_ms_per_frame'):>8} " f"{median('compositor_cpu_ms_per_frame'):>9} {median('machine_cpu_pct'):>9} " f"{median('rss_peak_mb'):>7} {int(median('bytes_p50')):>9}") if control: print(f"\ncontrol (nothing capturing): {control['system_cpu_pct']}% of one core, " f"compositor {control['compositor_cpu_ms']} ms/s") print(" subtract this from 'sys cpu' before attributing CPU to a route.") if failures: print(f"\nfailed cells: {', '.join(failures)}") print(f"\nraw runs: {args.results}/") return 0 if __name__ == "__main__": sys.exit(main())