Files
hyprland-headless-stream-pl…/spikes/capture-speed/bench.py
T
Hermenix bf0667303e spike: --resume for the runner, and run the matrix from a transient unit
Two attempts were reaped mid-run by agent lifecycle cleanup (exit 143 with no
error line); a nix-shell process tree of tens of minutes belongs in a systemd
transient unit, not an agent background process. --resume skips any cell whose
result file already exists, so a kill costs the cell in flight and nothing else.
2026-10-11 16:25:38 +00:00

219 lines
10 KiB
Python

#!/usr/bin/env python3
"""Run every route through the frozen contract and print a head-to-head.
The point of a runner rather than a shell loop: the *same* cells get run for
every route with identical arguments, the raw JSON of every run is kept, and the
table is derived from those files instead of from what anyone remembers seeing.
./bench.py --output hh-<session> --frames 300 --runs 2
./bench.py --output hh-<session> --only grim --cells output-jpeg
Routes are commands, so a route that is not built yet simply fails and is
reported as a failed cell rather than quietly omitted.
"""
from __future__ import annotations
import argparse
import json
import os
import shlex
import subprocess
import sys
import time
HERE = os.path.dirname(os.path.abspath(__file__))
PRODUCERS = os.path.join(HERE, "producers")
RUST = os.path.join(HERE, "001a-grim-rs", "target", "release", "grimrs-capture")
ROUTES = {
# name: (description, command builder)
"grim": (
"grim subprocess per frame (shipped)",
lambda o, cell, frames, warmup: [
sys.executable, os.path.join(PRODUCERS, "grim_producer.py"),
*(["--all"] if cell["all"] else ["--output", o]),
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
],
),
"py-screencopy": (
"pywayland in-process, zwlr-screencopy-v1",
lambda o, cell, frames, warmup: [
sys.executable, os.path.join(PRODUCERS, "wayland_producer.py"),
*(["--all"] if cell["all"] else ["--output", o]),
"--protocol", "screencopy",
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
],
),
"py-ext": (
"pywayland in-process, ext-image-copy-capture-v1",
lambda o, cell, frames, warmup: [
sys.executable, os.path.join(PRODUCERS, "wayland_producer.py"),
*(["--all"] if cell["all"] else ["--output", o]),
"--protocol", "ext",
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
],
),
"grim-rs": (
"grim-rs crate (Rust), long-lived process",
lambda o, cell, frames, warmup: [
RUST,
*(["--all"] if cell["all"] else ["--output", o]),
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
],
),
}
CELLS = [
{"name": "output-raw", "all": False, "mode": "raw"},
{"name": "output-jpeg", "all": False, "mode": "jpeg"},
{"name": "all-raw", "all": True, "mode": "raw"},
{"name": "all-jpeg", "all": True, "mode": "jpeg"},
]
def idle_control(seconds: float) -> dict:
"""What the machine burns with nothing capturing at all.
Without this the resource columns are unreadable: on a software-rendered
compositor an animating page can saturate both cores on its own, and a
per-frame CPU number that includes the animation would be attributed to the
capture route. This is the number to subtract -- and it is also the reason
the verdict has to talk about *what the machine was doing*, not just fps.
"""
sys.path.insert(0, PRODUCERS)
from resources import MachineSample # noqa: E402
before = MachineSample(children=False)
time.sleep(seconds)
after = MachineSample(children=False)
window_ms = seconds * 1000.0
system_ms = None
compositor_ms = None
if before.system_cpu is not None and after.system_cpu is not None:
system_ms = (after.system_cpu - before.system_cpu) * 1000.0
if before.compositor_cpu is not None and after.compositor_cpu is not None:
compositor_ms = (after.compositor_cpu - before.compositor_cpu) * 1000.0
return {
"window_ms": round(window_ms, 1),
"system_cpu_ms": None if system_ms is None else round(system_ms, 1),
"compositor_cpu_ms": None if compositor_ms is None else round(compositor_ms, 1),
"system_cpu_pct": None if system_ms is None else round(system_ms / window_ms * 100.0, 1),
"compositor_cpu_pct": None if compositor_ms is None else round(compositor_ms / window_ms * 100.0, 1),
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", required=True, help="the session container output to measure")
parser.add_argument("--frames", type=int, default=300)
parser.add_argument("--warmup", type=int, default=10)
parser.add_argument("--runs", type=int, default=2)
parser.add_argument("--only", action="append", help="restrict to these route names")
parser.add_argument("--cells", action="append", help="restrict to these cell names")
parser.add_argument("--results", default=os.path.join(HERE, "..", "results"))
parser.add_argument("--env", action="append", default=[], help="KEY=VALUE for the producers")
parser.add_argument("--control-seconds", type=float, default=0.0,
help="sample machine CPU with no captures for this long, as a control")
parser.add_argument("--resume", action="store_true",
help="skip any cell whose result file already exists (for re-running after a kill)")
args = parser.parse_args()
routes = {k: v for k, v in ROUTES.items() if not args.only or k in args.only}
cells = [c for c in CELLS if not args.cells or c["name"] in args.cells]
os.makedirs(args.results, exist_ok=True)
control = None
if args.control_seconds > 0:
print(f"control: sampling {args.control_seconds:.0f}s with nothing capturing...")
control = idle_control(args.control_seconds)
print(f" idle machine CPU: {control['system_cpu_pct']}% of one core "
f"({control['system_cpu_ms'] / control['window_ms'] * 1000:.0f} ms of CPU per second), "
f"compositor {control['compositor_cpu_pct']}% of one core")
with open(os.path.join(args.results, "control.json"), "w") as handle:
json.dump(control, handle, indent=2)
env = dict(os.environ)
for item in args.env:
key, _, value = item.partition("=")
env[key] = value
records: list[dict] = []
failures: list[str] = []
for route, (description, build) in routes.items():
for cell in cells:
for run in range(1, args.runs + 1):
command = build(args.output, cell, args.frames, args.warmup)
label = f"{route}/{cell['name']}/run{run}"
stem = os.path.join(args.results, f"{route}.{cell['name']}.run{run}")
if args.resume and os.path.exists(stem + ".stdout"):
print(f" {label}: already measured, skipped (--resume)")
try:
with open(stem + ".stdout") as handle:
lines = [line for line in handle.read().splitlines() if line.startswith("{")]
if lines:
records.append({"route": route, "description": description,
"cell": cell["name"], "run": run, "rc": 0,
**json.loads(lines[-1])})
except (OSError, json.JSONDecodeError):
pass
continue
started = time.monotonic()
proc = subprocess.run(command, capture_output=True, text=True, env=env)
elapsed = time.monotonic() - started
with open(stem + ".stdout", "w") as handle:
handle.write(proc.stdout)
with open(stem + ".stderr", "w") as handle:
handle.write(proc.stderr)
record = {"route": route, "description": description, "cell": cell["name"],
"run": run, "command": " ".join(shlex.quote(part) for part in command),
"rc": proc.returncode, "elapsed_s": round(elapsed, 1)}
last = [line for line in proc.stdout.splitlines() if line.startswith("{")]
if proc.returncode == 0 and last:
record.update(json.loads(last[-1]))
print(f" {label}: p50 {record['frame_ms']['p50']} ms, "
f"{record['fps_sustained']} fps, cpu {record['cpu_ms_per_frame']} ms")
else:
record["error"] = (proc.stderr or proc.stdout).strip().splitlines()[-1:] or ["no output"]
failures.append(label)
print(f" {label}: FAILED rc={proc.returncode} -- {record['error']}")
records.append(record)
with open(os.path.join(args.results, "results.json"), "w") as handle:
json.dump(records, handle, indent=2)
print("\n=== head-to-head (median of runs; jpeg cells are the ones the wire carries) ===")
print("speed | machine cost | footprint")
print(f"{'route':<15} {'cell':<12} {'p50 ms':>8} {'fps':>6} {'own cpu':>8} {'sys cpu':>8} "
f"{'comp cpu':>9} {'machine%':>9} {'rss MB':>7} {'bytes':>9}")
for route in routes:
for cell in cells:
group = [r for r in records if r.get("route") == route and r.get("cell") == cell["name"]]
good = [r for r in group if "frame_ms" in r]
if not good:
print(f"{route:<15} {cell['name']:<12} {'--':>8} {'--':>6} {'--':>8} {'--':>8} {'--':>9} {'--':>9} {'--':>7} (no successful run)")
continue
def median(key, default=0.0):
values = sorted(r.get(key) or 0 for r in good)
return values[len(values) // 2] if values else default
p50 = sorted(r["frame_ms"]["p50"] for r in good)[len(good) // 2]
print(f"{route:<15} {cell['name']:<12} {p50:>8} "
f"{sorted(r['fps_sustained'] for r in good)[len(good)//2]:>6} "
f"{median('cpu_ms_per_frame'):>8} {median('system_cpu_ms_per_frame'):>8} "
f"{median('compositor_cpu_ms_per_frame'):>9} {median('machine_cpu_pct'):>9} "
f"{median('rss_peak_mb'):>7} {int(median('bytes_p50')):>9}")
if control:
print(f"\ncontrol (nothing capturing): {control['system_cpu_pct']}% of one core, "
f"compositor {control['compositor_cpu_ms']} ms/s")
print(" subtract this from 'sys cpu' before attributing CPU to a route.")
if failures:
print(f"\nfailed cells: {', '.join(failures)}")
print(f"\nraw runs: {args.results}/")
return 0
if __name__ == "__main__":
sys.exit(main())