Four routes measured on one frozen contract at one session's container size, with RAM and CPU as first-class results: in-process pywayland is ~1.5x faster (78 vs 117 ms), uses ~2.8x less of our own CPU, and nearly halves the compositor's per-frame CPU (67 vs 116 ms). grim-rs buys nothing and costs 746 MB RSS. ext-image-copy-capture measures identical to screencopy and appears in a compositor crash stack, so the deprecated-but-working protocol is the pick. Also fixes the client's unclean exit (pywayland teardown segfaulted after the result was printed, 16 times, one batch coinciding with a compositor crash).
224 lines
11 KiB
Python
224 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Run every route through the frozen contract and print a head-to-head.
|
|
|
|
The point of a runner rather than a shell loop: the *same* cells get run for
|
|
every route with identical arguments, the raw JSON of every run is kept, and the
|
|
table is derived from those files instead of from what anyone remembers seeing.
|
|
|
|
./bench.py --output hh-<session> --frames 300 --runs 2
|
|
./bench.py --output hh-<session> --only grim --cells output-jpeg
|
|
|
|
Routes are commands, so a route that is not built yet simply fails and is
|
|
reported as a failed cell rather than quietly omitted.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
PRODUCERS = os.path.join(HERE, "producers")
|
|
RUST = os.path.join(HERE, "001a-grim-rs", "target", "release", "grimrs-capture")
|
|
|
|
ROUTES = {
|
|
# name: (description, command builder)
|
|
"grim": (
|
|
"grim subprocess per frame (shipped)",
|
|
lambda o, cell, frames, warmup: [
|
|
sys.executable, os.path.join(PRODUCERS, "grim_producer.py"),
|
|
*(["--all"] if cell["all"] else ["--output", o]),
|
|
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
|
|
],
|
|
),
|
|
"py-screencopy": (
|
|
"pywayland in-process, zwlr-screencopy-v1",
|
|
lambda o, cell, frames, warmup: [
|
|
sys.executable, os.path.join(PRODUCERS, "wayland_producer.py"),
|
|
*(["--all"] if cell["all"] else ["--output", o]),
|
|
"--protocol", "screencopy",
|
|
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
|
|
],
|
|
),
|
|
"py-ext": (
|
|
"pywayland in-process, ext-image-copy-capture-v1",
|
|
lambda o, cell, frames, warmup: [
|
|
sys.executable, os.path.join(PRODUCERS, "wayland_producer.py"),
|
|
*(["--all"] if cell["all"] else ["--output", o]),
|
|
"--protocol", "ext",
|
|
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
|
|
],
|
|
),
|
|
"grim-rs": (
|
|
"grim-rs crate (Rust), long-lived process",
|
|
lambda o, cell, frames, warmup: [
|
|
RUST,
|
|
*(["--all"] if cell["all"] else ["--output", o]),
|
|
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
|
|
],
|
|
),
|
|
}
|
|
|
|
CELLS = [
|
|
{"name": "output-raw", "all": False, "mode": "raw"},
|
|
{"name": "output-jpeg", "all": False, "mode": "jpeg"},
|
|
{"name": "all-raw", "all": True, "mode": "raw"},
|
|
{"name": "all-jpeg", "all": True, "mode": "jpeg"},
|
|
]
|
|
|
|
|
|
def idle_control(seconds: float) -> dict:
|
|
"""What the machine burns with nothing capturing at all.
|
|
|
|
Without this the resource columns are unreadable: on a software-rendered
|
|
compositor an animating page can saturate both cores on its own, and a
|
|
per-frame CPU number that includes the animation would be attributed to the
|
|
capture route. This is the number to subtract -- and it is also the reason
|
|
the verdict has to talk about *what the machine was doing*, not just fps.
|
|
"""
|
|
sys.path.insert(0, PRODUCERS)
|
|
from resources import MachineSample # noqa: E402
|
|
|
|
before = MachineSample(children=False)
|
|
time.sleep(seconds)
|
|
after = MachineSample(children=False)
|
|
window_ms = seconds * 1000.0
|
|
system_ms = None
|
|
compositor_ms = None
|
|
if before.system_cpu is not None and after.system_cpu is not None:
|
|
system_ms = (after.system_cpu - before.system_cpu) * 1000.0
|
|
if before.compositor_cpu is not None and after.compositor_cpu is not None:
|
|
compositor_ms = (after.compositor_cpu - before.compositor_cpu) * 1000.0
|
|
return {
|
|
"window_ms": round(window_ms, 1),
|
|
"system_cpu_ms": None if system_ms is None else round(system_ms, 1),
|
|
"compositor_cpu_ms": None if compositor_ms is None else round(compositor_ms, 1),
|
|
"system_cpu_pct": None if system_ms is None else round(system_ms / window_ms * 100.0, 1),
|
|
"compositor_cpu_pct": None if compositor_ms is None else round(compositor_ms / window_ms * 100.0, 1),
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output", required=True, help="the session container output to measure")
|
|
parser.add_argument("--frames", type=int, default=300)
|
|
parser.add_argument("--warmup", type=int, default=10)
|
|
parser.add_argument("--runs", type=int, default=2)
|
|
parser.add_argument("--only", action="append", help="restrict to these route names")
|
|
parser.add_argument("--cells", action="append", help="restrict to these cell names")
|
|
parser.add_argument("--results", default=os.path.join(HERE, "..", "results"))
|
|
parser.add_argument("--env", action="append", default=[], help="KEY=VALUE for the producers")
|
|
parser.add_argument("--control-seconds", type=float, default=0.0,
|
|
help="sample machine CPU with no captures for this long, as a control")
|
|
parser.add_argument("--resume", action="store_true",
|
|
help="skip any cell whose result file already exists (for re-running after a kill)")
|
|
args = parser.parse_args()
|
|
|
|
routes = {k: v for k, v in ROUTES.items() if not args.only or k in args.only}
|
|
cells = [c for c in CELLS if not args.cells or c["name"] in args.cells]
|
|
os.makedirs(args.results, exist_ok=True)
|
|
|
|
control = None
|
|
if args.control_seconds > 0:
|
|
print(f"control: sampling {args.control_seconds:.0f}s with nothing capturing...")
|
|
control = idle_control(args.control_seconds)
|
|
print(f" idle machine CPU: {control['system_cpu_pct']}% of one core "
|
|
f"({control['system_cpu_ms'] / control['window_ms'] * 1000:.0f} ms of CPU per second), "
|
|
f"compositor {control['compositor_cpu_pct']}% of one core")
|
|
with open(os.path.join(args.results, "control.json"), "w") as handle:
|
|
json.dump(control, handle, indent=2)
|
|
|
|
env = dict(os.environ)
|
|
for item in args.env:
|
|
key, _, value = item.partition("=")
|
|
env[key] = value
|
|
|
|
records: list[dict] = []
|
|
failures: list[str] = []
|
|
for route, (description, build) in routes.items():
|
|
for cell in cells:
|
|
for run in range(1, args.runs + 1):
|
|
command = build(args.output, cell, args.frames, args.warmup)
|
|
label = f"{route}/{cell['name']}/run{run}"
|
|
stem = os.path.join(args.results, f"{route}.{cell['name']}.run{run}")
|
|
if args.resume and os.path.exists(stem + ".stdout"):
|
|
# A *failed* cell also leaves a stdout file, so existence is
|
|
# not the test -- a parseable result line is. Otherwise one
|
|
# failure is skipped forever and the table shows a permanent
|
|
# gap that looks like it was never attempted.
|
|
with open(stem + ".stdout") as handle:
|
|
lines = [line for line in handle.read().splitlines() if line.startswith("{")]
|
|
if lines:
|
|
print(f" {label}: already measured, skipped (--resume)")
|
|
try:
|
|
records.append({"route": route, "description": description,
|
|
"cell": cell["name"], "run": run, "rc": 0,
|
|
**json.loads(lines[-1])})
|
|
except json.JSONDecodeError:
|
|
pass
|
|
continue
|
|
print(f" {label}: previous attempt left no result, re-running")
|
|
started = time.monotonic()
|
|
proc = subprocess.run(command, capture_output=True, text=True, env=env)
|
|
elapsed = time.monotonic() - started
|
|
with open(stem + ".stdout", "w") as handle:
|
|
handle.write(proc.stdout)
|
|
with open(stem + ".stderr", "w") as handle:
|
|
handle.write(proc.stderr)
|
|
record = {"route": route, "description": description, "cell": cell["name"],
|
|
"run": run, "command": " ".join(shlex.quote(part) for part in command),
|
|
"rc": proc.returncode, "elapsed_s": round(elapsed, 1)}
|
|
last = [line for line in proc.stdout.splitlines() if line.startswith("{")]
|
|
if proc.returncode == 0 and last:
|
|
record.update(json.loads(last[-1]))
|
|
print(f" {label}: p50 {record['frame_ms']['p50']} ms, "
|
|
f"{record['fps_sustained']} fps, cpu {record['cpu_ms_per_frame']} ms")
|
|
else:
|
|
record["error"] = (proc.stderr or proc.stdout).strip().splitlines()[-1:] or ["no output"]
|
|
failures.append(label)
|
|
print(f" {label}: FAILED rc={proc.returncode} -- {record['error']}")
|
|
records.append(record)
|
|
|
|
with open(os.path.join(args.results, "results.json"), "w") as handle:
|
|
json.dump(records, handle, indent=2)
|
|
|
|
print("\n=== head-to-head (median of runs; jpeg cells are the ones the wire carries) ===")
|
|
print("speed | machine cost | footprint")
|
|
print(f"{'route':<15} {'cell':<12} {'p50 ms':>8} {'fps':>6} {'own cpu':>8} {'sys cpu':>8} "
|
|
f"{'comp cpu':>9} {'machine%':>9} {'rss MB':>7} {'bytes':>9}")
|
|
for route in routes:
|
|
for cell in cells:
|
|
group = [r for r in records if r.get("route") == route and r.get("cell") == cell["name"]]
|
|
good = [r for r in group if "frame_ms" in r]
|
|
if not good:
|
|
print(f"{route:<15} {cell['name']:<12} {'--':>8} {'--':>6} {'--':>8} {'--':>8} {'--':>9} {'--':>9} {'--':>7} (no successful run)")
|
|
continue
|
|
|
|
def median(key, default=0.0):
|
|
values = sorted(r.get(key) or 0 for r in good)
|
|
return values[len(values) // 2] if values else default
|
|
|
|
p50 = sorted(r["frame_ms"]["p50"] for r in good)[len(good) // 2]
|
|
print(f"{route:<15} {cell['name']:<12} {p50:>8} "
|
|
f"{sorted(r['fps_sustained'] for r in good)[len(good)//2]:>6} "
|
|
f"{median('cpu_ms_per_frame'):>8} {median('system_cpu_ms_per_frame'):>8} "
|
|
f"{median('compositor_cpu_ms_per_frame'):>9} {median('machine_cpu_pct'):>9} "
|
|
f"{median('rss_peak_mb'):>7} {int(median('bytes_p50')):>9}")
|
|
if control:
|
|
print(f"\ncontrol (nothing capturing): {control['system_cpu_pct']}% of one core, "
|
|
f"compositor {control['compositor_cpu_ms']} ms/s")
|
|
print(" subtract this from 'sys cpu' before attributing CPU to a route.")
|
|
if failures:
|
|
print(f"\nfailed cells: {', '.join(failures)}")
|
|
print(f"\nraw runs: {args.results}/")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|