The question is which capture route to ship for live streaming and agent eyes. Speed alone does not decide it, so every route now also reports producer CPU and RSS, machine CPU from /proc/stat, the compositor's own CPU, and a per-frame buffer footprint -- all sampled around the measured loop, with an idle control sample so an animating page's cost is not attributed to the capture. The baseline forks a process per frame and spends 95% of each frame waiting, so the interesting finding is not which route is faster but which one leaves the compositor able to do its job: in-process acquisition roughly halves the compositor's per-frame CPU even though the machine saturates either way.
203 lines
9.2 KiB
Python
203 lines
9.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Run every route through the frozen contract and print a head-to-head.
|
|
|
|
The point of a runner rather than a shell loop: the *same* cells get run for
|
|
every route with identical arguments, the raw JSON of every run is kept, and the
|
|
table is derived from those files instead of from what anyone remembers seeing.
|
|
|
|
./bench.py --output hh-<session> --frames 300 --runs 2
|
|
./bench.py --output hh-<session> --only grim --cells output-jpeg
|
|
|
|
Routes are commands, so a route that is not built yet simply fails and is
|
|
reported as a failed cell rather than quietly omitted.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
PRODUCERS = os.path.join(HERE, "producers")
|
|
RUST = os.path.join(HERE, "..", "..", "001a-grim-rs", "target", "release", "grimrs-capture")
|
|
|
|
ROUTES = {
|
|
# name: (description, command builder)
|
|
"grim": (
|
|
"grim subprocess per frame (shipped)",
|
|
lambda o, cell, frames, warmup: [
|
|
sys.executable, os.path.join(HERE, "grim_producer.py"),
|
|
*(["--all"] if cell["all"] else ["--output", o]),
|
|
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
|
|
],
|
|
),
|
|
"py-screencopy": (
|
|
"pywayland in-process, zwlr-screencopy-v1",
|
|
lambda o, cell, frames, warmup: [
|
|
sys.executable, os.path.join(HERE, "wayland_producer.py"),
|
|
*(["--all"] if cell["all"] else ["--output", o]),
|
|
"--protocol", "screencopy",
|
|
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
|
|
],
|
|
),
|
|
"py-ext": (
|
|
"pywayland in-process, ext-image-copy-capture-v1",
|
|
lambda o, cell, frames, warmup: [
|
|
sys.executable, os.path.join(HERE, "wayland_producer.py"),
|
|
*(["--all"] if cell["all"] else ["--output", o]),
|
|
"--protocol", "ext",
|
|
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
|
|
],
|
|
),
|
|
"grim-rs": (
|
|
"grim-rs crate (Rust), long-lived process",
|
|
lambda o, cell, frames, warmup: [
|
|
RUST,
|
|
*(["--all"] if cell["all"] else ["--output", o]),
|
|
"--frames", str(frames), "--warmup", str(warmup), "--mode", cell["mode"], "--json",
|
|
],
|
|
),
|
|
}
|
|
|
|
CELLS = [
|
|
{"name": "output-raw", "all": False, "mode": "raw"},
|
|
{"name": "output-jpeg", "all": False, "mode": "jpeg"},
|
|
{"name": "all-raw", "all": True, "mode": "raw"},
|
|
{"name": "all-jpeg", "all": True, "mode": "jpeg"},
|
|
]
|
|
|
|
|
|
def idle_control(seconds: float) -> dict:
|
|
"""What the machine burns with nothing capturing at all.
|
|
|
|
Without this the resource columns are unreadable: on a software-rendered
|
|
compositor an animating page can saturate both cores on its own, and a
|
|
per-frame CPU number that includes the animation would be attributed to the
|
|
capture route. This is the number to subtract -- and it is also the reason
|
|
the verdict has to talk about *what the machine was doing*, not just fps.
|
|
"""
|
|
sys.path.insert(0, PRODUCERS)
|
|
from resources import MachineSample # noqa: E402
|
|
|
|
before = MachineSample(children=False)
|
|
time.sleep(seconds)
|
|
after = MachineSample(children=False)
|
|
window_ms = seconds * 1000.0
|
|
system_ms = None
|
|
compositor_ms = None
|
|
if before.system_cpu is not None and after.system_cpu is not None:
|
|
system_ms = (after.system_cpu - before.system_cpu) * 1000.0
|
|
if before.compositor_cpu is not None and after.compositor_cpu is not None:
|
|
compositor_ms = (after.compositor_cpu - before.compositor_cpu) * 1000.0
|
|
return {
|
|
"window_ms": round(window_ms, 1),
|
|
"system_cpu_ms": None if system_ms is None else round(system_ms, 1),
|
|
"compositor_cpu_ms": None if compositor_ms is None else round(compositor_ms, 1),
|
|
"system_cpu_pct": None if system_ms is None else round(system_ms / window_ms * 100.0, 1),
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output", required=True, help="the session container output to measure")
|
|
parser.add_argument("--frames", type=int, default=300)
|
|
parser.add_argument("--warmup", type=int, default=10)
|
|
parser.add_argument("--runs", type=int, default=2)
|
|
parser.add_argument("--only", action="append", help="restrict to these route names")
|
|
parser.add_argument("--cells", action="append", help="restrict to these cell names")
|
|
parser.add_argument("--results", default=os.path.join(HERE, "..", "results"))
|
|
parser.add_argument("--env", action="append", default=[], help="KEY=VALUE for the producers")
|
|
parser.add_argument("--control-seconds", type=float, default=0.0,
|
|
help="sample machine CPU with no captures for this long, as a control")
|
|
args = parser.parse_args()
|
|
|
|
routes = {k: v for k, v in ROUTES.items() if not args.only or k in args.only}
|
|
cells = [c for c in CELLS if not args.cells or c["name"] in args.cells]
|
|
os.makedirs(args.results, exist_ok=True)
|
|
|
|
control = None
|
|
if args.control_seconds > 0:
|
|
print(f"control: sampling {args.control_seconds:.0f}s with nothing capturing...")
|
|
control = idle_control(args.control_seconds)
|
|
print(f" idle machine CPU: {control['system_cpu_pct']}% of one core "
|
|
f"(compositor {control['compositor_cpu_ms']} ms/s)")
|
|
with open(os.path.join(args.results, "control.json"), "w") as handle:
|
|
json.dump(control, handle, indent=2)
|
|
|
|
env = dict(os.environ)
|
|
for item in args.env:
|
|
key, _, value = item.partition("=")
|
|
env[key] = value
|
|
|
|
records: list[dict] = []
|
|
failures: list[str] = []
|
|
for route, (description, build) in routes.items():
|
|
for cell in cells:
|
|
for run in range(1, args.runs + 1):
|
|
command = build(args.output, cell, args.frames, args.warmup)
|
|
label = f"{route}/{cell['name']}/run{run}"
|
|
started = time.monotonic()
|
|
proc = subprocess.run(command, capture_output=True, text=True, env=env)
|
|
elapsed = time.monotonic() - started
|
|
stem = os.path.join(args.results, f"{route}.{cell['name']}.run{run}")
|
|
with open(stem + ".stdout", "w") as handle:
|
|
handle.write(proc.stdout)
|
|
with open(stem + ".stderr", "w") as handle:
|
|
handle.write(proc.stderr)
|
|
record = {"route": route, "description": description, "cell": cell["name"],
|
|
"run": run, "command": " ".join(shlex.quote(part) for part in command),
|
|
"rc": proc.returncode, "elapsed_s": round(elapsed, 1)}
|
|
last = [line for line in proc.stdout.splitlines() if line.startswith("{")]
|
|
if proc.returncode == 0 and last:
|
|
record.update(json.loads(last[-1]))
|
|
print(f" {label}: p50 {record['frame_ms']['p50']} ms, "
|
|
f"{record['fps_sustained']} fps, cpu {record['cpu_ms_per_frame']} ms")
|
|
else:
|
|
record["error"] = (proc.stderr or proc.stdout).strip().splitlines()[-1:] or ["no output"]
|
|
failures.append(label)
|
|
print(f" {label}: FAILED rc={proc.returncode} -- {record['error']}")
|
|
records.append(record)
|
|
|
|
with open(os.path.join(args.results, "results.json"), "w") as handle:
|
|
json.dump(records, handle, indent=2)
|
|
|
|
print("\n=== head-to-head (median of runs; jpeg cells are the ones the wire carries) ===")
|
|
print("speed | machine cost | footprint")
|
|
print(f"{'route':<15} {'cell':<12} {'p50 ms':>8} {'fps':>6} {'own cpu':>8} {'sys cpu':>8} "
|
|
f"{'comp cpu':>9} {'machine%':>9} {'rss MB':>7} {'bytes':>9}")
|
|
for route in routes:
|
|
for cell in cells:
|
|
group = [r for r in records if r.get("route") == route and r.get("cell") == cell["name"]]
|
|
good = [r for r in group if "frame_ms" in r]
|
|
if not good:
|
|
print(f"{route:<15} {cell['name']:<12} {'--':>8} {'--':>6} {'--':>8} {'--':>8} {'--':>9} {'--':>9} {'--':>7} (no successful run)")
|
|
continue
|
|
|
|
def median(key, default=0.0):
|
|
values = sorted(r.get(key) or 0 for r in good)
|
|
return values[len(values) // 2] if values else default
|
|
|
|
p50 = sorted(r["frame_ms"]["p50"] for r in good)[len(good) // 2]
|
|
print(f"{route:<15} {cell['name']:<12} {p50:>8} "
|
|
f"{sorted(r['fps_sustained'] for r in good)[len(good)//2]:>6} "
|
|
f"{median('cpu_ms_per_frame'):>8} {median('system_cpu_ms_per_frame'):>8} "
|
|
f"{median('compositor_cpu_ms_per_frame'):>9} {median('machine_cpu_pct'):>9} "
|
|
f"{median('rss_peak_mb'):>7} {int(median('bytes_p50')):>9}")
|
|
if control:
|
|
print(f"\ncontrol (nothing capturing): {control['system_cpu_pct']}% of one core, "
|
|
f"compositor {control['compositor_cpu_ms']} ms/s")
|
|
print(" subtract this from 'sys cpu' before attributing CPU to a route.")
|
|
if failures:
|
|
print(f"\nfailed cells: {', '.join(failures)}")
|
|
print(f"\nraw runs: {args.results}/")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|