Wires a `checks` output into the flake, one file per check under checks/, all three asserting docs/CONTRACT.md rather than the implementation: - unit: pytest over tests/unit/ in the build sandbox, against the nix/package.nix build. Refuses an empty suite -- pytest exits 0 when every test was skipped or deselected, so checks/lib/check_report.py parses the JUnit report and fails on zero collected or zero executed. - vm-basic: nixosTest with captureCommand pointed at tests/fake_capture.py, which emits one fixed deterministic PNG. Because the image never changes, "two frames arrived and they are the same bytes" is an assertion about the framing rather than a coincidence. - vm-hyprland: the same machine with no captureCommand override, so grim captures the real compositor. Deliberately impatient -- bounded timeouts, no retry loops, and a full diagnostics dump (compositor log, hyprctl monitors, both journals) before any failure propagates. tests/assert_stream.py is a second, independent reader of the contract: it shares no code with hhstream.client, so a framing bug present in both the server's writer and the shipped client's reader still fails. The unit suite's negative tests use a `rejects` fixture instead of pytest.raises(Exception), which an unimplemented stub's NotImplementedError would otherwise satisfy. Three defects found in code this branch does not own, recorded in README.dev.md rather than worked around: - nix/module.nix has no config side, so neither unit exists and both VM checks fail at their first wait; - services.hyprland-headless-stream.package is declared with no default, against docs/CONTRACT.md; - src/hhstream/*.py is all stubs, so 66 of 70 unit tests fail. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017YXMoCNxzScTaKNr43GQNz
91 lines
3.0 KiB
Python
91 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Turn a pytest JUnit report into a build verdict.
|
|
|
|
Two jobs pytest's exit code alone does not do well inside a Nix build:
|
|
|
|
* say out loud how many tests ran, and which ones failed, in the build log;
|
|
* refuse a suite that collected nothing. An empty suite exits 5, but it also
|
|
exits 0 if every test was deselected or skipped away, and a check that passes
|
|
because it tested nothing is worse than no check at all.
|
|
|
|
Exit codes: 0 all good, 1 the suite is empty or the report is unusable,
|
|
2 the suite ran and some tests failed. The caller distinguishes them so the
|
|
build log does not accuse an empty suite of being a failing one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
if len(argv) != 2:
|
|
print("usage: check_report.py <junit.xml>", file=sys.stderr)
|
|
return 2
|
|
path = argv[1]
|
|
|
|
try:
|
|
root = ET.parse(path).getroot()
|
|
except FileNotFoundError:
|
|
print(
|
|
f"ERROR: pytest wrote no report at {path}: it collected nothing, or died "
|
|
"before it could run a single test.",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
except ET.ParseError as exc:
|
|
print(f"ERROR: {path} is not parseable JUnit XML: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
suites = root.findall("testsuite") or ([root] if root.tag == "testsuite" else [])
|
|
total = errors = failures = skipped = 0
|
|
failing: list[str] = []
|
|
for suite in suites:
|
|
total += int(suite.get("tests", 0))
|
|
errors += int(suite.get("errors", 0))
|
|
failures += int(suite.get("failures", 0))
|
|
skipped += int(suite.get("skipped", 0))
|
|
for case in suite.iter("testcase"):
|
|
for kind in ("failure", "error"):
|
|
for node in case.findall(kind):
|
|
name = f"{case.get('classname', '')}::{case.get('name', '')}"
|
|
message = (node.get("message") or "").splitlines()
|
|
head = message[0][:160] if message else node.get("type", kind)
|
|
failing.append(f" {kind.upper():7} {name}\n {head}")
|
|
|
|
executed = total - skipped
|
|
print(
|
|
f"unit suite: {total} collected, {executed} executed, "
|
|
f"{failures} failed, {errors} errored, {skipped} skipped"
|
|
)
|
|
|
|
if total == 0:
|
|
print(
|
|
"ERROR: the unit suite collected no tests. An empty suite is not a "
|
|
"passing suite -- check that tests/unit/ is in the build and that the "
|
|
"files are named test_*.py.",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
if executed == 0:
|
|
print(
|
|
f"ERROR: all {total} collected tests were skipped; nothing was actually "
|
|
"exercised.",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
|
|
if failing:
|
|
print(f"\n{len(failing)} failing test(s):")
|
|
for line in failing:
|
|
print(line)
|
|
return 2
|
|
|
|
print("unit suite: all executed tests passed")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv))
|