#!/usr/bin/env python3 """Turn a pytest JUnit report into a build verdict. Two jobs pytest's exit code alone does not do well inside a Nix build: * say out loud how many tests ran, and which ones failed, in the build log; * refuse a suite that collected nothing. An empty suite exits 5, but it also exits 0 if every test was deselected or skipped away, and a check that passes because it tested nothing is worse than no check at all. Exit codes: 0 all good, 1 the suite is empty or the report is unusable, 2 the suite ran and some tests failed. The caller distinguishes them so the build log does not accuse an empty suite of being a failing one. """ from __future__ import annotations import sys import xml.etree.ElementTree as ET def main(argv: list[str]) -> int: if len(argv) != 2: print("usage: check_report.py ", file=sys.stderr) return 2 path = argv[1] try: root = ET.parse(path).getroot() except FileNotFoundError: print( f"ERROR: pytest wrote no report at {path}: it collected nothing, or died " "before it could run a single test.", file=sys.stderr, ) return 1 except ET.ParseError as exc: print(f"ERROR: {path} is not parseable JUnit XML: {exc}", file=sys.stderr) return 1 suites = root.findall("testsuite") or ([root] if root.tag == "testsuite" else []) total = errors = failures = skipped = 0 failing: list[str] = [] for suite in suites: total += int(suite.get("tests", 0)) errors += int(suite.get("errors", 0)) failures += int(suite.get("failures", 0)) skipped += int(suite.get("skipped", 0)) for case in suite.iter("testcase"): for kind in ("failure", "error"): for node in case.findall(kind): name = f"{case.get('classname', '')}::{case.get('name', '')}" message = (node.get("message") or "").splitlines() head = message[0][:160] if message else node.get("type", kind) failing.append(f" {kind.upper():7} {name}\n {head}") executed = total - skipped print( f"unit suite: {total} collected, {executed} executed, " f"{failures} failed, {errors} errored, {skipped} skipped" ) if total == 0: print( "ERROR: the unit suite collected no tests. An empty suite is not a " "passing suite -- check that tests/unit/ is in the build and that the " "files are named test_*.py.", file=sys.stderr, ) return 1 if executed == 0: print( f"ERROR: all {total} collected tests were skipped; nothing was actually " "exercised.", file=sys.stderr, ) return 1 if failing: print(f"\n{len(failing)} failing test(s):") for line in failing: print(line) return 2 print("unit suite: all executed tests passed") return 0 if __name__ == "__main__": sys.exit(main(sys.argv))