diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..d2f4001 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,8 @@ +# Lenient on purpose: catches broken structure, not prose style. +default: true +MD013: false # line length +MD024: false # duplicate headings +MD033: false # inline HTML +MD036: false # emphasis used as heading +MD041: false # first line heading +MD060: false # table column style diff --git a/dev/hooks/run.sh b/dev/hooks/run.sh new file mode 100755 index 0000000..50c9236 --- /dev/null +++ b/dev/hooks/run.sh @@ -0,0 +1,242 @@ +#!/usr/bin/env bash +# Hook runner. One definition of every check, used by the devenv git-hooks +# entries and by the Nix flake checks. +# +# run.sh pre-commit staged files -> fast checks +# run.sh pre-push commits being pushed -> tests and slow checks +# run.sh lint-all every file in the tree, lint checks only (Nix) +# run.sh check NAME [FILE...] run one check +# +# Missing optional tools are skipped with a warning unless +# HOOKS_REQUIRE_TOOLS=1 (set by the Nix checks, where they are all provided). +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +root=$(cd "$here/../.." && pwd) +orig=$PWD +cd "$root" + +fail=0 +note() { printf ' %s\n' "$*" >&2; } + +need() { + if command -v "$1" >/dev/null 2>&1; then return 0; fi + if [ "${HOOKS_REQUIRE_TOOLS:-0}" = 1 ]; then + echo "required tool missing: $1" >&2 + exit 1 + fi + note "skip: $1 not installed" + return 1 +} + +# Keep only the arguments matching a case pattern, one per line. +filter() { + local pat=$1 f + shift + for f in "$@"; do + # shellcheck disable=SC2254 + case $f in $pat) [ -f "$f" ] && printf '%s\n' "$f" ;; esac + done + return 0 +} + +chk_hygiene() { + local f bad=0 + for f in "$@"; do + [ -f "$f" ] || continue + case $f in Cargo.lock | flake.lock | devenv.lock) continue ;; esac + if [ "$(wc -c <"$f")" -gt 524288 ]; then + note "$f: larger than 512 KiB" + bad=1 + fi + grep -qI . "$f" || continue + if grep -nE '^(<<<<<<< |>>>>>>> )' "$f" >&2; then + note "$f: merge conflict marker" + bad=1 + fi + case $f in + *.md) ;; + *) + if grep -nE '[[:blank:]]+$' "$f" >&2; then + note "$f: trailing whitespace" + bad=1 + fi + ;; + esac + if [ -s "$f" ] && [ -n "$(tail -c1 "$f")" ]; then + note "$f: no newline at end of file" + bad=1 + fi + done + return $bad +} + +# Credentials must never reach the repo (AGENTS.md Safety). A line carrying +# the marker "secret-scan: allow" is exempt, for deliberate test fixtures. +chk_secrets() { + local f bad=0 + local re='-----BEGIN ([A-Z]+ )?PRIVATE KEY-----' + re+='|[Bb]earer[[:space:]]+[A-Za-z0-9._~+/=-]{24,}' + re+='|gh[pousr]_[A-Za-z0-9]{30,}|AKIA[0-9A-Z]{16}|xox[abprs]-[A-Za-z0-9-]{10,}' + re+='|sk-[A-Za-z0-9]{32,}|glpat-[A-Za-z0-9_-]{20,}' + for f in "$@"; do + [ -f "$f" ] || continue + case $f in + *.pem | *.key | *.p12 | *.pfx | .env | .env.* | .mcp.json | *environment-file* | *.secret) + note "$f: file type that holds secrets must not be committed" + bad=1 + continue + ;; + esac + case $f in dev/hooks/*) continue ;; esac + if grep -nIE -e "$re" "$f" | grep -vE 'secret-scan: allow|(.)\1{9,}' >&2; then + note "$f: possible secret" + bad=1 + fi + done + return $bad +} + +# Repo rules from AGENTS.md that clippy cannot see. +chk_rules() { + local f bad=0 + for f in "$@"; do + [ -f "$f" ] || continue + # Workstream, task and decision tags in comments. + if grep -nE '//.*\b(WS-[A-Za-z0-9]+|DEC-[0-9]+|G[0-9]+)\b' "$f" >&2; then + note "$f: workstream/task/decision tag in a comment" + bad=1 + fi + # axum 0.8 panics at startup on ":name" path params. + if grep -nE '"/(:[A-Za-z_]|[^"]*/:[A-Za-z_])' "$f" >&2; then + note "$f: axum 0.8 route uses :name, write /{name}" + bad=1 + fi + done + return $bad +} + +chk_rust-fmt() { cargo fmt --check; } +chk_clippy() { cargo clippy --all-targets -- -D warnings; } + +# The service must stay rustls-only. +chk_openssl() { + local out + out=$(cargo tree -i openssl 2>/dev/null || true) + if [ -n "$out" ]; then + printf '%s\n' "$out" >&2 + note "openssl is in the dependency tree" + return 1 + fi +} + +chk_toml() { + need taplo || return 0 + taplo check "$@" +} +chk_yaml() { + need yamllint || return 0 + yamllint -d '{extends: relaxed, rules: {line-length: disable}}' "$@" +} +chk_markdown() { + need markdownlint || return 0 + markdownlint "$@" +} +chk_shell() { + need shellcheck || return 0 + shellcheck "$@" +} +chk_nushell() { + need nu || return 0 + local f + for f in "$@"; do nu -c "nu-check '$f'" >/dev/null || return 1; done +} +chk_nix-parse() { + need nix-instantiate || return 0 + local f + for f in "$@"; do nix-instantiate --parse "$f" >/dev/null || return 1; done +} + +# Evaluates every flake output, including the checks, without building. +chk_nix-eval() { + need nix || return 0 + nix flake check --no-build --no-update-lock-file +} +chk_hooks-selftest() { "$here/test-select.sh"; } + +chk_test-all() { cargo test; } +chk_test-unit() { cargo test --lib "$1::"; } +chk_test-it() { cargo test --test "$1"; } + +# Dispatch a selected check name (maybe "name:arg") with a file list. +dispatch() { + local name=$1 files=("${@:2}") arg="" + if [[ $name == *:* ]]; then + arg=${name#*:} + name=${name%%:*} + fi + local sel=() filebased=1 + case $name in + test-* | clippy | rust-fmt | openssl | nix-eval | hooks-selftest) filebased=0 ;; + hygiene | secrets) sel=("${files[@]}") ;; + rules) mapfile -t sel < <(filter '*.rs' "${files[@]}") ;; + toml) mapfile -t sel < <(filter '*.toml' "${files[@]}") ;; + yaml) mapfile -t sel < <(filter '*.y*ml' "${files[@]}") ;; + markdown) mapfile -t sel < <(filter '*.md' "${files[@]}") ;; + shell) mapfile -t sel < <(filter '*.sh' "${files[@]}") ;; + nushell) mapfile -t sel < <(filter '*.nu' "${files[@]}") ;; + nix-parse) mapfile -t sel < <(filter '*.nix' "${files[@]}") ;; + esac + if [ "$filebased" = 1 ]; then + [ ${#sel[@]} -gt 0 ] || return 0 + echo "== $name" >&2 + "chk_$name" "${sel[@]}" || fail=1 + else + echo "== $name${arg:+ $arg}" >&2 + "chk_$name" ${arg:+"$arg"} || fail=1 + fi +} + +changed_files() { + case $1 in + pre-commit) git diff --cached --name-only --diff-filter=ACMR ;; + pre-push) + local base + base=$(git rev-parse --verify -q '@{u}' || git merge-base HEAD main || git rev-parse HEAD~1) + git diff --name-only --diff-filter=ACMR "$base"...HEAD + ;; + esac +} + +mode=${1:?usage: run.sh pre-commit|pre-push|lint-all|check NAME [FILE...]} +case $mode in + pre-commit | pre-push) + mapfile -t files < <(changed_files "$mode") + mapfile -t checks < <(printf '%s\n' "${files[@]}" | "$here/select.sh" "$mode") + for c in "${checks[@]}"; do + [ -n "$c" ] && dispatch "$c" "${files[@]}" + done + ;; + lint-all) + if git rev-parse --git-dir >/dev/null 2>&1; then + mapfile -t files < <(git ls-files) + else + mapfile -t files < <(find . -type f -not -path './target/*' -not -path './.git/*' | sed 's|^\./||') + fi + for c in hygiene secrets rules toml yaml markdown shell nushell nix-parse hooks-selftest; do + dispatch "$c" "${files[@]}" + done + ;; + check) + name=${2:?check NAME} + shift 2 + # File arguments are relative to the caller's directory. + cd "$orig" + dispatch "$name" "$@" + ;; + *) + echo "unknown mode $mode" >&2 + exit 2 + ;; +esac +exit $fail diff --git a/dev/hooks/select.sh b/dev/hooks/select.sh new file mode 100755 index 0000000..176a5c5 --- /dev/null +++ b/dev/hooks/select.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Map changed paths (stdin, one per line) to the checks that must run. +# Usage: select.sh pre-commit|pre-push < paths +# Output: one check name per line, sorted and deduplicated. run.sh executes +# the names. HOOKS_TESTS_DIR redirects the tests/*.rs lookup (used by +# the fixture test). +set -euo pipefail + +stage=${1:?usage: select.sh pre-commit|pre-push} +case $stage in + pre-commit | pre-push) ;; + *) echo "unknown stage $stage" >&2; exit 2 ;; +esac +tests_dir=${HOOKS_TESTS_DIR:-tests} +out=() +add() { out+=("$@"); } + +# Integration test files exercised by a src module. Unknown modules fall back +# to tests/*.rs when such files exist. +integration_for() { + case $1 in + api | mcp | auth | sources | db | telemetry) echo integration ;; + tls) echo tls ;; + config) echo plain ;; + esac + local f + for f in "$tests_dir/$1"*.rs; do + if [ -e "$f" ]; then basename "$f" .rs; fi + done + return 0 +} + +# Prefix pc: runs at pre-commit (fast), pp: at pre-push (tests, slow lint). +while IFS= read -r p; do + [ -n "$p" ] || continue + case $p in + *.rs) add pc:rust-fmt pc:rules pc:clippy ;; + Cargo.toml | Cargo.lock | rustfmt.toml | clippy.toml) add pc:clippy ;; + esac + case $p in + Cargo.toml | Cargo.lock) add pc:openssl pp:test-all ;; + src/lib.rs | src/main.rs | src/state.rs | src/error.rs | src/server.rs) add pp:test-all ;; + tests/common/*) add pp:test-all ;; + tests/*.rs) add "pp:test-it:$(basename "$p" .rs)" ;; + src/*) + rest=${p#src/} + mod=${rest%%/*} + mod=${mod%.rs} + add "pp:test-unit:$mod" + while IFS= read -r t; do + if [ -n "$t" ]; then add "pp:test-it:$t"; fi + done < <(integration_for "$mod") + ;; + migrations/*) add pp:test-unit:db pp:test-it:integration ;; + config.example.toml) add pp:test-unit:config pp:test-it:plain ;; + nix/* | flake.nix | flake.lock | devenv.hooks.nix) add pp:nix-eval ;; + esac + case $p in + dev/hooks/*) add pp:hooks-selftest pc:shell ;; + esac + case $p in + *.toml) add pc:toml ;; + *.yaml | *.yml) add pc:yaml ;; + *.md) add pc:markdown ;; + *.sh) add pc:shell ;; + *.nu) add pc:nushell ;; + *.nix) add pc:nix-parse ;; + esac + add pc:hygiene pc:secrets +done + +((${#out[@]})) || exit 0 + +all=$(printf '%s\n' "${out[@]}" | sort -u) +# test-all subsumes the narrower test selections. +if grep -qx 'pp:test-all' <<<"$all"; then + all=$(grep -vE '^pp:test-(unit|it):' <<<"$all") +fi + +if [ "$stage" = pre-commit ]; then pat='^pc:'; else pat='^pp:'; fi +{ grep -E "$pat" <<<"$all" || true; } | sed -E 's/^(pc|pp):/ /; s/^ //' diff --git a/dev/hooks/test-select.sh b/dev/hooks/test-select.sh new file mode 100755 index 0000000..d4e9c09 --- /dev/null +++ b/dev/hooks/test-select.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Fixture test for select.sh: given changed paths, which checks run. +# Also exercises the secrets, rules and hygiene checks on synthetic files. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +# Fake tests dir so the tests/*.rs fallback is deterministic. +mkdir -p "$tmp/tests" +touch "$tmp/tests/peers.rs" "$tmp/tests/peers_auth.rs" "$tmp/tests/integration.rs" +export HOOKS_TESTS_DIR="$tmp/tests" + +failures=0 +# expect STAGE "paths (space separated)" "expected checks (space separated)" +expect() { + local stage=$1 paths=$2 want=$3 got + # shellcheck disable=SC2086 + got=$(printf '%s\n' $paths | "$here/select.sh" "$stage" | tr '\n' ' ' | sed 's/ $//') + if [ "$got" != "$want" ]; then + echo "FAIL [$stage] $paths" + echo " want: $want" + echo " got: $got" + failures=$((failures + 1)) + fi +} + +# Docs only: markdown lint and generic hygiene, never Rust. +expect pre-commit "docs/api.md README.md" "hygiene markdown secrets" +expect pre-push "docs/api.md README.md" "" + +# Rust source in a module with its own integration tests. +expect pre-commit "src/peers/registry.rs" "clippy hygiene rules rust-fmt secrets" +expect pre-push "src/peers/registry.rs" "test-it:peers test-it:peers_auth test-unit:peers" +expect pre-push "src/api/docker.rs" "test-it:integration test-unit:api" +expect pre-push "src/tls.rs" "test-it:tls test-unit:tls" + +# Cross-cutting files run everything. +expect pre-push "src/server.rs src/api/docker.rs" "test-all" +expect pre-push "Cargo.lock" "test-all" +expect pre-commit "Cargo.toml" "clippy hygiene openssl secrets toml" + +# Tests, migrations, config. +expect pre-push "tests/tls.rs" "test-it:tls" +expect pre-push "migrations/0002_x.sql" "test-it:integration test-unit:db" +expect pre-push "config.example.toml" "test-it:plain test-unit:config" +expect pre-commit "config.example.toml" "hygiene secrets toml" + +# Nix: parse at commit, flake evaluation at push, no Rust. +expect pre-commit "nix/module.nix" "hygiene nix-parse secrets" +expect pre-push "nix/module.nix flake.nix" "nix-eval" + +# Other file types. +expect pre-commit "dev/otelcol.yaml" "hygiene secrets yaml" +expect pre-commit "dev/test.nu" "hygiene nushell secrets" +expect pre-push "dev/hooks/run.sh" "hooks-selftest" + +# --- check behaviour on synthetic files --------------------------------- +run() { "$here/run.sh" check "$@" 2>/dev/null; } +expect_rc() { + local want=$1 name=$2 + shift 2 + local rc=0 + run "$name" "$@" >/dev/null || rc=$? + if { [ "$want" = ok ] && [ "$rc" != 0 ]; } || { [ "$want" = bad ] && [ "$rc" = 0 ]; }; then + echo "FAIL check $name $* (wanted $want, rc=$rc)" + failures=$((failures + 1)) + fi +} + +cd "$tmp" +# Fake credentials; dev/hooks is exempt from the secret scan. +printf 'token = "%s"\n' "AKIAQ7XK2M9PD4LZ8RWB" >aws.txt +printf 'curl -H "Authorization: Bearer %s"\n' "q7Xk2M9pD4lZ8rWb3NvC5tYh1JfGs6Ue" >bearer.sh +printf -- '-----BEGIN %s KEY-----\n' "RSA PRIVATE" >key.txt +printf 'hello\n' >clean.txt +printf 'ok = "secret-scan: allow %s"\n' "Bearer $(printf 'y%.0s' {1..32})" >allowed.txt +expect_rc bad secrets aws.txt +expect_rc bad secrets bearer.sh +expect_rc bad secrets key.txt +: >server.key +expect_rc bad secrets server.key +expect_rc ok secrets clean.txt allowed.txt + +printf '// WS-%s tag\nfn a() {}\n' "A" >tag.rs +printf '// see ARCHITECTURE.md\nfn a() {}\n' >notag.rs +printf 'fn r() { Router::new().route("/units/:name", x); }\n' >colon.rs +printf 'fn r() { Router::new().route("/units/{name}", x); }\n' >brace.rs +expect_rc bad rules tag.rs +expect_rc bad rules colon.rs +expect_rc ok rules notag.rs brace.rs + +printf 'a \nb\n' >trail.txt +printf 'no newline' >noeol.txt +printf '<<<<<<< HEAD\nx\n' >conflict.txt +expect_rc bad hygiene trail.txt +expect_rc bad hygiene noeol.txt +expect_rc bad hygiene conflict.txt +expect_rc ok hygiene clean.txt + +if [ "$failures" -gt 0 ]; then + echo "$failures failure(s)" + exit 1 +fi +echo "hook selection tests passed"