Files
skyg-linux-tracker/dev/dev-test.nu
T

1044 lines
41 KiB
Nu
Executable File

#!/usr/bin/env nu
# The single source of truth for names and flags: drives dispatch, zsh/bash
# completion and the self-check. Nushell completion still comes from the
# signatures below; the self-check keeps both in step.
const command_table = [
{ name: "unit", usage: "cargo test --lib --bins", flags: [] }
{ name: "integration", usage: "cargo test --test '*' [filter] [--nocapture]", flags: ["--nocapture"] }
{ name: "lint", usage: "fmt, clippy and the openssl check", flags: [] }
{ name: "vm", usage: "nix build of a NixOS VM test: vm or peers", flags: [] }
{ name: "all", usage: "lint, unit, integration; stops at the first failure", flags: [] }
{ name: "ci", usage: "all plus the vm test", flags: [] }
{ name: "completions", usage: "print a zsh or bash completion script", flags: [] }
{ name: "self-check", usage: "check that the table, commands and help agree", flags: [] }
]
# Baked into dry-run spawns. Docs and help use the repo-relative path.
const SCRIPT_PATH = path self
const SCRIPT_REL = "dev/dev-test.nu"
# File stem. Nushell binds `main` to this name on `use`, so every subcommand
# is `dev-test <sub>` or tab completion of `dev-test` does not list them.
const RUNNER = "dev-test"
# Same name in both generated scripts, so the self-check can call it.
const COMP_FN = "_dev_test"
# Completers cannot update the caller's environment, so the filter list is a file.
# Repo-relative: `repo-root` turns it absolute, so a completion started in a
# subdirectory writes the same file under the same key.
const FILTER_CACHE_REL = "target/skyg-test-filters.cache"
# One entry point for the repository's checks.
#
# Subcommands:
# unit cargo test --lib --bins
# integration cargo test --test '*' [filter] [--nocapture]
# lint cargo fmt --check, cargo clippy --all-targets -- -D warnings, cargo tree -i openssl (empty)
# vm nix build of a NixOS VM test: vm (checks) or peers (tests.<system>.peers)
# all lint, unit, integration; stops at the first failure
# ci all plus the vm test
# completions print a zsh or bash completion script
# self-check fail when the command table, the commands and the help text diverge
#
# Loaded as a module (`use dev/dev-test.nu *`) the file stem is `dev-test`, so
# the subcommands have to be named `dev-test <sub>` or Nushell will not
# complete them.
#
# The checks are read-only against the host: no /etc, systemd or Docker changes.
#
# Script mode (`nu dev/dev-test.nu`) is `--wrapped` so flags reach the parser below.
# Nushell would otherwise accept `--nocapture` on every subcommand and reject
# the rest before `main` can explain them.
export def --wrapped main [...args: string] {
# Both script mode (`nu dev/dev-test.nu <sub>`) and module mode land here:
# `use ... *` binds the bare name `dev-test` to this entry point, so a typo
# such as `dev-test bogus` calls it too. Raising keeps a module-mode caller
# alive; an uncaught error already gives script mode a non-zero status, so
# never `exit` here.
dispatch-script-args $args
}
def dispatch-script-args [args: list<string>] {
let parsed = (parse-script-args $args)
match $parsed.cmd {
"help" => { print-help }
"unit" => { dev-test unit }
"integration" => {
if ($parsed.positional | is-empty) {
dev-test integration --nocapture=$parsed.nocapture
} else {
dev-test integration $parsed.positional --nocapture=$parsed.nocapture
}
}
"lint" => { dev-test lint }
"vm" => {
if ($parsed.positional | is-empty) {
dev-test vm
} else {
dev-test vm $parsed.positional
}
}
"all" => { dev-test all }
"ci" => { dev-test ci }
"completions" => {
if ($parsed.positional | is-empty) {
dev-test completions
} else {
dev-test completions $parsed.positional
}
}
"self-check" => { dev-test self-check }
_ => { error make { msg: $"subcommand '($parsed.cmd)' has no dispatch arm" } }
}
}
# Run the unit tests: no sockets, no host access.
export def "dev-test unit" [] {
run-steps (unit-steps)
}
# Run the integration tests (plain, tls, integration, binary_smoke, peers).
export def "dev-test integration" [
filter?: string@"nu-complete dev-test filters" # cargo name filter, e.g. a test name or file
--nocapture # do not capture test output
] {
run-steps (integration-steps $filter $nocapture)
}
# Formatting, lints and the dependency check.
export def "dev-test lint" [] {
run-steps (lint-steps)
}
# Run a NixOS VM test. Slow: it builds and boots VMs.
export def "dev-test vm" [
name?: string@"nu-complete dev-test vm" # vm (checks) or peers (tests.<system>.peers)
] {
let name = ($name | default "vm")
if $name not-in (nu-complete dev-test vm) {
error make { msg: $"unknown VM test '($name)'; expected one of: (nu-complete dev-test vm | str join ', ')" }
}
run-steps (vm-steps $name)
}
# lint, unit and integration; stops at the first failure.
export def "dev-test all" [] {
run-steps ((lint-steps) | append (unit-steps) | append (integration-steps null false))
}
# `dev-test all` plus the VM test.
export def "dev-test ci" [] {
run-steps ((lint-steps) | append (unit-steps) | append (integration-steps null false) | append (vm-steps "vm"))
}
# Print a completion script for a shell that cannot load this module (zsh, bash).
export def "dev-test completions" [
shell?: string@"nu-complete dev-test shells" # zsh or bash
] {
let shell = ($shell | default "zsh")
match $shell {
"zsh" => { print (completion-script zsh) }
"bash" => { print (completion-script bash) }
_ => { error make { msg: $"unknown shell '($shell)'; expected one of: (nu-complete dev-test shells | str join ', ')" } }
}
}
# Fail if the command table, the defined commands or the help text diverge.
export def "dev-test self-check" [] {
let problems = (
(check-consistency)
++ (check-integration-args)
++ (check-dispatch)
++ (check-filter-cache)
++ (check-openssl-verdict)
++ (check-kvm-step)
++ (check-module-mode)
++ (check-shell-completion)
++ (check-nu-completions)
++ (check-docs)
)
if not ($problems | is-empty) {
print -e "dev-test self-check failed:"
$problems | each {|p| print -e $" - ($p)" } | ignore
# Catchable, so a module-mode caller survives; script mode still exits 1.
error make { msg: "dev-test self-check failed; see the problems above" }
}
print "dev-test self-check: ok"
}
def "nu-complete dev-test vm" [] {
["vm" "peers"]
}
def "nu-complete dev-test shells" [] {
["zsh" "bash"]
}
# `cargo test --test '*'` matches integration targets only, same as the integration step.
def "nu-complete dev-test filters" [] {
let key = (try { source-mtime-key } catch { "unknown" })
try {
cached-lines (filter-cache-path) $key {|| list-test-names }
} catch {
[]
}
}
def filter-cache-path [] {
let override = ($env.SKYG_TEST_FILTER_CACHE? | default "")
if ($override | is-empty) { repo-root | path join $FILTER_CACHE_REL } else { $override }
}
# The globs and the cache are anchored here, so the key does not depend on the
# completion's working directory (from `src/` the relative globs failed, which
# made the key "unknown" and the entry never stale).
def repo-root [] {
$SCRIPT_PATH | path dirname | path dirname
}
def source-mtime-key [] {
let root = (repo-root | path expand | into string)
# `ls` only expands globs written literally in the source, so the anchored
# patterns go through `glob`.
let rows = (
(glob ($root + "/src/**/*.rs"))
++ (glob ($root + "/tests/**/*.rs"))
++ (glob ($root + "/Cargo.toml"))
++ (glob ($root + "/Cargo.lock"))
)
if ($rows | is-empty) {
return "none"
}
$rows | each {|p| ls $p | get 0.modified } | math max | format date "%s.%f"
}
# Null means a miss. An empty list is a hit that listed no tests.
def read-cached-lines [path: string, key: string] {
if not ($path | path exists) {
return null
}
let lines = (open --raw $path | lines | where {|l| not ($l | is-empty) })
if ($lines | is-empty) or (($lines | first) != $key) {
return null
}
$lines | skip 1
}
def write-cached-lines [path: string, key: string, names: list<string>] {
mkdir ($path | path dirname)
([$key] | append $names | str join (char nl)) | save -f $path
}
def cached-lines [path: string, key: string, produce: closure] {
# A key that could not be computed ("unknown") or found no sources ("none")
# never changes, so caching under it would serve stale names forever.
if $key in ["unknown" "none"] {
return (do $produce)
}
let hit = (read-cached-lines $path $key)
if $hit != null {
return $hit
}
let names = (do $produce)
if not ($names | is-empty) {
write-cached-lines $path $key $names
}
$names
}
def list-test-names [] {
# Self-check points this at a counter file to prove Tab does not re-list.
let stub = ($env.SKYG_TEST_LIST_STUB? | default "")
if not ($stub | is-empty) {
let n = (if ($stub | path exists) {
open --raw $stub | str trim | into int
} else {
0
})
($n + 1 | into string) | save -f $stub
return ["stub-test"]
}
let listed = (try {
run-external "cargo" "test" "--test" "*" "--" "--list" | complete
} catch {
{stdout: "", stderr: "", exit_code: 1}
})
if $listed.exit_code != 0 {
return []
}
$listed.stdout
| lines
| where {|l| $l =~ ': (test|benchmark)$' }
| each {|l| $l | str replace -r ': (test|benchmark)$' '' }
| uniq
| sort
}
def step-record [label: string, command: string, body: closure] {
{ label: $label, command: $command, body: $body }
}
def unit-steps [] {
[(step-record "unit" "cargo test --lib --bins" {||
run-checked "cargo" ["test" "--lib" "--bins"]
})]
}
def kvm-available [] {
# Self-check sets this to prove a missing device fails inside the step.
if ($env.SKYG_TEST_ASSUME_NO_KVM? | default "") == "1" {
return false
}
"/dev/kvm" | path exists
}
def kvm-refusal [] {
if (kvm-available) {
null
} else {
"/dev/kvm is missing: NixOS VM tests need hardware virtualisation. Run on a Linux host with KVM, or in CI with KVM exposed; there is no software fallback."
}
}
def vm-steps [name: string] {
# The KVM check runs in the step body. Raising it here would abort `dev-test ci`
# before lint, unit and integration, and the summary table would not print.
let attr = (match $name {
"peers" => "tests"
_ => "checks"
})
let system = $"($nu.os-info.arch)-($nu.os-info.name)"
[(step-record $"vm ($name)" $"nix build .#($attr).($system).($name) -L" {||
let why = (kvm-refusal)
if $why != null {
error make { msg: $why }
}
print "NixOS VM tests are slow: expect several minutes."
run-checked "nix" ["build" $".#($attr).($system).($name)" "-L"]
})]
}
# Libtest flags must follow `--`. A bare `--nocapture` is a cargo flag and cargo rejects it.
def integration-cargo-args [filter: any, nocapture: bool] {
mut tail = []
if not ($filter | is-empty) {
$tail = ($tail | append ($filter | into string))
}
if $nocapture {
$tail = ($tail | append "--nocapture")
}
let args = ["test" "--test" "*"]
if ($tail | is-empty) {
$args
} else {
$args | append "--" | append $tail
}
}
def integration-steps [filter: any, nocapture: bool] {
let args = (integration-cargo-args $filter $nocapture)
let label = (if ($filter | is-empty) { "integration" } else { $"integration ($filter)" })
[(step-record $label $"cargo ($args | str join ' ')" {|| run-checked "cargo" $args })]
}
# Empty stdout is success only when cargo actually ran. A lockfile or registry
# error also leaves stdout empty and must fail the step.
def openssl-tree-ok [stdout: string, stderr: string, exit_code: int] {
let stdout_empty = (($stdout | str trim) | is-empty)
let no_match = ($stderr | str contains "did not match any packages")
($exit_code == 0 and $stdout_empty) or $no_match
}
def lint-steps [] {
[
(step-record "fmt" "cargo fmt --check" {||
run-checked "cargo" ["fmt" "--check"]
})
(step-record "clippy" "cargo clippy --all-targets -- -D warnings" {||
run-checked "cargo" ["clippy" "--all-targets" "--" "-D" "warnings"]
})
(step-record "openssl" "cargo tree -i openssl (must be empty)" {||
let found = (run-external "cargo" "tree" "-i" "openssl" | complete)
if not (openssl-tree-ok $found.stdout $found.stderr $found.exit_code) {
let detail = (if not (($found.stdout | str trim) | is-empty) {
$"openssl is in the dependency tree:\n($found.stdout)"
} else {
$"cargo tree failed with status ($found.exit_code): ($found.stderr | str trim)"
})
error make { msg: $detail }
}
})
]
}
# Run each step in order, stop at the first failure, always print the summary
# table and raise if anything failed (the process then exits non-zero).
def run-steps [steps: list] {
mut results = []
for s in $steps {
let r = (run-step $s)
$results = ($results | append $r)
if $r.result != "ok" { break }
}
let width = ([26 ($results | each {|r| $r.step | str length } | math max)] | math max)
print ""
print "Summary:"
print ((pad "step" $width "left") + (pad "result" 8 "left") + (pad "seconds" 9 "right") + " command")
for r in $results {
let status = (pad $r.result 8 "left")
print ((pad $r.step $width "left") + (if $r.result == "ok" { $"(ansi green)($status)(ansi reset)" } else { $"(ansi red)($status)(ansi reset)" }) + (pad $"(($r.seconds))s" 9 "right") + $" ($r.command)")
}
print ""
let failed = ($results | where result != "ok" | get step)
if not ($failed | is-empty) {
# A catchable error, not `exit`: with `use dev/dev-test.nu *` in a Nushell
# config, `exit` ends the caller's session and cannot be caught.
error make { msg: $"($failed | str join ', ') failed; see the summary above" }
}
}
def pad [text: string, width: int, alignment: string] {
$text | fill -a $alignment -w $width
}
def run-step [s: record] {
print ""
print $"(ansi cyan)==> ($s.label)(ansi reset) ($s.command)"
let started = (date now)
let ok = (try {
do $s.body | ignore
true
} catch {|e|
print -e $"(ansi red)($s.label) failed: ($e.msg)(ansi reset)"
false
})
{
step: $s.label
command: $s.command
result: (if $ok { "ok" } else { "FAILED" })
seconds: ((((date now) - $started) / 1sec) | math round --precision 1)
}
}
# Redirect straight to stdout so long builds and tests show progress; Nushell
# raises on a non-zero exit, which the enclosing step records as a failure.
def run-checked [cmd: string, args: list<string>] {
# Self-check spawns the runner with this set to read the argv without the suite.
if ($env.SKYG_TEST_DRY_RUN? | default "") == "1" {
print $"DRY-ARGS: ([$cmd] | append $args | str join '|')"
return
}
run-external $cmd ...$args o> /dev/stdout
}
def help-footer [] {
[
$"Run `devenv shell -- ($RUNNER) <subcommand>` or `nu ($SCRIPT_REL) <subcommand>`."
"An interactive `devenv shell` installs bash completion for this command."
# Single quotes: `(...)` inside a double-quoted string runs the command.
('zsh: after compinit, add `source <(nu ' + $SCRIPT_REL + ' completions zsh)` to .zshrc. direnv will not source it.')
$"Nushell: `use ($SCRIPT_REL) *`."
] | str join "\n"
}
def print-help [] {
print "Run the repository's checks."
print ""
print $"Usage: devenv shell -- ($RUNNER) <subcommand> [args]"
print $" nu ($SCRIPT_REL) <subcommand> [args]"
print ""
$command_table | each {|c| {subcommand: $c.name, flags: ($c.flags | str join " "), what: $c.usage} } | table -e | print
print ""
print (help-footer)
}
def completion-script [shell: string] {
let subcommands = ($command_table | get name | str join " ")
let flags = ($command_table | get flags | flatten | uniq | str join " ")
let replacements = {
"__SUBCOMMANDS__": $subcommands
"__FLAGS__": $flags
"__VM__": (nu-complete dev-test vm | str join " ")
"__SHELLS__": (nu-complete dev-test shells | str join " ")
"__FN__": $COMP_FN
"__RUNNER__": $RUNNER
}
let template = (match $shell {
"zsh" => (zsh-template)
_ => (bash-template)
})
$replacements | transpose key value | reduce --fold $template {|it, acc| $acc | str replace -a $it.key $it.value }
}
def zsh-template [] {
# `_describe` emits nothing unless a completion widget is already active, so
# the candidate list was empty. `compadd` writes the words itself.
# A raw string cannot start with `#` in Nushell, so the template starts on the next line.
r#'
#compdef __RUNNER__
# zsh completion for the dev-test runner. The command is an external program,
# so this file only registers completion; it does not define a dev-test function.
# Install after compinit (devenv's hook runs in bash, and direnv will not source
# it for you):
# source <(nu dev/dev-test.nu completions zsh)
__FN__() {
local -a subcommands
subcommands=(__SUBCOMMANDS__)
if (( CURRENT == 2 )); then
compadd -- "${subcommands[@]}"
return
fi
case "${words[2]}" in
integration) compadd -- __FLAGS__ ;;
vm) compadd -- __VM__ ;;
completions) compadd -- __SHELLS__ ;;
esac
}
if (( $+functions[compdef] )); then
compdef __FN__ __RUNNER__
fi
'# | str trim --left
}
def bash-template [] {
r#'
# bash completion for the dev-test runner. The command is an external program,
# so this file only registers completion; it does not define a dev-test function.
# source <(nu dev/dev-test.nu completions bash)
__FN__() {
local cur sub i
cur="${COMP_WORDS[COMP_CWORD]}"
sub=""
for ((i=1; i<COMP_CWORD; i++)); do
case "${COMP_WORDS[i]}" in
-*) ;;
*) sub="${COMP_WORDS[i]}"; break ;;
esac
done
if [[ -z "$sub" ]]; then
COMPREPLY=( $(compgen -W "__SUBCOMMANDS__" -- "$cur") )
return 0
fi
case "$sub" in
integration) COMPREPLY=( $(compgen -W "__FLAGS__" -- "$cur") ) ;;
vm) COMPREPLY=( $(compgen -W "__VM__" -- "$cur") ) ;;
completions) COMPREPLY=( $(compgen -W "__SHELLS__" -- "$cur") ) ;;
*) COMPREPLY=() ;;
esac
return 0
}
complete -F __FN__ __RUNNER__
'# | str trim --left
}
# Script-mode argv is one list. Swallowing a flag or a spare word made
# `dev-test unit --nocapture` and `dev-test integration a b` look successful.
def parse-script-args [args: list<string>] {
if ($args | is-empty) or ($args | any {|a| ($a == "--help") or ($a == "-h") }) {
return {cmd: "help", positional: null, nocapture: false}
}
let sub = ($args | first)
if $sub not-in ($command_table | get name) {
error make { msg: $"unknown subcommand '($sub)'; run '($RUNNER) --help' for the list" }
}
let rest = ($args | skip 1)
let allowed = ($command_table | where name == $sub | get 0.flags)
mut positionals = []
mut nocapture = false
for a in $rest {
if ($a | str starts-with "-") {
if $a not-in $allowed {
error make { msg: $"unknown flag '($a)' for '($RUNNER) ($sub)'" }
}
if $a == "--nocapture" {
$nocapture = true
}
} else {
$positionals = ($positionals | append $a)
}
}
let max_pos = (if $sub in ["integration" "vm" "completions"] { 1 } else { 0 })
if ($positionals | length) > $max_pos {
let what = (if $max_pos == 0 { "no arguments" } else { "at most one argument" })
error make { msg: $"($RUNNER) ($sub) takes ($what); got '($positionals | str join ' ')'" }
}
{cmd: $sub, positional: ($positionals | get 0?), nocapture: $nocapture}
}
# Compare the table with the commands Nushell actually parsed from this file and
# with the help block above; every entry must agree in all three places.
def check-consistency [] {
let prefix = $"($RUNNER) "
let defined = (
scope commands
| where {|c| $c.name == $RUNNER or ($c.name | str starts-with $prefix)}
| get name
| where {|n| $n != $RUNNER}
| each {|n| $n | str substring ($prefix | str length).. }
)
let table = ($command_table | get name)
# The entry point is `dev-test` once the file is used as a module and `main`
# when it runs as a script; both modes must agree with the table.
let entry = (scope commands | where {|c| $c.name in [$RUNNER "main"]} | get 0?)
let usage = (if $entry == null { "" } else { $entry.extra_description | default "" })
mut problems = []
for name in $table {
if $name not-in $defined {
$problems = ($problems | append $"table lists '($name)' but there is no '($RUNNER) ($name)' command")
}
if not ($usage =~ ('(?m)^\s+' + $name + '\s')) {
$problems = ($problems | append $"help text does not list '($name)'")
}
let cmd = (scope commands | where name == $"($RUNNER) ($name)" | get 0?)
let flags = (if $cmd == null { [] } else { $cmd.signatures | get any | where {|p| $p.parameter_type in ["switch" "named"]} | each {|p| $"--($p.parameter_name)" } | sort })
let expected = (($command_table | where name == $name | get 0.flags) | sort)
if $flags != $expected {
$problems = ($problems | append $"flags of '($RUNNER) ($name)' are ($flags | to json -r), table says ($expected | to json -r)")
}
}
for name in $defined {
if $name not-in $table {
$problems = ($problems | append $"command '($RUNNER) ($name)' is not in the command table")
}
}
let footer = (help-footer)
if ($footer | str contains "shell builtin") or ($footer | str contains "builtin test") {
$problems = ($problems | append "help still warns about the test builtin")
}
if not ($footer | str contains $"nu ($SCRIPT_REL) completions") {
$problems = ($problems | append $"help does not show 'nu ($SCRIPT_REL) completions'")
}
# `(...)` in a double-quoted string runs the command; that would inline the script.
if ($footer | str contains "#compdef") or ($footer | str contains "complete -F") {
$problems = ($problems | append "help footer embeds a completion script; quote the <(...) example literally")
}
$problems
}
def check-integration-args [] {
mut problems = []
let nocap = (integration-steps null true | first | get command)
if $nocap != "cargo test --test * -- --nocapture" {
$problems = ($problems | append $"integration --nocapture argv is '($nocap)'")
}
let both = (integration-steps "peers" true | first | get command)
if $both != "cargo test --test * -- peers --nocapture" {
$problems = ($problems | append $"integration filter+nocapture argv is '($both)'")
}
let plain = (integration-steps null false | first | get command)
if $plain != "cargo test --test *" {
$problems = ($problems | append $"integration argv is '($plain)'")
}
# The runner must emit that argv, not only build it in memory.
let child = (with-env { SKYG_TEST_DRY_RUN: "1" } {
run-external "nu" $SCRIPT_PATH "integration" "--nocapture" | complete
})
let want = "DRY-ARGS: cargo|test|--test|*|--|--nocapture"
if $child.exit_code != 0 or not ($child.stdout | str contains $want) {
$problems = ($problems | append $"CLI integration --nocapture exit ($child.exit_code), wanted '($want)': ($child.stdout)\n($child.stderr)")
}
let filtered = (with-env { SKYG_TEST_DRY_RUN: "1" } {
run-external "nu" $SCRIPT_PATH "integration" "peers" "--nocapture" | complete
})
let want_both = "DRY-ARGS: cargo|test|--test|*|--|peers|--nocapture"
if $filtered.exit_code != 0 or not ($filtered.stdout | str contains $want_both) {
$problems = ($problems | append $"CLI integration peers --nocapture exit ($filtered.exit_code), wanted '($want_both)'")
}
# Cargo itself must accept `--nocapture` with no filter. That is the bug.
let listed = (run-external "cargo" "test" "--test" "*" "--" "--nocapture" "--list" | complete)
if $listed.exit_code != 0 {
$problems = ($problems | append $"cargo rejected --nocapture after --: ($listed.stderr | str trim)")
}
$problems
}
def check-dispatch [] {
mut problems = []
let cases = [
{args: ["unit" "--nocapture"], needle: "unknown flag"}
{args: ["integration" "a" "b"], needle: "at most one"}
{args: ["integration" "--bogus"], needle: "unknown flag"}
{args: ["lint" "-x"], needle: "unknown flag"}
{args: ["vm" "peers" "extra"], needle: "at most one"}
{args: ["all" "--nocapture"], needle: "unknown flag"}
{args: ["ci" "extra"], needle: "no arguments"}
]
for c in $cases {
let got = (try { parse-script-args $c.args | ignore; "ok" } catch {|e| $e.msg })
if not ($got | str contains $c.needle) {
$problems = ($problems | append $"parse ($c.args | str join ' ') -> '($got)', expected '($c.needle)'")
}
}
let ok = (parse-script-args ["integration" "peers" "--nocapture"])
if $ok.cmd != "integration" or $ok.positional != "peers" or (not $ok.nocapture) {
$problems = ($problems | append $"parse integration peers --nocapture -> ($ok | to nuon)")
}
let bare = (parse-script-args ["integration" "--nocapture"])
if (not $bare.nocapture) or (not ($bare.positional | is-empty)) {
$problems = ($problems | append $"parse integration --nocapture -> ($bare | to nuon)")
}
let cli = (run-external "nu" $SCRIPT_PATH "unit" "--nocapture" | complete)
if $cli.exit_code == 0 or not ($cli.stderr | str contains "unknown flag") {
$problems = ($problems | append $"CLI '($RUNNER) unit --nocapture' exit ($cli.exit_code): ($cli.stderr | str trim) ($cli.stdout | str trim)")
}
let extra = (run-external "nu" $SCRIPT_PATH "integration" "a" "b" | complete)
if $extra.exit_code == 0 or not ($extra.stderr | str contains "at most one") {
$problems = ($problems | append $"CLI '($RUNNER) integration a b' exit ($extra.exit_code): ($extra.stderr | str trim)")
}
$problems
}
def check-filter-cache [] {
mut problems = []
let root = (repo-root)
let rel = (filter-cache-path | path relative-to $root)
if $rel != $FILTER_CACHE_REL {
$problems = ($problems | append $"filter cache is '($rel)', expected '($FILTER_CACHE_REL)' under the repo root")
}
# A completion started in a subdirectory must key and store the same way.
let root_key = (source-mtime-key)
let from_sub = (do {
cd ($root | path join "src")
{key: (source-mtime-key), cache: (filter-cache-path)}
})
if $from_sub.key != $root_key or $from_sub.key in ["unknown" "none"] {
$problems = ($problems | append $"source mtime key from src/ is '($from_sub.key)', from the repo root '($root_key)'")
}
if $from_sub.cache != ($from_sub.cache | path expand) {
$problems = ($problems | append $"filter cache path from src/ is relative: '($from_sub.cache)'")
}
let dir = ($nu.temp-dir | path join $"filter-cache-($nu.pid)")
rm -rf $dir
mkdir $dir
let counter = ($dir | path join "calls")
let cache = ($dir | path join "cache")
"0" | save -f $counter
let ran = (with-env { SKYG_TEST_LIST_STUB: $counter, SKYG_TEST_FILTER_CACHE: $cache } {
let first = (nu-complete dev-test filters)
let second = (nu-complete dev-test filters)
let n = (open --raw $counter | str trim | into int)
let key = (source-mtime-key)
let stored = (if ($cache | path exists) { open --raw $cache | lines | first } else { "" })
# A stale key must list again. Rewriting the key is the mtime changing.
if ($cache | path exists) {
let names = (open --raw $cache | lines | skip 1)
(["stale"] | append $names | str join (char nl)) | save -f $cache
}
let third = (nu-complete dev-test filters)
let n2 = (open --raw $counter | str trim | into int)
{first: $first, second: $second, third: $third, n: $n, n2: $n2, key: $key, stored: $stored}
})
if $ran.n != 1 {
$problems = ($problems | append $"filter completer listed ($ran.n) times; a warm cache should list once")
}
if $ran.first != $ran.second or $ran.first != ["stub-test"] {
$problems = ($problems | append $"filter cache returned ($ran.first | to nuon) then ($ran.second | to nuon)")
}
if $ran.stored != $ran.key {
$problems = ($problems | append $"filter cache key is '($ran.stored)', source mtime key is '($ran.key)'")
}
if $ran.n2 != 2 {
$problems = ($problems | append $"stale filter cache listed ($ran.n2) times after a stale key")
}
# An "unknown" key cannot go stale, so it must not be stored.
let unknown_cache = ($dir | path join "unknown-cache")
let produced = (cached-lines $unknown_cache "unknown" {|| ["x"] })
if ($unknown_cache | path exists) {
$problems = ($problems | append "an unknown cache key was written; that entry would never go stale")
}
if $produced != ["x"] {
$problems = ($problems | append $"an unknown cache key returned ($produced | to nuon), expected a fresh listing")
}
rm -rf $dir
$problems
}
# `exit` in a module ends the caller's Nushell session and cannot be caught, so
# no code path may call it, not even the entry point: `use ... *` binds the bare
# name to `main` and `dev-test bogus` reaches it. A failing check raises instead.
def check-module-mode [] {
mut problems = []
let lines = (open --raw $SCRIPT_PATH | lines)
# The scan has to see an inline `catch { …; exit 1 }` too, not just a line that
# starts with `exit`; comment lines are dropped first so prose cannot trip it.
let exits = ($lines | enumerate | where {|r| ($r.item | str replace --regex '^\s*#.*$' "") =~ '(^|[;{])\s*exit\b'} | get index)
for i in $exits {
$problems = ($problems | append $"line ($i + 1) calls exit, which ends a module-mode session: ($lines | get $i | str trim)")
}
# `(exit ...)` inside a double-quoted string is a subexpression that ends the
# process instead of building the message.
for r in ($lines | enumerate | where {|r| $r.item =~ '\$".*\(\s*exit\b'}) {
$problems = ($problems | append $"line ($r.index + 1) interpolates a call to exit: ($r.item | str trim)")
}
# The failure path itself: catch it in module mode and keep going.
let probe = (with-env { SKYG_TEST_ASSUME_NO_KVM: "1" } {
run-external "nu" "--no-config-file" "-c" $"use ($SCRIPT_PATH) *; try { ($RUNNER) vm } catch { print 'caught' }; print 'after'" | complete
})
if $probe.exit_code != 0 or not ($probe.stdout | str contains "caught") or not ($probe.stdout | str contains "after") {
$problems = ($problems | append $"module mode: a failing step was not catchable, exit code ($probe.exit_code); wanted 'caught' and 'after' in: ($probe.stdout | str trim) ($probe.stderr | str trim)")
}
# Dispatch errors reach `main` too (`use ... *` binds the bare name), so a typo
# must be catchable as well.
let typo = (run-external "nu" "--no-config-file" "-c" $"use ($SCRIPT_PATH) *; try { ($RUNNER) bogus } catch { print 'caught' }; print 'after'" | complete)
if $typo.exit_code != 0 or not ($typo.stdout | str contains "caught") or not ($typo.stdout | str contains "after") {
$problems = ($problems | append $"module mode: an unknown subcommand was not catchable, exit code ($typo.exit_code); wanted 'caught' and 'after' in: ($typo.stdout | str trim) ($typo.stderr | str trim)")
}
# Script mode still has to fail.
let script = (with-env { SKYG_TEST_ASSUME_NO_KVM: "1" } {
run-external "nu" "--no-config-file" $SCRIPT_PATH "vm" | complete
})
if $script.exit_code == 0 or not ($script.stderr | str contains "/dev/kvm") {
$problems = ($problems | append $"script mode: a failing step exited ($script.exit_code), expected non-zero with the /dev/kvm reason: ($script.stderr | str trim)")
}
# The dispatch error is no longer caught in `main`, so script mode relies on
# an uncaught error for its failure status.
let script_typo = (run-external "nu" "--no-config-file" $SCRIPT_PATH "bogus" | complete)
if $script_typo.exit_code == 0 or not ($script_typo.stderr | str contains "unknown subcommand") {
$problems = ($problems | append $"script mode: an unknown subcommand exited ($script_typo.exit_code), expected non-zero with the unknown-subcommand reason: ($script_typo.stderr | str trim)")
}
$problems
}
def check-openssl-verdict [] {
mut problems = []
let cases = [
{out: "", err: "error: package ID specification `openssl` did not match any packages\n", code: 101, ok: true}
{out: "", err: "", code: 0, ok: true}
{out: "openssl v1.0.0\n", err: "", code: 0, ok: false}
{out: "", err: "error: failed to load source for a dependency", code: 101, ok: false}
]
for c in $cases {
let got = (openssl-tree-ok $c.out $c.err $c.code)
if $got != $c.ok {
$problems = ($problems | append $"openssl-tree-ok exit ($c.code) = ($got), expected ($c.ok)")
}
}
$problems
}
def check-kvm-step [] {
let built = (with-env { SKYG_TEST_ASSUME_NO_KVM: "1" } {
try {
let steps = (vm-steps "vm")
let msg = (try {
do ($steps | first | get body)
"no-error"
} catch {|e| $e.msg })
{built: true, msg: $msg}
} catch {|e|
{built: false, msg: $e.msg}
}
})
if not $built.built {
return [$"vm-steps aborted while building the step list: ($built.msg)"]
}
if not ($built.msg | str contains "/dev/kvm") {
return [$"missing kvm should fail the vm step, got: ($built.msg)"]
}
[]
}
# Words after a `TAG:` line, dropping the `--` that ends compadd's options.
def tagged-words [text: string, tag: string] {
let line = ($text | lines | where {|l| $l | str starts-with $"($tag):"} | get 0?)
if $line == null {
return []
}
$line
| str substring ($"($tag):" | str length)..
| split row " "
| where {|w| not ($w | is-empty) and $w != "--" }
}
def check-shell-completion [] {
mut problems = []
let dir = ($nu.temp-dir | path join $"runner-complete-($nu.pid)")
rm -rf $dir
mkdir $dir
let subcommands = ($command_table | get name)
for shell in (nu-complete dev-test shells) {
let script = (completion-script $shell)
let file = ($dir | path join $"completions.($shell)")
$script | save -f $file
if ($script | str contains "builtin") {
$problems = ($problems | append $"($shell) completion still mentions the test builtin")
}
let wrappers = ($script | lines | where {|l| $l =~ '^(test|dev-test)\(\)' })
if not ($wrappers | is-empty) {
$problems = ($problems | append $"($shell) completion defines a command function")
}
if not ($script | str contains $COMP_FN) or not ($script | str contains $RUNNER) {
$problems = ($problems | append $"($shell) completion does not register ($COMP_FN) for ($RUNNER)")
}
if (which $shell | is-empty) {
continue
}
let syntax_args = (if $shell == "zsh" { ["-f" "-n" $file] } else { ["--noprofile" "--norc" "-n" $file] })
let syntax = (run-external $shell ...$syntax_args | complete)
if $syntax.exit_code != 0 {
$problems = ($problems | append $"($shell) completion script does not parse: ($syntax.stderr | str trim)")
continue
}
let probe = ($dir | path join $"probe.($shell)")
let body = (if $shell == "zsh" {
[
$"source '($file)'"
"typeset -a got"
'compadd() { got+=("$*"); }'
("words=(" + $RUNNER + " '')")
"CURRENT=2"
$COMP_FN
'print -r -- "SUB:${got[*]}"'
"got=()"
("words=(" + $RUNNER + " vm '')")
"CURRENT=3"
$COMP_FN
'print -r -- "VM:${got[*]}"'
"got=()"
("words=(" + $RUNNER + " integration '')")
"CURRENT=3"
$"($COMP_FN)"
'print -r -- "FLAG:${got[*]}"'
"funcs="
'(( $+functions[dev-test] )) && funcs="${funcs}dev-test"'
'(( $+functions[test] )) && funcs="${funcs} test"'
'print -r -- "FUNCS:${funcs}"'
]
} else {
[
$"source '($file)'"
("COMP_WORDS=(" + $RUNNER + " '')")
"COMP_CWORD=1"
$COMP_FN
'printf "SUB:%s\n" "${COMPREPLY[*]}"'
("COMP_WORDS=(" + $RUNNER + " vm '')")
"COMP_CWORD=2"
$COMP_FN
'printf "VM:%s\n" "${COMPREPLY[*]}"'
("COMP_WORDS=(" + $RUNNER + " integration '')")
"COMP_CWORD=2"
$COMP_FN
'printf "FLAG:%s\n" "${COMPREPLY[*]}"'
"funcs="
'declare -F dev-test >/dev/null && funcs="${funcs}dev-test"'
'declare -F test >/dev/null && funcs="${funcs} test"'
'printf "FUNCS:%s\n" "$funcs"'
]
})
$body | str join (char nl) | save -f $probe
let args = (if $shell == "zsh" { ["-f" $probe] } else { ["--noprofile" "--norc" $probe] })
let out = (run-external $shell ...$args | complete)
if $out.exit_code != 0 {
$problems = ($problems | append $"($shell) completion probe exited ($out.exit_code): ($out.stderr | str trim) ($out.stdout | str trim)")
continue
}
let sub_words = (tagged-words $out.stdout "SUB")
if ($sub_words | is-empty) {
$problems = ($problems | append $"($shell) completion output is empty: ($out.stdout | str trim)")
}
for name in $subcommands {
if $name not-in $sub_words {
$problems = ($problems | append $"($shell) completion missing '($name)': ($sub_words | str join ' ')")
}
}
let vm_words = (tagged-words $out.stdout "VM")
if ("vm" not-in $vm_words) or ("peers" not-in $vm_words) {
$problems = ($problems | append $"($shell) vm completion is ($vm_words | str join ' ')")
}
let flag_words = (tagged-words $out.stdout "FLAG")
if "--nocapture" not-in $flag_words {
$problems = ($problems | append $"($shell) integration completion is ($flag_words | str join ' ')")
}
let funcs = (tagged-words $out.stdout "FUNCS")
if not ($funcs | is-empty) {
$problems = ($problems | append $"($shell) defined a command function: ($funcs | str join ' ')")
}
}
rm -rf $dir
$problems
}
# Cursor sits just after `marker` in the raw file, before any trailing newline.
def complete-at [body: string, marker: string] {
let file = ($nu.temp-dir | path join $"dev-test-ide-($nu.pid)-($marker | str replace ' ' '_').nu")
$body | save -f $file
let raw = (open --raw $file)
let idx = ($raw | str index-of $marker)
let pos = (if $idx == null or $idx < 0 { -1 } else { $idx + ($marker | str length) })
let got = (if $pos < 0 {
{stdout: "", stderr: "marker not found", exit_code: 1}
} else {
run-external "nu" "--no-config-file" "--ide-complete" ($pos | into string) $file | complete
})
rm -f $file
let comps = (if $got.exit_code != 0 {
[]
} else {
try { $got.stdout | from json | get completions } catch { [] }
})
{pos: $pos, comps: $comps, stdout: $got.stdout, stderr: $got.stderr, exit_code: $got.exit_code}
}
def check-nu-completions [] {
mut problems = []
let sub = (complete-at $"use ($SCRIPT_PATH) *\n($RUNNER) " $"($RUNNER) ")
let want = ($command_table | get name | each {|n| $"($RUNNER) ($n)" })
let missing = ($want | where {|n| $n not-in $sub.comps })
if ($sub.comps | is-empty) or ("alias" in $sub.comps) or (not ($missing | is-empty)) or (($sub.comps | length) != ($want | length)) {
$problems = ($problems | append $"nushell completion output is empty or wrong at ($sub.pos): ($sub.comps | to json -r) ($sub.stderr | str trim)")
}
let vm = (complete-at $"use ($SCRIPT_PATH) *\n($RUNNER) vm " $"($RUNNER) vm ")
if ($vm.comps | is-empty) or ("vm" not-in $vm.comps) or ("peers" not-in $vm.comps) or ("alias" in $vm.comps) {
$problems = ($problems | append $"nushell vm completion is empty or wrong: ($vm.comps | to json -r)")
}
$problems
}
def check-docs [] {
mut problems = []
let rel = $"nu ($SCRIPT_REL) completions"
for f in ["AGENTS.md" "README.md" "docs/consolidator.md" "devenv.nix"] {
let text = (open --raw $f)
if ($text | str contains "dev/test.nu") {
$problems = ($problems | append $"($f) still names dev/test.nu")
}
if ($text | str contains "shell builtin") or ($text | str contains "builtin test") {
$problems = ($problems | append $"($f) still documents the test builtin workaround")
}
if ($text | str contains "devenv shell -- test") {
$problems = ($problems | append $"($f) still invokes 'devenv shell -- test'")
}
}
for f in ["AGENTS.md" "README.md" "docs/consolidator.md"] {
let text = (open --raw $f)
if not ($text | str contains $rel) {
$problems = ($problems | append $"($f) does not show '($rel)'")
}
if not ($text | str contains $"devenv shell -- ($RUNNER)") {
$problems = ($problems | append $"($f) does not show 'devenv shell -- ($RUNNER)'")
}
}
let hook = (open --raw devenv.nix)
if not ($hook | str contains 'scripts."dev-test"') {
$problems = ($problems | append "devenv.nix does not define scripts.\"dev-test\"")
}
if ($hook | str contains "scripts.test") {
$problems = ($problems | append "devenv.nix still defines scripts.test")
}
if not ($hook | str contains $"($SCRIPT_REL) completions bash") {
$problems = ($problems | append "enterShell does not install bash completion")
}
if ($hook | str contains "completions zsh") {
$problems = ($problems | append "enterShell tries to install zsh completion from bash")
}
if not (($hook | str contains "case $-") and ($hook | str contains "*i*")) {
$problems = ($problems | append "enterShell completion is not limited to interactive shells")
}
if ($hook | str contains 'echo "') {
$problems = ($problems | append "enterShell prints; non-interactive stdout must stay the command's own output")
}
$problems
}