From 49c6d6e6cc9426ec4ae0b4ff20aca2148998feed Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Thu, 17 Sep 2026 14:38:27 -0400 Subject: [PATCH] testing --- README.md | 40 ++++++++++++++++++++-- bin/runtest | 27 +++++++++++++++ bin/runtest-debug | 29 ++++++++++++++++ bin/test | 14 -------- devenv.nix | 4 ++- flake.nix | 6 ++++ guest/vm-test.nix | 84 +++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 187 insertions(+), 17 deletions(-) create mode 100755 bin/runtest create mode 100755 bin/runtest-debug delete mode 100755 bin/test create mode 100644 guest/vm-test.nix diff --git a/README.md b/README.md index a1175d4..4872f3e 100644 --- a/README.md +++ b/README.md @@ -836,9 +836,45 @@ The devshell provides nushell, nufmt, and pre-commit hooks. Enter it once: direnv allow ``` -On first entry devenv installs the pre-commit hooks automatically. Two hooks -run on every commit: +On first entry devenv installs the pre-commit hooks automatically. Hooks run +on every commit, including: - **nufmt** — formats all `*.nu` files in place - **gen-completions** — reruns `nu/gen-completions.nu` and stages `nu/completions.nu` whenever a usage block in the source files changes +- **nu-tests** — runs the CLI test suite (see below) whenever anything + under `nu/` changes + +### Testing + +Two test suites. Neither is ever run for you by an agent working in this +repo — they're for you (or CI) to run. + +**CLI tests** (`nu/tests/`), via [nutest](https://github.com/vyadh/nutest): + +```bash +runtest # everything (bin/ is on PATH inside the devShell) +runtest vm_test # just nu/tests/vm_test.nu +runtest-debug vm_test # load vm_test.nu into an interactive nu session instead +``` + +Covers the CLI's pure helper functions (path/env defaults, `shquote`, +`normalize-size`, the `gen-completions.nu` parser) and a smoke test that the +whole `hermenix`/`vm`/`ctl` subcommand tree parses and dispatches correctly +— without ever invoking a subcommand's real body (no ssh, no disk writes, +no `nix build`). + +**Guest VM test** (`guest/vm-test.nix`): boots `guest/common.nix` — the same +module set `hermes-guest-image` is built from — in an ephemeral, throwaway +NixOS test VM. No real disk under `/var/lib/hermes-vm/**` is ever touched: + +```bash +nix build .#checks.x86_64-linux.guest-vm --impure -L +``` + +Checks first-boot seeding (`~/.hermes`, `~/workspace`, the `/etc/nixos` +symlink), that seeding is idempotent across a reboot (the whole point of +the `.seeded` marker), the `hermes` user/sudo setup, and SSH. It +deliberately does not cover the real `hermes-install` script (no internet +route inside the test VM) or `modules/host.nix`'s own `hermes-vm` QEMU +service — both stay manual, operator-run checks. diff --git a/bin/runtest b/bin/runtest new file mode 100755 index 0000000..c492d82 --- /dev/null +++ b/bin/runtest @@ -0,0 +1,27 @@ +#!/usr/bin/env nu +# +# runtest — run the hermenix CLI test suite (nu/tests/). +# +# Requires the nutest module on NU_LIB_DIRS, which `nix develop`'s devShell +# sets up (that devShell also puts bin/ on PATH, so `runtest` works bare). +# Runs everything by default; pass a suite name to run just one file +# instead: +# runtest # everything +# runtest vm_test # only nu/tests/vm_test.nu +# +# To step through a single suite interactively instead of via nutest's +# runner, use runtest-debug. + +use nutest + +def main [ + suite?: string # only run nu/tests/.nu, e.g. vm_test +] { + let repo_root = ($env.FILE_PWD | path dirname) + let tests_path = ($repo_root | path join "nu" "tests") + if $suite == null { + nutest run-tests --path $tests_path --fail --returns table + } else { + nutest run-tests --path $tests_path --fail --match-suites $suite --returns table + } +} diff --git a/bin/runtest-debug b/bin/runtest-debug new file mode 100755 index 0000000..67d55b8 --- /dev/null +++ b/bin/runtest-debug @@ -0,0 +1,29 @@ +#!/usr/bin/env nu +# +# runtest-debug — interactively debug one CLI test file. +# +# Sources the given suite (e.g. `runtest-debug vm_test`) and drops you into +# an interactive nushell session with it loaded, instead of running it +# through nutest. Call any @test-tagged command by its full name to run +# just that test on its own and inspect what happens — nutest's runner +# captures output and runs tests concurrently, which gets in the way of +# stepping through a failure by hand. +# +# Test files don't need NU_LIB_DIRS/nutest themselves (only runtest's own +# runner does), so this works without the devShell's nutest wiring too. + +def main [suite: string] { + let repo_root = ($env.FILE_PWD | path dirname) + let file = ($repo_root | path join "nu" "tests" $"($suite).nu") + if not ($file | path exists) { + print -e $"no such test file: ($file)" + exit 1 + } + let child_script = ( + "source '" + $file + "'\n" + + "print 'Loaded. Custom/test commands now in scope:'\n" + + "scope commands | where type == 'custom' | get name | each {|n| print (' ' + $n)}\n" + + "print 'Call one by its full name to run just that test.'" + ) + ^nu --execute $child_script +} diff --git a/bin/test b/bin/test deleted file mode 100755 index 10bf230..0000000 --- a/bin/test +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env nu -# -# bin/test — run the full hermenix CLI test suite (nu/tests/). -# -# Requires the nutest module on NU_LIB_DIRS, which `nix develop`'s devShell -# sets up. For filtering to specific suites/tests, run -# `nutest run-tests --path nu/tests --match-tests ` directly. - -use nutest - -def main [] { - let repo_root = ($env.FILE_PWD | path dirname) - nutest run-tests --path ($repo_root | path join "nu" "tests") --fail -} diff --git a/devenv.nix b/devenv.nix index 9ff5ca7..6ed2a96 100644 --- a/devenv.nix +++ b/devenv.nix @@ -24,8 +24,10 @@ in ]; env.NU_LIB_DIRS = "${inputs.nutest}"; + env.SHELL = "nu"; enterShell = '' + export PATH="$DEVENV_ROOT/bin:$PATH" echo "Welcome to the Hermenix devshell!" ''; @@ -75,7 +77,7 @@ in enable = true; name = "nu-tests"; description = "Run the hermenix CLI test suite"; - entry = "${nu} -I ${inputs.nutest} bin/test"; + entry = "${nu} -I ${inputs.nutest} bin/runtest"; language = "system"; pass_filenames = false; files = "^nu/"; diff --git a/flake.nix b/flake.nix index 967371f..6845f6f 100644 --- a/flake.nix +++ b/flake.nix @@ -114,5 +114,11 @@ inherit inputs pkgs; modules = [ ./devenv.nix ]; }; + + # Never run by an agent — operator/CI only: + # nix build .#checks.x86_64-linux.guest-vm --impure -L + checks.${system} = { + guest-vm = import ./guest/vm-test.nix { inherit pkgs determinate; }; + }; }; } diff --git a/guest/vm-test.nix b/guest/vm-test.nix new file mode 100644 index 0000000..07435c9 --- /dev/null +++ b/guest/vm-test.nix @@ -0,0 +1,84 @@ +# Guest-config tier VM test: boots guest/common.nix (the same module set +# used to build hermes-guest-image, see flake.nix) in an ephemeral, +# throwaway NixOS test VM, and checks seeding/services. This never touches +# a real disk under /var/lib/hermes-vm/** and never runs `hermenix setup`. +# +# guest/hardware.nix (the real disk-label file) is intentionally not +# imported here, same as hermes-guest-image's own build — nixos-generators +# and, here, the NixOS test framework both supply their own bootloader and +# root filesystem instead. +# +# Build with (never run by an agent, only by the operator/CI): +# nix build .#checks.x86_64-linux.guest-vm --impure -L +# +# --impure is required because guest/common.nix reads /etc/hermenix/local.nix +# if present (see its own comment) — a pre-existing property of common.nix, +# not something introduced by this test. +{ pkgs, determinate }: + +pkgs.testers.runNixOSTest { + name = "hermenix-guest-config"; + + nodes.machine = + { lib, ... }: + { + imports = [ + determinate.nixosModules.default + ../guest/common.nix + ]; + + # A real, separate, auto-formatted virtual disk for the state dir — + # not tmpfs. tmpfs would not survive the in-test reboot below, which + # would make the idempotency check below pass for the wrong reason + # (nothing to re-seed over, rather than correctly skipping re-seeding). + virtualisation.emptyDiskImages = [ 1024 ]; + fileSystems."/var/lib/hermes" = lib.mkForce { + device = "/dev/vdb"; + fsType = "ext4"; + autoFormat = true; + neededForBoot = true; + }; + + # nixosTest VMs have no route to the internet by design, so the real + # hermes-install body (a third-party curl | bash installer) can't run + # hermetically here. Stubbed to just flag itself done; the real + # installer flow stays an operator-run manual smoke test. + systemd.services.hermes-install.script = lib.mkForce '' + touch /var/lib/hermes/.hermes-installed + ''; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + + # seedHermesState (guest/common.nix) ran on first boot. + machine.succeed("test -e /var/lib/hermes/.seeded") + machine.succeed("test -f /var/lib/hermes/.hermes/SOUL.md") + machine.succeed("test -d /var/lib/hermes/workspace") + machine.succeed("test -d /var/lib/hermes/nixos/.git") + machine.succeed("readlink /etc/nixos | grep -q '^/var/lib/hermes/nixos$'") + + # The single most important invariant in guest/common.nix (per its own + # comments): seeding must never re-run over an already-seeded state + # disk, or every rebuild would quietly wipe the agent's own edits to + # SOUL.md, memories, etc. + machine.succeed("echo IDEMPOTENCY_MARKER >> /var/lib/hermes/.hermes/SOUL.md") + machine.reboot() + machine.wait_for_unit("multi-user.target") + machine.succeed("grep -q IDEMPOTENCY_MARKER /var/lib/hermes/.hermes/SOUL.md") + + # The agent's account. + machine.succeed("id hermes") + machine.succeed("groups hermes | grep -q wheel") + machine.succeed("su - hermes -c 'sudo -n true'") + + # SSH is up. + machine.wait_for_unit("sshd.service") + machine.wait_for_open_port(22) + + # nix.custom.conf reflects the no-host-cache default (no local.nix in + # this hermetic build). + machine.succeed("grep -q cache.flakehub.com /etc/nix/nix.custom.conf") + machine.fail("grep -q 'http://10.77.0.1:5000' /etc/nix/nix.custom.conf") + ''; +}