fixing test failures
This commit is contained in:
1 parent
c8ff8b1f25
commit
42e903909c
14 files changed
+823
-111
No files matched your search
@@ -0,0 +1,104 @@
|
||||
# Shared building blocks for the two guest-config VM tests in this
|
||||
# directory: test_unbridge.nix and test_bridge.nix.
|
||||
#
|
||||
# The guest node's configuration and every assertion that must hold in both
|
||||
# networking modes are defined here, once; each mode test imports this file,
|
||||
# uses guestNode as its machine, and appends only the checks that apply to
|
||||
# its mode (the isolated-bridge NAT setup vs. the DHCP-served LAN bridge).
|
||||
#
|
||||
# Both tests boot guest/common.nix — the same module set hermes-guest-image
|
||||
# is built from — in an ephemeral, throwaway NixOS test VM. No real disk
|
||||
# under /var/lib/hermes-vm/** is ever touched, and `hermenix setup` is never
|
||||
# run. guest/hardware.nix (the real disk-label file) is intentionally not
|
||||
# imported, same as hermes-guest-image's own build — nixos-generators and,
|
||||
# here, the NixOS test framework both supply their own bootloader and root
|
||||
# filesystem instead.
|
||||
#
|
||||
# Build with (never run by an agent, only by the operator/CI):
|
||||
# nix build .#checks.x86_64-linux.guest-vm-unbridged --impure -L
|
||||
# nix build .#checks.x86_64-linux.guest-vm-bridged --impure -L
|
||||
#
|
||||
# --impure is required because guest/common.nix reads /etc/hermenix/local.nix
|
||||
# if present (see its own comment) — a pre-existing property of common.nix,
|
||||
# not something the tests introduce.
|
||||
{ determinate }:
|
||||
|
||||
{
|
||||
# The guest node, exactly as both tests boot it.
|
||||
guestNode =
|
||||
{ lib, ... }:
|
||||
{
|
||||
imports = [
|
||||
determinate.nixosModules.default
|
||||
../common.nix
|
||||
];
|
||||
|
||||
# A real, separate, auto-formatted virtual disk for the state dir —
|
||||
# not tmpfs. tmpfs would not survive the in-test reboot below, which
|
||||
# would make the idempotency check pass for the wrong reason (nothing
|
||||
# to re-seed over, rather than correctly skipping re-seeding).
|
||||
virtualisation.emptyDiskImages = [ 1024 ];
|
||||
fileSystems."/var/lib/hermes" = lib.mkForce {
|
||||
device = "/dev/vdb";
|
||||
fsType = "ext4";
|
||||
autoFormat = true;
|
||||
neededForBoot = true;
|
||||
};
|
||||
|
||||
# The test framework's default topology is not the real guest's: it
|
||||
# adds a SLiRP user-mode NIC (10.0.2.0/24, with its own DHCP server)
|
||||
# alongside the shared vlan, and the guest's networkd config matches
|
||||
# any ethernet NIC. Strip the user-mode NIC and keep only the shared
|
||||
# vlan (which udev names eth0) so the test machine has the production
|
||||
# shape: one virtual NIC, and no stray DHCP server to confuse the
|
||||
# mode-specific networking assertions.
|
||||
virtualisation.qemu.networkingOptions = lib.mkForce [ ];
|
||||
virtualisation.vlans = [ 1 ];
|
||||
|
||||
# nixosTest VMs have no route to the internet by design, so the real
|
||||
# hermes-install body (a third-party curl | bash installer) can't run
|
||||
# hermetically here. Stubbed to just flag itself done; the real
|
||||
# installer flow stays an operator-run manual smoke test.
|
||||
systemd.services.hermes-install.script = lib.mkForce ''
|
||||
touch /var/lib/hermes/.hermes-installed
|
||||
'';
|
||||
};
|
||||
|
||||
# Assertions that hold in both modes: first-boot seeding, seeding
|
||||
# idempotency across a reboot, the agent's account, and SSH.
|
||||
commonScript = ''
|
||||
# allow_reboot=True: without it, the VM starts with qemu's -no-reboot,
|
||||
# so the guest-initiated machine.reboot() below would just kill the VM
|
||||
# instead of restarting it, and the backdoor shell would never
|
||||
# reconnect. Must happen before anything else touches `machine`, which
|
||||
# would otherwise auto-start it with the (False) default first.
|
||||
machine.start(allow_reboot=True)
|
||||
machine.wait_for_unit("multi-user.target")
|
||||
|
||||
# seedHermesState (guest/common.nix) ran on first boot.
|
||||
machine.succeed("test -e /var/lib/hermes/.seeded")
|
||||
machine.succeed("test -f /var/lib/hermes/.hermes/SOUL.md")
|
||||
machine.succeed("test -d /var/lib/hermes/workspace")
|
||||
machine.succeed("test -d /var/lib/hermes/nixos/.git")
|
||||
machine.succeed("readlink /etc/nixos | grep -q '^/var/lib/hermes/nixos$'")
|
||||
|
||||
# The single most important invariant in guest/common.nix (per its own
|
||||
# comments): seeding must never re-run over an already-seeded state
|
||||
# disk, or every rebuild would quietly wipe the agent's own edits to
|
||||
# SOUL.md, memories, etc.
|
||||
machine.succeed("echo IDEMPOTENCY_MARKER >> /var/lib/hermes/.hermes/SOUL.md")
|
||||
machine.reboot()
|
||||
machine.wait_for_unit("multi-user.target")
|
||||
machine.succeed("grep -q IDEMPOTENCY_MARKER /var/lib/hermes/.hermes/SOUL.md")
|
||||
|
||||
# The agent's account.
|
||||
machine.succeed("id hermes")
|
||||
machine.succeed("groups hermes | grep -q wheel")
|
||||
machine.succeed("su - hermes -c 'sudo -n true'")
|
||||
|
||||
# SSH is up (checked over the test backdoor, so it is independent of
|
||||
# whatever address configuration the mode under test produces).
|
||||
machine.wait_for_unit("sshd.service")
|
||||
machine.wait_for_open_port(22)
|
||||
'';
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
# guest/vm-tests/default.nix — the granular guest VM test suite.
|
||||
#
|
||||
# Aggregates the three runNixOSTest checks into a single import surface for
|
||||
# flake.nix's checks.${system}. common.nix holds the guest node config and
|
||||
# the mode-independent assertions shared by the two host-networking-mode
|
||||
# tests; test_unbridge.nix / test_bridge.nix add each mode's own checks, and
|
||||
# test_migrate_connections.nix covers `hermenix migrate connections`.
|
||||
#
|
||||
# Run by the operator/CI, never by an agent (see GUIDELINES.md):
|
||||
# nix build .#checks.x86_64-linux.guest-vm-unbridged --impure -L
|
||||
# nix build .#checks.x86_64-linux.guest-vm-bridged --impure -L
|
||||
# nix build .#checks.x86_64-linux.migrate-connections-vm --impure -L
|
||||
#
|
||||
# --impure is required because guest/common.nix reads
|
||||
# /etc/hermenix/local.nix if present (see its own comment) — a pre-existing
|
||||
# property of common.nix, not something the tests introduce.
|
||||
{
|
||||
pkgs,
|
||||
determinate,
|
||||
hermenix,
|
||||
}:
|
||||
|
||||
{
|
||||
guest-vm-unbridged = import ./test_unbridge.nix { inherit pkgs determinate; };
|
||||
guest-vm-bridged = import ./test_bridge.nix { inherit pkgs determinate; };
|
||||
migrate-connections-vm = import ./test_migrate_connections.nix { inherit pkgs hermenix; };
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
# Guest-config VM test, bridged mode: the guest on a LAN served by DHCP,
|
||||
# with a second node acting as the LAN (a DHCP server, a throwaway HTTP
|
||||
# service, and a plain device that connects back to the guest). This is the
|
||||
# guest-facing half of services.hermenix.network.bridged = true: the module's
|
||||
# own bridge/enforcement rules are covered statically by
|
||||
# tests/network-modes-check.nix.
|
||||
#
|
||||
# The guest node and the mode-independent assertions (seeding, idempotency,
|
||||
# account, SSH) are shared with guest-vm-unbridged via
|
||||
# common.nix.
|
||||
#
|
||||
# Build with (never run by an agent, only by the operator/CI):
|
||||
# nix build .#checks.x86_64-linux.guest-vm-bridged --impure -L
|
||||
{ pkgs, determinate }:
|
||||
|
||||
let
|
||||
common = import ./common.nix { inherit determinate; };
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "hermenix-guest-config-bridged";
|
||||
|
||||
# Boots once, reboots once, and waits on DHCP lease negotiation plus a few
|
||||
# cross-node checks - still only minutes. Bound it far below the
|
||||
# framework's 1-hour default so a genuine hang fails fast.
|
||||
globalTimeout = 900;
|
||||
|
||||
nodes = {
|
||||
machine = common.guestNode;
|
||||
|
||||
# The "LAN": serves DHCP to the guest and hosts a throwaway HTTP
|
||||
# service, standing in for the LAN router and another LAN device. Its
|
||||
# firewall is off so the guest<->LAN checks exercise the network path
|
||||
# itself, not this throwaway node's own firewall.
|
||||
lan =
|
||||
{ pkgs, lib, ... }:
|
||||
{
|
||||
networking.firewall.enable = false;
|
||||
|
||||
# Same production-shape networking as the guest node: kill the test
|
||||
# framework's SLiRP user-mode NIC so the only NIC left is the one
|
||||
# shared with the guest.
|
||||
virtualisation.qemu.networkingOptions = lib.mkForce [ ];
|
||||
virtualisation.vlans = [ 1 ];
|
||||
|
||||
# The framework's own name-based static assignment missed the
|
||||
# shared NIC when this suite actually ran (the NIC came up as eth1,
|
||||
# not the eth0 the framework expected), and real interface names
|
||||
# vary anyway - and node numbering is attr order, which is not
|
||||
# something to trust. The shared NIC's MAC is deterministic
|
||||
# (52:54:00:12:<vlan>:<node>, see qemu-common.nix), so pin the
|
||||
# address in networkd by MAC. Two candidate rules cover either node
|
||||
# numbering; the one that matches this node's actual NIC wins, and
|
||||
# nothing depends on an interface name.
|
||||
networking.useNetworkd = true;
|
||||
systemd.network.networks."10-hermes-lan-1" = {
|
||||
matchConfig.MACAddress = "52:54:00:12:01:01"; # vlan 1, node #1
|
||||
address = [ "192.168.1.2/24" ];
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
};
|
||||
systemd.network.networks."10-hermes-lan-2" = {
|
||||
matchConfig.MACAddress = "52:54:00:12:01:02"; # vlan 1, node #2
|
||||
address = [ "192.168.1.2/24" ];
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
};
|
||||
|
||||
# DHCP only (port = 0 disables dnsmasq's DNS half; the guest keeps
|
||||
# its own QUAD9 resolvers regardless). Bound to the node's static
|
||||
# vlan address rather than an interface name - the framework's
|
||||
# interface naming is not something this test should depend on.
|
||||
services.dnsmasq = {
|
||||
enable = true;
|
||||
settings = {
|
||||
port = 0;
|
||||
listen-address = "192.168.1.2";
|
||||
bind-interfaces = true;
|
||||
dhcp-range = "192.168.1.100,192.168.1.200,12h";
|
||||
dhcp-option = [
|
||||
"3,192.168.1.2" # default gateway -> this node
|
||||
"6,192.168.1.2" # DNS -> this node
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
environment.systemPackages = [ pkgs.python3 ];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
# Bring the LAN up early so the guest's DHCP client (which retries for
|
||||
# as long as it runs) can get a lease as soon as the guest is online.
|
||||
lan.start()
|
||||
''
|
||||
+ common.commonScript
|
||||
+ ''
|
||||
# --- bridged-mode networking ---------------------------------------
|
||||
# The LAN's DHCP gave the guest a lease alongside its static
|
||||
# service-net address. (Both test nodes share one vlan, standing in for
|
||||
# the bridged uplink on a real host.)
|
||||
machine.wait_until_succeeds("ip -4 route show default | grep -q 'metric 100'")
|
||||
|
||||
addrs = machine.succeed("ip -4 -o addr show | grep -v '^1: lo'").strip().splitlines()
|
||||
guest_lan_ip = None
|
||||
for line in addrs:
|
||||
ip = line.split()[3].split("/")[0]
|
||||
if not ip.startswith("10.77."):
|
||||
guest_lan_ip = ip
|
||||
assert any("10.77.0.2/24" in line for line in addrs), f"static service-net address missing in bridged mode: {addrs}"
|
||||
assert guest_lan_ip is not None, f"guest never took a LAN DHCP lease: {addrs}"
|
||||
|
||||
# Both default routes are in place: the LAN's (metric 100, in use) and
|
||||
# the host service-net fallback (metric 200) that NAT mode relies on.
|
||||
machine.succeed("ip -4 route show default | grep -q 'default via 192.168.1.2'")
|
||||
machine.succeed("ip -4 route show default | grep -q 'default via 10.77.0.1'")
|
||||
|
||||
# Guest -> LAN: the guest reaches a service on the LAN node.
|
||||
lan.succeed("systemd-run --collect -u lanhttpd python3 -m http.server 8000 --bind 0.0.0.0")
|
||||
machine.wait_until_succeeds("curl -fsS http://192.168.1.2:8000/ >/dev/null")
|
||||
|
||||
# LAN -> guest: a plain device on the LAN can connect to a port the
|
||||
# guest exposes - the guest's own firewall still decides what answers
|
||||
# (here, sshd on 22).
|
||||
lan.wait_until_succeeds(f"bash -c 'exec 3<>/dev/tcp/{guest_lan_ip}/22'")
|
||||
'';
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
# VM test for `hermenix migrate connections` — local (folder-to-folder) mode.
|
||||
#
|
||||
# Boots a minimal NixOS VM with the hermenix binary available, creates a fake
|
||||
# source HERMES_HOME from Nix-store fixtures, runs the command, and asserts:
|
||||
#
|
||||
# 1. Every requested file lands in the destination.
|
||||
# 2. Hostname rewrites are applied in the destination.
|
||||
# 3. The source is completely untouched.
|
||||
# 4. Identity files (SOUL.md) are NOT copied.
|
||||
# 5. Absent optional files are silently skipped without crashing.
|
||||
# 6. Subdirectories (--extra-dirs) are copied recursively.
|
||||
#
|
||||
# Run with:
|
||||
# nix build .#checks.x86_64-linux.migrate-connections-vm -L
|
||||
#
|
||||
# Fixture files are defined as pkgs.writeText values so their content lives
|
||||
# entirely in Nix string literals — no heredocs in the testScript. This
|
||||
# keeps every line in the testScript at consistent indentation so Nix's ''
|
||||
# string stripping produces valid top-level Python.
|
||||
{ pkgs, hermenix }:
|
||||
|
||||
let
|
||||
# Source fixtures — contain the stale Docker hostname that should be
|
||||
# rewritten in the destination but preserved in the source.
|
||||
srcConfigYaml = pkgs.writeText "src-config.yaml" ''
|
||||
model:
|
||||
provider: openai
|
||||
connections:
|
||||
- name: desktop
|
||||
host: host.docker.internal
|
||||
port: 3000
|
||||
'';
|
||||
srcAuthJson = pkgs.writeText "src-auth.json" ''
|
||||
{"endpoint": "http://host.docker.internal:8080/auth"}
|
||||
'';
|
||||
srcDotEnv = pkgs.writeText "src-dot-env" ''
|
||||
OPENAI_API_KEY=sk-test
|
||||
GATEWAY_URL=http://host.docker.internal:9000
|
||||
'';
|
||||
srcMcpJson = pkgs.writeText "src-mcp.json" ''
|
||||
{"url":"http://host.docker.internal:5000"}
|
||||
'';
|
||||
# Identity file — must never be copied by migrate connections.
|
||||
srcSoulMd = pkgs.writeText "src-SOUL.md" ''
|
||||
I am Hermes.
|
||||
'';
|
||||
in
|
||||
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "hermenix-migrate-connections";
|
||||
|
||||
# This test only boots once and copies a few small files locally — it
|
||||
# should finish in well under a minute. Bound it far below the framework's
|
||||
# 1-hour default so a regression (e.g. a prompt silently blocking on
|
||||
# terminal input instead of reading piped stdin) fails fast instead of
|
||||
# burning a full hour before being reported.
|
||||
globalTimeout = 300;
|
||||
|
||||
nodes.machine = {
|
||||
environment.systemPackages = [ hermenix ];
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
machine.wait_for_unit("multi-user.target")
|
||||
|
||||
# ------------------------------------------------------------------ setup
|
||||
# Copy Nix-store fixtures into a writable fake source HERMES_HOME so that
|
||||
# sed -i (which needs a writable file) works during the rewrite step.
|
||||
machine.succeed("mkdir -p /tmp/src/connections")
|
||||
machine.succeed("install -m644 ${srcConfigYaml} /tmp/src/config.yaml")
|
||||
machine.succeed("install -m644 ${srcAuthJson} /tmp/src/auth.json")
|
||||
machine.succeed("install -m644 ${srcDotEnv} /tmp/src/.env")
|
||||
machine.succeed("install -m644 ${srcMcpJson} /tmp/src/connections/mcp.json")
|
||||
machine.succeed("install -m644 ${srcSoulMd} /tmp/src/SOUL.md")
|
||||
|
||||
# ------------------------------------------------------- run the command
|
||||
# `echo copy` satisfies the confirm-word prompt non-interactively.
|
||||
# Default --files=[config.yaml auth.json .env] and --extra-dirs=[connections]
|
||||
# are used; list-typed flags cannot be passed from bash to a Nu binary.
|
||||
machine.succeed("echo copy | hermenix migrate connections /tmp/src /tmp/dst")
|
||||
|
||||
# ------------------------------------------- 1. files land in destination
|
||||
machine.succeed("test -f /tmp/dst/config.yaml")
|
||||
machine.succeed("test -f /tmp/dst/auth.json")
|
||||
machine.succeed("test -f /tmp/dst/.env")
|
||||
machine.succeed("test -f /tmp/dst/connections/mcp.json")
|
||||
|
||||
# --------------------------------------- 2. hostname rewrites applied
|
||||
machine.succeed("grep -q '10.77.0.1' /tmp/dst/config.yaml")
|
||||
machine.fail( "grep -q 'host.docker.internal' /tmp/dst/config.yaml")
|
||||
machine.succeed("grep -q '10.77.0.1' /tmp/dst/auth.json")
|
||||
machine.fail( "grep -q 'host.docker.internal' /tmp/dst/auth.json")
|
||||
machine.succeed("grep -q '10.77.0.1' /tmp/dst/.env")
|
||||
machine.fail( "grep -q 'host.docker.internal' /tmp/dst/.env")
|
||||
machine.succeed("grep -q '10.77.0.1' /tmp/dst/connections/mcp.json")
|
||||
machine.fail( "grep -q 'host.docker.internal' /tmp/dst/connections/mcp.json")
|
||||
|
||||
# ------------------------------------------ 3. source is completely untouched
|
||||
machine.succeed("grep -q 'host.docker.internal' /tmp/src/config.yaml")
|
||||
machine.succeed("grep -q 'host.docker.internal' /tmp/src/auth.json")
|
||||
machine.succeed("grep -q 'host.docker.internal' /tmp/src/.env")
|
||||
machine.succeed("grep -q 'host.docker.internal' /tmp/src/connections/mcp.json")
|
||||
|
||||
# ------------------------------------ 4. identity file was NOT copied
|
||||
machine.fail("test -f /tmp/dst/SOUL.md")
|
||||
|
||||
# ------------------- 5. absent optional files are silently skipped
|
||||
# Use a minimal source that only has config.yaml; auth.json and .env are
|
||||
# absent. The command must succeed and must not create the missing files.
|
||||
machine.succeed("mkdir -p /tmp/src2")
|
||||
machine.succeed("install -m644 ${srcConfigYaml} /tmp/src2/config.yaml")
|
||||
machine.succeed("echo copy | hermenix migrate connections /tmp/src2 /tmp/dst2")
|
||||
machine.succeed("test -f /tmp/dst2/config.yaml")
|
||||
machine.fail( "test -f /tmp/dst2/auth.json")
|
||||
machine.fail( "test -f /tmp/dst2/.env")
|
||||
'';
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
# Guest-config VM test, NAT (unbridged) mode: guest/common.nix as it sits on
|
||||
# the host's isolated bridge, where there is no DHCP server and the guest
|
||||
# must be fully served by its static service-net configuration.
|
||||
#
|
||||
# The guest node and the mode-independent assertions (seeding, idempotency,
|
||||
# account, SSH) are shared with guest-vm-bridged via common.nix;
|
||||
# the networking assertions here only make sense in this mode.
|
||||
#
|
||||
# Build with (never run by an agent, only by the operator/CI):
|
||||
# nix build .#checks.x86_64-linux.guest-vm-unbridged --impure -L
|
||||
{ pkgs, determinate }:
|
||||
|
||||
let
|
||||
common = import ./common.nix { inherit determinate; };
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "hermenix-guest-config-unbridged";
|
||||
|
||||
# Boots once, reboots once, and runs only quick local checks — a few
|
||||
# minutes is generous. Bound it far below the framework's 1-hour default so
|
||||
# a genuine hang (boot failure, a blocked prompt, a stuck reboot) fails
|
||||
# fast instead of burning a full hour before being reported.
|
||||
globalTimeout = 600;
|
||||
|
||||
nodes.machine = common.guestNode;
|
||||
|
||||
testScript = common.commonScript + ''
|
||||
# --- NAT-mode networking -------------------------------------------
|
||||
# The isolated vlan has no DHCP server, so the guest's NIC must be
|
||||
# self-sufficient: exactly the one static service-net address, and
|
||||
# exactly one default route - via the host - because the DHCP client
|
||||
# found nobody to talk to and contributed nothing. The interface name
|
||||
# is deliberately not hard-coded (the real guest's NIC name varies too;
|
||||
# only the address layout is asserted).
|
||||
addrs = machine.succeed("ip -4 -o addr show | grep -v '^1: lo'").strip().splitlines()
|
||||
assert len(addrs) == 1, f"expected only the static service-net address, got: {addrs}"
|
||||
assert "10.77.0.2/24" in addrs[0], f"static service-net address missing: {addrs}"
|
||||
|
||||
machine.succeed("test \"$(ip -4 route show default | wc -l)\" -eq 1")
|
||||
machine.succeed("ip -4 route show default | grep -q 'default via 10.77.0.1'")
|
||||
'';
|
||||
}
|
||||
Reference in new issue
Block a user