diff --git a/flake.nix b/flake.nix index d6fde92..5ee6b1f 100644 --- a/flake.nix +++ b/flake.nix @@ -21,22 +21,33 @@ lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.default; }; - # Acceptance test written ahead of the consolidator feature; fails until - # it lands, so it stays out of `checks`. Build: nix build .#tests.x86_64-linux.peers -L - tests.${system}.peers = import ./nix/tests/peers.nix { - pkgs = nixpkgs.legacyPackages.${system}; - module = self.nixosModules.default; - }; - - checks.${system} = import ./nix/checks.nix { - pkgs = nixpkgs.legacyPackages.${system}; - package = self.packages.${system}.default; - } // { - package = self.packages.${system}.default; - vm = import ./nix/tests/vm.nix { + checks.${system} = + let pkgs = nixpkgs.legacyPackages.${system}; - module = self.nixosModules.default; + in + { + package = self.packages.${system}.default; + vm = import ./nix/tests/vm.nix { + inherit pkgs; + module = self.nixosModules.default; + }; + vm-peers = import ./nix/tests/peers.nix { + inherit pkgs; + module = self.nixosModules.default; + }; + # Evaluation-only: every case renders the module's config and unit, + # so forcing the list either lists the passed checks or throws. The + # output file is only there to give `nix flake check` a derivation. + module-peers = pkgs.runCommand "skyg-linux-tracker-module-peers-eval" { } '' + printf '%s\n' ${ + nixpkgs.lib.escapeShellArg ( + builtins.concatStringsSep "\n" (import ./nix/tests/module-peers.nix { + inherit pkgs; + module = self.nixosModules.default; + }) + ) + } > $out + ''; }; - }; }; } diff --git a/nix/tests/peers.nix b/nix/tests/peers.nix index 7b3983e..4983059 100644 --- a/nix/tests/peers.nix +++ b/nix/tests/peers.nix @@ -1,23 +1,21 @@ { pkgs, module }: -# Consolidator acceptance test, written ahead of the feature: it is expected to -# fail at the first peers assertion until the peers feature lands. Everything -# before that (boot, healthz, token minting, marker units on each node's own -# API) must already pass against the current binary. +# Consolidator acceptance test. Three machines run the real slinuxm: alpha +# consolidates beta over plain HTTP and the TLS-only gamma through a private +# CA, so one consolidator fans in two peers with different transports; beta +# also declares alpha, so both directions are covered. # -# alpha and beta each run the real slinuxm and declare the other as a peer. -# gamma is a small TLS-only third node that alpha declares as a peer with a -# private CA (ca_file), covering the TLS variant. -# -# Peer config uses the module's freeform `settings` and a LoadCredential -# override, so the module needs no peers options yet. The token files below are -# root-only 0400 stand-ins for agenix paths: they exist from boot with a -# placeholder and the test script overwrites them after minting real tokens. +# Peers are declared with the module's first-class options +# (`peers.enable`, `peers.peers..tokenFile`), which render the TOML +# `token_file` paths and one `LoadCredential=peer-` per peer. The token +# files below are root-only 0400 stand-ins for agenix paths: they exist from +# boot with a placeholder and the test overwrites them after minting real +# tokens on each peer. let + inherit (pkgs) lib; port = 1984; tokenDir = "/var/lib/peer-tokens"; - credDir = "/run/credentials/skyg-linux-tracker.service"; # Private CA plus a leaf for host `gamma`. certs = pkgs.runCommand "skyg-peers-certs" { nativeBuildInputs = [ pkgs.openssl ]; } '' @@ -33,10 +31,9 @@ let rm ca.key leaf.csr ext.cnf ca.srl ''; - # Timers are short so the test observes flips in seconds; timeouts stay - # below their intervals as the config validation will require. + # Short timers so the test observes flips in seconds; timeouts stay below + # their intervals as the config validation requires. peerTuning = { - enabled = true; tcp_interval_secs = 2; tcp_timeout_secs = 1; api_interval_secs = 3; @@ -45,38 +42,40 @@ let request_timeout_secs = 4; }; - peerEntry = name: extra: { - inherit name; - url = "http://${name}:${toString port}"; - token_file = "${credDir}/peer-${name}"; - allow_plain_http = true; - } // extra; - - # Service-side wiring shared by every node: token stand-ins and LoadCredential. + # Stand-ins for agenix files: a root-only 0400 file per peer, read by the + # module's LoadCredential line. tokenFiles = names: { systemd.tmpfiles.rules = [ "d ${tokenDir} 0700 root root -" ] ++ map (n: "f ${tokenDir}/${n} 0400 root root - pending") names; - systemd.services.skyg-linux-tracker.serviceConfig = { - LoadCredential = map (n: "peer-${n}:${tokenDir}/${n}") names; - # Credentials are copied at start; refresh them on SIGHUP so a rewritten - # token file is picked up without a restart. - RefreshOnReload = "credentials"; - }; }; + # The test network gives every node an IPv4 and an IPv6 address in + # /etc/hosts, and name resolution returns the IPv6 one first, so the nodes + # listen on both families. A peer that only listens on one of them is + # reported as unreachable. mkNode = { peers }: { ... }: { imports = [ module ]; services.skyg-linux-tracker = { enable = true; inherit port; - bind = "0.0.0.0"; + bind = "::"; openFirewall = true; - settings = pkgs.lib.optionalAttrs (peers != [ ]) { - peers = peerTuning // { peer = peers; }; + # A node without peers keeps the feature off: `peers.enable` requires + # at least one peer. + peers = lib.optionalAttrs (peers != { }) { + enable = true; + settings = peerTuning; + inherit peers; }; }; }; + + plainPeer = name: { + url = "http://${name}:${toString port}"; + tokenFile = "${tokenDir}/${name}"; + allowPlainHttp = true; + }; in pkgs.testers.runNixOSTest { name = "skyg-linux-tracker-peers"; @@ -84,13 +83,14 @@ pkgs.testers.runNixOSTest { nodes.alpha = { ... }: { imports = [ (mkNode { - peers = [ - (peerEntry "beta" { }) - ((peerEntry "gamma" { + peers = { + beta = plainPeer "beta"; + gamma = { url = "https://gamma:${toString port}"; - ca_file = "${certs}/ca.pem"; - }) // { allow_plain_http = false; }) - ]; + tokenFile = "${tokenDir}/gamma"; + caFile = "${certs}/ca.pem"; + }; + }; }) (tokenFiles [ "beta" "gamma" ]) ]; @@ -102,7 +102,7 @@ pkgs.testers.runNixOSTest { nodes.beta = { ... }: { imports = [ - (mkNode { peers = [ (peerEntry "alpha" { }) ]; }) + (mkNode { peers = { alpha = plainPeer "alpha"; }; }) (tokenFiles [ "alpha" ]) ]; systemd.services.marker-beta = { @@ -117,7 +117,7 @@ pkgs.testers.runNixOSTest { }; nodes.gamma = { ... }: { - imports = [ (mkNode { peers = [ ]; }) ]; + imports = [ (mkNode { peers = { }; }) ]; services.skyg-linux-tracker.tls = { enable = true; certFile = "${certs}/cert.pem"; @@ -130,7 +130,9 @@ pkgs.testers.runNixOSTest { }; testScript = '' + import base64 import json + import shlex import time CLI = "sudo -u skyg-linux-tracker slinuxm" @@ -138,6 +140,7 @@ pkgs.testers.runNixOSTest { INIT = '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"vm","version":"0"}}}' TOKDIR = "${tokenDir}" PORT = ${toString port} + CONFIG = "/etc/skyg-linux-tracker.toml" def bearer(tok): return f"-H 'Authorization: Bearer {tok}'" @@ -158,19 +161,35 @@ pkgs.testers.runNixOSTest { return json.loads(body) def poll(fn, what, timeout=60): + """Wait until fn() returns a truthy (ok, observed) pair. + + The observed value of the last attempt, usually the peer's status + JSON, is part of the failure so a timeout says what was wrong. + """ deadline = time.monotonic() + timeout last = None while time.monotonic() < deadline: try: - if fn(): - return + ok, last = fn() except Exception as e: # noqa: BLE001 - last = e + last = f"{type(e).__name__}: {e}" + else: + if ok: + return time.sleep(1) - raise AssertionError(f"timed out waiting for: {what} (last error: {last})") + raise AssertionError(f"timed out waiting for: {what} (last observed: {last})") def wait_status(m, tok, peer, check, what, timeout=60): - poll(lambda: check(peer_status(m, tok, peer)), f"{peer}: {what}", timeout) + def probe(): + status = peer_status(m, tok, peer) + return check(status), json.dumps(status, sort_keys=True) + poll(probe, f"{peer}: {what}", timeout) + + def wait_gone(m, tok, peer, timeout=30): + def probe(): + status, body = local(m, f"/api/v1/peers/{peer}", tok) + return status == 404, f"{status}: {body}" + poll(probe, f"{peer}: gone", timeout) def mcp_call(m, tok, method, params): """One-shot MCP session: initialize, initialized, then the call.""" @@ -187,10 +206,20 @@ pkgs.testers.runNixOSTest { def install_token(m, peer, tok): # Stand-in for an agenix secret: root-only 0400, then reload (SIGHUP) - # so the credential is refreshed without a restart. + # so the credential copy is refreshed without a restart. m.succeed(f"printf '%s\\n' '{tok}' > {TOKDIR}/{peer}.new && chmod 0400 {TOKDIR}/{peer}.new && mv {TOKDIR}/{peer}.new {TOKDIR}/{peer}") m.succeed("systemctl reload skyg-linux-tracker.service") + def main_pid(m): + return m.succeed("systemctl show -p MainPID --value skyg-linux-tracker.service").strip() + + def drop_peer(config_text, name): + """Remove one [[peers.peer]] block from a rendered config.""" + head, *blocks = config_text.split("[[peers.peer]]") + kept = [b for b in blocks if f'name = "{name}"' not in b] + assert len(kept) == len(blocks) - 1, f"no peer block for {name}" + return "[[peers.peer]]".join([head] + kept) + start_all() # ---- Phase 1: nothing here depends on the peers feature ---------------- @@ -206,7 +235,7 @@ pkgs.testers.runNixOSTest { assert http(m, f"{url}/healthz")[0] == 200 assert http(alpha, f"http://beta:{PORT}/healthz")[0] == 200, "alpha cannot reach beta" assert http(beta, f"http://alpha:{PORT}/healthz")[0] == 200, "beta cannot reach alpha" - assert http(gamma, f"https://127.0.0.1:{PORT}/healthz", extra="--cacert ${certs}/ca.pem --resolve gamma:1984:127.0.0.1 -k")[0] == 200 + assert http(gamma, f"https://127.0.0.1:{PORT}/healthz", extra="--cacert ${certs}/ca.pem --resolve gamma:${toString port}:127.0.0.1 -k")[0] == 200 with subtest("tokens: minted per consolidator by the service user"): # Each node mints the token its consolidator will use against it, plus @@ -233,7 +262,7 @@ pkgs.testers.runNixOSTest { for m, p in ((alpha, "beta"), (alpha, "gamma"), (beta, "alpha")): assert m.succeed(f"stat -c '%a %U' {TOKDIR}/{p}").strip() == "400 root" - # ---- Phase 2: peers (expected to fail here until the feature lands) ---- + # ---- Phase 2: peers ---------------------------------------------------- with subtest("peers: declared but token is still a placeholder -> auth rejected, then recovery"): # Before the real token is installed the peer API check must report @@ -241,8 +270,12 @@ pkgs.testers.runNixOSTest { wait_status(alpha, A, "beta", lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected", "server up, api up, auth rejected with the placeholder token") + wait_status(alpha, A, "gamma", + lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected", + "TLS peer up with the placeholder token") with subtest("peers: install real tokens, status recovers without a restart"): + pid = main_pid(alpha) install_token(alpha, "beta", tok_beta_for_alpha) install_token(alpha, "gamma", tok_gamma_for_alpha) install_token(beta, "alpha", tok_alpha_for_beta) @@ -250,11 +283,23 @@ pkgs.testers.runNixOSTest { wait_status(m, tok, peer, lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "ok", "server up, api up, auth ok") + assert main_pid(alpha) == pid, "rotation must not restart the service" status, body = local(alpha, "/api/v1/peers", A) assert status == 200 and "beta" in body and "gamma" in body, body status, body = local(beta, "/api/v1/peers", B) assert status == 200 and "alpha" in body, body + with subtest("fan-in: one consolidator reports both peers with their own version"): + status, body = local(alpha, "/api/v1/peers", A) + assert status == 200, body + peers = {p["name"]: p for p in json.loads(body)["peers"]} + assert sorted(peers) == ["beta", "gamma"], sorted(peers) + for name in ("beta", "gamma"): + assert peers[name]["enabled"] and peers[name]["server"]["up"] and peers[name]["api"]["up"], peers[name] + assert peers[name]["api"]["auth"] == "ok", peers[name] + assert peers[name]["api"]["peer_version"], peers[name] + assert peers["gamma"]["url"].startswith("https://"), peers["gamma"] + with subtest("proxy: each side returns the other's data, not its own"): for m, tok, peer, mine, theirs in ((alpha, A, "beta", "marker-alpha", "marker-beta"), (beta, B, "alpha", "marker-beta", "marker-alpha")): @@ -267,6 +312,14 @@ pkgs.testers.runNixOSTest { # beta's deliberately failing unit is only visible through the proxy. assert "broken-beta.service" in local(alpha, "/api/v1/peers/beta/systemd/units", A)[1] + with subtest("proxy: the peer's answer is the peer's own data"): + _, direct = local(beta, "/api/v1/systemd/units", B) + status, proxied = local(alpha, "/api/v1/peers/beta/systemd/units", A) + assert status == 200, proxied + names = lambda body: sorted(u["name"] for u in json.loads(body)) + assert names(proxied) == names(direct), (names(proxied), names(direct)) + assert names(proxied) != names(local(alpha, "/api/v1/systemd/units", A)[1]) + with subtest("mcp: list_peers and list_systemd_units with peer match REST"): out = mcp_tool(alpha, A, "list_peers") assert "beta" in out and "gamma" in out, out @@ -288,6 +341,7 @@ pkgs.testers.runNixOSTest { assert http(alpha, f"http://127.0.0.1:{PORT}/api/v1/peers/nope", A)[0] == 404 with subtest("token rejected: revoke on beta, then restore with a new token file"): + pid = main_pid(alpha) beta.succeed(f"{CLI} token revoke consolidator-alpha") wait_status(alpha, A, "beta", lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected", @@ -300,6 +354,25 @@ pkgs.testers.runNixOSTest { install_token(alpha, "beta", new_tok) wait_status(alpha, A, "beta", lambda s: s["api"]["auth"] == "ok", "auth ok after new token") assert local(alpha, "/api/v1/peers/beta/systemd/units", A)[0] == 200 + assert main_pid(alpha) == pid, "revoke and rotation must not restart the service" + + with subtest("reload: removing a peer from the config drops it on SIGHUP"): + # The module renders the config into the store, so replace the /etc + # symlink with a writable copy first: SIGHUP re-reads this path. + rendered = alpha.succeed(f"readlink -f {CONFIG}").strip() + text = alpha.succeed(f"cat {rendered}") + edited = drop_peer(text, "gamma") + encoded = base64.b64encode(edited.encode()).decode() + alpha.succeed(f"printf '%s' {shlex.quote(encoded)} | base64 -d > {CONFIG}.new && mv {CONFIG}.new {CONFIG}") + assert "gamma" not in alpha.succeed(f"cat {CONFIG}") + pid = main_pid(alpha) + alpha.succeed("systemctl reload skyg-linux-tracker.service") + wait_gone(alpha, A, "gamma") + status, body = local(alpha, "/api/v1/peers", A) + assert status == 200 and "gamma" not in body and "beta" in body, body + assert main_pid(alpha) == pid, "removing a peer must not restart the service" + assert alpha.succeed(f"stat -c '%a' {TOKDIR}/gamma").strip() == "400", "the removed peer's file is untouched" + assert peer_status(alpha, A, "beta")["api"]["auth"] == "ok", "beta must survive the reload" with subtest("outage: stopping beta's service flips server and api, alpha stays healthy"): beta.systemctl("stop skyg-linux-tracker.service") @@ -308,7 +381,6 @@ pkgs.testers.runNixOSTest { "server and api down", timeout=30) assert http(alpha, f"http://127.0.0.1:{PORT}/healthz")[0] == 200 assert local(alpha, "/api/v1/systemd/units", A)[0] == 200 - assert peer_status(alpha, A, "gamma")["api"]["up"], "gamma must be unaffected" beta.systemctl("start skyg-linux-tracker.service") beta.wait_for_open_port(PORT) wait_status(alpha, A, "beta", diff --git a/plan/phase-5-consolidator/contracts.md b/plan/phase-5-consolidator/contracts.md index 23b2d6c..8496796 100644 --- a/plan/phase-5-consolidator/contracts.md +++ b/plan/phase-5-consolidator/contracts.md @@ -313,14 +313,16 @@ services.skyg-linux-tracker.peers = { Rendering: one `[[peers.peer]]` per enabled peer with `token_file = "/run/credentials/skyg-linux-tracker.service/peer-"`, `LoadCredential=peer-:`, `RefreshOnReload=credentials`, `ExecReload` = SIGHUP, assertions as in F1. -## 10. Test config contract (assumed by `nix/tests/peers.nix`, WS-T) +## 10. Test config contract (`nix/tests/peers.nix`) -The VM test is written against the pre-F1 configuration style, so the TOML contract above must stay compatible with it: +The VM test declares its peers with the module's first-class options (F2), so the TOML contract above must stay compatible with what they render: -- Hosts: `alpha` (peers `beta` over plain HTTP, `gamma` over HTTPS with `ca_file`), `beta` (peer `alpha`), `gamma` (TLS-only, no peers). Port 1984, bind `0.0.0.0`. -- Config comes through the module's freeform `settings.peers = { enabled = true; tcp_interval_secs = 2; tcp_timeout_secs = 1; api_interval_secs = 3; api_timeout_secs = 2; fail_threshold = 2; request_timeout_secs = 4; peer = [ ... ]; }`, i.e. `[peers]` scalars plus `[[peers.peer]]` with keys `name`, `url`, `token_file`, `allow_plain_http`, optional `ca_file`. -- `token_file = /run/credentials/skyg-linux-tracker.service/peer-` (via `LoadCredential=peer-:/var/lib/peer-tokens/`); the file holds the placeholder `pending` until the test installs a real token, so the placeholder must pass A3 loading (non-empty, under `$CREDENTIALS_DIRECTORY` so the permission check is skipped) and be reported as `auth = "rejected"`, not as a config error. +- Hosts: `alpha` (peers `beta` over plain HTTP, `gamma` over HTTPS with `ca_file`), `beta` (peer `alpha`), `gamma` (TLS-only, no peers). Port 1984, bind `::` (dual stack, see the address note below). +- Config comes through the module: `peers.enable = true`, `peers.settings = { tcp_interval_secs = 2; tcp_timeout_secs = 1; api_interval_secs = 3; api_timeout_secs = 2; fail_threshold = 2; request_timeout_secs = 4; }` and `peers.peers. = { url; tokenFile; caFile ? null; allowPlainHttp }`. The module renders `[peers]` plus one `[[peers.peer]]` per enabled peer with keys `name`, `url`, `token_file`, `allow_plain_http`, optional `ca_file`. +- `token_file = /run/credentials/skyg-linux-tracker.service/peer-` (the module adds `LoadCredential=peer-:/var/lib/peer-tokens/` and `RefreshOnReload=credentials`); the file holds the placeholder `pending` until the test installs a real token, so the placeholder must pass A3 loading (non-empty, under `$CREDENTIALS_DIRECTORY` so the permission check is skipped) and be reported as `auth = "rejected"`, not as a config error. +- Address note: a test node's name resolves to both its IPv4 and its IPv6 address, and resolution returns the IPv6 one first, so each node binds `::` and the peer URLs keep the hostnames. A peer reachable only on IPv4 is reported as `unreachable` (the transport connects to the first resolved address); see the ws-f Log. - Token rotation = `mv` a new file into place and `systemctl reload skyg-linux-tracker.service` (SIGHUP with `RefreshOnReload=credentials`); status must recover without a restart. SIGHUP must therefore re-read token files and keep the registry (B4). - Timeouts (1 s / 2 s) are below intervals (2 s / 3 s), as validation requires; `request_timeout_secs = 4` and the test's `elapsed < 15` bound the proxy call when a peer VM is down. - Assertions the contract must satisfy: `GET /api/v1/peers/

` JSON has `server.up`, `api.up`, `api.auth` in {`ok`,`rejected`}; `GET /api/v1/peers` body contains each peer name; proxied `GET /api/v1/peers/

/systemd/units[/.service]` returns the peer's data with response header `x-slinuxm-peer:

`; `POST`/`DELETE` on a proxy path -> 405; `x-slinuxm-hop: 1` -> 4xx; unknown peer -> 404; peer rejected/down -> 503 with the peer name in the body and the caller's own token still valid; MCP `list_peers` output contains the peer names; MCP `list_systemd_units` with `{"peer": "

"}` returns the peer's units. - The peer's `/api/v1/ping` must return 401 for a bad token (placeholder case) and 200 for a good one; the test relies on `api.up = true` with `auth = "rejected"` for the 401 case (D4). +- Removal: the test replaces the rendered `/etc/skyg-linux-tracker.toml` symlink with an edited copy (one `[[peers.peer]]` block dropped), sends SIGHUP, and expects the peer to disappear from `GET /api/v1/peers` and to 404 on `GET /api/v1/peers/

`, with the service's `MainPID` unchanged. Rotation and revocation likewise leave `MainPID` alone. diff --git a/plan/phase-5-consolidator/ws-f-nixos-docs.md b/plan/phase-5-consolidator/ws-f-nixos-docs.md index b185dbe..51d806c 100644 --- a/plan/phase-5-consolidator/ws-f-nixos-docs.md +++ b/plan/phase-5-consolidator/ws-f-nixos-docs.md @@ -64,3 +64,15 @@ Shippable, documented, verified. - Eval cases: 40, the new ones proving a peer `url` change and an `api_interval_secs` change move only `reloadTriggers` while `port`, `auth`, `deny_cidrs` and `max_concurrent_total` changes move `restartTriggers`, plus the new assertions and both enabled-conflict directions. - Recorded for F2: `nix/tests/module-peers.nix` is now in the Owned files row, and F2 must add it as `checks.module-peers` so `nix flake check` and `dev-test ci` run it. - Still open: the two VM tests, see the "Not run" note above. +- 2026-10-20: rebased `ws/F` onto `main` (9e4fb38, D merged). D's MCP peer tools (`list_peers`, the `peer` argument) were not on the branch, so the test's MCP subtest could not pass; the rebase is mechanical, F1's commits are unchanged. +- 2026-10-20: **F2 done** (commit `F: F2 promote the peers VM test and the module eval test into checks`). + - **Root cause of the known failure** (the first peers subtest timing out while `/api/v1/peers/beta` answered 200): not the JSON shape. `contracts.md` section 3 and the real binary agree (`server.up`, `api.up`, and `api.auth` = `"rejected"` with a 401). It was address selection: the test network writes **both** an IPv4 and an IPv6 address for every node into `/etc/hosts` (`nixos/lib/testing/network.nix`), name resolution returns the IPv6 one first, and the nodes bound `0.0.0.0`. `ReqwestTransport::resolve_target` takes `addrs.first()` (the IPv6 address), so both probes failed with `unreachable` (`server.up` false, `api.up` false, `api.auth` `"unknown"`) while curl reached the IPv4 address. Reproduced locally with `url = "http://localhost:1985"` against a peer bound to `127.0.0.1`: `/api/v1/peers/beta` reported `server.up: false, api.up: false, api.last_error: "unreachable"`. With the peer bound to `::` the same setup reports `up: true, up: true, auth: "rejected"`. + - Test fix (the binary is unchanged): every node binds `::` (dual stack; Linux accepts IPv4-mapped connections, verified against the release binary on both `127.0.0.1` and `::1`) and the peer URLs keep their hostnames, so the TLS peer still verifies its `DNS:gamma` SAN. Reported to WS-B in the Handoff: a peer that listens on IPv4 only is reported as `unreachable` when its name also resolves to IPv6, because the transport connects to the first resolved address only. A candidate fix (WS-B, `src/peers/client.rs`) is to try the resolved addresses in order, or to prefer the family the peer actually answers on. + - `poll()` now raises with the **last observed value**: `wait_status` hands it the status JSON of every attempt, so a timeout prints e.g. `last observed: {"api": {"auth": "unknown", ...}}` instead of `(last error: None)`. New `wait_gone` for the removal case. + - Switched to the first-class options: `peers.enable`, `peers.settings` (intervals/timeouts only), `peers.peers. = { url; tokenFile; caFile; allowPlainHttp }`. The test's hand-written `LoadCredential` and `RefreshOnReload` override are gone; the module renders `token_file = /run/credentials/skyg-linux-tracker.service/peer-`, one `LoadCredential=peer-:/var/lib/peer-tokens/` per enabled peer and `RefreshOnReload=credentials`. Token stand-ins stay root-only 0400 files under `/var/lib/peer-tokens` (agenix stand-ins). gamma keeps `peers` off (a peer-less node), since `peers.enable` requires a peer. + - New assertions: fan-in (alpha's `GET /api/v1/peers` lists exactly beta and gamma, both with `server.up`, `api.up`, `auth == "ok"` and a `peer_version`; gamma's display URL is `https://`); proxied unit names equal a **direct** call to the peer's own API (and differ from alpha's own); placeholder-token rejection also covers the TLS peer; token install, revoke and rotation leave `MainPID` unchanged; **removing a peer** (the rendered `/etc/skyg-linux-tracker.toml` symlink is replaced by an edited copy with gamma's `[[peers.peer]]` block dropped, then SIGHUP) makes gamma 404 on `GET /api/v1/peers/gamma`, drop out of `GET /api/v1/peers`, leaves beta healthy, leaves gamma's token file at 0400 and leaves `MainPID` unchanged. The outage subtests run last, because gamma is no longer configured after the removal. + - `flake.nix`: `tests.${system}.peers` is gone; `checks.${system}` now has `package`, `vm`, `vm-peers` (the same VM test) and `module-peers`. `module-peers` wraps the eval test in `pkgs.runCommand` whose interpolation forces the returned list, so a failing case throws during **evaluation** of the derivation and `nix flake check` fails without a VM (a bare list is not a derivation and `nix flake check` rejects it). + - `contracts.md` section 10 updated (first-class options, dual-stack bind and the address note, the removal scenario). That file is outside the Owned files row but the F2 task text assigns section 10 to F2; no other section changed. + - Verified here (no nix **builds** in this worktree, per AGENTS.md): `nix eval` of `nix/tests/module-peers.nix` after the rebase, 40/40 cases; `nix eval` of `checks.x86_64-linux.{module-peers,vm-peers}.drvPath` (both instantiate, so the test nodes, the module assertions and the first-class rendering are all evaluated); `nix-instantiate --parse` on `flake.nix`, `nix/tests/peers.nix`; `python3 -m py_compile` on the assembled `testScript`. + - **For the orchestrator to run** (the nix gate): `nix flake check -L` (now includes `checks.vm-peers` and `checks.module-peers`), or `nix build .#checks.x86_64-linux.vm-peers -L` on its own. + - **Seam, blocking for `dev-test vm peers` (WS-T owns `dev/dev-test.nu`)**: `vm-steps` builds `.#($attr).($system).($name)` with `"peers" => "tests"`, so after the move it must build `.#checks.($system).vm-peers`. Suggested change: keep `let attr = "checks"` and map the build name (`"peers" => "vm-peers"`, `_ => $name`). `dev-test ci` is unaffected (it runs `vm`), as is `nix flake check`.