{ pkgs, module }: # Consolidator acceptance test. Three machines run the real slinuxm: alpha # consolidates beta over plain HTTP and the TLS-only gamma through a private # CA, so one consolidator fans in two peers with different transports; beta # also declares alpha, so both directions are covered. # # Peers are declared with the module's first-class options # (`peers.enable`, `peers.peers..tokenFile`), which render the TOML # `token_file` paths and one `LoadCredential=peer-` per peer. The token # files below are root-only 0400 stand-ins for agenix paths: they exist from # boot with a placeholder and the test overwrites them after minting real # tokens on each peer. let inherit (pkgs) lib; port = 1984; tokenDir = "/var/lib/peer-tokens"; # Private CA plus a leaf for host `gamma`. certs = pkgs.runCommand "skyg-peers-certs" { nativeBuildInputs = [ pkgs.openssl ]; } '' mkdir $out && cd $out openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 \ -keyout ca.key -out ca.pem -subj '/CN=skyg-test-ca' \ -addext 'basicConstraints=critical,CA:TRUE' -addext 'keyUsage=critical,keyCertSign,cRLSign' openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ -keyout key.pem -out leaf.csr -subj '/CN=gamma' printf 'subjectAltName=DNS:gamma\nbasicConstraints=CA:FALSE\n' > ext.cnf openssl x509 -req -in leaf.csr -CA ca.pem -CAkey ca.key -CAcreateserial -days 3650 \ -extfile ext.cnf -out cert.pem rm ca.key leaf.csr ext.cnf ca.srl ''; # Short timers so the test observes flips in seconds; timeouts stay below # their intervals as the config validation requires. peerTuning = { tcp_interval_secs = 2; tcp_timeout_secs = 1; api_interval_secs = 3; api_timeout_secs = 2; fail_threshold = 2; request_timeout_secs = 4; }; # Stand-ins for agenix files: a root-only 0400 file per peer, read by the # module's LoadCredential line. tokenFiles = names: { systemd.tmpfiles.rules = [ "d ${tokenDir} 0700 root root -" ] ++ map (n: "f ${tokenDir}/${n} 0400 root root - pending") names; }; # The test network gives every node an IPv4 and an IPv6 address in # /etc/hosts, and name resolution returns the IPv6 one first, so the nodes # listen on both families. A peer that only listens on one of them is # reported as unreachable. mkNode = { peers }: { ... }: { imports = [ module ]; services.skyg-linux-tracker = { enable = true; inherit port; bind = "::"; openFirewall = true; # A node without peers keeps the feature off: `peers.enable` requires # at least one peer. peers = lib.optionalAttrs (peers != { }) { enable = true; settings = peerTuning; inherit peers; }; }; }; plainPeer = name: { url = "http://${name}:${toString port}"; tokenFile = "${tokenDir}/${name}"; allowPlainHttp = true; }; in pkgs.testers.runNixOSTest { name = "skyg-linux-tracker-peers"; nodes.alpha = { ... }: { imports = [ (mkNode { peers = { beta = plainPeer "beta"; gamma = { url = "https://gamma:${toString port}"; tokenFile = "${tokenDir}/gamma"; caFile = "${certs}/ca.pem"; }; }; }) (tokenFiles [ "beta" "gamma" ]) ]; systemd.services.marker-alpha = { wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; }; }; }; nodes.beta = { ... }: { imports = [ (mkNode { peers = { alpha = plainPeer "alpha"; }; }) (tokenFiles [ "alpha" ]) ]; systemd.services.marker-beta = { wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; }; }; # Deliberately failing unit, so beta's data differs from alpha's. systemd.services.broken-beta = { wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; ExecStart = "${pkgs.coreutils}/bin/false"; }; }; }; nodes.gamma = { ... }: { imports = [ (mkNode { peers = { }; }) ]; services.skyg-linux-tracker.tls = { enable = true; certFile = "${certs}/cert.pem"; keyFile = "${certs}/key.pem"; }; systemd.services.marker-gamma = { wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; }; }; }; testScript = '' import base64 import json import shlex import time CLI = "sudo -u skyg-linux-tracker slinuxm" JSON = "-H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream'" INIT = '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"vm","version":"0"}}}' TOKDIR = "${tokenDir}" PORT = ${toString port} CONFIG = "/etc/skyg-linux-tracker.toml" def bearer(tok): return f"-H 'Authorization: Bearer {tok}'" def http(m, url, tok=None, extra=""): """Returns (status_code, body).""" auth = bearer(tok) if tok else "" out = m.succeed(f"curl -s -w '\\n%{{http_code}}' {auth} {extra} {url}") body, _, status = out.rpartition("\n") return int(status), body def local(m, path, tok): return http(m, f"http://127.0.0.1:{PORT}{path}", tok) def peer_status(m, tok, peer): status, body = local(m, f"/api/v1/peers/{peer}", tok) assert status == 200, f"/api/v1/peers/{peer} -> {status}: {body}" return json.loads(body) def poll(fn, what, timeout=60): """Wait until fn() returns a truthy (ok, observed) pair. The observed value of the last attempt, usually the peer's status JSON, is part of the failure so a timeout says what was wrong. """ deadline = time.monotonic() + timeout last = None while time.monotonic() < deadline: try: ok, last = fn() except Exception as e: # noqa: BLE001 last = f"{type(e).__name__}: {e}" else: if ok: return time.sleep(1) raise AssertionError(f"timed out waiting for: {what} (last observed: {last})") def wait_status(m, tok, peer, check, what, timeout=60): def probe(): status = peer_status(m, tok, peer) return check(status), json.dumps(status, sort_keys=True) poll(probe, f"{peer}: {what}", timeout) def wait_gone(m, tok, peer, timeout=30): def probe(): status, body = local(m, f"/api/v1/peers/{peer}", tok) return status == 404, f"{status}: {body}" poll(probe, f"{peer}: gone", timeout) def mcp_call(m, tok, method, params): """One-shot MCP session: initialize, initialized, then the call.""" url = f"http://127.0.0.1:{PORT}/mcp" m.succeed(f"curl -sf -D /tmp/h {bearer(tok)} {JSON} -d '{INIT}' {url} >/dev/null") sid = m.succeed("awk -F': ' 'tolower($1)==\"mcp-session-id\"{print $2}' /tmp/h | tr -d '\\r'").strip() sess = f"-H 'Mcp-Session-Id: {sid}'" if sid else "" m.succeed(f"curl -sf {bearer(tok)} {sess} {JSON} -d '{{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}}' {url} >/dev/null") body = json.dumps({"jsonrpc": "2.0", "id": 2, "method": method, "params": params}) return m.succeed(f"curl -sf {bearer(tok)} {sess} {JSON} -d '{body}' {url}") def mcp_tool(m, tok, name, args=None): return mcp_call(m, tok, "tools/call", {"name": name, "arguments": args or {}}) def install_token(m, peer, tok): # Stand-in for an agenix secret: root-only 0400, then reload (SIGHUP) # so the credential copy is refreshed without a restart. m.succeed(f"printf '%s\\n' '{tok}' > {TOKDIR}/{peer}.new && chmod 0400 {TOKDIR}/{peer}.new && mv {TOKDIR}/{peer}.new {TOKDIR}/{peer}") m.succeed("systemctl reload skyg-linux-tracker.service") def main_pid(m): return m.succeed("systemctl show -p MainPID --value skyg-linux-tracker.service").strip() def drop_peer(config_text, name): """Remove one [[peers.peer]] block from a rendered config.""" head, *blocks = config_text.split("[[peers.peer]]") kept = [b for b in blocks if f'name = "{name}"' not in b] assert len(kept) == len(blocks) - 1, f"no peer block for {name}" return "[[peers.peer]]".join([head] + kept) start_all() # ---- Phase 1: nothing here depends on the peers feature ---------------- with subtest("boot: all services up, healthz public"): for m in (alpha, beta, gamma): m.wait_for_unit("skyg-linux-tracker.service") m.wait_for_open_port(PORT) alpha.wait_for_unit("marker-alpha.service") beta.wait_for_unit("marker-beta.service") gamma.wait_for_unit("marker-gamma.service") for m, url in ((alpha, f"http://127.0.0.1:{PORT}"), (beta, f"http://127.0.0.1:{PORT}")): assert http(m, f"{url}/healthz")[0] == 200 assert http(alpha, f"http://beta:{PORT}/healthz")[0] == 200, "alpha cannot reach beta" assert http(beta, f"http://alpha:{PORT}/healthz")[0] == 200, "beta cannot reach alpha" assert http(gamma, f"https://127.0.0.1:{PORT}/healthz", extra="--cacert ${certs}/ca.pem --resolve gamma:${toString port}:127.0.0.1 -k")[0] == 200 with subtest("tokens: minted per consolidator by the service user"): # Each node mints the token its consolidator will use against it, plus # a separate token for local assertions. tok_alpha_for_beta = alpha.succeed(f"{CLI} token create consolidator-beta").strip() tok_beta_for_alpha = beta.succeed(f"{CLI} token create consolidator-alpha").strip() tok_gamma_for_alpha = gamma.succeed(f"{CLI} token create consolidator-alpha").strip() A = alpha.succeed(f"{CLI} token create test-alpha").strip() B = beta.succeed(f"{CLI} token create test-beta").strip() G = gamma.succeed(f"{CLI} token create test-gamma").strip() for t in (tok_alpha_for_beta, tok_beta_for_alpha, tok_gamma_for_alpha, A, B, G): assert t, "empty token" with subtest("data: marker units are visible on each node's own API only"): _, a_units = local(alpha, "/api/v1/systemd/units", A) _, b_units = local(beta, "/api/v1/systemd/units", B) assert "marker-alpha.service" in a_units and "marker-beta.service" not in a_units assert "marker-beta.service" in b_units and "marker-alpha.service" not in b_units assert "broken-beta.service" in b_units and "broken-beta.service" not in a_units status, one = local(beta, "/api/v1/systemd/units/marker-beta.service", B) assert status == 200 and "marker-beta.service" in one with subtest("token files: placeholders exist root-only 0400 before any token is installed"): for m, p in ((alpha, "beta"), (alpha, "gamma"), (beta, "alpha")): assert m.succeed(f"stat -c '%a %U' {TOKDIR}/{p}").strip() == "400 root" # ---- Phase 2: peers ---------------------------------------------------- with subtest("peers: declared but token is still a placeholder -> auth rejected, then recovery"): # Before the real token is installed the peer API check must report # server up, api up (it answered 401), auth rejected. wait_status(alpha, A, "beta", lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected", "server up, api up, auth rejected with the placeholder token") wait_status(alpha, A, "gamma", lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected", "TLS peer up with the placeholder token") with subtest("peers: install real tokens, status recovers without a restart"): pid = main_pid(alpha) install_token(alpha, "beta", tok_beta_for_alpha) install_token(alpha, "gamma", tok_gamma_for_alpha) install_token(beta, "alpha", tok_alpha_for_beta) for m, tok, peer in ((alpha, A, "beta"), (alpha, A, "gamma"), (beta, B, "alpha")): wait_status(m, tok, peer, lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "ok", "server up, api up, auth ok") assert main_pid(alpha) == pid, "rotation must not restart the service" status, body = local(alpha, "/api/v1/peers", A) assert status == 200 and "beta" in body and "gamma" in body, body status, body = local(beta, "/api/v1/peers", B) assert status == 200 and "alpha" in body, body with subtest("fan-in: one consolidator reports both peers with their own version"): status, body = local(alpha, "/api/v1/peers", A) assert status == 200, body peers = {p["name"]: p for p in json.loads(body)["peers"]} assert sorted(peers) == ["beta", "gamma"], sorted(peers) for name in ("beta", "gamma"): assert peers[name]["enabled"] and peers[name]["server"]["up"] and peers[name]["api"]["up"], peers[name] assert peers[name]["api"]["auth"] == "ok", peers[name] assert peers[name]["api"]["peer_version"], peers[name] assert peers["gamma"]["url"].startswith("https://"), peers["gamma"] with subtest("proxy: each side returns the other's data, not its own"): for m, tok, peer, mine, theirs in ((alpha, A, "beta", "marker-alpha", "marker-beta"), (beta, B, "alpha", "marker-beta", "marker-alpha")): status, body = local(m, f"/api/v1/peers/{peer}/systemd/units/{theirs}.service", tok) assert status == 200 and f"{theirs}.service" in body, (status, body) status, body = local(m, f"/api/v1/peers/{peer}/systemd/units", tok) assert status == 200 and f"{theirs}.service" in body and f"{mine}.service" not in body, body _, hdrs = m.execute(f"curl -s -D - -o /dev/null {bearer(tok)} http://127.0.0.1:{PORT}/api/v1/peers/{peer}/systemd/units") assert f"x-slinuxm-peer: {peer}" in hdrs.lower(), hdrs # beta's deliberately failing unit is only visible through the proxy. assert "broken-beta.service" in local(alpha, "/api/v1/peers/beta/systemd/units", A)[1] with subtest("proxy: the peer's answer is the peer's own data"): _, direct = local(beta, "/api/v1/systemd/units", B) status, proxied = local(alpha, "/api/v1/peers/beta/systemd/units", A) assert status == 200, proxied names = lambda body: sorted(u["name"] for u in json.loads(body)) assert names(proxied) == names(direct), (names(proxied), names(direct)) assert names(proxied) != names(local(alpha, "/api/v1/systemd/units", A)[1]) with subtest("mcp: list_peers and list_systemd_units with peer match REST"): out = mcp_tool(alpha, A, "list_peers") assert "beta" in out and "gamma" in out, out out = mcp_tool(alpha, A, "list_systemd_units", {"peer": "beta"}) assert "marker-beta.service" in out and "marker-alpha.service" not in out, out out = mcp_tool(beta, B, "list_systemd_units", {"peer": "alpha"}) assert "marker-alpha.service" in out and "marker-beta.service" not in out, out with subtest("tls: gamma is reached over HTTPS through ca_file"): status, body = local(alpha, "/api/v1/peers/gamma/systemd/units/marker-gamma.service", A) assert status == 200 and "marker-gamma.service" in body, (status, body) with subtest("read-only and no chaining"): base = f"http://127.0.0.1:{PORT}/api/v1/peers/beta/systemd/units" assert http(alpha, base, A, "-X POST")[0] == 405 assert http(alpha, base, A, "-X DELETE")[0] == 405 status, _ = http(alpha, base, A, "-H 'x-slinuxm-hop: 1'") assert 400 <= status < 500, status assert http(alpha, f"http://127.0.0.1:{PORT}/api/v1/peers/nope", A)[0] == 404 with subtest("token rejected: revoke on beta, then restore with a new token file"): pid = main_pid(alpha) beta.succeed(f"{CLI} token revoke consolidator-alpha") wait_status(alpha, A, "beta", lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected", "auth rejected after revoke") status, body = local(alpha, "/api/v1/peers/beta/systemd/units", A) assert status == 503 and "beta" in body, (status, body) # alpha's own credentials still work: the caller never sees a 401. assert local(alpha, "/api/v1/systemd/units", A)[0] == 200 new_tok = beta.succeed(f"{CLI} token create consolidator-alpha-2").strip() install_token(alpha, "beta", new_tok) wait_status(alpha, A, "beta", lambda s: s["api"]["auth"] == "ok", "auth ok after new token") assert local(alpha, "/api/v1/peers/beta/systemd/units", A)[0] == 200 assert main_pid(alpha) == pid, "revoke and rotation must not restart the service" with subtest("reload: removing a peer from the config drops it on SIGHUP"): # The module renders the config into the store, so replace the /etc # symlink with a writable copy first: SIGHUP re-reads this path. rendered = alpha.succeed(f"readlink -f {CONFIG}").strip() text = alpha.succeed(f"cat {rendered}") edited = drop_peer(text, "gamma") encoded = base64.b64encode(edited.encode()).decode() alpha.succeed(f"printf '%s' {shlex.quote(encoded)} | base64 -d > {CONFIG}.new && mv {CONFIG}.new {CONFIG}") assert "gamma" not in alpha.succeed(f"cat {CONFIG}") pid = main_pid(alpha) alpha.succeed("systemctl reload skyg-linux-tracker.service") wait_gone(alpha, A, "gamma") status, body = local(alpha, "/api/v1/peers", A) assert status == 200 and "gamma" not in body and "beta" in body, body assert main_pid(alpha) == pid, "removing a peer must not restart the service" assert alpha.succeed(f"stat -c '%a' {TOKDIR}/gamma").strip() == "400", "the removed peer's file is untouched" assert peer_status(alpha, A, "beta")["api"]["auth"] == "ok", "beta must survive the reload" with subtest("outage: stopping beta's service flips server and api, alpha stays healthy"): beta.systemctl("stop skyg-linux-tracker.service") wait_status(alpha, A, "beta", lambda s: not s["server"]["up"] and not s["api"]["up"], "server and api down", timeout=30) assert http(alpha, f"http://127.0.0.1:{PORT}/healthz")[0] == 200 assert local(alpha, "/api/v1/systemd/units", A)[0] == 200 beta.systemctl("start skyg-linux-tracker.service") beta.wait_for_open_port(PORT) wait_status(alpha, A, "beta", lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "ok", "recovery after restart") with subtest("outage: beta VM down, alpha requests are bounded by timeouts"): beta.shutdown() t0 = time.monotonic() status, body = http(alpha, f"http://127.0.0.1:{PORT}/api/v1/peers/beta/systemd/units", A, "--max-time 30") elapsed = time.monotonic() - t0 assert status == 503 and "beta" in body, (status, body) assert elapsed < 15, f"proxy call took {elapsed:.1f}s, expected it bounded by request_timeout_secs" wait_status(alpha, A, "beta", lambda s: not s["server"]["up"], "server down with the VM off", timeout=30) assert local(alpha, "/api/v1/systemd/units", A)[0] == 200 ''; }