F: F2 promote the peers VM test and the module eval test into checks
The test nodes declare peers through the first-class module options, so LoadCredential and RefreshOnReload come from the module. Every node binds :: because the test network resolves node names to IPv6 first. poll() raises with the last observed status JSON.
This commit is contained in:
1 parent
4f5fa589ac
commit
ca000d973f
4 files changed
+167
-70
No files matched your search
+122
-50
@@ -1,23 +1,21 @@
|
||||
{ pkgs, module }:
|
||||
|
||||
# Consolidator acceptance test, written ahead of the feature: it is expected to
|
||||
# fail at the first peers assertion until the peers feature lands. Everything
|
||||
# before that (boot, healthz, token minting, marker units on each node's own
|
||||
# API) must already pass against the current binary.
|
||||
# Consolidator acceptance test. Three machines run the real slinuxm: alpha
|
||||
# consolidates beta over plain HTTP and the TLS-only gamma through a private
|
||||
# CA, so one consolidator fans in two peers with different transports; beta
|
||||
# also declares alpha, so both directions are covered.
|
||||
#
|
||||
# alpha and beta each run the real slinuxm and declare the other as a peer.
|
||||
# gamma is a small TLS-only third node that alpha declares as a peer with a
|
||||
# private CA (ca_file), covering the TLS variant.
|
||||
#
|
||||
# Peer config uses the module's freeform `settings` and a LoadCredential
|
||||
# override, so the module needs no peers options yet. The token files below are
|
||||
# root-only 0400 stand-ins for agenix paths: they exist from boot with a
|
||||
# placeholder and the test script overwrites them after minting real tokens.
|
||||
# Peers are declared with the module's first-class options
|
||||
# (`peers.enable`, `peers.peers.<name>.tokenFile`), which render the TOML
|
||||
# `token_file` paths and one `LoadCredential=peer-<name>` per peer. The token
|
||||
# files below are root-only 0400 stand-ins for agenix paths: they exist from
|
||||
# boot with a placeholder and the test overwrites them after minting real
|
||||
# tokens on each peer.
|
||||
|
||||
let
|
||||
inherit (pkgs) lib;
|
||||
port = 1984;
|
||||
tokenDir = "/var/lib/peer-tokens";
|
||||
credDir = "/run/credentials/skyg-linux-tracker.service";
|
||||
|
||||
# Private CA plus a leaf for host `gamma`.
|
||||
certs = pkgs.runCommand "skyg-peers-certs" { nativeBuildInputs = [ pkgs.openssl ]; } ''
|
||||
@@ -33,10 +31,9 @@ let
|
||||
rm ca.key leaf.csr ext.cnf ca.srl
|
||||
'';
|
||||
|
||||
# Timers are short so the test observes flips in seconds; timeouts stay
|
||||
# below their intervals as the config validation will require.
|
||||
# Short timers so the test observes flips in seconds; timeouts stay below
|
||||
# their intervals as the config validation requires.
|
||||
peerTuning = {
|
||||
enabled = true;
|
||||
tcp_interval_secs = 2;
|
||||
tcp_timeout_secs = 1;
|
||||
api_interval_secs = 3;
|
||||
@@ -45,38 +42,40 @@ let
|
||||
request_timeout_secs = 4;
|
||||
};
|
||||
|
||||
peerEntry = name: extra: {
|
||||
inherit name;
|
||||
url = "http://${name}:${toString port}";
|
||||
token_file = "${credDir}/peer-${name}";
|
||||
allow_plain_http = true;
|
||||
} // extra;
|
||||
|
||||
# Service-side wiring shared by every node: token stand-ins and LoadCredential.
|
||||
# Stand-ins for agenix files: a root-only 0400 file per peer, read by the
|
||||
# module's LoadCredential line.
|
||||
tokenFiles = names: {
|
||||
systemd.tmpfiles.rules =
|
||||
[ "d ${tokenDir} 0700 root root -" ]
|
||||
++ map (n: "f ${tokenDir}/${n} 0400 root root - pending") names;
|
||||
systemd.services.skyg-linux-tracker.serviceConfig = {
|
||||
LoadCredential = map (n: "peer-${n}:${tokenDir}/${n}") names;
|
||||
# Credentials are copied at start; refresh them on SIGHUP so a rewritten
|
||||
# token file is picked up without a restart.
|
||||
RefreshOnReload = "credentials";
|
||||
};
|
||||
};
|
||||
|
||||
# The test network gives every node an IPv4 and an IPv6 address in
|
||||
# /etc/hosts, and name resolution returns the IPv6 one first, so the nodes
|
||||
# listen on both families. A peer that only listens on one of them is
|
||||
# reported as unreachable.
|
||||
mkNode = { peers }: { ... }: {
|
||||
imports = [ module ];
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
inherit port;
|
||||
bind = "0.0.0.0";
|
||||
bind = "::";
|
||||
openFirewall = true;
|
||||
settings = pkgs.lib.optionalAttrs (peers != [ ]) {
|
||||
peers = peerTuning // { peer = peers; };
|
||||
# A node without peers keeps the feature off: `peers.enable` requires
|
||||
# at least one peer.
|
||||
peers = lib.optionalAttrs (peers != { }) {
|
||||
enable = true;
|
||||
settings = peerTuning;
|
||||
inherit peers;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
plainPeer = name: {
|
||||
url = "http://${name}:${toString port}";
|
||||
tokenFile = "${tokenDir}/${name}";
|
||||
allowPlainHttp = true;
|
||||
};
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "skyg-linux-tracker-peers";
|
||||
@@ -84,13 +83,14 @@ pkgs.testers.runNixOSTest {
|
||||
nodes.alpha = { ... }: {
|
||||
imports = [
|
||||
(mkNode {
|
||||
peers = [
|
||||
(peerEntry "beta" { })
|
||||
((peerEntry "gamma" {
|
||||
peers = {
|
||||
beta = plainPeer "beta";
|
||||
gamma = {
|
||||
url = "https://gamma:${toString port}";
|
||||
ca_file = "${certs}/ca.pem";
|
||||
}) // { allow_plain_http = false; })
|
||||
];
|
||||
tokenFile = "${tokenDir}/gamma";
|
||||
caFile = "${certs}/ca.pem";
|
||||
};
|
||||
};
|
||||
})
|
||||
(tokenFiles [ "beta" "gamma" ])
|
||||
];
|
||||
@@ -102,7 +102,7 @@ pkgs.testers.runNixOSTest {
|
||||
|
||||
nodes.beta = { ... }: {
|
||||
imports = [
|
||||
(mkNode { peers = [ (peerEntry "alpha" { }) ]; })
|
||||
(mkNode { peers = { alpha = plainPeer "alpha"; }; })
|
||||
(tokenFiles [ "alpha" ])
|
||||
];
|
||||
systemd.services.marker-beta = {
|
||||
@@ -117,7 +117,7 @@ pkgs.testers.runNixOSTest {
|
||||
};
|
||||
|
||||
nodes.gamma = { ... }: {
|
||||
imports = [ (mkNode { peers = [ ]; }) ];
|
||||
imports = [ (mkNode { peers = { }; }) ];
|
||||
services.skyg-linux-tracker.tls = {
|
||||
enable = true;
|
||||
certFile = "${certs}/cert.pem";
|
||||
@@ -130,7 +130,9 @@ pkgs.testers.runNixOSTest {
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
import base64
|
||||
import json
|
||||
import shlex
|
||||
import time
|
||||
|
||||
CLI = "sudo -u skyg-linux-tracker slinuxm"
|
||||
@@ -138,6 +140,7 @@ pkgs.testers.runNixOSTest {
|
||||
INIT = '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"vm","version":"0"}}}'
|
||||
TOKDIR = "${tokenDir}"
|
||||
PORT = ${toString port}
|
||||
CONFIG = "/etc/skyg-linux-tracker.toml"
|
||||
|
||||
def bearer(tok):
|
||||
return f"-H 'Authorization: Bearer {tok}'"
|
||||
@@ -158,19 +161,35 @@ pkgs.testers.runNixOSTest {
|
||||
return json.loads(body)
|
||||
|
||||
def poll(fn, what, timeout=60):
|
||||
"""Wait until fn() returns a truthy (ok, observed) pair.
|
||||
|
||||
The observed value of the last attempt, usually the peer's status
|
||||
JSON, is part of the failure so a timeout says what was wrong.
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if fn():
|
||||
return
|
||||
ok, last = fn()
|
||||
except Exception as e: # noqa: BLE001
|
||||
last = e
|
||||
last = f"{type(e).__name__}: {e}"
|
||||
else:
|
||||
if ok:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise AssertionError(f"timed out waiting for: {what} (last error: {last})")
|
||||
raise AssertionError(f"timed out waiting for: {what} (last observed: {last})")
|
||||
|
||||
def wait_status(m, tok, peer, check, what, timeout=60):
|
||||
poll(lambda: check(peer_status(m, tok, peer)), f"{peer}: {what}", timeout)
|
||||
def probe():
|
||||
status = peer_status(m, tok, peer)
|
||||
return check(status), json.dumps(status, sort_keys=True)
|
||||
poll(probe, f"{peer}: {what}", timeout)
|
||||
|
||||
def wait_gone(m, tok, peer, timeout=30):
|
||||
def probe():
|
||||
status, body = local(m, f"/api/v1/peers/{peer}", tok)
|
||||
return status == 404, f"{status}: {body}"
|
||||
poll(probe, f"{peer}: gone", timeout)
|
||||
|
||||
def mcp_call(m, tok, method, params):
|
||||
"""One-shot MCP session: initialize, initialized, then the call."""
|
||||
@@ -187,10 +206,20 @@ pkgs.testers.runNixOSTest {
|
||||
|
||||
def install_token(m, peer, tok):
|
||||
# Stand-in for an agenix secret: root-only 0400, then reload (SIGHUP)
|
||||
# so the credential is refreshed without a restart.
|
||||
# so the credential copy is refreshed without a restart.
|
||||
m.succeed(f"printf '%s\\n' '{tok}' > {TOKDIR}/{peer}.new && chmod 0400 {TOKDIR}/{peer}.new && mv {TOKDIR}/{peer}.new {TOKDIR}/{peer}")
|
||||
m.succeed("systemctl reload skyg-linux-tracker.service")
|
||||
|
||||
def main_pid(m):
|
||||
return m.succeed("systemctl show -p MainPID --value skyg-linux-tracker.service").strip()
|
||||
|
||||
def drop_peer(config_text, name):
|
||||
"""Remove one [[peers.peer]] block from a rendered config."""
|
||||
head, *blocks = config_text.split("[[peers.peer]]")
|
||||
kept = [b for b in blocks if f'name = "{name}"' not in b]
|
||||
assert len(kept) == len(blocks) - 1, f"no peer block for {name}"
|
||||
return "[[peers.peer]]".join([head] + kept)
|
||||
|
||||
start_all()
|
||||
|
||||
# ---- Phase 1: nothing here depends on the peers feature ----------------
|
||||
@@ -206,7 +235,7 @@ pkgs.testers.runNixOSTest {
|
||||
assert http(m, f"{url}/healthz")[0] == 200
|
||||
assert http(alpha, f"http://beta:{PORT}/healthz")[0] == 200, "alpha cannot reach beta"
|
||||
assert http(beta, f"http://alpha:{PORT}/healthz")[0] == 200, "beta cannot reach alpha"
|
||||
assert http(gamma, f"https://127.0.0.1:{PORT}/healthz", extra="--cacert ${certs}/ca.pem --resolve gamma:1984:127.0.0.1 -k")[0] == 200
|
||||
assert http(gamma, f"https://127.0.0.1:{PORT}/healthz", extra="--cacert ${certs}/ca.pem --resolve gamma:${toString port}:127.0.0.1 -k")[0] == 200
|
||||
|
||||
with subtest("tokens: minted per consolidator by the service user"):
|
||||
# Each node mints the token its consolidator will use against it, plus
|
||||
@@ -233,7 +262,7 @@ pkgs.testers.runNixOSTest {
|
||||
for m, p in ((alpha, "beta"), (alpha, "gamma"), (beta, "alpha")):
|
||||
assert m.succeed(f"stat -c '%a %U' {TOKDIR}/{p}").strip() == "400 root"
|
||||
|
||||
# ---- Phase 2: peers (expected to fail here until the feature lands) ----
|
||||
# ---- Phase 2: peers ----------------------------------------------------
|
||||
|
||||
with subtest("peers: declared but token is still a placeholder -> auth rejected, then recovery"):
|
||||
# Before the real token is installed the peer API check must report
|
||||
@@ -241,8 +270,12 @@ pkgs.testers.runNixOSTest {
|
||||
wait_status(alpha, A, "beta",
|
||||
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected",
|
||||
"server up, api up, auth rejected with the placeholder token")
|
||||
wait_status(alpha, A, "gamma",
|
||||
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected",
|
||||
"TLS peer up with the placeholder token")
|
||||
|
||||
with subtest("peers: install real tokens, status recovers without a restart"):
|
||||
pid = main_pid(alpha)
|
||||
install_token(alpha, "beta", tok_beta_for_alpha)
|
||||
install_token(alpha, "gamma", tok_gamma_for_alpha)
|
||||
install_token(beta, "alpha", tok_alpha_for_beta)
|
||||
@@ -250,11 +283,23 @@ pkgs.testers.runNixOSTest {
|
||||
wait_status(m, tok, peer,
|
||||
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "ok",
|
||||
"server up, api up, auth ok")
|
||||
assert main_pid(alpha) == pid, "rotation must not restart the service"
|
||||
status, body = local(alpha, "/api/v1/peers", A)
|
||||
assert status == 200 and "beta" in body and "gamma" in body, body
|
||||
status, body = local(beta, "/api/v1/peers", B)
|
||||
assert status == 200 and "alpha" in body, body
|
||||
|
||||
with subtest("fan-in: one consolidator reports both peers with their own version"):
|
||||
status, body = local(alpha, "/api/v1/peers", A)
|
||||
assert status == 200, body
|
||||
peers = {p["name"]: p for p in json.loads(body)["peers"]}
|
||||
assert sorted(peers) == ["beta", "gamma"], sorted(peers)
|
||||
for name in ("beta", "gamma"):
|
||||
assert peers[name]["enabled"] and peers[name]["server"]["up"] and peers[name]["api"]["up"], peers[name]
|
||||
assert peers[name]["api"]["auth"] == "ok", peers[name]
|
||||
assert peers[name]["api"]["peer_version"], peers[name]
|
||||
assert peers["gamma"]["url"].startswith("https://"), peers["gamma"]
|
||||
|
||||
with subtest("proxy: each side returns the other's data, not its own"):
|
||||
for m, tok, peer, mine, theirs in ((alpha, A, "beta", "marker-alpha", "marker-beta"),
|
||||
(beta, B, "alpha", "marker-beta", "marker-alpha")):
|
||||
@@ -267,6 +312,14 @@ pkgs.testers.runNixOSTest {
|
||||
# beta's deliberately failing unit is only visible through the proxy.
|
||||
assert "broken-beta.service" in local(alpha, "/api/v1/peers/beta/systemd/units", A)[1]
|
||||
|
||||
with subtest("proxy: the peer's answer is the peer's own data"):
|
||||
_, direct = local(beta, "/api/v1/systemd/units", B)
|
||||
status, proxied = local(alpha, "/api/v1/peers/beta/systemd/units", A)
|
||||
assert status == 200, proxied
|
||||
names = lambda body: sorted(u["name"] for u in json.loads(body))
|
||||
assert names(proxied) == names(direct), (names(proxied), names(direct))
|
||||
assert names(proxied) != names(local(alpha, "/api/v1/systemd/units", A)[1])
|
||||
|
||||
with subtest("mcp: list_peers and list_systemd_units with peer match REST"):
|
||||
out = mcp_tool(alpha, A, "list_peers")
|
||||
assert "beta" in out and "gamma" in out, out
|
||||
@@ -288,6 +341,7 @@ pkgs.testers.runNixOSTest {
|
||||
assert http(alpha, f"http://127.0.0.1:{PORT}/api/v1/peers/nope", A)[0] == 404
|
||||
|
||||
with subtest("token rejected: revoke on beta, then restore with a new token file"):
|
||||
pid = main_pid(alpha)
|
||||
beta.succeed(f"{CLI} token revoke consolidator-alpha")
|
||||
wait_status(alpha, A, "beta",
|
||||
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected",
|
||||
@@ -300,6 +354,25 @@ pkgs.testers.runNixOSTest {
|
||||
install_token(alpha, "beta", new_tok)
|
||||
wait_status(alpha, A, "beta", lambda s: s["api"]["auth"] == "ok", "auth ok after new token")
|
||||
assert local(alpha, "/api/v1/peers/beta/systemd/units", A)[0] == 200
|
||||
assert main_pid(alpha) == pid, "revoke and rotation must not restart the service"
|
||||
|
||||
with subtest("reload: removing a peer from the config drops it on SIGHUP"):
|
||||
# The module renders the config into the store, so replace the /etc
|
||||
# symlink with a writable copy first: SIGHUP re-reads this path.
|
||||
rendered = alpha.succeed(f"readlink -f {CONFIG}").strip()
|
||||
text = alpha.succeed(f"cat {rendered}")
|
||||
edited = drop_peer(text, "gamma")
|
||||
encoded = base64.b64encode(edited.encode()).decode()
|
||||
alpha.succeed(f"printf '%s' {shlex.quote(encoded)} | base64 -d > {CONFIG}.new && mv {CONFIG}.new {CONFIG}")
|
||||
assert "gamma" not in alpha.succeed(f"cat {CONFIG}")
|
||||
pid = main_pid(alpha)
|
||||
alpha.succeed("systemctl reload skyg-linux-tracker.service")
|
||||
wait_gone(alpha, A, "gamma")
|
||||
status, body = local(alpha, "/api/v1/peers", A)
|
||||
assert status == 200 and "gamma" not in body and "beta" in body, body
|
||||
assert main_pid(alpha) == pid, "removing a peer must not restart the service"
|
||||
assert alpha.succeed(f"stat -c '%a' {TOKDIR}/gamma").strip() == "400", "the removed peer's file is untouched"
|
||||
assert peer_status(alpha, A, "beta")["api"]["auth"] == "ok", "beta must survive the reload"
|
||||
|
||||
with subtest("outage: stopping beta's service flips server and api, alpha stays healthy"):
|
||||
beta.systemctl("stop skyg-linux-tracker.service")
|
||||
@@ -308,7 +381,6 @@ pkgs.testers.runNixOSTest {
|
||||
"server and api down", timeout=30)
|
||||
assert http(alpha, f"http://127.0.0.1:{PORT}/healthz")[0] == 200
|
||||
assert local(alpha, "/api/v1/systemd/units", A)[0] == 200
|
||||
assert peer_status(alpha, A, "gamma")["api"]["up"], "gamma must be unaffected"
|
||||
beta.systemctl("start skyg-linux-tracker.service")
|
||||
beta.wait_for_open_port(PORT)
|
||||
wait_status(alpha, A, "beta",
|
||||
|
||||
Reference in new issue
Block a user