329 lines
16 KiB
Nix
329 lines
16 KiB
Nix
{ pkgs, module }:
|
|
|
|
# Consolidator acceptance test, written ahead of the feature: it is expected to
|
|
# fail at the first peers assertion until the peers feature lands. Everything
|
|
# before that (boot, healthz, token minting, marker units on each node's own
|
|
# API) must already pass against the current binary.
|
|
#
|
|
# alpha and beta each run the real slinuxm and declare the other as a peer.
|
|
# gamma is a small TLS-only third node that alpha declares as a peer with a
|
|
# private CA (ca_file), covering the TLS variant.
|
|
#
|
|
# Peer config uses the module's freeform `settings` and a LoadCredential
|
|
# override, so the module needs no peers options yet. The token files below are
|
|
# root-only 0400 stand-ins for agenix paths: they exist from boot with a
|
|
# placeholder and the test script overwrites them after minting real tokens.
|
|
|
|
let
|
|
port = 1984;
|
|
tokenDir = "/var/lib/peer-tokens";
|
|
credDir = "/run/credentials/skyg-linux-tracker.service";
|
|
|
|
# Private CA plus a leaf for host `gamma`.
|
|
certs = pkgs.runCommand "skyg-peers-certs" { nativeBuildInputs = [ pkgs.openssl ]; } ''
|
|
mkdir $out && cd $out
|
|
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 \
|
|
-keyout ca.key -out ca.pem -subj '/CN=skyg-test-ca' \
|
|
-addext 'basicConstraints=critical,CA:TRUE' -addext 'keyUsage=critical,keyCertSign,cRLSign'
|
|
openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
|
|
-keyout key.pem -out leaf.csr -subj '/CN=gamma'
|
|
printf 'subjectAltName=DNS:gamma\nbasicConstraints=CA:FALSE\n' > ext.cnf
|
|
openssl x509 -req -in leaf.csr -CA ca.pem -CAkey ca.key -CAcreateserial -days 3650 \
|
|
-extfile ext.cnf -out cert.pem
|
|
rm ca.key leaf.csr ext.cnf ca.srl
|
|
'';
|
|
|
|
# Timers are short so the test observes flips in seconds; timeouts stay
|
|
# below their intervals as the config validation will require.
|
|
peerTuning = {
|
|
enabled = true;
|
|
tcp_interval_secs = 2;
|
|
tcp_timeout_secs = 1;
|
|
api_interval_secs = 3;
|
|
api_timeout_secs = 2;
|
|
fail_threshold = 2;
|
|
request_timeout_secs = 4;
|
|
};
|
|
|
|
peerEntry = name: extra: {
|
|
inherit name;
|
|
url = "http://${name}:${toString port}";
|
|
token_file = "${credDir}/peer-${name}";
|
|
allow_plain_http = true;
|
|
} // extra;
|
|
|
|
# Service-side wiring shared by every node: token stand-ins and LoadCredential.
|
|
tokenFiles = names: {
|
|
systemd.tmpfiles.rules =
|
|
[ "d ${tokenDir} 0700 root root -" ]
|
|
++ map (n: "f ${tokenDir}/${n} 0400 root root - pending") names;
|
|
systemd.services.skyg-linux-tracker.serviceConfig = {
|
|
LoadCredential = map (n: "peer-${n}:${tokenDir}/${n}") names;
|
|
# Credentials are copied at start; refresh them on SIGHUP so a rewritten
|
|
# token file is picked up without a restart.
|
|
RefreshOnReload = "credentials";
|
|
};
|
|
};
|
|
|
|
mkNode = { peers }: { ... }: {
|
|
imports = [ module ];
|
|
services.skyg-linux-tracker = {
|
|
enable = true;
|
|
inherit port;
|
|
bind = "0.0.0.0";
|
|
openFirewall = true;
|
|
settings = pkgs.lib.optionalAttrs (peers != [ ]) {
|
|
peers = peerTuning // { peer = peers; };
|
|
};
|
|
};
|
|
};
|
|
in
|
|
pkgs.testers.runNixOSTest {
|
|
name = "skyg-linux-tracker-peers";
|
|
|
|
nodes.alpha = { ... }: {
|
|
imports = [
|
|
(mkNode {
|
|
peers = [
|
|
(peerEntry "beta" { })
|
|
((peerEntry "gamma" {
|
|
url = "https://gamma:${toString port}";
|
|
ca_file = "${certs}/ca.pem";
|
|
}) // { allow_plain_http = false; })
|
|
];
|
|
})
|
|
(tokenFiles [ "beta" "gamma" ])
|
|
];
|
|
systemd.services.marker-alpha = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; };
|
|
};
|
|
};
|
|
|
|
nodes.beta = { ... }: {
|
|
imports = [
|
|
(mkNode { peers = [ (peerEntry "alpha" { }) ]; })
|
|
(tokenFiles [ "alpha" ])
|
|
];
|
|
systemd.services.marker-beta = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; };
|
|
};
|
|
# Deliberately failing unit, so beta's data differs from alpha's.
|
|
systemd.services.broken-beta = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = { Type = "oneshot"; ExecStart = "${pkgs.coreutils}/bin/false"; };
|
|
};
|
|
};
|
|
|
|
nodes.gamma = { ... }: {
|
|
imports = [ (mkNode { peers = [ ]; }) ];
|
|
services.skyg-linux-tracker.tls = {
|
|
enable = true;
|
|
certFile = "${certs}/cert.pem";
|
|
keyFile = "${certs}/key.pem";
|
|
};
|
|
systemd.services.marker-gamma = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; };
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
import json
|
|
import time
|
|
|
|
CLI = "sudo -u skyg-linux-tracker slinuxm"
|
|
JSON = "-H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream'"
|
|
INIT = '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"vm","version":"0"}}}'
|
|
TOKDIR = "${tokenDir}"
|
|
PORT = ${toString port}
|
|
|
|
def bearer(tok):
|
|
return f"-H 'Authorization: Bearer {tok}'"
|
|
|
|
def http(m, url, tok=None, extra=""):
|
|
"""Returns (status_code, body)."""
|
|
auth = bearer(tok) if tok else ""
|
|
out = m.succeed(f"curl -s -w '\\n%{{http_code}}' {auth} {extra} {url}")
|
|
body, _, status = out.rpartition("\n")
|
|
return int(status), body
|
|
|
|
def local(m, path, tok):
|
|
return http(m, f"http://127.0.0.1:{PORT}{path}", tok)
|
|
|
|
def peer_status(m, tok, peer):
|
|
status, body = local(m, f"/api/v1/peers/{peer}", tok)
|
|
assert status == 200, f"/api/v1/peers/{peer} -> {status}: {body}"
|
|
return json.loads(body)
|
|
|
|
def poll(fn, what, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if fn():
|
|
return
|
|
except Exception as e: # noqa: BLE001
|
|
last = e
|
|
time.sleep(1)
|
|
raise AssertionError(f"timed out waiting for: {what} (last error: {last})")
|
|
|
|
def wait_status(m, tok, peer, check, what, timeout=60):
|
|
poll(lambda: check(peer_status(m, tok, peer)), f"{peer}: {what}", timeout)
|
|
|
|
def mcp_call(m, tok, method, params):
|
|
"""One-shot MCP session: initialize, initialized, then the call."""
|
|
url = f"http://127.0.0.1:{PORT}/mcp"
|
|
m.succeed(f"curl -sf -D /tmp/h {bearer(tok)} {JSON} -d '{INIT}' {url} >/dev/null")
|
|
sid = m.succeed("awk -F': ' 'tolower($1)==\"mcp-session-id\"{print $2}' /tmp/h | tr -d '\\r'").strip()
|
|
sess = f"-H 'Mcp-Session-Id: {sid}'" if sid else ""
|
|
m.succeed(f"curl -sf {bearer(tok)} {sess} {JSON} -d '{{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}}' {url} >/dev/null")
|
|
body = json.dumps({"jsonrpc": "2.0", "id": 2, "method": method, "params": params})
|
|
return m.succeed(f"curl -sf {bearer(tok)} {sess} {JSON} -d '{body}' {url}")
|
|
|
|
def mcp_tool(m, tok, name, args=None):
|
|
return mcp_call(m, tok, "tools/call", {"name": name, "arguments": args or {}})
|
|
|
|
def install_token(m, peer, tok):
|
|
# Stand-in for an agenix secret: root-only 0400, then reload (SIGHUP)
|
|
# so the credential is refreshed without a restart.
|
|
m.succeed(f"printf '%s\\n' '{tok}' > {TOKDIR}/{peer}.new && chmod 0400 {TOKDIR}/{peer}.new && mv {TOKDIR}/{peer}.new {TOKDIR}/{peer}")
|
|
m.succeed("systemctl reload skyg-linux-tracker.service")
|
|
|
|
start_all()
|
|
|
|
# ---- Phase 1: nothing here depends on the peers feature ----------------
|
|
|
|
with subtest("boot: all services up, healthz public"):
|
|
for m in (alpha, beta, gamma):
|
|
m.wait_for_unit("skyg-linux-tracker.service")
|
|
m.wait_for_open_port(PORT)
|
|
alpha.wait_for_unit("marker-alpha.service")
|
|
beta.wait_for_unit("marker-beta.service")
|
|
gamma.wait_for_unit("marker-gamma.service")
|
|
for m, url in ((alpha, f"http://127.0.0.1:{PORT}"), (beta, f"http://127.0.0.1:{PORT}")):
|
|
assert http(m, f"{url}/healthz")[0] == 200
|
|
assert http(alpha, f"http://beta:{PORT}/healthz")[0] == 200, "alpha cannot reach beta"
|
|
assert http(beta, f"http://alpha:{PORT}/healthz")[0] == 200, "beta cannot reach alpha"
|
|
assert http(gamma, f"https://127.0.0.1:{PORT}/healthz", extra="--cacert ${certs}/ca.pem --resolve gamma:1984:127.0.0.1 -k")[0] == 200
|
|
|
|
with subtest("tokens: minted per consolidator by the service user"):
|
|
# Each node mints the token its consolidator will use against it, plus
|
|
# a separate token for local assertions.
|
|
tok_alpha_for_beta = alpha.succeed(f"{CLI} token create consolidator-beta").strip()
|
|
tok_beta_for_alpha = beta.succeed(f"{CLI} token create consolidator-alpha").strip()
|
|
tok_gamma_for_alpha = gamma.succeed(f"{CLI} token create consolidator-alpha").strip()
|
|
A = alpha.succeed(f"{CLI} token create test-alpha").strip()
|
|
B = beta.succeed(f"{CLI} token create test-beta").strip()
|
|
G = gamma.succeed(f"{CLI} token create test-gamma").strip()
|
|
for t in (tok_alpha_for_beta, tok_beta_for_alpha, tok_gamma_for_alpha, A, B, G):
|
|
assert t, "empty token"
|
|
|
|
with subtest("data: marker units are visible on each node's own API only"):
|
|
_, a_units = local(alpha, "/api/v1/systemd/units", A)
|
|
_, b_units = local(beta, "/api/v1/systemd/units", B)
|
|
assert "marker-alpha.service" in a_units and "marker-beta.service" not in a_units
|
|
assert "marker-beta.service" in b_units and "marker-alpha.service" not in b_units
|
|
assert "broken-beta.service" in b_units and "broken-beta.service" not in a_units
|
|
status, one = local(beta, "/api/v1/systemd/units/marker-beta.service", B)
|
|
assert status == 200 and "marker-beta.service" in one
|
|
|
|
with subtest("token files: placeholders exist root-only 0400 before any token is installed"):
|
|
for m, p in ((alpha, "beta"), (alpha, "gamma"), (beta, "alpha")):
|
|
assert m.succeed(f"stat -c '%a %U' {TOKDIR}/{p}").strip() == "400 root"
|
|
|
|
# ---- Phase 2: peers (expected to fail here until the feature lands) ----
|
|
|
|
with subtest("peers: declared but token is still a placeholder -> auth rejected, then recovery"):
|
|
# Before the real token is installed the peer API check must report
|
|
# server up, api up (it answered 401), auth rejected.
|
|
wait_status(alpha, A, "beta",
|
|
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected",
|
|
"server up, api up, auth rejected with the placeholder token")
|
|
|
|
with subtest("peers: install real tokens, status recovers without a restart"):
|
|
install_token(alpha, "beta", tok_beta_for_alpha)
|
|
install_token(alpha, "gamma", tok_gamma_for_alpha)
|
|
install_token(beta, "alpha", tok_alpha_for_beta)
|
|
for m, tok, peer in ((alpha, A, "beta"), (alpha, A, "gamma"), (beta, B, "alpha")):
|
|
wait_status(m, tok, peer,
|
|
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "ok",
|
|
"server up, api up, auth ok")
|
|
status, body = local(alpha, "/api/v1/peers", A)
|
|
assert status == 200 and "beta" in body and "gamma" in body, body
|
|
status, body = local(beta, "/api/v1/peers", B)
|
|
assert status == 200 and "alpha" in body, body
|
|
|
|
with subtest("proxy: each side returns the other's data, not its own"):
|
|
for m, tok, peer, mine, theirs in ((alpha, A, "beta", "marker-alpha", "marker-beta"),
|
|
(beta, B, "alpha", "marker-beta", "marker-alpha")):
|
|
status, body = local(m, f"/api/v1/peers/{peer}/systemd/units/{theirs}.service", tok)
|
|
assert status == 200 and f"{theirs}.service" in body, (status, body)
|
|
status, body = local(m, f"/api/v1/peers/{peer}/systemd/units", tok)
|
|
assert status == 200 and f"{theirs}.service" in body and f"{mine}.service" not in body, body
|
|
_, hdrs = m.execute(f"curl -s -D - -o /dev/null {bearer(tok)} http://127.0.0.1:{PORT}/api/v1/peers/{peer}/systemd/units")
|
|
assert f"x-slinuxm-peer: {peer}" in hdrs.lower(), hdrs
|
|
# beta's deliberately failing unit is only visible through the proxy.
|
|
assert "broken-beta.service" in local(alpha, "/api/v1/peers/beta/systemd/units", A)[1]
|
|
|
|
with subtest("mcp: list_peers and list_systemd_units with peer match REST"):
|
|
out = mcp_tool(alpha, A, "list_peers")
|
|
assert "beta" in out and "gamma" in out, out
|
|
out = mcp_tool(alpha, A, "list_systemd_units", {"peer": "beta"})
|
|
assert "marker-beta.service" in out and "marker-alpha.service" not in out, out
|
|
out = mcp_tool(beta, B, "list_systemd_units", {"peer": "alpha"})
|
|
assert "marker-alpha.service" in out and "marker-beta.service" not in out, out
|
|
|
|
with subtest("tls: gamma is reached over HTTPS through ca_file"):
|
|
status, body = local(alpha, "/api/v1/peers/gamma/systemd/units/marker-gamma.service", A)
|
|
assert status == 200 and "marker-gamma.service" in body, (status, body)
|
|
|
|
with subtest("read-only and no chaining"):
|
|
base = f"http://127.0.0.1:{PORT}/api/v1/peers/beta/systemd/units"
|
|
assert http(alpha, base, A, "-X POST")[0] == 405
|
|
assert http(alpha, base, A, "-X DELETE")[0] == 405
|
|
status, _ = http(alpha, base, A, "-H 'x-slinuxm-hop: 1'")
|
|
assert 400 <= status < 500, status
|
|
assert http(alpha, f"http://127.0.0.1:{PORT}/api/v1/peers/nope", A)[0] == 404
|
|
|
|
with subtest("token rejected: revoke on beta, then restore with a new token file"):
|
|
beta.succeed(f"{CLI} token revoke consolidator-alpha")
|
|
wait_status(alpha, A, "beta",
|
|
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected",
|
|
"auth rejected after revoke")
|
|
status, body = local(alpha, "/api/v1/peers/beta/systemd/units", A)
|
|
assert status == 503 and "beta" in body, (status, body)
|
|
# alpha's own credentials still work: the caller never sees a 401.
|
|
assert local(alpha, "/api/v1/systemd/units", A)[0] == 200
|
|
new_tok = beta.succeed(f"{CLI} token create consolidator-alpha-2").strip()
|
|
install_token(alpha, "beta", new_tok)
|
|
wait_status(alpha, A, "beta", lambda s: s["api"]["auth"] == "ok", "auth ok after new token")
|
|
assert local(alpha, "/api/v1/peers/beta/systemd/units", A)[0] == 200
|
|
|
|
with subtest("outage: stopping beta's service flips server and api, alpha stays healthy"):
|
|
beta.systemctl("stop skyg-linux-tracker.service")
|
|
wait_status(alpha, A, "beta",
|
|
lambda s: not s["server"]["up"] and not s["api"]["up"],
|
|
"server and api down", timeout=30)
|
|
assert http(alpha, f"http://127.0.0.1:{PORT}/healthz")[0] == 200
|
|
assert local(alpha, "/api/v1/systemd/units", A)[0] == 200
|
|
assert peer_status(alpha, A, "gamma")["api"]["up"], "gamma must be unaffected"
|
|
beta.systemctl("start skyg-linux-tracker.service")
|
|
beta.wait_for_open_port(PORT)
|
|
wait_status(alpha, A, "beta",
|
|
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "ok",
|
|
"recovery after restart")
|
|
|
|
with subtest("outage: beta VM down, alpha requests are bounded by timeouts"):
|
|
beta.shutdown()
|
|
t0 = time.monotonic()
|
|
status, body = http(alpha, f"http://127.0.0.1:{PORT}/api/v1/peers/beta/systemd/units", A, "--max-time 30")
|
|
elapsed = time.monotonic() - t0
|
|
assert status == 503 and "beta" in body, (status, body)
|
|
assert elapsed < 15, f"proxy call took {elapsed:.1f}s, expected it bounded by request_timeout_secs"
|
|
wait_status(alpha, A, "beta", lambda s: not s["server"]["up"], "server down with the VM off", timeout=30)
|
|
assert local(alpha, "/api/v1/systemd/units", A)[0] == 200
|
|
'';
|
|
}
|