A peer whose name resolves to both families but which listens on only one of them was reported unreachable, which the documented 0.0.0.0 binding produces on a dual-stack LAN and in the NixOS test network. resolve_target now returns the whole address set (still refusing the name if any address is denied), the HTTP client pins it with resolve_to_addrs so the connector falls through to the next address, and tcp_check tries each address in order with a per-attempt share of the remaining deadline. ClientKey keys on the set. Regression cover: a client test that resolves localhost against a single-family listener, a tests/peers.rs case declaring the peer as http://localhost:<port>, and the VM test's TLS peer gamma now binds 0.0.0.0 (IPv4 only) while alpha and beta stay dual stack. Docs and the contracts section 10 address note describe the new behaviour.
402 lines
20 KiB
Nix
402 lines
20 KiB
Nix
{ pkgs, module }:
|
|
|
|
# Consolidator acceptance test. Three machines run the real slinuxm: alpha
|
|
# consolidates beta over plain HTTP and the TLS-only gamma through a private
|
|
# CA, so one consolidator fans in two peers with different transports; beta
|
|
# also declares alpha, so both directions are covered.
|
|
#
|
|
# Peers are declared with the module's first-class options
|
|
# (`peers.enable`, `peers.peers.<name>.tokenFile`), which render the TOML
|
|
# `token_file` paths and one `LoadCredential=peer-<name>` per peer. The token
|
|
# files below are root-only 0400 stand-ins for agenix paths: they exist from
|
|
# boot with a placeholder and the test overwrites them after minting real
|
|
# tokens on each peer.
|
|
|
|
let
|
|
inherit (pkgs) lib;
|
|
port = 1984;
|
|
tokenDir = "/var/lib/peer-tokens";
|
|
|
|
# Private CA plus a leaf for host `gamma`.
|
|
certs = pkgs.runCommand "skyg-peers-certs" { nativeBuildInputs = [ pkgs.openssl ]; } ''
|
|
mkdir $out && cd $out
|
|
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 \
|
|
-keyout ca.key -out ca.pem -subj '/CN=skyg-test-ca' \
|
|
-addext 'basicConstraints=critical,CA:TRUE' -addext 'keyUsage=critical,keyCertSign,cRLSign'
|
|
openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
|
|
-keyout key.pem -out leaf.csr -subj '/CN=gamma'
|
|
printf 'subjectAltName=DNS:gamma\nbasicConstraints=CA:FALSE\n' > ext.cnf
|
|
openssl x509 -req -in leaf.csr -CA ca.pem -CAkey ca.key -CAcreateserial -days 3650 \
|
|
-extfile ext.cnf -out cert.pem
|
|
rm ca.key leaf.csr ext.cnf ca.srl
|
|
'';
|
|
|
|
# Short timers so the test observes flips in seconds; timeouts stay below
|
|
# their intervals as the config validation requires.
|
|
peerTuning = {
|
|
tcp_interval_secs = 2;
|
|
tcp_timeout_secs = 1;
|
|
api_interval_secs = 3;
|
|
api_timeout_secs = 2;
|
|
fail_threshold = 2;
|
|
request_timeout_secs = 4;
|
|
};
|
|
|
|
# Stand-ins for agenix files: a root-only 0400 file per peer, read by the
|
|
# module's LoadCredential line.
|
|
tokenFiles = names: {
|
|
systemd.tmpfiles.rules =
|
|
[ "d ${tokenDir} 0700 root root -" ]
|
|
++ map (n: "f ${tokenDir}/${n} 0400 root root - pending") names;
|
|
};
|
|
|
|
# The test network gives every node an IPv4 and an IPv6 address in
|
|
# /etc/hosts, so a peer name resolves to two addresses. The consolidators
|
|
# listen on both families; gamma listens on IPv4 only, which is the case that
|
|
# used to read as unreachable when the transport pinned the first address.
|
|
mkNode = { peers, bind ? "::" }: { ... }: {
|
|
imports = [ module ];
|
|
services.skyg-linux-tracker = {
|
|
enable = true;
|
|
inherit port bind;
|
|
openFirewall = true;
|
|
# A node without peers keeps the feature off: `peers.enable` requires
|
|
# at least one peer.
|
|
peers = lib.optionalAttrs (peers != { }) {
|
|
enable = true;
|
|
settings = peerTuning;
|
|
inherit peers;
|
|
};
|
|
};
|
|
};
|
|
|
|
plainPeer = name: {
|
|
url = "http://${name}:${toString port}";
|
|
tokenFile = "${tokenDir}/${name}";
|
|
allowPlainHttp = true;
|
|
};
|
|
in
|
|
pkgs.testers.runNixOSTest {
|
|
name = "skyg-linux-tracker-peers";
|
|
|
|
nodes.alpha = { ... }: {
|
|
imports = [
|
|
(mkNode {
|
|
peers = {
|
|
beta = plainPeer "beta";
|
|
gamma = {
|
|
url = "https://gamma:${toString port}";
|
|
tokenFile = "${tokenDir}/gamma";
|
|
caFile = "${certs}/ca.pem";
|
|
};
|
|
};
|
|
})
|
|
(tokenFiles [ "beta" "gamma" ])
|
|
];
|
|
systemd.services.marker-alpha = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; };
|
|
};
|
|
};
|
|
|
|
nodes.beta = { ... }: {
|
|
imports = [
|
|
(mkNode { peers = { alpha = plainPeer "alpha"; }; })
|
|
(tokenFiles [ "alpha" ])
|
|
];
|
|
systemd.services.marker-beta = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; };
|
|
};
|
|
# Deliberately failing unit, so beta's data differs from alpha's.
|
|
systemd.services.broken-beta = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = { Type = "oneshot"; ExecStart = "${pkgs.coreutils}/bin/false"; };
|
|
};
|
|
};
|
|
|
|
nodes.gamma = { ... }: {
|
|
# IPv4 only: alpha still reaches it by name, because every resolved
|
|
# address is tried.
|
|
imports = [ (mkNode { peers = { }; bind = "0.0.0.0"; }) ];
|
|
services.skyg-linux-tracker.tls = {
|
|
enable = true;
|
|
certFile = "${certs}/cert.pem";
|
|
keyFile = "${certs}/key.pem";
|
|
};
|
|
systemd.services.marker-gamma = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; };
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
import base64
|
|
import json
|
|
import shlex
|
|
import time
|
|
|
|
CLI = "sudo -u skyg-linux-tracker slinuxm"
|
|
JSON = "-H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream'"
|
|
INIT = '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"vm","version":"0"}}}'
|
|
TOKDIR = "${tokenDir}"
|
|
PORT = ${toString port}
|
|
CONFIG = "/etc/skyg-linux-tracker.toml"
|
|
|
|
def bearer(tok):
|
|
return f"-H 'Authorization: Bearer {tok}'"
|
|
|
|
def http(m, url, tok=None, extra=""):
|
|
"""Returns (status_code, body)."""
|
|
auth = bearer(tok) if tok else ""
|
|
out = m.succeed(f"curl -s -w '\\n%{{http_code}}' {auth} {extra} {url}")
|
|
body, _, status = out.rpartition("\n")
|
|
return int(status), body
|
|
|
|
def local(m, path, tok):
|
|
return http(m, f"http://127.0.0.1:{PORT}{path}", tok)
|
|
|
|
def peer_status(m, tok, peer):
|
|
status, body = local(m, f"/api/v1/peers/{peer}", tok)
|
|
assert status == 200, f"/api/v1/peers/{peer} -> {status}: {body}"
|
|
return json.loads(body)
|
|
|
|
def poll(fn, what, timeout=60):
|
|
"""Wait until fn() returns a truthy (ok, observed) pair.
|
|
|
|
The observed value of the last attempt, usually the peer's status
|
|
JSON, is part of the failure so a timeout says what was wrong.
|
|
"""
|
|
deadline = time.monotonic() + timeout
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
ok, last = fn()
|
|
except Exception as e: # noqa: BLE001
|
|
last = f"{type(e).__name__}: {e}"
|
|
else:
|
|
if ok:
|
|
return
|
|
time.sleep(1)
|
|
raise AssertionError(f"timed out waiting for: {what} (last observed: {last})")
|
|
|
|
def wait_status(m, tok, peer, check, what, timeout=60):
|
|
def probe():
|
|
status = peer_status(m, tok, peer)
|
|
return check(status), json.dumps(status, sort_keys=True)
|
|
poll(probe, f"{peer}: {what}", timeout)
|
|
|
|
def wait_gone(m, tok, peer, timeout=30):
|
|
def probe():
|
|
status, body = local(m, f"/api/v1/peers/{peer}", tok)
|
|
return status == 404, f"{status}: {body}"
|
|
poll(probe, f"{peer}: gone", timeout)
|
|
|
|
def mcp_call(m, tok, method, params):
|
|
"""One-shot MCP session: initialize, initialized, then the call."""
|
|
url = f"http://127.0.0.1:{PORT}/mcp"
|
|
m.succeed(f"curl -sf -D /tmp/h {bearer(tok)} {JSON} -d '{INIT}' {url} >/dev/null")
|
|
sid = m.succeed("awk -F': ' 'tolower($1)==\"mcp-session-id\"{print $2}' /tmp/h | tr -d '\\r'").strip()
|
|
sess = f"-H 'Mcp-Session-Id: {sid}'" if sid else ""
|
|
m.succeed(f"curl -sf {bearer(tok)} {sess} {JSON} -d '{{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}}' {url} >/dev/null")
|
|
body = json.dumps({"jsonrpc": "2.0", "id": 2, "method": method, "params": params})
|
|
return m.succeed(f"curl -sf {bearer(tok)} {sess} {JSON} -d '{body}' {url}")
|
|
|
|
def mcp_tool(m, tok, name, args=None):
|
|
return mcp_call(m, tok, "tools/call", {"name": name, "arguments": args or {}})
|
|
|
|
def install_token(m, peer, tok):
|
|
# Stand-in for an agenix secret: root-only 0400, then reload (SIGHUP)
|
|
# so the credential copy is refreshed without a restart.
|
|
m.succeed(f"printf '%s\\n' '{tok}' > {TOKDIR}/{peer}.new && chmod 0400 {TOKDIR}/{peer}.new && mv {TOKDIR}/{peer}.new {TOKDIR}/{peer}")
|
|
m.succeed("systemctl reload skyg-linux-tracker.service")
|
|
|
|
def main_pid(m):
|
|
return m.succeed("systemctl show -p MainPID --value skyg-linux-tracker.service").strip()
|
|
|
|
def drop_peer(config_text, name):
|
|
"""Remove one [[peers.peer]] block from a rendered config."""
|
|
head, *blocks = config_text.split("[[peers.peer]]")
|
|
kept = [b for b in blocks if f'name = "{name}"' not in b]
|
|
assert len(kept) == len(blocks) - 1, f"no peer block for {name}"
|
|
return "[[peers.peer]]".join([head] + kept)
|
|
|
|
start_all()
|
|
|
|
# ---- Phase 1: nothing here depends on the peers feature ----------------
|
|
|
|
with subtest("boot: all services up, healthz public"):
|
|
for m in (alpha, beta, gamma):
|
|
m.wait_for_unit("skyg-linux-tracker.service")
|
|
m.wait_for_open_port(PORT)
|
|
alpha.wait_for_unit("marker-alpha.service")
|
|
beta.wait_for_unit("marker-beta.service")
|
|
gamma.wait_for_unit("marker-gamma.service")
|
|
for m, url in ((alpha, f"http://127.0.0.1:{PORT}"), (beta, f"http://127.0.0.1:{PORT}")):
|
|
assert http(m, f"{url}/healthz")[0] == 200
|
|
assert http(alpha, f"http://beta:{PORT}/healthz")[0] == 200, "alpha cannot reach beta"
|
|
assert http(beta, f"http://alpha:{PORT}/healthz")[0] == 200, "beta cannot reach alpha"
|
|
assert http(gamma, f"https://127.0.0.1:{PORT}/healthz", extra="--cacert ${certs}/ca.pem --resolve gamma:${toString port}:127.0.0.1 -k")[0] == 200
|
|
|
|
with subtest("tokens: minted per consolidator by the service user"):
|
|
# Each node mints the token its consolidator will use against it, plus
|
|
# a separate token for local assertions.
|
|
tok_alpha_for_beta = alpha.succeed(f"{CLI} token create consolidator-beta").strip()
|
|
tok_beta_for_alpha = beta.succeed(f"{CLI} token create consolidator-alpha").strip()
|
|
tok_gamma_for_alpha = gamma.succeed(f"{CLI} token create consolidator-alpha").strip()
|
|
A = alpha.succeed(f"{CLI} token create test-alpha").strip()
|
|
B = beta.succeed(f"{CLI} token create test-beta").strip()
|
|
G = gamma.succeed(f"{CLI} token create test-gamma").strip()
|
|
for t in (tok_alpha_for_beta, tok_beta_for_alpha, tok_gamma_for_alpha, A, B, G):
|
|
assert t, "empty token"
|
|
|
|
with subtest("data: marker units are visible on each node's own API only"):
|
|
_, a_units = local(alpha, "/api/v1/systemd/units", A)
|
|
_, b_units = local(beta, "/api/v1/systemd/units", B)
|
|
assert "marker-alpha.service" in a_units and "marker-beta.service" not in a_units
|
|
assert "marker-beta.service" in b_units and "marker-alpha.service" not in b_units
|
|
assert "broken-beta.service" in b_units and "broken-beta.service" not in a_units
|
|
status, one = local(beta, "/api/v1/systemd/units/marker-beta.service", B)
|
|
assert status == 200 and "marker-beta.service" in one
|
|
|
|
with subtest("token files: placeholders exist root-only 0400 before any token is installed"):
|
|
for m, p in ((alpha, "beta"), (alpha, "gamma"), (beta, "alpha")):
|
|
assert m.succeed(f"stat -c '%a %U' {TOKDIR}/{p}").strip() == "400 root"
|
|
|
|
# ---- Phase 2: peers ----------------------------------------------------
|
|
|
|
with subtest("peers: declared but token is still a placeholder -> auth rejected, then recovery"):
|
|
# Before the real token is installed the peer API check must report
|
|
# server up, api up (it answered 401), auth rejected.
|
|
wait_status(alpha, A, "beta",
|
|
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected",
|
|
"server up, api up, auth rejected with the placeholder token")
|
|
wait_status(alpha, A, "gamma",
|
|
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected",
|
|
"TLS peer up with the placeholder token")
|
|
|
|
with subtest("peers: install real tokens, status recovers without a restart"):
|
|
pid = main_pid(alpha)
|
|
install_token(alpha, "beta", tok_beta_for_alpha)
|
|
install_token(alpha, "gamma", tok_gamma_for_alpha)
|
|
install_token(beta, "alpha", tok_alpha_for_beta)
|
|
for m, tok, peer in ((alpha, A, "beta"), (alpha, A, "gamma"), (beta, B, "alpha")):
|
|
wait_status(m, tok, peer,
|
|
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "ok",
|
|
"server up, api up, auth ok")
|
|
assert main_pid(alpha) == pid, "rotation must not restart the service"
|
|
status, body = local(alpha, "/api/v1/peers", A)
|
|
assert status == 200 and "beta" in body and "gamma" in body, body
|
|
status, body = local(beta, "/api/v1/peers", B)
|
|
assert status == 200 and "alpha" in body, body
|
|
|
|
with subtest("fan-in: one consolidator reports both peers with their own version"):
|
|
status, body = local(alpha, "/api/v1/peers", A)
|
|
assert status == 200, body
|
|
peers = {p["name"]: p for p in json.loads(body)["peers"]}
|
|
assert sorted(peers) == ["beta", "gamma"], sorted(peers)
|
|
for name in ("beta", "gamma"):
|
|
assert peers[name]["enabled"] and peers[name]["server"]["up"] and peers[name]["api"]["up"], peers[name]
|
|
assert peers[name]["api"]["auth"] == "ok", peers[name]
|
|
assert peers[name]["api"]["peer_version"], peers[name]
|
|
assert peers["gamma"]["url"].startswith("https://"), peers["gamma"]
|
|
|
|
with subtest("proxy: each side returns the other's data, not its own"):
|
|
for m, tok, peer, mine, theirs in ((alpha, A, "beta", "marker-alpha", "marker-beta"),
|
|
(beta, B, "alpha", "marker-beta", "marker-alpha")):
|
|
status, body = local(m, f"/api/v1/peers/{peer}/systemd/units/{theirs}.service", tok)
|
|
assert status == 200 and f"{theirs}.service" in body, (status, body)
|
|
status, body = local(m, f"/api/v1/peers/{peer}/systemd/units", tok)
|
|
assert status == 200 and f"{theirs}.service" in body and f"{mine}.service" not in body, body
|
|
_, hdrs = m.execute(f"curl -s -D - -o /dev/null {bearer(tok)} http://127.0.0.1:{PORT}/api/v1/peers/{peer}/systemd/units")
|
|
assert f"x-slinuxm-peer: {peer}" in hdrs.lower(), hdrs
|
|
# beta's deliberately failing unit is only visible through the proxy.
|
|
assert "broken-beta.service" in local(alpha, "/api/v1/peers/beta/systemd/units", A)[1]
|
|
|
|
with subtest("proxy: the peer's answer is the peer's own data"):
|
|
_, direct = local(beta, "/api/v1/systemd/units", B)
|
|
status, proxied = local(alpha, "/api/v1/peers/beta/systemd/units", A)
|
|
assert status == 200, proxied
|
|
names = lambda body: sorted(u["name"] for u in json.loads(body))
|
|
assert names(proxied) == names(direct), (names(proxied), names(direct))
|
|
assert names(proxied) != names(local(alpha, "/api/v1/systemd/units", A)[1])
|
|
|
|
with subtest("mcp: list_peers and list_systemd_units with peer match REST"):
|
|
out = mcp_tool(alpha, A, "list_peers")
|
|
assert "beta" in out and "gamma" in out, out
|
|
out = mcp_tool(alpha, A, "list_systemd_units", {"peer": "beta"})
|
|
assert "marker-beta.service" in out and "marker-alpha.service" not in out, out
|
|
out = mcp_tool(beta, B, "list_systemd_units", {"peer": "alpha"})
|
|
assert "marker-alpha.service" in out and "marker-beta.service" not in out, out
|
|
|
|
with subtest("tls: gamma is reached over HTTPS through ca_file"):
|
|
status, body = local(alpha, "/api/v1/peers/gamma/systemd/units/marker-gamma.service", A)
|
|
assert status == 200 and "marker-gamma.service" in body, (status, body)
|
|
|
|
with subtest("read-only and no chaining"):
|
|
base = f"http://127.0.0.1:{PORT}/api/v1/peers/beta/systemd/units"
|
|
assert http(alpha, base, A, "-X POST")[0] == 405
|
|
assert http(alpha, base, A, "-X DELETE")[0] == 405
|
|
status, _ = http(alpha, base, A, "-H 'x-slinuxm-hop: 1'")
|
|
assert 400 <= status < 500, status
|
|
assert http(alpha, f"http://127.0.0.1:{PORT}/api/v1/peers/nope", A)[0] == 404
|
|
|
|
with subtest("token rejected: revoke on beta, then restore with a new token file"):
|
|
pid = main_pid(alpha)
|
|
beta.succeed(f"{CLI} token revoke consolidator-alpha")
|
|
wait_status(alpha, A, "beta",
|
|
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected",
|
|
"auth rejected after revoke")
|
|
status, body = local(alpha, "/api/v1/peers/beta/systemd/units", A)
|
|
assert status == 503 and "beta" in body, (status, body)
|
|
# alpha's own credentials still work: the caller never sees a 401.
|
|
assert local(alpha, "/api/v1/systemd/units", A)[0] == 200
|
|
new_tok = beta.succeed(f"{CLI} token create consolidator-alpha-2").strip()
|
|
install_token(alpha, "beta", new_tok)
|
|
wait_status(alpha, A, "beta", lambda s: s["api"]["auth"] == "ok", "auth ok after new token")
|
|
assert local(alpha, "/api/v1/peers/beta/systemd/units", A)[0] == 200
|
|
assert main_pid(alpha) == pid, "revoke and rotation must not restart the service"
|
|
|
|
with subtest("reload: removing a peer from the config drops it on SIGHUP"):
|
|
# The module renders the config into the store, so replace the /etc
|
|
# symlink with a writable copy first: SIGHUP re-reads this path.
|
|
rendered = alpha.succeed(f"readlink -f {CONFIG}").strip()
|
|
text = alpha.succeed(f"cat {rendered}")
|
|
edited = drop_peer(text, "gamma")
|
|
encoded = base64.b64encode(edited.encode()).decode()
|
|
alpha.succeed(f"printf '%s' {shlex.quote(encoded)} | base64 -d > {CONFIG}.new && mv {CONFIG}.new {CONFIG}")
|
|
assert "gamma" not in alpha.succeed(f"cat {CONFIG}")
|
|
pid = main_pid(alpha)
|
|
alpha.succeed("systemctl reload skyg-linux-tracker.service")
|
|
wait_gone(alpha, A, "gamma")
|
|
status, body = local(alpha, "/api/v1/peers", A)
|
|
assert status == 200 and "gamma" not in body and "beta" in body, body
|
|
assert main_pid(alpha) == pid, "removing a peer must not restart the service"
|
|
assert alpha.succeed(f"stat -c '%a' {TOKDIR}/gamma").strip() == "400", "the removed peer's file is untouched"
|
|
assert peer_status(alpha, A, "beta")["api"]["auth"] == "ok", "beta must survive the reload"
|
|
|
|
with subtest("outage: stopping beta's service flips server and api, alpha stays healthy"):
|
|
beta.systemctl("stop skyg-linux-tracker.service")
|
|
wait_status(alpha, A, "beta",
|
|
lambda s: not s["server"]["up"] and not s["api"]["up"],
|
|
"server and api down", timeout=30)
|
|
assert http(alpha, f"http://127.0.0.1:{PORT}/healthz")[0] == 200
|
|
assert local(alpha, "/api/v1/systemd/units", A)[0] == 200
|
|
beta.systemctl("start skyg-linux-tracker.service")
|
|
beta.wait_for_open_port(PORT)
|
|
wait_status(alpha, A, "beta",
|
|
lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "ok",
|
|
"recovery after restart")
|
|
|
|
with subtest("outage: beta VM down, alpha requests are bounded by timeouts"):
|
|
beta.shutdown()
|
|
t0 = time.monotonic()
|
|
status, body = http(alpha, f"http://127.0.0.1:{PORT}/api/v1/peers/beta/systemd/units", A, "--max-time 30")
|
|
elapsed = time.monotonic() - t0
|
|
assert status == 503 and "beta" in body, (status, body)
|
|
assert elapsed < 15, f"proxy call took {elapsed:.1f}s, expected it bounded by request_timeout_secs"
|
|
wait_status(alpha, A, "beta", lambda s: not s["server"]["up"], "server down with the VM off", timeout=30)
|
|
assert local(alpha, "/api/v1/systemd/units", A)[0] == 200
|
|
'';
|
|
}
|