From 782e5dc4409733082b5b78965b8ea2c8b15acf22 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Mon, 19 Oct 2026 16:43:35 -0400 Subject: [PATCH] next phase --- flake.nix | 7 + nix/tests/peers.nix | 328 +++++++++++++++++++ plan/README.md | 32 +- plan/TEMPLATE.md | 89 +++-- plan/phase-5-consolidator/README.md | 133 ++++++++ plan/phase-5-consolidator/contracts.md | 326 ++++++++++++++++++ plan/phase-5-consolidator/decisions.md | 41 +++ plan/phase-5-consolidator/ws-a-config.md | 53 +++ plan/phase-5-consolidator/ws-b-registry.md | 52 +++ plan/phase-5-consolidator/ws-c-rest.md | 45 +++ plan/phase-5-consolidator/ws-d-mcp.md | 44 +++ plan/phase-5-consolidator/ws-e-telemetry.md | 42 +++ plan/phase-5-consolidator/ws-f-nixos-docs.md | 49 +++ plan/phase-5-consolidator/ws-t-tests.md | 66 ++++ 14 files changed, 1280 insertions(+), 27 deletions(-) create mode 100644 nix/tests/peers.nix create mode 100644 plan/phase-5-consolidator/README.md create mode 100644 plan/phase-5-consolidator/contracts.md create mode 100644 plan/phase-5-consolidator/decisions.md create mode 100644 plan/phase-5-consolidator/ws-a-config.md create mode 100644 plan/phase-5-consolidator/ws-b-registry.md create mode 100644 plan/phase-5-consolidator/ws-c-rest.md create mode 100644 plan/phase-5-consolidator/ws-d-mcp.md create mode 100644 plan/phase-5-consolidator/ws-e-telemetry.md create mode 100644 plan/phase-5-consolidator/ws-f-nixos-docs.md create mode 100644 plan/phase-5-consolidator/ws-t-tests.md diff --git a/flake.nix b/flake.nix index 5c95b5e..990e8a7 100644 --- a/flake.nix +++ b/flake.nix @@ -21,6 +21,13 @@ lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.default; }; + # Acceptance test written ahead of the consolidator feature; fails until + # it lands, so it stays out of `checks`. Build: nix build .#tests.x86_64-linux.peers -L + tests.${system}.peers = import ./nix/tests/peers.nix { + pkgs = nixpkgs.legacyPackages.${system}; + module = self.nixosModules.default; + }; + checks.${system} = { package = self.packages.${system}.default; vm = import ./nix/tests/vm.nix { diff --git a/nix/tests/peers.nix b/nix/tests/peers.nix new file mode 100644 index 0000000..7b3983e --- /dev/null +++ b/nix/tests/peers.nix @@ -0,0 +1,328 @@ +{ pkgs, module }: + +# Consolidator acceptance test, written ahead of the feature: it is expected to +# fail at the first peers assertion until the peers feature lands. Everything +# before that (boot, healthz, token minting, marker units on each node's own +# API) must already pass against the current binary. +# +# alpha and beta each run the real slinuxm and declare the other as a peer. +# gamma is a small TLS-only third node that alpha declares as a peer with a +# private CA (ca_file), covering the TLS variant. +# +# Peer config uses the module's freeform `settings` and a LoadCredential +# override, so the module needs no peers options yet. The token files below are +# root-only 0400 stand-ins for agenix paths: they exist from boot with a +# placeholder and the test script overwrites them after minting real tokens. + +let + port = 1984; + tokenDir = "/var/lib/peer-tokens"; + credDir = "/run/credentials/skyg-linux-tracker.service"; + + # Private CA plus a leaf for host `gamma`. + certs = pkgs.runCommand "skyg-peers-certs" { nativeBuildInputs = [ pkgs.openssl ]; } '' + mkdir $out && cd $out + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 \ + -keyout ca.key -out ca.pem -subj '/CN=skyg-test-ca' \ + -addext 'basicConstraints=critical,CA:TRUE' -addext 'keyUsage=critical,keyCertSign,cRLSign' + openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ + -keyout key.pem -out leaf.csr -subj '/CN=gamma' + printf 'subjectAltName=DNS:gamma\nbasicConstraints=CA:FALSE\n' > ext.cnf + openssl x509 -req -in leaf.csr -CA ca.pem -CAkey ca.key -CAcreateserial -days 3650 \ + -extfile ext.cnf -out cert.pem + rm ca.key leaf.csr ext.cnf ca.srl + ''; + + # Timers are short so the test observes flips in seconds; timeouts stay + # below their intervals as the config validation will require. + peerTuning = { + enabled = true; + tcp_interval_secs = 2; + tcp_timeout_secs = 1; + api_interval_secs = 3; + api_timeout_secs = 2; + fail_threshold = 2; + request_timeout_secs = 4; + }; + + peerEntry = name: extra: { + inherit name; + url = "http://${name}:${toString port}"; + token_file = "${credDir}/peer-${name}"; + allow_plain_http = true; + } // extra; + + # Service-side wiring shared by every node: token stand-ins and LoadCredential. + tokenFiles = names: { + systemd.tmpfiles.rules = + [ "d ${tokenDir} 0700 root root -" ] + ++ map (n: "f ${tokenDir}/${n} 0400 root root - pending") names; + systemd.services.skyg-linux-tracker.serviceConfig = { + LoadCredential = map (n: "peer-${n}:${tokenDir}/${n}") names; + # Credentials are copied at start; refresh them on SIGHUP so a rewritten + # token file is picked up without a restart. + RefreshOnReload = "credentials"; + }; + }; + + mkNode = { peers }: { ... }: { + imports = [ module ]; + services.skyg-linux-tracker = { + enable = true; + inherit port; + bind = "0.0.0.0"; + openFirewall = true; + settings = pkgs.lib.optionalAttrs (peers != [ ]) { + peers = peerTuning // { peer = peers; }; + }; + }; + }; +in +pkgs.testers.runNixOSTest { + name = "skyg-linux-tracker-peers"; + + nodes.alpha = { ... }: { + imports = [ + (mkNode { + peers = [ + (peerEntry "beta" { }) + ((peerEntry "gamma" { + url = "https://gamma:${toString port}"; + ca_file = "${certs}/ca.pem"; + }) // { allow_plain_http = false; }) + ]; + }) + (tokenFiles [ "beta" "gamma" ]) + ]; + systemd.services.marker-alpha = { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; }; + }; + }; + + nodes.beta = { ... }: { + imports = [ + (mkNode { peers = [ (peerEntry "alpha" { }) ]; }) + (tokenFiles [ "alpha" ]) + ]; + systemd.services.marker-beta = { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; }; + }; + # Deliberately failing unit, so beta's data differs from alpha's. + systemd.services.broken-beta = { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { Type = "oneshot"; ExecStart = "${pkgs.coreutils}/bin/false"; }; + }; + }; + + nodes.gamma = { ... }: { + imports = [ (mkNode { peers = [ ]; }) ]; + services.skyg-linux-tracker.tls = { + enable = true; + certFile = "${certs}/cert.pem"; + keyFile = "${certs}/key.pem"; + }; + systemd.services.marker-gamma = { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${pkgs.coreutils}/bin/true"; }; + }; + }; + + testScript = '' + import json + import time + + CLI = "sudo -u skyg-linux-tracker slinuxm" + JSON = "-H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream'" + INIT = '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"vm","version":"0"}}}' + TOKDIR = "${tokenDir}" + PORT = ${toString port} + + def bearer(tok): + return f"-H 'Authorization: Bearer {tok}'" + + def http(m, url, tok=None, extra=""): + """Returns (status_code, body).""" + auth = bearer(tok) if tok else "" + out = m.succeed(f"curl -s -w '\\n%{{http_code}}' {auth} {extra} {url}") + body, _, status = out.rpartition("\n") + return int(status), body + + def local(m, path, tok): + return http(m, f"http://127.0.0.1:{PORT}{path}", tok) + + def peer_status(m, tok, peer): + status, body = local(m, f"/api/v1/peers/{peer}", tok) + assert status == 200, f"/api/v1/peers/{peer} -> {status}: {body}" + return json.loads(body) + + def poll(fn, what, timeout=60): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + try: + if fn(): + return + except Exception as e: # noqa: BLE001 + last = e + time.sleep(1) + raise AssertionError(f"timed out waiting for: {what} (last error: {last})") + + def wait_status(m, tok, peer, check, what, timeout=60): + poll(lambda: check(peer_status(m, tok, peer)), f"{peer}: {what}", timeout) + + def mcp_call(m, tok, method, params): + """One-shot MCP session: initialize, initialized, then the call.""" + url = f"http://127.0.0.1:{PORT}/mcp" + m.succeed(f"curl -sf -D /tmp/h {bearer(tok)} {JSON} -d '{INIT}' {url} >/dev/null") + sid = m.succeed("awk -F': ' 'tolower($1)==\"mcp-session-id\"{print $2}' /tmp/h | tr -d '\\r'").strip() + sess = f"-H 'Mcp-Session-Id: {sid}'" if sid else "" + m.succeed(f"curl -sf {bearer(tok)} {sess} {JSON} -d '{{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}}' {url} >/dev/null") + body = json.dumps({"jsonrpc": "2.0", "id": 2, "method": method, "params": params}) + return m.succeed(f"curl -sf {bearer(tok)} {sess} {JSON} -d '{body}' {url}") + + def mcp_tool(m, tok, name, args=None): + return mcp_call(m, tok, "tools/call", {"name": name, "arguments": args or {}}) + + def install_token(m, peer, tok): + # Stand-in for an agenix secret: root-only 0400, then reload (SIGHUP) + # so the credential is refreshed without a restart. + m.succeed(f"printf '%s\\n' '{tok}' > {TOKDIR}/{peer}.new && chmod 0400 {TOKDIR}/{peer}.new && mv {TOKDIR}/{peer}.new {TOKDIR}/{peer}") + m.succeed("systemctl reload skyg-linux-tracker.service") + + start_all() + + # ---- Phase 1: nothing here depends on the peers feature ---------------- + + with subtest("boot: all services up, healthz public"): + for m in (alpha, beta, gamma): + m.wait_for_unit("skyg-linux-tracker.service") + m.wait_for_open_port(PORT) + alpha.wait_for_unit("marker-alpha.service") + beta.wait_for_unit("marker-beta.service") + gamma.wait_for_unit("marker-gamma.service") + for m, url in ((alpha, f"http://127.0.0.1:{PORT}"), (beta, f"http://127.0.0.1:{PORT}")): + assert http(m, f"{url}/healthz")[0] == 200 + assert http(alpha, f"http://beta:{PORT}/healthz")[0] == 200, "alpha cannot reach beta" + assert http(beta, f"http://alpha:{PORT}/healthz")[0] == 200, "beta cannot reach alpha" + assert http(gamma, f"https://127.0.0.1:{PORT}/healthz", extra="--cacert ${certs}/ca.pem --resolve gamma:1984:127.0.0.1 -k")[0] == 200 + + with subtest("tokens: minted per consolidator by the service user"): + # Each node mints the token its consolidator will use against it, plus + # a separate token for local assertions. + tok_alpha_for_beta = alpha.succeed(f"{CLI} token create consolidator-beta").strip() + tok_beta_for_alpha = beta.succeed(f"{CLI} token create consolidator-alpha").strip() + tok_gamma_for_alpha = gamma.succeed(f"{CLI} token create consolidator-alpha").strip() + A = alpha.succeed(f"{CLI} token create test-alpha").strip() + B = beta.succeed(f"{CLI} token create test-beta").strip() + G = gamma.succeed(f"{CLI} token create test-gamma").strip() + for t in (tok_alpha_for_beta, tok_beta_for_alpha, tok_gamma_for_alpha, A, B, G): + assert t, "empty token" + + with subtest("data: marker units are visible on each node's own API only"): + _, a_units = local(alpha, "/api/v1/systemd/units", A) + _, b_units = local(beta, "/api/v1/systemd/units", B) + assert "marker-alpha.service" in a_units and "marker-beta.service" not in a_units + assert "marker-beta.service" in b_units and "marker-alpha.service" not in b_units + assert "broken-beta.service" in b_units and "broken-beta.service" not in a_units + status, one = local(beta, "/api/v1/systemd/units/marker-beta.service", B) + assert status == 200 and "marker-beta.service" in one + + with subtest("token files: placeholders exist root-only 0400 before any token is installed"): + for m, p in ((alpha, "beta"), (alpha, "gamma"), (beta, "alpha")): + assert m.succeed(f"stat -c '%a %U' {TOKDIR}/{p}").strip() == "400 root" + + # ---- Phase 2: peers (expected to fail here until the feature lands) ---- + + with subtest("peers: declared but token is still a placeholder -> auth rejected, then recovery"): + # Before the real token is installed the peer API check must report + # server up, api up (it answered 401), auth rejected. + wait_status(alpha, A, "beta", + lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected", + "server up, api up, auth rejected with the placeholder token") + + with subtest("peers: install real tokens, status recovers without a restart"): + install_token(alpha, "beta", tok_beta_for_alpha) + install_token(alpha, "gamma", tok_gamma_for_alpha) + install_token(beta, "alpha", tok_alpha_for_beta) + for m, tok, peer in ((alpha, A, "beta"), (alpha, A, "gamma"), (beta, B, "alpha")): + wait_status(m, tok, peer, + lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "ok", + "server up, api up, auth ok") + status, body = local(alpha, "/api/v1/peers", A) + assert status == 200 and "beta" in body and "gamma" in body, body + status, body = local(beta, "/api/v1/peers", B) + assert status == 200 and "alpha" in body, body + + with subtest("proxy: each side returns the other's data, not its own"): + for m, tok, peer, mine, theirs in ((alpha, A, "beta", "marker-alpha", "marker-beta"), + (beta, B, "alpha", "marker-beta", "marker-alpha")): + status, body = local(m, f"/api/v1/peers/{peer}/systemd/units/{theirs}.service", tok) + assert status == 200 and f"{theirs}.service" in body, (status, body) + status, body = local(m, f"/api/v1/peers/{peer}/systemd/units", tok) + assert status == 200 and f"{theirs}.service" in body and f"{mine}.service" not in body, body + _, hdrs = m.execute(f"curl -s -D - -o /dev/null {bearer(tok)} http://127.0.0.1:{PORT}/api/v1/peers/{peer}/systemd/units") + assert f"x-slinuxm-peer: {peer}" in hdrs.lower(), hdrs + # beta's deliberately failing unit is only visible through the proxy. + assert "broken-beta.service" in local(alpha, "/api/v1/peers/beta/systemd/units", A)[1] + + with subtest("mcp: list_peers and list_systemd_units with peer match REST"): + out = mcp_tool(alpha, A, "list_peers") + assert "beta" in out and "gamma" in out, out + out = mcp_tool(alpha, A, "list_systemd_units", {"peer": "beta"}) + assert "marker-beta.service" in out and "marker-alpha.service" not in out, out + out = mcp_tool(beta, B, "list_systemd_units", {"peer": "alpha"}) + assert "marker-alpha.service" in out and "marker-beta.service" not in out, out + + with subtest("tls: gamma is reached over HTTPS through ca_file"): + status, body = local(alpha, "/api/v1/peers/gamma/systemd/units/marker-gamma.service", A) + assert status == 200 and "marker-gamma.service" in body, (status, body) + + with subtest("read-only and no chaining"): + base = f"http://127.0.0.1:{PORT}/api/v1/peers/beta/systemd/units" + assert http(alpha, base, A, "-X POST")[0] == 405 + assert http(alpha, base, A, "-X DELETE")[0] == 405 + status, _ = http(alpha, base, A, "-H 'x-slinuxm-hop: 1'") + assert 400 <= status < 500, status + assert http(alpha, f"http://127.0.0.1:{PORT}/api/v1/peers/nope", A)[0] == 404 + + with subtest("token rejected: revoke on beta, then restore with a new token file"): + beta.succeed(f"{CLI} token revoke consolidator-alpha") + wait_status(alpha, A, "beta", + lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "rejected", + "auth rejected after revoke") + status, body = local(alpha, "/api/v1/peers/beta/systemd/units", A) + assert status == 503 and "beta" in body, (status, body) + # alpha's own credentials still work: the caller never sees a 401. + assert local(alpha, "/api/v1/systemd/units", A)[0] == 200 + new_tok = beta.succeed(f"{CLI} token create consolidator-alpha-2").strip() + install_token(alpha, "beta", new_tok) + wait_status(alpha, A, "beta", lambda s: s["api"]["auth"] == "ok", "auth ok after new token") + assert local(alpha, "/api/v1/peers/beta/systemd/units", A)[0] == 200 + + with subtest("outage: stopping beta's service flips server and api, alpha stays healthy"): + beta.systemctl("stop skyg-linux-tracker.service") + wait_status(alpha, A, "beta", + lambda s: not s["server"]["up"] and not s["api"]["up"], + "server and api down", timeout=30) + assert http(alpha, f"http://127.0.0.1:{PORT}/healthz")[0] == 200 + assert local(alpha, "/api/v1/systemd/units", A)[0] == 200 + assert peer_status(alpha, A, "gamma")["api"]["up"], "gamma must be unaffected" + beta.systemctl("start skyg-linux-tracker.service") + beta.wait_for_open_port(PORT) + wait_status(alpha, A, "beta", + lambda s: s["server"]["up"] and s["api"]["up"] and s["api"]["auth"] == "ok", + "recovery after restart") + + with subtest("outage: beta VM down, alpha requests are bounded by timeouts"): + beta.shutdown() + t0 = time.monotonic() + status, body = http(alpha, f"http://127.0.0.1:{PORT}/api/v1/peers/beta/systemd/units", A, "--max-time 30") + elapsed = time.monotonic() - t0 + assert status == 503 and "beta" in body, (status, body) + assert elapsed < 15, f"proxy call took {elapsed:.1f}s, expected it bounded by request_timeout_secs" + wait_status(alpha, A, "beta", lambda s: not s["server"]["up"], "server down with the VM off", timeout=30) + assert local(alpha, "/api/v1/systemd/units", A)[0] == 200 + ''; +} diff --git a/plan/README.md b/plan/README.md index 8f1f5c8..be1f6f0 100644 --- a/plan/README.md +++ b/plan/README.md @@ -11,15 +11,33 @@ Where other things live: | Long-form docs | [`../docs/`](../docs/README.md) | | Records of finished workstreams | [`../docs/workstreams/`](../docs/workstreams/) | +## Phase folder layout + +Each phase is a folder `plan//` (for example `plan/phase-5-consolidator/`) so workstreams can be worked in parallel by separate agents or people: + +| File | Purpose | +|---|---| +| `README.md` | Phase overview: goal, scope, grounding in code, workstream index (ID, file, owner, depends on, can start when, owned files), dependency graph, parallel lanes, risks, verification, exit criteria | +| `contracts.md` | Shared interfaces (types, traits, fakes, REST/MCP/OTel shapes, config surface) that let streams start against fakes before their dependency lands | +| `decisions.md` | Open-decisions table and decision records | +| `ws--.md` | One file per workstream: header table, goal, tasks, inputs, done when, handoff, log | + +Rules: + +- **Do not touch**: a workstream edits only the files in its "Owned files" row. Other streams' files are in its "Do not touch" row. For the few shared files (for example `src/state.rs`, `src/api/mod.rs`, `flake.nix`) the header names the integration seam and which stream edits it first; prefer one stream wiring the seam up front so the rest only fill their own files. +- **Contracts change only through `contracts.md`**: edit it first, then add a dated note to the Log of every affected workstream file. +- **Claim a workstream** by setting its `Owner` field (name or agent id) and committing that change before starting; set `Status` as you go. Do not start a stream whose "Can start when" is not met unless you work against the contracts and fakes. +- Log progress, discoveries and handoffs in the workstream's own `Log` (dated), not in other files. + ## Proposing a phase plan -1. Copy [`TEMPLATE.md`](TEMPLATE.md) to `plan/.md` (for example `plan/phase-5-alerting.md`). -2. Fill in: +1. Copy the folder layout from [`TEMPLATE.md`](TEMPLATE.md) to `plan//` (for example `plan/phase-5-alerting/`). +2. Fill in the phase `README.md`: - **Goal**: what changes for users or operators, in a few sentences. - **Scope**: what is in, and explicitly what is out. - - **Workstreams**: independent tracks with owned files, dependencies and done criteria, so they can run in parallel without merge conflicts. - - **Open decisions**: questions that need a human call, each with options and a recommendation. + - **Workstream index** and **parallel lanes**: independent tracks with owned files, dependencies and what can run from day one, so they run in parallel without merge conflicts. - **Exit criteria**: how you will know the phase is done (commands to run, behaviour to demonstrate). -3. Review and settle the open decisions in the plan file. -4. When the plan is accepted, move the work into [`../TODO.md`](../TODO.md): add the phase milestone and one checklist per workstream with stable task IDs. Fold the accepted decisions into the Decisions section of [`../ARCHITECTURE.md`](../ARCHITECTURE.md). -5. When the phase finishes, move the plan file (and any per-workstream notes) to `docs/workstreams/` as an archived record and collapse its tasks in `TODO.md` into the Completed table. +3. Write `contracts.md` for anything two streams share, and `decisions.md` for questions that need a human call, each with options and a recommendation. If a stream would block on another, add a small first task that lands the shared stubs (for example a fake). +4. Review and settle the open decisions in `decisions.md`. +5. When the plan is accepted, move the work into [`../TODO.md`](../TODO.md): add the phase milestone and one checklist per workstream with stable task IDs. Fold the accepted decisions into the Decisions section of [`../ARCHITECTURE.md`](../ARCHITECTURE.md). +6. When the phase finishes, move the whole phase folder to `docs/workstreams/` as an archived record and collapse its tasks in `TODO.md` into the Completed table. diff --git a/plan/TEMPLATE.md b/plan/TEMPLATE.md index c7fb7d9..3bc0ebd 100644 --- a/plan/TEMPLATE.md +++ b/plan/TEMPLATE.md @@ -1,3 +1,18 @@ +# Phase folder template + +Copy this layout to `plan//`. See [`README.md`](README.md) for the rules (Do not touch, contracts, claiming a workstream). + +``` +plan// + README.md phase overview (template below) + contracts.md shared interface contracts (omit if streams share nothing) + decisions.md open decisions and records + ws--.md one per workstream (template below) +``` + +## `README.md` (phase overview) + +```markdown # Phase N: | Field | Value | @@ -8,44 +23,78 @@ | Depends on | | ## Goal - What changes for users or operators once this phase is done, in a few sentences. ## Scope - In scope: - - - Out of scope: - - -## Workstreams +## Grounding in the current code +The files and types the work builds on. -One block per independent track. Name the files each owns so tracks do not collide. +## Workstream index +| ID | File | Owner | Depends on | Can start when | Owned files | +|---|---|---|---|---|---| -### WS-X: +### Dependency graph +(ascii) -- Goal: -- Owned files: -- Depends on: -- Tasks (IDs to be ticked in `TODO.md`): - - **X1** -- Done when: +### Parallel lanes +Which workstreams may run concurrently from day one, and what each later wave unlocks. -## Open decisions +## Risks +- +## Verification plan +Numbered, concrete steps. + +## Exit criteria +Concrete, checkable. Commands to run and behaviour to demonstrate, for example `cargo clippy --all-targets -- -D warnings`, `cargo test`, and a live check on a real host. +- +``` + +## `contracts.md` + +Rust signatures, JSON shapes, config keys and fakes shared by more than one stream; a "change rule" stating that contracts change only here, with a log note in each affected workstream file. + +## `decisions.md` + +```markdown | ID | Question | Options | Recommendation | Decision | |---|---|---|---|---| | | | | | | +``` -## Risks +## `ws--.md` (workstream file) -- +```markdown +# WS-X: -## Exit criteria +| Field | Value | +|---|---| +| Status | Not started / In progress / Blocked / Done | +| Owner | | +| Depends on | | +| Blocks | | +| Can start when | | +| Owned files | | +| Do not touch | files owned by other streams; for shared files, the exact integration seam and which stream edits it first | -Concrete, checkable. Commands to run and behaviour to demonstrate, for example `cargo clippy --all-targets -- -D warnings`, `cargo test`, and a live check on a real host. +## Goal -- +## Tasks (IDs to be ticked in `TODO.md`) +- **X1** + +## Inputs +Which `contracts.md` sections this stream relies on; which fakes it uses. + +## Done when + +## Handoff +What this stream must publish for others. + +## Log +- YYYY-MM-DD: +``` diff --git a/plan/phase-5-consolidator/README.md b/plan/phase-5-consolidator/README.md new file mode 100644 index 0000000..580fec8 --- /dev/null +++ b/plan/phase-5-consolidator/README.md @@ -0,0 +1,133 @@ +# Phase 5: Consolidator (peers) + +| Field | Value | +|---|---| +| Status | Draft | +| Owner | | +| Date | 2026-10-19 | +| Depends on | Phase 4 / WS-H (NixOS module and VM test) done | + + +This phase is a folder so workstreams can be claimed and worked in parallel by separate agents or people. Start here, then read [contracts.md](contracts.md) and your workstream file only. Rules: every workstream edits only its own files (see its "Do not touch" list); shared interfaces change only through contracts.md. How the folder works: [../README.md](../README.md). + +## Goal + +One `slinuxm` binary can also act as a consolidator. An operator declares other machines running `slinuxm` as **peers** in configuration, declaratively: on NixOS via `services.skyg-linux-tracker.peers.`, with each peer's bearer token supplied by an agenix-managed file. The consolidator then (1) health-checks each peer in two ways, "server up" (TCP connect) and "API running" (authenticated API call), (2) exposes that status over REST, MCP and OTel, and (3) proxies read-only requests to a peer so one MCP endpoint or one REST base URL covers the fleet. A peer-less instance (the default, `[peers] enabled = false`) behaves exactly as today: same routes, same 8 MCP tools, same metrics, no new background tasks, no new required config. + +## Scope + +In scope: + +- `[peers]` config section with a declarative peer list (`[[peers.peer]]`: name, url, `token_file`, optional `ca_file`), a peer registry in `AppState`, a read-only `slinuxm peer list|check` diagnostic CLI. No peers table, no migration, no secrets in the DB or the config file. +- Per-peer TCP and API health loops with in-memory state and last-change timestamps. +- REST `GET /api/v1/peers`, `GET /api/v1/peers/{peer}`, and a GET-only pass-through under `/api/v1/peers/{peer}/...`. +- MCP: optional `peer` argument on the existing tools plus `list_peers` and `get_peer`. +- OTel gauges for peer health, trace propagation to peers. +- NixOS module options (`peers.peers.` with `tokenFile`, e.g. `config.age.secrets..path`), VM test with a third machine, an agenix example in the docs, README, `config.example.toml`, ARCHITECTURE.md decisions (folded in by the orchestrator after acceptance). + +Out of scope (Backlog with reasons when accepted): + +- Fleet-wide aggregation or merged views (see D8), persisted health history, alerting, peer discovery. +- Chained consolidators (a peer that is itself a consolidator, D12). +- Re-exporting a peer's OTel signals (each peer exports its own telemetry). +- Any write path to a peer or to the host beyond the state dir. Proxying is GET only. +- Runtime peer management (`peer add|remove`, a DB table): peers are declarative only in v1 (D1). `--insecure` TLS skip. + +## Grounding in the current code + +- `AppState` (`state.rs`) holds `Arc`s, `db: Option` (None when `auth.enabled = false`) and the shutdown token. Add `peers: Option>`; `None` means the feature is off and every new route and tool is absent. Peers need no DB. +- Errors are `AppError` (`error.rs`): `NotFound`, `BadRequest`, `Unauthorized`, `Unavailable`, `Internal`; MCP maps via `From for ErrorData`. Peer failures must fit these variants (D9). +- The token CLI and server share SQLite for inbound tokens; peers do not touch it. SIGHUP is already taken by TLS reload; the shared signal handler can fan out to a peers reload (B4). +- `/healthz` is public and does not validate a token, so it cannot prove "API running with our credentials". No lightweight authenticated endpoint exists today. +- `reqwest` (blocking plus rustls) is already in `Cargo.lock` via `opentelemetry-otlp`; `hyper-util`, `tokio-rustls`, `rcgen` (dev) are direct deps. The unit and module already allow `AF_INET`/`AF_INET6`, so outbound sockets need no sandbox change. +- Background tasks live in `telemetry::spawn_exporters`-style spawns and select on `shutdown`; peer loops follow the same rule (AGENTS.md). + +## Files in this phase + +- [contracts.md](contracts.md): shared Rust, REST, MCP, OTel, config and NixOS contracts, fakes, and the VM test's config contract. +- [decisions.md](decisions.md): open-decisions table (D1 to D15) and the DEC-13 section (outbound peer credentials are external files). The config surface ("Config surface") is section 6 of contracts.md. +- Workstreams: [ws-t-tests.md](ws-t-tests.md), [ws-a-config.md](ws-a-config.md), [ws-b-registry.md](ws-b-registry.md), [ws-c-rest.md](ws-c-rest.md), [ws-d-mcp.md](ws-d-mcp.md), [ws-e-telemetry.md](ws-e-telemetry.md), [ws-f-nixos-docs.md](ws-f-nixos-docs.md). + + +## Workstream index + +Claim a workstream by putting your name in its Owner field (header table of its file) and committing that change. Task IDs (T1..T7, A0..A4, B1..B5, C1..C4, D1..D3, E1..E2, F1..F4) are ticked in `TODO.md` once the plan is accepted. + +| ID | File | Owner | Depends on | Can start when | Owned files (summary) | +|---|---|---|---|---|---| +| WS-T | [ws-t-tests.md](ws-t-tests.md) | | none | day one | `nix/tests/peers.nix` (written), `dev/test.nu`, `devenv.nix`, `flake.nix` peers output, command lists in `AGENTS.md`/`README.md`, `docs/consolidator.md` Testing section | +| WS-A | [ws-a-config.md](ws-a-config.md) | | none | day one (A0 first) | `src/peers/{mod,types,error,transport,testing,config,secrets,cli}.rs`, `src/config.rs`, `src/main.rs` (`Command::Peer`), `config.example.toml` `[peers]` | +| WS-B | [ws-b-registry.md](ws-b-registry.md) | | A0 (A1 to A3 for real config) | A0 merged | `src/peers/{client,health,registry}.rs`, `src/api/ping.rs`, `Cargo.toml`, `src/main.rs` `serve()`, `tests/peers_registry.rs` | +| WS-C | [ws-c-rest.md](ws-c-rest.md) | | A0 (B for the final e2e) | A0 merged | `src/api/peers.rs`, `src/peers/proxy.rs`, `docs/api.md` | +| WS-D | [ws-d-mcp.md](ws-d-mcp.md) | | A0 (B and C2 for the final e2e) | A0 merged (rmcp spike earlier) | `src/mcp/**` | +| WS-E | [ws-e-telemetry.md](ws-e-telemetry.md) | | A0 (B for the final e2e) | A0 merged | `src/telemetry/{mod,metrics_export}.rs` | +| WS-F | [ws-f-nixos-docs.md](ws-f-nixos-docs.md) | | C, D, E (F1/F4 drafts after A1) | F1: A1 settled; F2: T, C, D, E green | `nix/module.nix`, `nix/tests/vm.nix`, `systemd/*`, `README.md` peers section, `docs/consolidator.md`, `tests/peers.rs`, `tests/common/mod.rs` | + +Original dependency statement, kept for reference: **T first** (tests written ahead of the feature) -> A -> B -> {C, D, E} (parallel) -> F. T goes green at the end of E. Tests for each stream ship with it. B defines a `PeerTransport` trait with an in-memory fake (same pattern as DEC-2) so C, D and E test without sockets. In this split the trait, the fake and the shared types land in task A0, so C, D and E no longer wait for B. + +### Dependency graph + +``` + WS-T (tests, runner, peers.nix) -------------------------------+--> goes green when A..E land + | + WS-A: A0 (contract stubs) --+--> A1 -> A2 -> A3 -> A4* | + | | + +--> WS-B (client, health, registry)+--+ + +--> WS-C (REST, proxy.rs) | (against FakeTransport, + +--> WS-D (MCP) | real B at integration) + +--> WS-E (telemetry) ---+ + \ | | / + v v v v + WS-F (module, VM check, e2e, docs) + + * A4 `peer check` is wired to the real transport when B1/B2 land. +``` + +### Parallel lanes + +Day one, no coordination needed (three lanes at once): + +1. **WS-T**: T7 runner, docs Testing section; `peers.nix` is done. +2. **WS-A**: A0 first (small PR: types, trait, fake, seams), then A1 to A4. +3. **Contract stubs**: A0 is the contract-stubs step; everyone else can also begin reading code and drafting against [contracts.md](contracts.md) in their branches, and WS-D can run the rmcp conditional-tools spike. + +After A0 merges, the next wave runs fully in parallel, all against `FakeTransport` / `registry_with` / `static_registry`: + +- **WS-B** (real client, loops, registry, ping), **WS-C** (REST, `proxy.rs`), **WS-D** (MCP), **WS-E** (telemetry); WS-A continues A1 to A4; WS-F can start F1 (module) and the doc drafts once A1 has settled the config keys. +- Real dependencies that remain: D1 and C2 share `PeerRegistry::get` (D uses the A0 forwarder until C2 lands, then picks up the hardened one with no code change); C/D/E "done" is proved against the fake and rechecked with the real registry once B lands; A4 `peer check` needs B1/B2. + +What each wave unlocks: + +- **A0 merged**: B, C, D, E compile and can merge; F1 can reference real option/config names. +- **B merged**: end-to-end checks for C, D, E with real loops; `slinuxm peer check` works; `tests/peers_registry.rs`. +- **C2 merged**: D1 uses the hardened proxy. +- **E merged (with A to D)**: WS-T's `peers.nix` should pass. +- **T green + C, D, E merged**: F2 promotes the VM test to `checks`, F3/F4 finish; phase exit. + +Contract changes during the phase follow the rule at the top of [contracts.md](contracts.md): edit contracts.md first, log in the affected workstream files. + +## Risks + +- Tool-list growth and optional parameters change MCP schemas; clients caching the list see a different one only when peers are enabled. Mitigated by gating and the snapshot test. +- A consolidator token on a peer is a lateral-movement asset (read access to journals and containers). Mitigated by per-consolidator tokens, agenix-encrypted at rest, delivered via `LoadCredential`, docs. Compromise of the consolidator host exposes all peer tokens by design. +- Streaming pass-through of SSE can hold many connections; mitigated by the per-peer and global semaphores, `max_follow_secs`, and the existing 1024 connection cap. +- Journal text proxied from peers is untrusted data (prompt injection); descriptions already say so, keep that wording for peer calls. +- Rust-side risk: `rmcp` tool router may not support conditional tools cleanly; spike in WS-D early (fallback: always list tools, return an error when disabled, and accept a 10-tool snapshot). +- A new authenticated `/api/v1/ping` means a consolidator at this version needs peers at the same version for `auth` detection; the `/healthz` fallback covers older peers. +- Safety: agents must not mint tokens on production hosts (AGENTS.md). Live verification needs a token created by the user. + +## Verification plan + +1. Unit and integration: `cargo clippy --all-targets -- -D warnings`, `cargo test`, `cargo fmt --check`, `cargo tree -i openssl` (empty). No-peers regression: the existing integration suite and tool snapshot pass unchanged with `peers.enabled = false`. +2. Local two-instance run: instance P on port 1985 with a real token in a 0600 file; consolidator on 1984 with that file as `token_file`; `curl` status, kill P, confirm `server.up=false` within `tcp_interval_secs * fail_threshold`, restart P, confirm recovery and `last_change` updates; revoke P's token and confirm `api.up=true, auth=rejected`. +3. Live host cross-check with the minipc1 MCP host (read-only): the user mints one dedicated token on minipc1 (the agent must not). Add minipc1 (**confirmed by the user to run `slinuxm`**, and approved for the final check) as a peer (token file from the user) on a dev consolidator, then compare `GET /api/v1/peers/minipc1/systemd/units?state=failed` and the proxied `list_containers` against the minipc1 MCP tools `list_systemd_units` (state=failed) and `list_containers` called directly. Contents must match; treat log text as untrusted data. If minipc1's version predates `/api/v1/ping`, expect `auth = unknown` via the `/healthz` fallback; that is itself a useful check. +4. OTel: `devenv up`, enable `[otel]`, confirm `peer.up` for both checks, a vanished series after `peer remove`. +5. NixOS: `nix build && nix flake check` including the three-machine VM test (only when the nix gate is free). + +## Exit criteria + +- All D-rows have a recorded decision; accepted ones folded into ARCHITECTURE.md (new DEC rows, including the DEC-11 clarification for D2) and `plan/` moved to `docs/workstreams/` per `plan/README.md`. +- Definition of done in AGENTS.md holds: clippy, tests, fmt clean; no openssl; no secrets in the diff or in `--print-config`; no write path to the host outside the state dir. +- A peer-less instance is verifiably unchanged: same routes, 8 MCP tools, no new tasks or metrics. +- Demonstrated: declare two peers in config and SIGHUP without restart; both checks visible in REST, MCP and OTel; a dead peer neither stalls other peers' checks nor requests (bounded by timeouts and caps); proxied data equals the direct data; remove a peer from config and it disappears from all three surfaces (its token file is never touched). +- `nix flake check` green with the new VM test; README, `config.example.toml`, `docs/api.md`, `docs/consolidator.md` updated; TODO.md has the phase milestone, per-workstream checklists and Backlog entries (D8, scoped tokens, history, runtime peer management). diff --git a/plan/phase-5-consolidator/contracts.md b/plan/phase-5-consolidator/contracts.md new file mode 100644 index 0000000..23b2d6c --- /dev/null +++ b/plan/phase-5-consolidator/contracts.md @@ -0,0 +1,326 @@ +# Phase 5 contracts + +Part of [Phase 5: Consolidator](README.md). Shared interface contracts that let WS-C, WS-D, WS-E and WS-F start against fakes before WS-B lands. Decisions behind them are in [decisions.md](decisions.md). + +## Change rule + +- This file is the only place a shared interface changes. Do not alter a signature, JSON shape, metric name or option name in code first. +- A contract change = edit this file in its own small commit/PR, then add a dated note to the `Log` of every affected workstream file ([ws-a](ws-a-config.md), [ws-b](ws-b-registry.md), [ws-c](ws-c-rest.md), [ws-d](ws-d-mcp.md), [ws-e](ws-e-telemetry.md), [ws-f](ws-f-nixos-docs.md), [ws-t](ws-t-tests.md)). +- The WS-T test (`nix/tests/peers.nix`) is the executable form of the REST/config parts. If a change breaks it, the change is wrong or WS-T must be updated in the same PR. +- Signatures below are the intended shape, not frozen syntax: names, field types and the JSON in sections 3 to 5 are the contract. Lifetimes, derives and module-private helpers are the implementer's call. + +## Landing order of the stubs + +Task **A0** (in [ws-a-config.md](ws-a-config.md)) lands sections 1, 2 and 6 as compilable stubs in `src/peers/` plus the `AppState.peers` field and route/tool seams. Until A0 is merged, C/D/E/F code against this file and rebase onto A0 afterwards. + +## 1. Rust types (`src/peers/types.rs`, `src/peers/error.rs`) + +```rust +/// Validated peer name, regex ^[a-z0-9][a-z0-9_-]{0,31}$ (A2). Cheap to clone. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct PeerName(String); +impl PeerName { pub fn parse(s: &str) -> Result; pub fn as_str(&self) -> &str; } + +/// Token wrapper with redacted Debug/Display. Never Serialize (A3). +pub struct Secret(/* private */); +impl Secret { pub fn expose(&self) -> &str; } // only the transport calls this + +/// One `[[peers.peer]]` entry after validation (src/peers/config.rs). +#[derive(Debug, Clone, PartialEq)] +pub struct PeerConfig { + pub name: PeerName, + pub url: url::Url, // http/https, no userinfo/query/fragment (D11) + pub token_file: PathBuf, // absolute; contents never held here + pub ca_file: Option, + pub allow_plain_http: bool, // default false + pub enabled: bool, // default true +} + +/// `[peers]` section (src/peers/config.rs); defaults in section 6. +pub struct PeersConfig { + pub enabled: bool, + pub tcp_interval_secs: u64, pub tcp_timeout_secs: u64, + pub api_interval_secs: u64, pub api_timeout_secs: u64, + pub fail_threshold: u32, pub request_timeout_secs: u64, + pub max_response_bytes: u64, + pub max_concurrent_per_peer: usize, pub max_concurrent_total: usize, + pub deny_cidrs: Vec, + pub peer: Vec, +} +// Added to Config as `pub peers: PeersConfig` (A1, `#[serde(default)]`). +``` + +### PeerStatus model (serialized as-is by REST, `GET /api/v1/peers[/{peer}]`) + +```rust +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PeerStatus { + pub name: PeerName, + pub enabled: bool, + pub url: String, // display form; no userinfo by construction + pub server: ServerStatus, // "server up": TCP connect + pub api: ApiStatus, // "API running": authenticated GET /api/v1/ping +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ServerStatus { + pub up: bool, + pub since: Option, // last change; None until first check; resets on restart + pub last_checked: Option, + pub last_error: Option, // class string, no secrets, no URL + pub latency_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ApiStatus { + pub up: bool, // true also when the peer answered 401 (D4) + pub since: Option, + pub last_checked: Option, + pub last_error: Option, + pub latency_ms: Option, + pub auth: AuthState, // "ok" | "rejected" | "unknown" + pub peer_version: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuthState { Ok, Rejected, Unknown } + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CheckClass { Ok, Unauthorized, HttpError, Timeout, Unreachable, TlsError } // B2 +``` + +`Timestamp` is RFC 3339 UTC in JSON (same type the existing DTOs use for times; if none exists, `time::OffsetDateTime`/`chrono` as already in the tree, decided in A0 and noted in the A log). Initial state before the first check: `up = false`, `auth = unknown`, timestamps `null`. + +## 2. Transport (`src/peers/transport.rs`) + +```rust +/// One GET to a peer. `path` is already validated by the proxy layer (C2); query is pre-filtered. +pub struct PeerRequest { + pub path: String, // e.g. "/api/v1/systemd/units" + pub query: Vec<(String, String)>, + pub headers: Vec<(HeaderName, HeaderValue)>, // traceparent, x-slinuxm-hop; never Authorization + pub accept: Option, // "text/event-stream" for follow +} + +pub struct PeerResponse { + pub status: u16, + pub content_type: Option, + pub body: PeerBody, // Bytes (capped) for JSON, or a byte stream for SSE +} +pub enum PeerBody { Bytes(Bytes), Stream(BoxStream<'static, Result>) } + +#[async_trait] +pub trait PeerTransport: Send + Sync { + /// TCP connect to host:port under `timeout`. Returns connect latency. + async fn tcp_check(&self, peer: &PeerConfig, timeout: Duration) -> Result; + /// Authenticated GET. Reads the token file on each call (rotation needs no restart). + /// Enforces redirect-off, size cap and per-peer/global concurrency (B1). + async fn get(&self, peer: &PeerConfig, req: PeerRequest, timeout: Duration) + -> Result; +} +``` + +`PeerError` (`src/peers/error.rs`): + +```rust +#[derive(Debug, Clone)] +pub enum PeerError { + Rejected, // peer 401/403: our credential was refused + NotFound, // peer 404 + BadRequest(String), // peer 400/422 + Http(u16), // other 5xx / unexpected status + Timeout, Unreachable, Dns, Tls, Oversize, + Malformed(String), // peer JSON could not be parsed + TokenFile(String), // missing/empty/oversize token file + Saturated, // concurrency limit hit +} +impl PeerError { pub fn class(&self) -> CheckClass; } // for health classification +``` + +### Fake behaviour (`src/peers/testing.rs`, `#[cfg(any(test, feature = "test-support"))]` or plain `pub` if integration tests need it; A0 decides, noted in log) + +`FakeTransport` is configured per peer name and mirrors the DEC-2 fake sources: + +- `FakeTransport::new()`; `.peer(name)` returns a builder with: + - `.tcp(Ok(latency) | Err(PeerError::..))` + - `.respond(path_prefix, status, content_type, body)` scripted GET responses (first matching prefix wins); `.respond_err(path_prefix, PeerError)`; `.stream(path_prefix, Vec)` for SSE. + - `.delay(Duration)` to simulate slowness (uses `tokio::time`, pauseable). +- Default for an unscripted peer: tcp ok (1 ms); `GET /api/v1/ping` returns `200 {"status":"ok","version":"0.0.0-fake"}`; any other path `404`. +- Records every call (`fake.calls()` -> `Vec<(PeerName, PeerRequest)>`) so tests assert that `Authorization` is never forwarded, `x-slinuxm-hop: 1` is added and `traceparent` is injected. +- Scripted `Rejected` on `/api/v1/ping` reproduces the "token revoked" state for D4/D9/T4-style tests. + +Helpers (also in `testing.rs`): + +```rust +pub fn fake_peers_config(names: &[&str]) -> PeersConfig; // enabled, short intervals, plain-http allowed +pub fn registry_with(fake: Arc, cfg: PeersConfig) -> Arc; // no loops spawned +pub fn static_registry(statuses: Vec) -> Arc; // fixed snapshot, for REST/MCP/OTel tests +pub fn fake_state_with_peers(config: Config, reg: Arc) -> AppState; // wraps sources::fake::fake_state +``` + +`registry_with` + `FakeTransport` lets C and D test full proxy behaviour; `static_registry` lets C1/D2/E1 test status rendering and metrics without any transport. + +## 3. PeerRegistry read API (`src/peers/registry.rs`, field `AppState.peers: Option>`) + +`None` = feature off: no routes, no tools, no tasks, no metrics. + +```rust +impl PeerRegistry { + /// Sorted by name. Includes disabled peers (enabled = false). + pub fn snapshot(&self) -> Vec; + pub fn status(&self, name: &str) -> Result; // NotFound if unknown + pub fn contains(&self, name: &str) -> bool; + pub fn config(&self) -> Arc; // limits for C2/E (max_follow comes from Config.limits) + /// Used by REST C2 and MCP D1: single proxy implementation (src/peers/proxy.rs, owned by WS-C). + pub async fn get(&self, name: &str, path: &str, query: &[(String, String)], opts: ProxyOpts) + -> Result; // already error-mapped (below) + /// Lifecycle (WS-B): start loops, reload, shutdown. Not used by surfaces. + pub async fn reload(&self, new: PeersConfig) -> Result<(), PeerConfigError>; + pub async fn shutdown(&self); +} +pub struct ProxyOpts { pub traceparent: Option, pub accept_sse: bool } +``` + +Rules for `get` (implemented by C2 in `proxy.rs`, the A0 stub forwards without allowlist): allowlist check, `..`/encoded-slash rejection, query allowlist, hop header, no caller credentials, peer error mapping. MCP D1 calls exactly this; no second implementation. For typed use: `PeerResponse` JSON bodies are returned to the MCP tool verbatim (the tool result is the peer's JSON text, wrapped in the usual cap/truncation). + +Registry reads are lock-light (`ArcSwap` or `RwLock` clone of a `Vec`); surfaces must never await on a peer to render status. + +## 4. Error mapping (`src/peers/error.rs`, implements D9) + +```rust +/// Pure function; the only place peer failures become AppError. +pub fn peer_error_to_app(peer: &PeerName, e: &PeerError) -> AppError; +``` + +| PeerError | AppError | Message form | +|---|---|---| +| `NotFound` | `NotFound` | `peer : not found on peer` | +| `BadRequest(m)` | `BadRequest` | `peer : ` | +| `Rejected` | `Unavailable` (never `Unauthorized`) | `peer : peer rejected credentials` | +| `Timeout`, `Unreachable`, `Dns`, `Tls`, `Http(5xx)`, `Oversize`, `Saturated`, `TokenFile` | `Unavailable` | `peer : ` (class: timeout, unreachable, dns, tls_error, http_, oversize, saturated, token_file) | +| `Malformed` | `Internal` | `peer : malformed response` | +| unknown `{peer}` (registry) | `NotFound` | `peer : unknown peer` | + +Messages always contain the peer name (WS-T asserts `"beta" in body` on 503). No new `AppError` variant (D9). The message never contains URLs, tokens or token-file paths. An optional extra `peer` field in the REST error body is not part of this contract (see [ws-c-rest.md](ws-c-rest.md), C4 note). + +## 5. REST contract + +All under the existing inbound auth; absent entirely when `state.peers` is `None` (route-level 404 as for any unknown route). + +### `GET /api/v1/peers` -> 200 + +```json +{ + "peers": [ + { + "name": "beta", + "enabled": true, + "url": "http://beta:1984", + "server": { "up": true, "since": "2026-10-19T10:00:00Z", "last_checked": "2026-10-19T10:00:09Z", "last_error": null, "latency_ms": 1 }, + "api": { "up": true, "since": "2026-10-19T10:00:00Z", "last_checked": "2026-10-19T10:00:05Z", "last_error": null, "latency_ms": 7, + "auth": "ok", "peer_version": "0.1.0" } + } + ] +} +``` + +Sorted by `name`. `auth` is `"ok" | "rejected" | "unknown"`. The WS-T test reads `s["server"]["up"]`, `s["api"]["up"]`, `s["api"]["auth"]` from the single-peer response and string-matches peer names in the list. + +### `GET /api/v1/peers/{peer}` -> 200 one object (the element above, not wrapped); unknown peer -> 404 `{"error":{"code":"not_found",...}}`. + +### `GET /api/v1/peers/{peer}/{*path}` (pass-through) + +- Allowlisted path prefixes (peer-side routes): `systemd/...`, `journal`, `journal/follow`, `docker/...`. Excluded: `ping`, `peers/...`. +- Methods other than GET -> `405` `method_not_allowed` (existing shape). +- Request headers added toward the peer: `Authorization: Bearer ` (never the caller's), `x-slinuxm-hop: 1`, `traceparent` when present. Request headers rejected from the caller: any `x-slinuxm-hop` -> `400 bad_request` (`peer : chained requests are not allowed`). +- Response: peer's status and body unchanged for 2xx; header `x-slinuxm-peer: ` on every proxied response, including errors produced by the consolidator for that peer (503 etc.). Content-Type passes through (`application/json`, `text/event-stream`). +- Errors per section 4; a rejected peer token is `503`, never `401` for the caller. + +## 6. Config surface (TOML) + +```toml +[peers] +enabled = false # default; peer-less behaviour is byte-identical +tcp_interval_secs = 10 +tcp_timeout_secs = 3 +api_interval_secs = 30 +api_timeout_secs = 5 +fail_threshold = 2 +request_timeout_secs = 10 +max_response_bytes = 8388608 +max_concurrent_per_peer = 4 +max_concurrent_total = 16 +deny_cidrs = ["169.254.0.0/16", "0.0.0.0/8"] + +[[peers.peer]] +name = "minipc1" # ^[a-z0-9][a-z0-9_-]{0,31}$, unique, max 64 peers +url = "https://minipc1:1984" # http/https only; no userinfo/query/fragment +token_file = "/run/credentials/skyg-linux-tracker.service/peer-minipc1" # absolute +ca_file = "/etc/ssl/private-ca.pem" # optional +allow_plain_http = false # required true for http:// to a non-loopback host +enabled = true +``` + +Env overrides for scalars: `SKYG_PEERS__ENABLED`, `SKYG_PEERS__TCP_INTERVAL_SECS`, ... (peer entries are file-only). Validation: each timeout below its interval; `enabled` with zero peers is a warning. `--print-config` shows `token_file` paths, never contents. + +Peer-side endpoint added by WS-B: `GET /api/v1/ping` (authenticated) returns `200 {"status":"ok","version":""}` (D4). `GET /healthz` stays public and unchanged. + +## 7. MCP contract + +With `peers.enabled = false` the tool list is exactly today's 8 tools and their schemas are unchanged. + +With peers enabled: + +- Every one of the 8 existing tools gains an optional parameter `peer: Option` ("name of a configured peer; omit for this host"). Absent = local, identical behaviour. Present = same arguments forwarded to the peer through `PeerRegistry::get` (path and query built from the same params as the local REST route). +- New tool `list_peers` (no params) -> JSON `{"peers":[PeerStatus,...]}` (same shape as REST list). +- New tool `get_peer { name: String }` -> one `PeerStatus` JSON; unknown name -> tool error (`not_found` data code). +- Tool count 10. Errors: `AppError` -> existing `tool_error`/`ErrorData` mapping, so peer failures are tool errors with the section 4 messages. +- Resource `peers://status` -> same JSON as `list_peers`. `container_health_report` and `diagnose_unit` prompts gain an optional `peer` argument. +- Instructions/descriptions mention peers only when enabled, and state that peer-sourced text is untrusted data. + +Rust shape of the added parameter: each params struct gets `#[serde(default)] pub peer: Option`, plus `pub struct GetPeerParams { pub name: String }`. + +## 8. OTel contract (D10) + +All are observable f64 gauges read from `PeerRegistry::snapshot()` (no polling), registered only when `state.peers` is `Some` and `otel.metrics.enabled`. + +| Metric | Unit | Attributes | Value | +|---|---|---|---| +| `peer.up` | `1` | `peer.name`, `peer.check` = `server` or `api` | 0/1 | +| `peer.api.auth_ok` | `1` | `peer.name` | 0/1 (0 with server up + api up = "token rejected") | +| `peer.check.duration_seconds` | `s` | `peer.name`, `peer.check` | last check latency | +| `peer.last_change_timestamp_seconds` | `s` | `peer.name`, `peer.check` | Unix time of `since` | + +No URL or version attributes. Removed peers vanish because the callback only reports what is in the snapshot. Outbound spans: name `peer.request`, attribute `peer.name`; `traceparent` injected into the `PeerRequest.headers` by the proxy layer (E2 provides the helper `telemetry::inject_traceparent() -> Option`, which `ProxyOpts.traceparent` carries; WS-C/WS-D pass it through and need not know more). + +## 9. NixOS options contract + +```nix +services.skyg-linux-tracker.peers = { + enable = false; # bool + settings = { }; # freeform: intervals/caps, keys as in [peers] (snake_case) + peers. = { + url = "https://minipc1:1984"; # str + tokenFile = config.age.secrets.peer-minipc1.path; # str, never a path literal; store paths rejected by assertion + caFile = null; # null or str + enable = true; + allowPlainHttp = false; + }; +}; +``` + +Rendering: one `[[peers.peer]]` per enabled peer with `token_file = "/run/credentials/skyg-linux-tracker.service/peer-"`, `LoadCredential=peer-:`, `RefreshOnReload=credentials`, `ExecReload` = SIGHUP, assertions as in F1. + +## 10. Test config contract (assumed by `nix/tests/peers.nix`, WS-T) + +The VM test is written against the pre-F1 configuration style, so the TOML contract above must stay compatible with it: + +- Hosts: `alpha` (peers `beta` over plain HTTP, `gamma` over HTTPS with `ca_file`), `beta` (peer `alpha`), `gamma` (TLS-only, no peers). Port 1984, bind `0.0.0.0`. +- Config comes through the module's freeform `settings.peers = { enabled = true; tcp_interval_secs = 2; tcp_timeout_secs = 1; api_interval_secs = 3; api_timeout_secs = 2; fail_threshold = 2; request_timeout_secs = 4; peer = [ ... ]; }`, i.e. `[peers]` scalars plus `[[peers.peer]]` with keys `name`, `url`, `token_file`, `allow_plain_http`, optional `ca_file`. +- `token_file = /run/credentials/skyg-linux-tracker.service/peer-` (via `LoadCredential=peer-:/var/lib/peer-tokens/`); the file holds the placeholder `pending` until the test installs a real token, so the placeholder must pass A3 loading (non-empty, under `$CREDENTIALS_DIRECTORY` so the permission check is skipped) and be reported as `auth = "rejected"`, not as a config error. +- Token rotation = `mv` a new file into place and `systemctl reload skyg-linux-tracker.service` (SIGHUP with `RefreshOnReload=credentials`); status must recover without a restart. SIGHUP must therefore re-read token files and keep the registry (B4). +- Timeouts (1 s / 2 s) are below intervals (2 s / 3 s), as validation requires; `request_timeout_secs = 4` and the test's `elapsed < 15` bound the proxy call when a peer VM is down. +- Assertions the contract must satisfy: `GET /api/v1/peers/

` JSON has `server.up`, `api.up`, `api.auth` in {`ok`,`rejected`}; `GET /api/v1/peers` body contains each peer name; proxied `GET /api/v1/peers/

/systemd/units[/.service]` returns the peer's data with response header `x-slinuxm-peer:

`; `POST`/`DELETE` on a proxy path -> 405; `x-slinuxm-hop: 1` -> 4xx; unknown peer -> 404; peer rejected/down -> 503 with the peer name in the body and the caller's own token still valid; MCP `list_peers` output contains the peer names; MCP `list_systemd_units` with `{"peer": "

"}` returns the peer's units. +- The peer's `/api/v1/ping` must return 401 for a bad token (placeholder case) and 200 for a good one; the test relies on `api.up = true` with `auth = "rejected"` for the 401 case (D4). diff --git a/plan/phase-5-consolidator/decisions.md b/plan/phase-5-consolidator/decisions.md new file mode 100644 index 0000000..d34c6f4 --- /dev/null +++ b/plan/phase-5-consolidator/decisions.md @@ -0,0 +1,41 @@ +# Phase 5 decisions + +Part of [Phase 5: Consolidator](README.md). Record decisions here and nowhere else; workstream files link to row IDs. Interface details that follow from a decision live in [contracts.md](contracts.md). + +## DEC-13 (proposed): outbound peer credentials are external files + +- **DEC-11 today (inbound):** a token that clients present *to this service* is generated by `slinuxm token create`, shown once, and only its sha256 hash is stored. A stolen database yields nothing usable, because the service never needs the plaintext again. +- **Why that cannot apply to peers (outbound):** to call a peer, the consolidator must send the peer's *plaintext* bearer token. A hash is useless for that, so the plaintext has to exist on the consolidator somewhere. DEC-11's "never stored" cannot hold literally. +- **DEC-13 rule:** `slinuxm` itself never persists a peer token. It owns no copy: not in the DB, not in `config.toml`, not in `--print-config` or logs, not in the Nix store. The config holds only a `token_file` path. The secret lives in a file managed by something else (agenix on NixOS, decrypted at activation to `/run/agenix/...` with root-only permissions and delivered to the service through `LoadCredential`). `slinuxm` reads it into memory (redacted `Debug`) and re-reads it on each check, so rotation needs no restart. +- **What this does and does not protect:** an offline copy of the DB, a backup, the Nix store or a support bundle contains no peer secret, and the encrypted secret in git is decryptable only by the host key. It does not protect against root on the consolidator host, who can read the decrypted file; that is inherent to any scheme that must replay a bearer token. Mitigation: each consolidator uses its own read-only token on every peer (`consolidator-`), so one peer-side `token revoke` cuts that consolidator off alone. +- **Scope:** DEC-11 is amended to "inbound tokens are stored only as hashes"; DEC-13 covers outbound. Both go into ARCHITECTURE.md Decisions after acceptance. + +## Open decisions + +| ID | Question | Options | Recommendation | Decision | +|---|---|---|---|---| +| D1 | How are peers defined? | (a) CLI plus `peers` table, server polls the table; (b) config file plus SIGHUP reload; (c) both | **Decided by the user (2026-10-19): declarative.** Peers are defined in config (on NixOS, in the module); no DB table, no `peer add`. Reload via SIGHUP, shared with TLS. | Declarative | +| D2 | Where do outbound peer tokens live (DEC-11 forbids plaintext tokens)? | (a) plaintext column in the DB; (b) AES-GCM in the DB; (c) per-peer files written by a CLI; (d) config references a `token_file`, contents come from an external secret manager | **Decided by the user (2026-10-19): agenix.** (d): config holds only a path; on NixOS `config.age.secrets..path` passed through `LoadCredential`. The secret is never in the Nix store, the config, the DB or `--print-config`. New DEC-13: DEC-11 covers inbound tokens only; outbound tokens are externally managed files. Blast radius: use a read-only token per consolidator (`consolidator-`) so a peer can revoke it alone. | agenix `token_file` | +| D3 | Must the DB exist when `auth.enabled = false`? | n/a | Dropped: peers no longer use the DB. | Dropped | +| D4 | What is "API running"? | (a) public `/healthz` 200; (b) new authenticated `GET /api/v1/ping`; (c) existing cheap endpoint such as `systemd/timers` | (b) returning `{status, version}`; `/healthz` stays public. A 401 is reported as `api.up = true, auth = rejected` (the API is running, our credential is not), so the operator sees the real fault. Fallback for older peers without `/ping`: `/healthz` plus `auth = unknown`. Adds one route to every instance (small, authed, harmless). | | +| D5 | Intervals, timeouts, state? | Proposed: TCP 10 s / 3 s, API 30 s / 5 s, 2 consecutive failures to flip down, immediate recovery; state in memory only | Accept the proposal; `last_change` resets on restart (documented). Persisting history conflicts with "live reads only" (DEC-5 reasoning); history belongs to the OTel backend. | | +| D6 | REST proxy shape | (a) `/api/v1/peers/{peer}/...` pass-through of an allowlist; (b) typed endpoints per peer resource; (c) status only | (a). Zero duplicated DTOs, peers evolve independently, allowlist keeps it from becoming an open proxy. | | +| D7 | MCP shape | (a) optional `peer` arg on existing 8 tools plus `list_peers`/`get_peer` (10 tools); (b) separate `peer_*` tool copies (16+); (c) status tools only | (a). Tool count becomes 10 only when `peers.enabled`; snapshot test covers both. Keeps context small for agents. | | +| D8 | Aggregation (fan-out, e.g. "failed units on all peers") | (a) none in this phase; (b) `peer = "*"` on the list tools plus `GET /api/v1/fleet/failed`; (c) both with per-peer partial-failure reporting | (a) for this phase, (b) as the first follow-up: partial failures need a result shape decision (`{peer, ok, data|error}`) that is easier to get right once pass-through is proven. | | +| D9 | Peer error mapping | Peer 404 -> `NotFound`; 400/422 -> `BadRequest`; peer 401/403 (our token rejected) -> `Unavailable` "peer rejected credentials" (never `Unauthorized`, which would tell the caller to re-auth with us); timeout, refused, DNS, TLS, 5xx, oversize -> `Unavailable` with peer name and class; unknown `{peer}` -> `NotFound`; malformed peer JSON -> `Internal`. Optionally a new `BadGateway` variant (502/504). | Reuse `Unavailable` (503) and put the class in the message and an extra `peer` field; no new variant, so the error contract and MCP mapping are unchanged. Revisit if clients need to distinguish "consolidator down" from "peer down". | | +| D10 | OTel metrics | `peer.up` (0/1; attrs `peer.name`, `peer.check` = `server`\|`api`); `peer.api.auth_ok` (0/1; `peer.name`; **confirmed by the user as a first-class metric**: `server up` + `api up` + `auth_ok = 0` is the "token rejected" state, kept separate from "API down" so alerts can distinguish a revoked or rotated credential from an outage); `peer.check.duration_seconds` (`s`; `peer.name`, `peer.check`); `peer.last_change_timestamp_seconds` (`s`; `peer.name`, `peer.check`). All observable f64 gauges from the registry snapshot; no URL or version in attributes (cardinality, and URLs may embed hostnames the operator treats as sensitive). | Accept. | | +| D11 | SSRF and TLS to peers | Peers come only from configuration (root- or Nix-managed); no request parameter ever supplies a URL. `http`/`https` only, no userinfo, query or fragment; redirects never followed; `deny_cidrs` checked against the resolved IP at connect time (LAN/private ranges must stay allowed, so default denies only link-local/metadata and unspecified). HTTPS verified against the webpki roots plus an optional per-peer `ca_file`; no skip-verify flag. Plain `http` to a non-loopback host requires `allow_plain_http = true` on that peer and logs a warning (the token travels in clear, same stance as DEC-12). | Accept. Alternative to weigh: system roots (needs `rustls-native-certs`, one more dep) instead of webpki roots; recommend webpki plus custom CA since LAN peers will use a private CA anyway. | | +| D12 | May a peer be a consolidator (chaining)? | (a) no, hop header blocks it; (b) allow with depth limit | (a). Prevents loops and amplification; revisit with D8. | | +| D13 | Declarative peers (NixOS list, agenix token files)? | (a) not in v1; (b) module renders config entries | **Decided by the user: this is the primary and only mechanism** (see D1, D2). | Declarative | +| D14 | Who may read peer status or proxy? | (a) any valid inbound token; (b) new scoped tokens | (a). Tokens have no scopes today; scoping is a separate phase. Note in docs that a consolidator token reaches every peer's data. | | +| D15 | HTTP client crate | (a) `reqwest` direct dep (already in the tree via OTel), rustls with ring passed explicitly; (b) `hyper-util` client plus `tokio-rustls` | (a) if `cargo tree -i openssl` stays empty and no second crypto provider is pulled in; otherwise (b). Decide in B1, record the outcome. | | + +## Where each decision lands + +| Decision | Implemented in | +|---|---| +| D1, D2, D13 (declarative, agenix `token_file`) | [ws-a-config.md](ws-a-config.md) (A1 to A3), [ws-f-nixos-docs.md](ws-f-nixos-docs.md) (F1) | +| D4 (`/api/v1/ping`), D5 (intervals), D11, D15 | [ws-b-registry.md](ws-b-registry.md) | +| D6, D9, D12, D14 | [ws-c-rest.md](ws-c-rest.md), error table in [contracts.md](contracts.md) section 4 | +| D7, D8 | [ws-d-mcp.md](ws-d-mcp.md) | +| D10 | [ws-e-telemetry.md](ws-e-telemetry.md), [contracts.md](contracts.md) section 8 | diff --git a/plan/phase-5-consolidator/ws-a-config.md b/plan/phase-5-consolidator/ws-a-config.md new file mode 100644 index 0000000..63403ea --- /dev/null +++ b/plan/phase-5-consolidator/ws-a-config.md @@ -0,0 +1,53 @@ +# WS-A: Config, secrets, diagnostics CLI (and A0 contract stubs) + +Part of [Phase 5: Consolidator](README.md). Contracts: [contracts.md](contracts.md). + +| Field | Value | +|---|---| +| Status | Not started | +| Owner | | +| Depends on | none | +| Blocks | [WS-B](ws-b-registry.md), [WS-C](ws-c-rest.md), [WS-D](ws-d-mcp.md), [WS-E](ws-e-telemetry.md) can start before A0 lands but cannot compile or merge until it does; A4 `peer check` needs B1/B2 | +| Can start when | day one, in parallel with [WS-T](ws-t-tests.md). **A0 first**: it is small and everything else rebases onto it | +| Owned files | `src/peers/mod.rs`, `src/peers/types.rs`, `src/peers/error.rs`, `src/peers/transport.rs`, `src/peers/testing.rs` (A0 creates; after A0 they change only through [contracts.md](contracts.md)), `src/peers/config.rs`, `src/peers/secrets.rs`, `src/peers/cli.rs`, `src/config.rs` (`[peers]`, validation), `src/main.rs` (`Command::Peer` and its dispatch only), `config.example.toml` (the `[peers]` block), `tests/peers_config.rs` | +| Created by A0, then owned by | `src/peers/registry.rs`, `src/peers/health.rs`, `src/peers/client.rs` (WS-B); `src/peers/proxy.rs`, `src/api/peers.rs` (WS-C); `src/api/ping.rs` (WS-B) | +| Do not touch | `src/peers/client.rs`, `health.rs`, `registry.rs` bodies after A0 (WS-B); `src/peers/proxy.rs`, `src/api/peers.rs`, `docs/api.md` (WS-C); `src/mcp/**` (WS-D); `src/telemetry/**` (WS-E); `nix/**`, `systemd/**`, `README.md`, `docs/consolidator.md` (WS-F); `dev/test.nu`, `devenv.nix`, `flake.nix` (WS-T); `Cargo.toml` except as noted below | + +Shared-file seams (A0 wires them once, so later streams only fill their own files): + +- `src/state.rs`: A0 adds `pub peers: Option>` (always `None` in `AppState::live`) and fixes the other constructor, `sources::fake::fake_state` (`src/sources/fake.rs`), to set `peers: None`. WS-B later assigns the real registry where `serve()` builds the state (see ws-b seams). Nobody else edits `state.rs`. +- `src/lib.rs`: A0 adds `pub mod peers;`. +- `src/api/mod.rs`: A0 adds `pub mod peers; pub mod ping;` with stub modules and the merge lines in `router()`: `.merge(ping::routes())` (unconditional; the route is authenticated like the rest) and the peers routes merged only when `state.peers.is_some()`. WS-B fills `ping.rs`, WS-C fills `peers.rs`; neither edits `mod.rs`. +- `Cargo.toml`: A0 and A add nothing unless unavoidable (an `ipnet`-style CIDR crate for D11 `deny_cidrs` is allowed in A2; note it in the Log). WS-B owns `reqwest`/TLS dependency edits. +- `src/main.rs`: A edits the clap `Command` enum and its dispatch; WS-B edits `serve()` only. + +## Goal + +Peers are declared in config, validated, and loadable; the CLI can list and one-shot check them. No network loops yet. + +## Tasks + +- **A0** contract stubs (new, small, lands first; see [contracts.md](contracts.md) sections 1 to 4, 6). Add to `src/peers/`: `types.rs` (`PeerName`, `Secret`, `PeerConfig`, `PeersConfig`, `PeerStatus`/`ServerStatus`/`ApiStatus`/`AuthState`/`CheckClass`), `error.rs` (`PeerError`, `peer_error_to_app` with the D9 table), `transport.rs` (`PeerTransport`, `PeerRequest`, `PeerResponse`), `testing.rs` (`FakeTransport`, `fake_peers_config`, `registry_with`, `static_registry`, `fake_state_with_peers`), skeletons of `registry.rs` (`PeerRegistry` with `snapshot/status/contains/config/get` over a stored map and the transport, no loops), `proxy.rs` (`get` forwarding through the transport without allowlist; WS-C hardens it), `health.rs` (signature of `check_once`, a stub). Wire the seams listed above. Add `peers: PeersConfig` to `Config` with `#[serde(default)]` and `enabled = false`, so the peer-less default is unchanged. Tests: the D9 table (pure mapping), `FakeTransport` default and scripted behaviour, `static_registry` snapshot ordering. Done when `cargo build`, `cargo test`, clippy and fmt pass and every contract signature compiles; announce completion in the Log of A and of C, D, E. This task must not take more than a small PR; anything not needed to compile the contracts moves to A1 to A4 or B. +- **A1** `[peers]` config and `[[peers.peer]]` entries (keys in [contracts.md](contracts.md) section 6, "Config surface"), env `SKYG_PEERS__*` for the scalars, redaction check, `--print-config` (shows `token_file` paths, never contents). Also fills the `[peers]` block of `config.example.toml`. +- **A2** validation: name regex `^[a-z0-9][a-z0-9_-]{0,31}$`, unique names, URL rules (D11), `token_file` absolute and (when `peers.enabled`) readable at startup with a clear error naming the peer, max 64 peers. Also timeouts below their interval; `peers.enabled` with zero peers is a warning, not an error. A placeholder token file whose content is non-empty must load (see contracts section 10). +- **A3** secrets loader: read the token file at start and on reload; trim one trailing newline; reject empty, over 4 KiB, or world/group-readable files only when they are not under `$CREDENTIALS_DIRECTORY`; never log or `Debug`-print the value (wrapper type with redacted `Debug`). +- **A4** CLI `peer list` (config only) and `peer check ` (one-shot TCP plus API probe, prints classification and latency, never the token). Runs as any user that can read the config and token file. `peer list` is done within WS-A; `peer check` calls `peers::health::check_once` with the real transport, so it is wired when WS-B's B1/B2 land (until then it is tested with `FakeTransport` and returns "transport not available" in the binary). Record the wiring in the Log. + +## Inputs + +- [contracts.md](contracts.md) sections 1 to 4 and 6 (it defines what A0 builds); [decisions.md](decisions.md) D1, D2, D11, D13 and DEC-13. +- No fakes needed except `FakeTransport` for A4 tests. + +## Done when + +Unit tests for validation and loader; a peer-less default config is byte-identical in `--print-config` apart from the new `[peers]` block; the token never appears in stdout, logs, `--print-config` or `Debug` output. + +## Handoff + +- A0 merged: C, D, E, B compile against the stubs; post in all their Logs. +- `PeersConfig`/`PeerConfig`, the secrets loader (`Secret`, `load_token(&PeerConfig) -> Result`) and `config.example.toml` `[peers]` block for B and F. +- The `peer` subcommand for the docs (F4). + +## Log + +- 2026-10-19: Split from the single phase plan 2026-10-19. A0 added to cover the shared stubs so C/D/E/F can start against fakes; ownership of the stub files passes to B/C after A0 merges. diff --git a/plan/phase-5-consolidator/ws-b-registry.md b/plan/phase-5-consolidator/ws-b-registry.md new file mode 100644 index 0000000..184bd68 --- /dev/null +++ b/plan/phase-5-consolidator/ws-b-registry.md @@ -0,0 +1,52 @@ +# WS-B: Client, health, registry + +Part of [Phase 5: Consolidator](README.md). Contracts: [contracts.md](contracts.md) sections 1 to 4 and 8. + +| Field | Value | +|---|---| +| Status | Not started | +| Owner | | +| Depends on | A0 (stubs and seams) and A1 to A3 for real config/secrets; can start on B1 as soon as A0 lands | +| Blocks | integration of C, D, E (they are developed against the fake registry and finish for real once B lands); F | +| Can start when | A0 is merged (design and client code can begin earlier in a branch against contracts.md) | +| Owned files | `src/peers/client.rs`, `src/peers/health.rs`, `src/peers/registry.rs` (bodies; signatures per contracts), `src/api/ping.rs` (new authenticated `GET /api/v1/ping`, see D4), `Cargo.toml`/`Cargo.lock`, `src/main.rs` (`serve()` only), `src/tls.rs` (SIGHUP fan-out line only, if the handler stays there), `tests/peers_registry.rs`, `tests/common/mod.rs` (append new helpers only) | +| Do not touch | `src/peers/types.rs`, `transport.rs`, `error.rs`, `testing.rs` (change via contracts.md only); `src/peers/config.rs`, `secrets.rs`, `cli.rs`, `src/config.rs` (WS-A); `src/peers/proxy.rs`, `src/api/peers.rs`, `src/api/mod.rs`, `src/error.rs` (WS-C / A0); `src/mcp/**` (D); `src/telemetry/**` (E); `nix/**`, `docs/**`, `README.md` (F, T) | + +Shared-file seams: + +- `src/state.rs`: do not edit; A0 added `AppState.peers`. B assigns it in `serve()` in `src/main.rs` (state is built there by `AppState::live`, then `state.peers = peers::start(&state)` before `server::build_router`). One edit region, `serve()`; WS-A edits the `Command` enum elsewhere in the file. +- SIGHUP: `serve()` installs the handler and `tls::spawn_sighup_reload` consumes it. B adds the fan-out so the same signal also calls `PeerRegistry::reload` (and the plain-TLS-off branch no longer just logs when peers are enabled). Keep the TLS reload behaviour unchanged. +- `src/api/mod.rs`: untouched; `ping.rs` is already merged by A0. B only fills `ping::routes()`. +- `Cargo.toml`/`Cargo.lock`: B owns dependency edits (reqwest direct dep per D15, features, any CIDR crate not already added by A). Regenerate `Cargo.lock`, never hand-merge. +- `tests/common/mod.rs`: append-only helpers; WS-F appends too. + +## Goal + +A `PeerRegistry` built from config that keeps live status; a bounded HTTP client. + +## Tasks + +- **B1** client: connect/request timeouts, redirects disabled, GET only, response byte cap enforced while streaming (not just `Content-Length`), per-peer `Semaphore` plus a global one, custom CA per peer, rustls with the ring provider passed explicitly; `cargo tree -i openssl` stays empty. Implements `PeerTransport` for real (`ReqwestTransport`) and records the D15 outcome in the Log. +- **B2** health loops: one task per peer with its own child `CancellationToken` and start jitter; TCP check = `TcpStream::connect(host:port)` under timeout; API check = authenticated `GET /api/v1/ping`; classification `ok | unauthorized | http_error | timeout | unreachable | tls_error`. Provides `health::check_once` (shared with `slinuxm peer check`, A4) and the `/api/v1/ping` handler in `ping.rs` returning `{status, version}`; fallback for peers without `/ping` is `/healthz` plus `auth = unknown` (D4). +- **B3** status model (in memory only): `{server: {up, since, last_checked, last_error}, api: {up, since, last_checked, last_error, latency_ms, auth: ok|rejected|unknown, peer_version}}`; flips need `fail_threshold` consecutive failures, recovery is immediate; `since` is the last change time and resets on restart. +- **B4** reload: on startup build the registry; on SIGHUP (shared handler with TLS) re-read only the `[peers]` block and token files, start loops for new peers, cancel removed ones, restart changed ones; a bad reload keeps the old registry (same rule as TLS T3). Token files are re-read on every API check, so rotating an agenix secret needs no restart. Shutdown awaits all loops (bounded). +- **B5** `PeerTransport` trait and fake; SSRF guard (D11) applied at config load and again at connect time on the resolved address. The trait and fake already exist from A0 (contracts section 2); B5 is now the SSRF guard on the real transport plus any fake extensions the loops need (changed through contracts.md). + +## Inputs + +- [contracts.md](contracts.md) sections 1 to 4 (types, transport, registry API, error mapping), 6 (config), 10 (the test's timings and rotation flow). +- Fake: `FakeTransport` and `registry_with` for loop/classification unit tests with paused time; real in-process servers for the integration test. + +## Done when + +Integration test starts a second in-process server (fake sources) as a peer and observes up, down (port closed), api-down-but-server-up (listener that accepts and hangs), unauthorized (wrong token) and recovery; one hung peer does not delay another peer's check beyond its own timeout; clippy, fmt, `cargo tree -i openssl` clean. + +## Handoff + +- Real `PeerRegistry` with the contract API (`snapshot`, `status`, `get`, `reload`, `shutdown`) assigned to `AppState.peers` when `peers.enabled`. +- `GET /api/v1/ping`; `ReqwestTransport`; `health::check_once` for A4. +- Post in C, D, E, F Logs when the registry is wired, so they swap their fakes for real end-to-end checks. + +## Log + +- 2026-10-19: Split from the single phase plan 2026-10-19. The trait and fake moved to A0 so C/D/E are not blocked on this stream; B keeps the implementations. diff --git a/plan/phase-5-consolidator/ws-c-rest.md b/plan/phase-5-consolidator/ws-c-rest.md new file mode 100644 index 0000000..ee55e9f --- /dev/null +++ b/plan/phase-5-consolidator/ws-c-rest.md @@ -0,0 +1,45 @@ +# WS-C: REST surface + +Part of [Phase 5: Consolidator](README.md). Contracts: [contracts.md](contracts.md) sections 3 to 5. + +| Field | Value | +|---|---| +| Status | Not started | +| Owner | | +| Depends on | B (original plan). In practice: A0 for compiling, B only for the final end-to-end check; develop against `FakeTransport` + `registry_with` / `static_registry` | +| Blocks | [WS-D](ws-d-mcp.md) D1 (shares `PeerRegistry::get` / `proxy.rs`; D can stub it from A0 until C2 lands), F | +| Can start when | A0 is merged (earlier in a branch against contracts.md) | +| Owned files | `src/api/peers.rs`, `src/peers/proxy.rs` (the single proxy implementation: allowlist, header rules, error mapping calls), `src/error.rs` (only if D9 adds a variant), `docs/api.md`, `src/api/behaviour_tests.rs` (peer cases appended) | +| Do not touch | `src/api/mod.rs` (A0 wired `pub mod peers`, the conditional merge and the 404/405 fallbacks; stay out), `src/api/ping.rs` (B), `src/peers/{registry,health,client}.rs` (B), `src/peers/{types,error,transport,testing}.rs` (contracts.md only), `src/mcp/**` (D), `src/telemetry/**` (E), `src/state.rs` (A0), `nix/**`, `README.md` (F) | + +Shared-file seam: `src/api/mod.rs` is edited only by A0. WS-C fills `peers::routes()` (returns the `Router` merged when `state.peers` is `Some`). The contract for what a route returns is [contracts.md](contracts.md) section 5. + +## Goal + +Status and pass-through over REST under the existing auth. + +## Tasks + +- **C1** `GET /api/v1/peers` and `/{peer}`: status JSON (shape in contracts section 5), sorted by name; unknown peer is `NotFound`. +- **C2** pass-through `GET /api/v1/peers/{peer}/{*path}`: allowlist of the peer's documented routes (`systemd/...`, `journal`, `journal/follow`, `docker/...`; `ping` and `peers/...` excluded), reject `..`, encoded slashes and non-allowlisted query keys; forward only the allowlisted query; never forward the caller's `Authorization` or cookies, use the peer token; add `x-slinuxm-hop: 1` and reject requests that already carry it (no chaining); stream bodies through the size cap; SSE passes through bounded by `limits.max_follow_secs` and shutdown. Implemented in `src/peers/proxy.rs` as the body of `PeerRegistry::get`, which WS-D also calls; the route in `peers.rs` is a thin adapter. (Axum 0.8: routes use `/{peer}` and `/{*path}`.) +- **C3** response header `x-slinuxm-peer: `; body unchanged so clients can reuse local parsers. +- **C4** (note, no new task from the original) error mapping goes through `peer_error_to_app` (contracts section 4). Whether to add an extra `peer` field to the REST error body is optional; if done it is a contracts.md change plus `src/error.rs`. + +## Inputs + +- Contracts sections 3 (registry API), 4 (error table), 5 (REST shapes). +- Fakes: `FakeTransport` (records calls to assert hop header, no `Authorization`, filtered query), `registry_with`, `static_registry`, `fake_state_with_peers`. + +## Done when + +Behaviour tests with the fake transport for every error mapping row in D9; 405 on non-GET; traversal and header-smuggling cases rejected; SSE ends on shutdown. + +## Handoff + +- `PeerRegistry::get` fully implemented (allowlist, hop, mapping) for WS-D; post in D's Log when C2 merges. +- `docs/api.md` section for the peers endpoints for F4. +- Verified shapes that WS-T asserts (`x-slinuxm-peer`, 405, 4xx on hop header, 503 naming the peer). + +## Log + +- 2026-10-19: Split from the single phase plan 2026-10-19. The proxy logic is placed in `src/peers/proxy.rs` (not `api/peers.rs`) so REST and MCP share one implementation, as the original plan required. diff --git a/plan/phase-5-consolidator/ws-d-mcp.md b/plan/phase-5-consolidator/ws-d-mcp.md new file mode 100644 index 0000000..54f7d7e --- /dev/null +++ b/plan/phase-5-consolidator/ws-d-mcp.md @@ -0,0 +1,44 @@ +# WS-D: MCP surface + +Part of [Phase 5: Consolidator](README.md). Contracts: [contracts.md](contracts.md) sections 3, 4 and 7. + +| Field | Value | +|---|---| +| Status | Not started | +| Owner | | +| Depends on | B (and C2 client call shared, not duplicated). In practice: A0 for compiling; `PeerRegistry::get` from A0 (unhardened forwarder) until C2 lands; develop against `FakeTransport` + `registry_with` | +| Blocks | F | +| Can start when | A0 is merged (the rmcp conditional-tools spike can start on day one, it needs nothing from peers) | +| Owned files | `src/mcp/mod.rs`, `src/mcp/prompts.rs`, `src/mcp/resources.rs`, tool-list snapshot test (in `src/mcp/mod.rs`), a new `src/mcp/peers.rs` if the code is easier to keep apart (WS-D may add it) | +| Do not touch | `src/api/**` (C, B), `src/peers/**` (A, B, C), `src/state.rs`, `src/server.rs`, `src/telemetry/**`, `src/error.rs`, `docs/api.md`, `README.md`, `AGENTS.md` (F4 edits the tool-count line), `nix/**`, `Cargo.toml` | + +Shared-file seams: WS-D is the only editor of `src/mcp/**`. It reads `AppState.peers` (A0) and calls `PeerRegistry::get` / `snapshot` / `status`; it never builds URLs or headers itself. The MCP `service()` function must keep working unchanged when `state.peers` is `None`. + +## Goal + +Agents reach peers through the same endpoint. + +## Tasks + +- **D1** optional `peer` string on the 8 existing tools (absent = local); resolved through the same peer client function as REST C2, no logic copy (`PeerRegistry::get`, contracts section 3). +- **D2** new tools `list_peers` and `get_peer { name }`; with `peers.enabled = false` the tool list is exactly today's 8 (snapshot test has both variants). Risk from the plan: `rmcp` may not support conditional tools cleanly; spike this early (fallback: always list tools, return an error when disabled, and accept a 10-tool snapshot), and record the outcome in the Log and, if it changes the tool-count behaviour, in contracts section 7. +- **D3** server instructions and descriptions: mention peers only when enabled; state that peer text is untrusted data; add `peers://status` resource and extend `container_health_report`/`diagnose_unit` with an optional `peer`. + +## Inputs + +- Contracts section 7 (tool signatures), 3 (registry API), 4 (error mapping to `CallToolResult` errors). +- Fakes: `registry_with(FakeTransport, ...)` scripted with peer JSON for each tool's path; `static_registry` for `list_peers`/`get_peer`/`peers://status`; `fake_state_with_peers`. + +## Done when + +Both snapshot variants pass; each tool with `peer` returns the peer's data via the fake; peer errors surface as `CallToolResult` errors with actionable text. + +## Handoff + +- The tool list (10 tools when enabled) and descriptions for docs (F4: README, AGENTS.md tool count note). +- The rmcp conditional-tools finding for contracts section 7. +- Verified against WS-T's MCP assertions (`list_peers`, `list_systemd_units {"peer": ...}`). + +## Log + +- 2026-10-19: Split from the single phase plan 2026-10-19. diff --git a/plan/phase-5-consolidator/ws-e-telemetry.md b/plan/phase-5-consolidator/ws-e-telemetry.md new file mode 100644 index 0000000..712eda9 --- /dev/null +++ b/plan/phase-5-consolidator/ws-e-telemetry.md @@ -0,0 +1,42 @@ +# WS-E: Telemetry + +Part of [Phase 5: Consolidator](README.md). Contracts: [contracts.md](contracts.md) sections 3 and 8. + +| Field | Value | +|---|---| +| Status | Not started | +| Owner | | +| Depends on | B (registry). In practice: A0 for compiling; read the snapshot from `static_registry` / `registry_with` until B lands | +| Blocks | F; [WS-T](ws-t-tests.md) goes green at the end of E | +| Can start when | A0 is merged | +| Owned files | `src/telemetry/metrics_export.rs`, `src/telemetry/mod.rs` | +| Do not touch | `src/peers/**` (A, B, C), `src/api/**`, `src/mcp/**`, `src/state.rs`, `src/server.rs`, `src/main.rs`, `Cargo.toml` (B; if a new telemetry dependency is truly needed, ask B and log it), `nix/**`, `dev/otelcol.yaml` (E may add peer series examples only if needed, note in Log), `docs/**` | + +Shared-file seams: telemetry reads `state.peers` (the field exists from A0, set by B in `serve()`); E does not edit `state.rs`/`main.rs`. `spawn_exporters(state)` already receives `AppState`, so no signature change is expected. `inject_traceparent()` (E2) is exported from `telemetry/mod.rs` and called by `src/peers/proxy.rs`/client via `ProxyOpts.traceparent` (contracts section 8); until E2 lands `ProxyOpts.traceparent` is simply `None`. + +## Goal + +Peer health in OTel, peer calls in traces. + +## Tasks + +- **E1** observable gauges (f64) fed from the registry snapshot, so removed peers vanish (G11); no polling of peers by the exporter, it reads the registry. Metric table in D10 (names and attributes frozen in contracts section 8). +- **E2** outbound client spans with `peer.name`; inject `traceparent` into peer requests. + +## Inputs + +- Contracts section 8 (metric names, attributes, units) and 3 (`snapshot`); decisions D10. +- Fakes: `static_registry(Vec)` to feed the gauges; `FakeTransport` call records to assert `traceparent` injection. + +## Done when + +Metrics test with the fake registry; with `otel.enabled` and no peers nothing new is exported; `devenv up` collector shows `peer.up` series for two peers and drops one after `peer remove`. (Peers are declarative, so "peer remove" means removing the entry from config and sending SIGHUP; see D1.) + +## Handoff + +- Working gauges and `inject_traceparent` for WS-B/C wiring and WS-F's docs. +- Signal to WS-T and WS-F: with E, B, C, D merged the VM test should pass; F2 then promotes it. E also unlocks adding the `peer.api.auth_ok = 0` check from T4 if wanted. + +## Log + +- 2026-10-19: Split from the single phase plan 2026-10-19. Note added that "peer remove" in the collector check means removal from config plus SIGHUP (D1). diff --git a/plan/phase-5-consolidator/ws-f-nixos-docs.md b/plan/phase-5-consolidator/ws-f-nixos-docs.md new file mode 100644 index 0000000..321a3bf --- /dev/null +++ b/plan/phase-5-consolidator/ws-f-nixos-docs.md @@ -0,0 +1,49 @@ +# WS-F: NixOS, docs, end-to-end tests + +Part of [Phase 5: Consolidator](README.md). Contracts: [contracts.md](contracts.md) sections 5 to 10. + +| Field | Value | +|---|---| +| Status | Not started | +| Owner | | +| Depends on | C, D, E (original plan). F1 (module) and F4 (docs drafts) can begin once A1 settles the config keys, since they depend only on contracts sections 6 and 9; F2/F3 need the other streams | +| Blocks | phase exit | +| Can start when | F1 and doc drafts: after A0/A1 land. F2: after C, D, E and [WS-T](ws-t-tests.md) are green. F3: after B and C | +| Owned files | `nix/module.nix`, `nix/tests/vm.nix`, `systemd/skyg-linux-tracker.service`, `README.md` (peers section), `docs/consolidator.md` (new; except the `## Testing` section owned by WS-T), `tests/peers.rs`, `tests/common/mod.rs` (shared helpers, append-only with B), `config.example.toml` (merge with A1: comments and example peers, not the keys), `flake.nix` (only the move of `tests..peers` into `checks.vm-peers`, in F2), `AGENTS.md` (MCP tool count note) | +| Do not touch | `src/**` (all code streams; report bugs to the owning stream's Log), `nix/tests/peers.nix` except to adapt it to the first-class options in F2 (coordinate with WS-T, update contracts section 10), `dev/test.nu`, `devenv.nix` (T), `docs/api.md` (C), `src/config.rs` (A) | + +Shared-file seams: + +- `flake.nix`: WS-T owns it until F2; F2 then moves exactly one attribute into `checks`. +- `README.md` / `AGENTS.md`: WS-T owns the command lists; F4 owns the peers section and the MCP tool count note (tool count becomes "8, or 10 with peers enabled"). +- `config.example.toml`: A1 owns the keys and defaults; F reviews that it is in sync with `docs/consolidator.md` and adds only commentary/examples. +- `tests/common/mod.rs`: append-only; B may have added helpers already. + +## Goal + +Shippable, documented, verified. + +## Tasks + +- **F1** module: `peers.enable`; `peers.peers. = { url; tokenFile; caFile ? null; enable ? true; allowPlainHttp ? false; }`; `peers.settings` for intervals/caps. The module renders `[[peers.peer]]` entries into the TOML with the *paths* only and adds one `LoadCredential=peer-:` per peer, pointing each `token_file` at `/run/credentials/skyg-linux-tracker.service/peer-` so the service user needs no read access to `/run/agenix/*` (same mechanism as TLS, DEC-12). `tokenFile` is typed as a string (`config.age.secrets..path`), never a path literal, so secrets cannot enter the Nix store; an eval assertion rejects a store path. Config change reloads via `ExecReload` (SIGHUP) with `reloadIfChanged`, else restarts. Assertions: at least one peer when `peers.enable`; names valid and unique. Also add `RefreshOnReload=credentials` as `nix/tests/peers.nix` does (contracts section 10) if the systemd version supports it. +- **F2** promote WS-T to `checks`, switched to the first-class module options (`peers.peers..tokenFile`); plus a three-machine variant (consolidator, peer1, peer2) for fan-in. Mint tokens on peers, write them to root-owned 0400 files on the consolidator (stand-ins for agenix paths) and declare the peers through the module, assert status via REST and MCP, stop peer2's service and assert server/api flip, assert proxied `units` match a direct call, then rotate a token file and revoke the token on the peer: `auth` flips to rejected, then back after rewriting the file, with no restart. Remove a peer from the config, SIGHUP, assert the series is gone. +- **F3** `tests/peers.rs` end-to-end (plain, TLS with an `rcgen` CA via `--ca-file`, wrong CA rejected, redirect not followed, oversize body, slow-loris peer, 64-peer cap, cancel on shutdown). +- **F4** README section, `docs/consolidator.md` (threat model, token handling, firewall note: peers must allow the consolidator), a worked agenix example (`secrets.nix` entry, `age.secrets.peer-minipc1.file`, `peers.peers.minipc1.tokenFile = config.age.secrets.peer-minipc1.path`), AGENTS.md MCP tool count note. + +## Inputs + +- Contracts section 9 (Nix options), 6 (TOML), 10 (test contract), 5 and 7 (what the tests assert). Fakes: none for F2 (real binary); F3 uses in-process servers, not the fake transport. +- Constraint: nix builds only when the nix gate is free (AGENTS.md). + +## Done when + +`nix flake check` passes including the new VM test; docs and example config in sync. + +## Handoff + +- Phase exit evidence: `nix flake check` output, docs, example config, the Verification plan results in the [README](README.md#verification-plan). +- To the orchestrator: ARCHITECTURE.md decision rows (DEC-13 and the DEC-11 clarification), TODO.md checklists, and moving `plan/phase-5-consolidator/` to `docs/workstreams/`. + +## Log + +- 2026-10-19: Split from the single phase plan 2026-10-19. `tests/peers.rs` ownership moved here from WS-T; `docs/consolidator.md` shares one `## Testing` section with WS-T. diff --git a/plan/phase-5-consolidator/ws-t-tests.md b/plan/phase-5-consolidator/ws-t-tests.md new file mode 100644 index 0000000..f160a5a --- /dev/null +++ b/plan/phase-5-consolidator/ws-t-tests.md @@ -0,0 +1,66 @@ +# WS-T: Integration test suite (written first, user request) + +Part of [Phase 5: Consolidator](README.md). Contracts: [contracts.md](contracts.md). + +| Field | Value | +|---|---| +| Status | In progress: `nix/tests/peers.nix` is written; T7 runner and docs open | +| Owner | | +| Depends on | none (uses the module's freeform `settings` for `[peers]` and a systemd override for `LoadCredential`, so it needs no module changes). F1 later swaps this for the first-class module options | +| Blocks | nothing technically; it is the acceptance gate: goes green at the end of [WS-E](ws-e-telemetry.md) (with A, B, C, D landed), then F2 promotes it to `checks` | +| Can start when | day one | +| Owned files | `nix/tests/peers.nix` (done), `flake.nix` (the `tests..peers` output only), `dev/test.nu`, `devenv.nix` (nushell package, `test` script, completion hook), `AGENTS.md` and `README.md` (command lists only, see seams), `docs/consolidator.md` (creates the file with only a `## Testing` section, see seams) | +| Do not touch | `src/**` (all streams), `nix/module.nix`, `nix/tests/vm.nix`, `systemd/skyg-linux-tracker.service` (F), `config.example.toml` (A/F), `tests/peers.rs` and `tests/common/mod.rs` (F; in-process variants by A/B live in their own test files, see [ws-a](ws-a-config.md), [ws-b](ws-b-registry.md)), `docs/api.md` (C) | + +Shared-file seams: + +- `flake.nix`: T owns it until F2. F2 later moves exactly one attribute, `tests.${system}.peers`, into `checks` (rename `checks.vm-peers`). Nobody else edits `flake.nix`; it already carries the uncommitted peers output. +- `devenv.nix`: T only. +- `README.md`: T edits only the command/test list near the existing commands; F4 owns the new peers section. Different headings, so no overlap. `AGENTS.md`: T replaces the command list; F4 adds only the MCP tool-count note. +- `docs/consolidator.md`: T creates it with a single `## Testing` section (the one-line `use dev/test.nu *` and the completion setup). F4 adds every other section and must not edit `## Testing`. + +## Goal + +Executable acceptance criteria before the implementation. Two NixOS VMs each run the real `slinuxm` binary and consolidate each other: VM `alpha` declares `beta` as a peer, VM `beta` declares `alpha`, and each proxies the other's data. Expected to fail until WS-E lands, which is the point. + +Note on the written test: `nix/tests/peers.nix` also contains a third node `gamma` (TLS-only, no peers) that `alpha` declares with a private CA via `ca_file`, covering the TLS variant. It is already written and, by design, fails at the first peers assertion (phase 2, "declared but token is still a placeholder -> auth rejected") against the current binary; phase 1 (boot, healthz, token minting, marker units, token-file placeholders) already passes. + +## Tasks + +- **T1** two machines `alpha` and `beta` with distinct hostnames, each with a distinct fake workload (a marker systemd unit `marker-alpha.service` / `marker-beta.service`, and a deliberately failing unit on one side) so proxied data is distinguishable from local data. **Done** in `nix/tests/peers.nix` (failing unit is `broken-beta.service`; `gamma` with `marker-gamma.service` added). +- **T2** each mints a read-only token via `sudo -u skyg-linux-tracker slinuxm token create consolidator-`, writes the other's token to a root-only 0400 file (stand-in for an agenix path), and declares the other as a peer with that `token_file`. **Done** (files under `/var/lib/peer-tokens`, delivered via `LoadCredential`). +- **T3** assertions on both directions: `/api/v1/peers` shows the peer with server up, api up and auth ok; `GET /api/v1/peers//systemd/units/marker-.service` returns the other machine's unit and not a local one; MCP `list_peers` and a `list_systemd_units` call with `peer` return the same data; response header `x-slinuxm-peer`. **Done** in the test script. +- **T4** token rejected: revoke the token on `beta`; `alpha` reports server up, api up, `auth = rejected`, `peer.api.auth_ok = 0`, and proxy calls fail with 503 naming the peer, without becoming a 401 for the caller; restore with a new token file and no restart, status recovers. **Done** in the script except the `peer.api.auth_ok = 0` metric check (needs WS-E; add it to the script, or leave to F2 which may query the collector/metrics endpoint, note in Log). +- **T5** outage: stop `beta`'s service: `alpha` flips server and api after `fail_threshold`, `alpha`'s own endpoints stay healthy, and recovery is detected; stop `beta` entirely (VM down) and check that `alpha`'s requests are bounded by timeouts. **Done** in the script. +- **T6** no chaining and read-only: a request carrying `x-slinuxm-hop` is rejected; non-GET to a proxy path is 405; a TLS variant with a test CA via `ca_file`. **Done** (405 for POST/DELETE, hop header 4xx, `gamma` over HTTPS). +- **T7** a simple test runner CLI written in **Nushell**: `dev/test.nu` (devenv adds `pkgs.nushell`; also exposed as the devenv script `test`, so `devenv shell -- test unit` works from any shell). One entry point so contributors and agents do not memorise cargo/nix incantations. Structure it as a Nushell script with a `main` and `main ` subcommands (`def "main unit"`, ...) so `--help` is generated and completion comes from the typed signatures: + - `test unit` runs `cargo test --lib --bins` (no sockets, no host access). + - `test integration [filter]` runs `cargo test --tests` (plain, tls, integration, binary_smoke, `peers` once it exists). `filter` is passed to cargo as a name filter; `--nocapture` flag supported. + - `test lint` runs `cargo fmt --check`, `cargo clippy --all-targets -- -D warnings` and the `cargo tree -i openssl` emptiness check. + - `test vm [name]` runs a NixOS VM test. `name` is one of `vm` (from `checks`) or `peers` (`nix build .#tests.x86_64-linux.peers -L`; moves to `checks` when green). Refuses without `/dev/kvm` and says why; prints that it is slow. + - `test all` = lint, unit, integration, stopping at the first failure; `test ci` = `all` plus `vm`. No args prints help. + - Each step prints a header and a final table (step, result, seconds); exit code non-zero on any failure (use `error make`/`exit 1`, never swallow cargo's exit status). Read-only against the host: no `/etc`, systemd or Docker changes (live-host checks stay manual, see Verification plan in the [README](README.md#verification-plan)). + - **Autocomplete (must work, and is a done-criterion):** + - Nushell: subcommands and flags complete natively once the script is loaded as a module. Dynamic values use custom completers (`def "nu-complete test vm" [] { ["vm" "peers"] }` and `name: string@"nu-complete test vm"`; integration filters complete from `cargo test --tests -- --list` test names, cached for the session). The devenv `enterShell` prints/loads `use dev/test.nu *` for Nushell users, and `docs/consolidator.md` shows the one-line `use` for a permanent setup. + - Other shells (the repo owner's login shell is zsh): `test completions zsh|bash` prints a small static completion script (subcommands, `vm` names, flags) generated from the same command table in the script, so zsh/bash tab completion works through `devenv shell` too (`source <(test completions zsh)` in `enterShell`). + - Verified by a self-test: `test --help` and `test vm --help` list every subcommand/flag, and a check in `dev/test.nu` (`test self-check`) fails if the completion table and the defined subcommands diverge. + - AGENTS.md and README replace their scattered command lists with `test ...`, keeping the underlying commands visible in `--help`. + +## Inputs + +- [contracts.md](contracts.md) section 10 (test config contract) and sections 5 and 7 (REST/MCP shapes the script asserts). No fakes: the test runs the real binary. +- `nix/tests/peers.nix` is the source of truth for section 10; if the test must change, change it and update contracts.md in the same PR, then log in the affected streams. + +## Done when + +The test builds under `nix build .#tests.x86_64-linux.peers` (or equivalent) and evaluates, fails cleanly at the first peers assertion against the current binary, and is documented in `docs/consolidator.md`. `test unit|integration|lint|vm|all` each run and report correctly on the current tree, and tab completion works in Nushell, zsh and bash (T7). It must pass at the end of WS-E, then move to `checks.vm-peers` (F2 does the move). + +## Handoff + +- To everyone: the `test` runner and its subcommand names (referenced by AGENTS.md/README and by every stream's "Done when"). +- To WS-F: a passing `peers.nix` (ready to promote) and any changes made to its assumptions, recorded in contracts.md section 10. +- To WS-A/B: the `test integration` step picks up any `tests/peers*.rs` automatically. + +## Log + +- 2026-10-19: Split from the single phase plan 2026-10-19. `nix/tests/peers.nix` already written (T1 to T6) and failing at the first peers assertion by design; `flake.nix` exposes it as `tests..peers`. T7 not started. Ownership change from the single plan: `tests/peers.rs` moved to WS-F; WS-A/B in-process tests use their own files to avoid collisions.