From 4d83c2eff53fefbdb88927d36e2eafa7638edd17 Mon Sep 17 00:00:00 2001 From: ari Date: Tue, 20 Oct 2026 10:40:17 -0400 Subject: [PATCH] F: F4 peers docs, agenix example and tool-count note README peers section, docs/consolidator.md (threat model, tokens, firewall, worked agenix example, reload/restart), the MCP tool count in both tables, and config.example.toml commentary. --- AGENTS.md | 2 +- README.md | 53 ++++- config.example.toml | 5 + docs/consolidator.md | 221 +++++++++++++++++++ nix/module.nix | 5 +- plan/phase-5-consolidator/ws-f-nixos-docs.md | 7 + tests/peers.rs | 28 +-- 7 files changed, 301 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 612c718..db0bb50 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,7 +46,7 @@ Git hooks run fmt, clippy, lint and a secret scan on commit and path-filtered te - Read-only against the host: no code path may start, stop or modify units, containers or files outside the state dir. - Default port is 1984, default bind is loopback. Do not change defaults for convenience. - Comments: sparse, explain why not what. No workstream, task or review tags (`WS-x`, `G3`, `DEC-n`) in code comments; reference ARCHITECTURE.md by section if needed. -- The MCP tool count is 8; the tool-list snapshot test guards it. Change tools deliberately. +- The MCP tool count is 8 without peers and 10 with `peers.enabled` (`list_peers`, `get_peer`, and an optional `peer` argument on the other eight); the tool-list snapshot test guards both tables. Change tools deliberately. - Keep `config.example.toml` and `README.md` in sync with config or CLI changes. ## Safety diff --git a/README.md b/README.md index 8e59e4e..aaba528 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Full route table with parameters: [docs/api.md](docs/api.md). ## MCP -Endpoint: `http://127.0.0.1:1984/mcp` (Streamable HTTP, same bearer token). Eight tools (`list_systemd_units`, `get_systemd_unit`, `list_systemd_timers`, `get_journal_logs`, `tail_journal`, `list_containers`, `get_container`, `get_container_logs`), two prompts, two resources and two resource templates. Log content is untrusted data and is labelled as such. +Endpoint: `http://127.0.0.1:1984/mcp` (Streamable HTTP, same bearer token). Eight tools (`list_systemd_units`, `get_systemd_unit`, `list_systemd_timers`, `get_journal_logs`, `tail_journal`, `list_containers`, `get_container`, `get_container_logs`), two prompts, two resources and two resource templates. A [consolidator](#peers-consolidator) adds `list_peers` and `get_peer` and a `peer` argument on every tool, for ten tools. Log content is untrusted data and is labelled as such. Claude Code: `claude mcp add --transport http linux-tracker http://127.0.0.1:1984/mcp --header "Authorization: Bearer $TOKEN"`. @@ -110,6 +110,57 @@ key_file = "/etc/skyg/tls/key.pem" # PEM, PKCS#8, PKCS#1 or SEC1 REST and `/mcp` are then served over HTTPS only. Replace the files and send `SIGHUP` to rotate without dropping connections (a bad pair keeps the old certificate). rustls only; no openssl. Self-signed example, rotation caveats, ACME and sops paths: [docs/tls.md](docs/tls.md). +## Peers (consolidator) + +One `slinuxm` can watch other instances: declare them as **peers** and it health-checks each one and passes read-only requests through to it, so one MCP endpoint or one REST base URL covers the fleet. Off by default; a peer-less instance keeps exactly the same routes, eight MCP tools, metrics and background tasks as before. + +```toml +[peers] +enabled = true # other intervals, timeouts and caps: config.example.toml + +[[peers.peer]] +name = "minipc1" # ^[a-z0-9][a-z0-9_-]{0,31}$, at most 64 peers +url = "https://minipc1:1984" # http:// to a non-loopback host also needs allow_plain_http = true +token_file = "/run/credentials/skyg-linux-tracker.service/peer-minipc1" +ca_file = "/etc/skyg/private-ca.pem" # optional extra trust anchor + +[[peers.peer]] +name = "nas" +url = "http://nas:1984" +token_file = "/etc/skyg/peer-nas.token" # 0600, owned by the service user +allow_plain_http = true +``` + +`token_file` holds a path, never the token. A consolidator has to replay the peer's plaintext bearer token, so the token lives in a file managed outside `slinuxm` (agenix on NixOS), is read into memory on every check and request, and never appears in the config, `--print-config`, the database or the logs. Mint **one token per consolidator on every peer** and use it only there (`sudo -u slinuxm token create consolidator-`), so a single `token revoke` on a peer cuts that consolidator off and nothing else. Replacing the file is enough to rotate: no restart. + +Two checks per peer, kept in memory and reset on restart: + +- **server**: a TCP connect to the peer's port. +- **API**: an authenticated `GET /api/v1/ping`. A `401` means the API runs and our token does not: `api.up = true` with `api.auth = "rejected"`. That is reported to you as a `503` naming the peer, never as a `401` that would suggest re-authenticating with this host. A peer without the ping route falls back to public `/healthz` and reports `api.auth = "unknown"`. + +```sh +curl -H "Authorization: Bearer $TOKEN" http://127.0.0.1:1984/api/v1/peers +curl -H "Authorization: Bearer $TOKEN" 'http://127.0.0.1:1984/api/v1/peers/minipc1/systemd/units?state=failed' +sudo -u skyg slinuxm peer list --config /etc/skyg-linux-tracker.toml +sudo -u skyg slinuxm peer check minipc1 --config /etc/skyg-linux-tracker.toml +``` + +`GET /api/v1/peers` lists every peer, `GET /api/v1/peers/{peer}` one of them. `GET /api/v1/peers/{peer}/...` is a GET-only, allowlisted pass-through (`systemd/...`, `journal`, `journal/follow`, `docker/...`): the peer's answer comes back unchanged with `x-slinuxm-peer: `, a peer that is down, rejects our token or times out is a `503` naming it, `x-slinuxm-hop` from a caller is a `400`, and unknown peers are a `404`. With peers enabled MCP grows `list_peers` and `get_peer`, and every other tool accepts an optional `peer` argument (ten tools instead of eight). There is no aggregation across peers in this version: ask one peer at a time. OTel gains `peer.up`, `peer.api.auth_ok`, `peer.check.duration_seconds` and `peer.last_change_timestamp_seconds`. + +On NixOS the module declares peers and delivers each token through systemd `LoadCredential`, so the service user never needs to read the agenix output: + +```nix +services.skyg-linux-tracker.peers = { + enable = true; + peers.minipc1 = { + url = "https://minipc1:1984"; + tokenFile = config.age.secrets.peer-minipc1.path; # a string path, never a path literal + }; +}; +``` + +Worked agenix setup, the threat model, the firewall note (every peer must accept the consolidator's address on its port), reload and shutdown semantics: [docs/consolidator.md](docs/consolidator.md). The peers tests are `tests/peers.rs` (in-process, no VM) and the three-machine NixOS VM test: `devenv shell -- dev-test integration peers`, and for the VM test `devenv shell -- dev-test vm peers`, which is `nix build .#checks.x86_64-linux.vm-peers -L`. + ## NixOS diff --git a/config.example.toml b/config.example.toml index 13ca665..5b27380 100644 --- a/config.example.toml +++ b/config.example.toml @@ -60,6 +60,11 @@ max_follow_secs = 60 # max duration of a bounded follow (MCP tail_journal); S # Scalar keys accept SKYG_PEERS__* (SKYG_PEERS__ENABLED, SKYG_PEERS__TCP_INTERVAL_SECS, ...). # [[peers.peer]] entries are file-only. token_file is a path; the file is read # into memory and never written back, logged, or shown by --print-config. +# Mint one token per consolidator on each peer (consolidator-) and give +# each peer its own file, 0600 and owned by the service user. On NixOS the +# module does the plumbing through LoadCredential: +# services.skyg-linux-tracker.peers.peers..tokenFile = config.age.secrets..path; +# Threat model, agenix example and firewall note: docs/consolidator.md. [peers] enabled = false # SKYG_PEERS__ENABLED tcp_interval_secs = 10 # SKYG_PEERS__TCP_INTERVAL_SECS diff --git a/docs/consolidator.md b/docs/consolidator.md index 65fb7c0..d45a201 100644 --- a/docs/consolidator.md +++ b/docs/consolidator.md @@ -1,3 +1,224 @@ +# Consolidator (peers) + +One `slinuxm` can be a **consolidator** for other `slinuxm` instances. Each one it is told about is a *peer*: it health-checks the peer in two ways and passes read-only requests through to it, so one MCP endpoint or one REST base URL covers the fleet. The feature is off by default (`[peers] enabled = false`), and a peer-less instance behaves exactly as before: same routes, eight MCP tools, same metrics, no extra tasks or config. + +Everything here is declarative: peers come from the config file, nothing writes to a peer, and no request parameter can supply a URL. Runtime peer management (`peer add`, a peers table) does not exist. + +Tests: `tests/peers.rs` runs the whole path in-process (no VM), and a three-machine NixOS VM test covers boot, TLS, rotation and outages. Both go through the repository's runner, `devenv shell -- dev-test …`; see [Testing](#testing) below. + +## Model + +| Check | What it proves | How | +|---|---|---| +| `server` | The peer's port accepts a TCP connection | one connect per `tcp_interval_secs`, bounded by `tcp_timeout_secs` | +| `api` | The peer's REST API runs **and** accepts our token | authenticated `GET /api/v1/ping` every `api_interval_secs`, bounded by `api_timeout_secs` | + +A check flips down after `fail_threshold` consecutive failures and up again on the first success. All state is in memory (`since`, `last_checked`, `last_error`, `latency_ms`), so a restart of the consolidator resets it; history belongs in your OTel backend. + +`last_error` is a class, never peer text or a URL: `unreachable`, `timeout`, `dns`, `tls_error`, `http_`, `unauthorized`, `oversize`, `saturated`, `token_file`. + +### Reading `server` and `api` + +| `server.up` | `api.up` | `api.auth` | Meaning | +|---|---|---|---| +| `true` | `true` | `ok` | healthy | +| `true` | `true` | `rejected` | the API runs, **our token is refused**: revoked, rotated on the peer, or the file is still a placeholder. Mint a new token on the peer and replace the file (`auth_ok` is 0 in OTel while `peer.up` is 1 for both checks) | +| `true` | `true` | `unknown` | the peer predates `GET /api/v1/ping`; the check fell back to public `/healthz`, so liveness is proven but not our credential | +| `true` | `false` | `unknown` | the port answers but the API does not (`tls_error` for a certificate problem, `timeout`, `http_5xx`) | +| `false` | `false` | `unknown` | nothing answers on the port: the peer is down or unreachable (also the case when the name resolves to an address the peer does not listen on, see [Address families](#address-families)) | + +A peer that rejects our token is reported to *your* callers as a `503` naming the peer, never as a `401`: re-authenticating with this host would not help. + +## Configuration + +```toml +[peers] +enabled = false # default: no peer routes, tools, tasks or metrics +tcp_interval_secs = 10 +tcp_timeout_secs = 3 # must be below tcp_interval_secs +api_interval_secs = 30 +api_timeout_secs = 5 # must be below api_interval_secs +fail_threshold = 2 # consecutive failures before a check flips down +request_timeout_secs = 10 # one proxied request, connect to body +max_response_bytes = 8388608 # buffered bodies above this are refused +max_concurrent_per_peer = 4 # proxied requests in flight per peer +max_concurrent_total = 16 # ... across all peers +deny_cidrs = ["169.254.0.0/16", "0.0.0.0/8"] # resolved peer addresses in these ranges are refused + +[[peers.peer]] +name = "minipc1" # ^[a-z0-9][a-z0-9_-]{0,31}$, unique, at most 64 +url = "https://minipc1:1984" # http or https, no userinfo, query or fragment +token_file = "/run/credentials/skyg-linux-tracker.service/peer-minipc1" # absolute +ca_file = "/etc/skyg/private-ca.pem" # optional extra trust anchor +allow_plain_http = false # required true for http:// to a non-loopback host +enabled = true # false keeps the declaration without contacting the peer +``` + +Scalar keys accept environment overrides (`SKYG_PEERS__ENABLED`, `SKYG_PEERS__API_INTERVAL_SECS`, ...); `[[peers.peer]]` entries are file-only. `slinuxm --print-config` prints `token_file` paths, never their contents. Validation refuses: a timeout that is not below its interval, more than 64 peers, duplicate names, a bad name, a relative `token_file` or `ca_file`, a non-http(s) URL, userinfo/query/fragment in a URL, an IP literal in `deny_cidrs`, and plain `http` to a non-loopback host without `allow_plain_http = true` (which also logs a warning: the bearer token travels in cleartext). `enabled = true` with no peers logs a warning. + +`deny_cidrs` is checked against the **resolved** address at connect time, so a name that resolves into a denied range is refused as `unreachable`. Private LAN ranges are deliberately allowed; the default denies only link-local/metadata and the unspecified range. + +## REST and MCP + +All under the inbound auth of this host; nothing is mounted while peers are off. + +| Route | Behaviour | +|---|---| +| `GET /api/v1/peers` | `{"peers": [...]}`, sorted by name, disabled peers included | +| `GET /api/v1/peers/{peer}` | one status object; unknown peer is `404` | +| `GET /api/v1/peers/{peer}/...` | GET-only pass-through of an allowlisted route | + +The status object is `{name, enabled, url, server{up, since, last_checked, last_error, latency_ms}, api{up, since, last_checked, last_error, latency_ms, auth, peer_version}}`, times in RFC 3339 UTC. + +The pass-through allowlist is `systemd/...`, `journal`, `journal/follow`, `docker/...`, with the query parameters the local routes accept (`state`, `unit`, `priority`, `lines`, `since`, `all`, `tail`, `follow`). A caller may include the `/api/v1` prefix; it is stripped. Anything else is a `400`, `..` and encoded slashes are rejected, `ping`, `peers/...` and `/healthz` are not reachable through it, and methods other than GET are `405`. The peer's 2xx answer comes back unchanged (status, `Content-Type`, body; SSE stays a stream) with `x-slinuxm-peer: `; every error produced for a peer carries that header too, except the `405` for a non-GET method, which the router answers before the peer route runs. Consolidated failures use the existing error shape: + +| Peer outcome | Answer to your caller | +|---|---| +| peer `404` | `404 not_found`, `peer : not found on peer` | +| peer `400`/`422` | `400 bad_request` | +| peer rejects our token (`401`/`403`) | `503 unavailable`, `peer : peer rejected credentials` | +| timeout, refused, DNS, TLS, `5xx`, oversize, saturated, token file | `503 unavailable` with the class | +| malformed peer JSON | `500 internal` | +| unknown peer | `404 not_found` | + +No caller header is forwarded except `traceparent`; the Authorization header is always the peer's own token from `token_file`. A caller that sends `x-slinuxm-hop` is refused with `400` (no chained consolidators). + +With peers enabled, MCP exposes `list_peers`, `get_peer` and the resource `peers://status` (same JSON as the list), every one of the eight existing tools gains an optional `peer` argument, and the `container_health_report` and `diagnose_unit` prompts gain a `peer` argument. Ten tools instead of eight, and only on the instances that need them. Peer-sourced text is labelled untrusted data, as local log text is. + +## OpenTelemetry + +Four observable gauges, registered only with peers enabled and `otel.metrics.enabled`, read from the registry snapshot on each scrape: `peer.up` (attributes `peer.name`, `peer.check` = `server`/`api`), `peer.api.auth_ok` (`peer.name`), `peer.check.duration_seconds` (`peer.name`, `peer.check`) and `peer.last_change_timestamp_seconds` (`peer.name`, `peer.check`). No URL, version or token in attributes. A removed peer disappears from the next scrape. Outbound proxied requests carry a span named `peer.request` with `peer.name`, and the caller's `traceparent` is passed through, so a trace spans the fleet. + +## Credentials and threat model + +DEC-11 stores *inbound* tokens only as sha256 hashes. A consolidator cannot do that: it has to replay the peer's plaintext bearer token. DEC-13 therefore makes **outbound** peer tokens externally managed files: + +- `slinuxm` owns no copy of a peer token: not in SQLite, not in `config.toml`, not in `--print-config`, the logs or a support bundle. The config holds a path; the process reads the file into a redacted `Secret` on every check and request and drops it. +- On NixOS the file is an agenix secret decrypted at activation to `/run/agenix/...` (root-only) and handed to the service through systemd `LoadCredential`, so the service user never needs read access to the agenix output. Outside NixOS, put the file somewhere only the service user can read: a group- or world-readable token file is refused (`token_file` class) unless it sits in `$CREDENTIALS_DIRECTORY`. +- Rotation is a file replacement; the next check reads the new value. On NixOS also `systemctl reload` (SIGHUP) so systemd re-copies the credential (`RefreshOnReload=credentials`). +- Use **one token per consolidator on every peer** (`consolidator-`, read-only by nature since every token is read-only): `slinuxm token revoke consolidator-` on a peer cuts exactly that consolidator off, and nothing else. + +What it does not protect: root on the consolidator (it can read the decrypted file), and the consolidator's own token as an asset — it reaches every peer's data, which is why tokens are per consolidator, revocable, and why scoping tokens per peer is a follow-up, not a v1 feature. + +Peers are only ever declared by whoever manages the config (root or Nix). No route, parameter or MCP argument supplies a URL, so the consolidator cannot be steered at an arbitrary target. Redirects from a peer are never followed (`503 http_3xx`), TLS is verified against the webpki roots plus the optional `ca_file`, and there is no skip-verify switch. + +## NixOS with agenix + +The module renders the peers and the credentials; you only declare them: + +```nix +services.skyg-linux-tracker = { + enable = true; + bind = "0.0.0.0"; + openFirewall = true; # or firewallInterfaces = [ "enp3s0" ]; + peers = { + enable = true; + settings = { api_interval_secs = 30; fail_threshold = 2; }; # [peers] tuning, snake_case + peers = { + minipc1 = { url = "https://minipc1:1984"; tokenFile = config.age.secrets.peer-minipc1.path; }; + nas = { + url = "http://nas:1984"; + tokenFile = config.age.secrets.peer-nas.path; + allowPlainHttp = true; # the token then travels in cleartext in your LAN + }; + }; + }; +}; +``` + +A worked agenix setup for host `monitor`, which consolidates `minipc1` (HTTPS with a private CA) and `nas` (plain HTTP in a trusted LAN). + +On each peer, mint a token for this consolidator. It has to be a dedicated token: revoking it cuts off exactly this consolidator. + +```sh +# on minipc1 and on nas +sudo -u skyg-linux-tracker slinuxm token create consolidator-monitor +``` + +Register the encrypted files next to your other agenix secrets and encrypt them for the consolidator host. Only the ciphertext goes into git. + +```nix +# secrets/secrets.nix (agenix) +{ + # The token minipc1 issued to `monitor`. Only `monitor` needs to decrypt it: + # the consolidator replays it, no other host reads it. + "peer-minipc1.age".publicKeys = [ monitor ]; + "peer-nas.age".publicKeys = [ monitor ]; +} +``` + +```sh +agenix -e secrets/peer-minipc1.age # paste the token printed on minipc1 +agenix -e secrets/peer-nas.age # paste the token printed on nas +``` + +```nix +{ config, inputs, ... }: { + imports = [ inputs.agenix.nixosModules.default ]; + + # Decrypted at activation to /run/agenix/peer-, root-only. The module + # hands the file to the service through LoadCredential, so the + # `skyg-linux-tracker` user never reads it directly. + age.secrets.peer-minipc1 = { + file = ./secrets/peer-minipc1.age; + owner = "root"; + mode = "0400"; + }; + age.secrets.peer-nas = { file = ./secrets/peer-nas.age; owner = "root"; mode = "0400"; }; + + services.skyg-linux-tracker = { + enable = true; + bind = "0.0.0.0"; + openFirewall = true; + peers = { + enable = true; + peers = { + minipc1 = { + url = "https://minipc1:1984"; + tokenFile = config.age.secrets.peer-minipc1.path; # a string path, not a path literal + caFile = "/etc/ssl/private-ca.pem"; # readable by the service user + }; + nas = { + url = "http://nas:1984"; + tokenFile = config.age.secrets.peer-nas.path; + allowPlainHttp = true; + }; + }; + }; + }; +} +``` + +The module turns that into one `[[peers.peer]]` entry per enabled peer, with `token_file = /run/credentials/skyg-linux-tracker.service/peer-`, and one `LoadCredential=peer-:` line. Consequences worth knowing: + +- `tokenFile` must be a **string** (`config.age.secrets...path`), not a path literal: a `nix` path or `"${./secret}"` interpolates a store path and is rejected at evaluation. Such a secret has already been copied into the local store by then, so rotate it. +- `caFile` is **not** passed through `LoadCredential`: the service user reads it directly, so it must be readable by `skyg-linux-tracker`. A root-only file such as the agenix output makes every check and proxied request to that peer fail as `tls_error`. Store paths are fine for a public CA. +- Peers with `enable = false` are rendered nowhere, so no credential is shipped for them. +- `peers.settings` is freeform `[peers]` tuning; it must not contain `enabled`, `peer` or a token. + +### Reload or restart + +`switch-to-configuration` reloads (SIGHUP) when only the config file changed, and restarts when the unit file changed. SIGHUP re-reads `[peers]`, so editing a peer's `url`, `caFile` or the intervals applies without a restart, and systemd re-copies the credentials (`RefreshOnReload=credentials`), so a rotated token is picked up as well. Adding, removing, enabling or disabling a peer, or changing its `tokenFile`, changes the `LoadCredential=` lines and therefore **restarts** the service — the safe outcome. `enabled`, `deny_cidrs`, `max_response_bytes` and the two concurrency limits also need a restart (the routes are mounted once and the limits live in the transport), and the module's restart trigger makes a `switch` do that instead of silently keeping the old values. + +Outside NixOS the same split applies by hand: SIGHUP re-reads the TLS pair and `[peers]` (peer entries, intervals, timeouts, `fail_threshold`); everything else needs a restart. + +## Firewall and networking + +Every **peer** must accept the consolidator's address on its port; the consolidator only needs outbound access. On a peer's NixOS host that is `openFirewall = true` or `firewallInterfaces = [ "enp3s0" ]`; a peer bound to `127.0.0.1` is not reachable from another machine, so a consolidator needs the peer to bind a LAN address (with auth on, which the module enforces) and either TLS or a trusted network. + +### Address families + +A peer is reached at the address its name resolves to at connect time, and the first resolved address wins. A peer that listens on IPv4 only, with a name that also resolves to an IPv6 address (as a NixOS test network does), reads as `unreachable`; declare it by IP address or let it listen on both families (`bind = "::"` accepts IPv4-mapped connections too). DNS rebinding is not a risk here because the target is pinned to the address that was checked against `deny_cidrs`. + +## Limits and non-goals + +- No aggregation: `peer = "*"` and fleet-wide views are a follow-up, because partial failures need a result shape of their own. Ask one peer at a time. +- No chaining: a consolidator's request carries `x-slinuxm-hop`, so a peer that is itself a consolidator refuses it. This prevents loops and amplification. +- No write path: proxying is GET-only and allowlisted, and nothing here starts, stops or changes a unit or a container. +- No history: peer state is in memory; use OTel for trends. +- No runtime management: peers are declarative, reloaded on SIGHUP. +- Each peer exports its own telemetry; peers' OTel signals are not re-exported by the consolidator. + ## Testing The peers acceptance test is a NixOS VM test. Three machines run the real `slinuxm`: `alpha` and `beta` consolidate each other over plain HTTP, and `alpha` also declares the TLS-only `gamma` with a private CA. It needs KVM and takes minutes. diff --git a/nix/module.nix b/nix/module.nix index 0e01cf7..606486f 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -264,8 +264,9 @@ in Optional PEM CA bundle used to verify this peer's TLS certificate. Unlike `tokenFile` it is not passed through `LoadCredential`: the service reads it directly as the - `skyg-linux-tracker` user, so that user needs read access and a - root-only file such as `/run/agenix/*` fails at startup. + `skyg-linux-tracker` user, so that user needs read access. A + root-only file such as `/run/agenix/*` makes every check and + proxied request to that peer fail as `tls_error`. ''; }; enable = mkOption { diff --git a/plan/phase-5-consolidator/ws-f-nixos-docs.md b/plan/phase-5-consolidator/ws-f-nixos-docs.md index 0c74eab..7913096 100644 --- a/plan/phase-5-consolidator/ws-f-nixos-docs.md +++ b/plan/phase-5-consolidator/ws-f-nixos-docs.md @@ -81,3 +81,10 @@ Shippable, documented, verified. - Cases: plain peer (status `server.up`/`api.up`/`auth ok`, `peer_version`, list shape, 401 without a caller token, 404 for an unknown peer, 405 on POST, both checks flipping when the peer dies, proxy 503 naming the peer while the caller's token keeps working); TLS peer with an `rcgen` CA and a leaf for `127.0.0.1` via `ca_file` (`api.up` needs a verified handshake and our token); a peer signed by a different CA (`server.up` true, `api.up` false, `last_error = tls_error`, proxy 503 `tls_error`); a `302` (503 `http_302`, and the `Location` target sees **zero** connections); an oversize body (503 `oversize` with `max_response_bytes = 64`); a silent peer (503 `timeout`, bounded); the 64-peer cap (`Config::validate`, the path `serve` uses: 64 accepted, 65 refused); and shutdown ending a proxied follow. - **Blocking finding, reported to WS-C's Log (2026-10-20)**: the proxy hands the transport the stripped route (`/systemd/units`) while peers serve `/api/v1/systemd/units`, so every proxied call to a healthy peer answers `peer : not found on peer`. Health checks are fine (`/api/v1/ping`). Two tests (`a_plain_peer_serves_its_own_data_through_the_proxy`, `shutdown_ends_a_proxied_follow`) are `#[ignore]`d with that reason and state the contract; removing the attribute is the acceptance check once C (or B) prefixes the path. The same defect will fail the proxy subtest of `checks.vm-peers`. - Verified: `cargo test --test peers` → 7 passed, 2 ignored. +- 2026-10-20: **F4 done** (commit `F: F4 peers docs, agenix example and tool-count note`). + - `README.md`: new `## Peers (consolidator)` section between TLS and NixOS — TOML example, why `token_file` is a path (DEC-13) and why each consolidator needs its own token, the two checks with a server/api/auth table, REST/`peer list`/`peer check` examples, the pass-through rules, MCP 8-to-10 and OTel metric names, a NixOS snippet, and pointers to `docs/consolidator.md` and the test runner. The MCP section now says "eight tools (ten with a consolidator)"; the peers section names the underlying `nix build .#checks.x86_64-linux.vm-peers -L` next to `dev-test integration peers` / `dev-test vm peers`. + - `docs/consolidator.md`: every section except `## Testing`, which is byte-identical (checked). Title and intro; `## Model` (checks, thresholds, `last_error` classes, the `server`/`api`/`auth` reading table); `## Configuration` (all keys, `SKYG_PEERS__*`, validation rules, `deny_cidrs` against the resolved address); `## REST and MCP` (status shape, allowlist, hop header, error mapping table, the 405-has-no-peer-header exception); `## OpenTelemetry`; `## Credentials and threat model` (DEC-13, per-consolidator tokens, what it does not protect, the D14 note that a consolidator token reaches every peer's data); `## NixOS with agenix` (options, mint-on-the-peer, `secrets.nix`, `age.secrets`, the module block, `tokenFile`/`caFile` consequences, reload-vs-restart including the review-n5 note that adding or removing a peer or changing its `tokenFile` restarts on `switch`); `## Firewall and networking` plus `### Address families`; `## Limits and non-goals` (no aggregation, no chaining, GET-only, no history, declarative only). + - `AGENTS.md`: the tool-count line is now "8 without peers and 10 with `peers.enabled` (`list_peers`, `get_peer`, an optional `peer` argument on the other eight); the snapshot test guards both tables". Command lists untouched. + - `config.example.toml`: commentary only — one token per consolidator, the module's `tokenFile` option and a pointer to `docs/consolidator.md`. Keys and defaults unchanged and checked against `docs/consolidator.md` and `PeersConfig::default`. + - Corrected F1's `caFile` description in `nix/module.nix`: nothing opens the CA at startup, so "fails at startup" was wrong; a root-only file makes every check and proxied request to that peer fail as `tls_error`. + - Not an Owned file, so left for the fold: `docs/README.md`'s Reference table does not list `docs/consolidator.md` yet. diff --git a/tests/peers.rs b/tests/peers.rs index 386f714..1589664 100644 --- a/tests/peers.rs +++ b/tests/peers.rs @@ -391,8 +391,7 @@ async fn a_plain_peer_reports_up_and_auth_ok() { let dir = TempDir::new("plain"); let peer = Peer::start().await; let token = peer.token_file(&dir); - let consolidator = - Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await; + let consolidator = Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await; let mut status = consolidator .wait_status("beta", |s| s.server.up && s.api.up) @@ -455,8 +454,7 @@ async fn a_plain_peer_serves_its_own_data_through_the_proxy() { let dir = TempDir::new("proxy"); let peer = Peer::start().await; let token = peer.token_file(&dir); - let consolidator = - Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await; + let consolidator = Consolidator::start(one_peer("beta", peer.url("http"), None, &token)).await; consolidator .wait_status("beta", |s| s.server.up && s.api.up) .await; @@ -487,13 +485,8 @@ async fn a_tls_peer_is_trusted_through_its_ca_file() { let (ca_path, cert, key) = ca.leaf_for_loopback(&dir, "good"); let peer = Peer::start_tls(&cert, &key).await; let token = peer.token_file(&dir); - let consolidator = Consolidator::start(one_peer( - "beta", - peer.url("https"), - Some(ca_path), - &token, - )) - .await; + let consolidator = + Consolidator::start(one_peer("beta", peer.url("https"), Some(ca_path), &token)).await; // `api.up` needs a verified handshake and our token: that is the CA file // working end to end. @@ -515,8 +508,13 @@ async fn a_peer_signed_by_another_ca_is_a_tls_error() { let (_, cert, key) = other.leaf_for_loopback(&dir, "other"); let peer = Peer::start_tls(&cert, &key).await; let token = peer.token_file(&dir); - let consolidator = - Consolidator::start(one_peer("beta", peer.url("https"), Some(trusted_ca), &token)).await; + let consolidator = Consolidator::start(one_peer( + "beta", + peer.url("https"), + Some(trusted_ca), + &token, + )) + .await; // The port answers, the handshake does not: a certificate fault must not // read as an outage. @@ -561,9 +559,7 @@ async fn a_redirect_is_reported_and_never_followed() { // Let the health loop probe as well, then check the Location was never // acted on. tokio::time::sleep(Duration::from_millis(500)).await; - consolidator - .wait_status("beta", |s| !s.api.up) - .await; + consolidator.wait_status("beta", |s| !s.api.up).await; assert!(source.hits() > 0); assert_eq!(target.hits(), 0, "the redirect was followed"); consolidator.stop().await;