From 53e272367eed216107ee056d96975cd49411dade Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Wed, 21 Oct 2026 00:07:50 -0400 Subject: [PATCH] setup for peers --- bin/ABOUTME.md | 2 +- bin/lib/remote.nu | 11 ++++++-- flake.lock | 8 +++--- flake.nix | 6 ++++ hosts/hl-fwdesk/services/hermenix.nix | 6 ++-- hosts/hl-fwdesk/services/linux-tracker.nix | 27 ++++++++++++++++-- secrets.nix | 9 ++++++ secrets/ABOUTME.md | 33 +++++++++++----------- 8 files changed, 73 insertions(+), 29 deletions(-) diff --git a/bin/ABOUTME.md b/bin/ABOUTME.md index 89678b6..554f714 100644 --- a/bin/ABOUTME.md +++ b/bin/ABOUTME.md @@ -37,7 +37,7 @@ bin/ | `skyg os rollback` | Roll back to previous generation | | `skyg remote switch ` | Deploy to a remote host | | `skyg remote install ` | Fresh-install a remote host via nixos-anywhere, using `hosts/.disko.nix`. Always wipes the disks in that file. Aborts if the file is missing, if the host already boots NixOS from disk (unless `--force-wipe`), or if you do not type the short hostname. | -| `skyg remote status ` | Check status of a single remote host: booted / current / next-boot generation. `reboot_pending` = booted != next boot (`/nix/var/nix/profiles/system`), `needs_build` = local build != next boot. So hosts deployed with `boot` show up correctly. | +| `skyg remote status ` | Check status of a single remote host: booted / current / next-boot generation. `reboot_pending` = booted != next boot (`/nix/var/nix/profiles/system`), `needs_build` = local build != next boot (or `"unknown"` if the local eval fails). So hosts deployed with `boot` show up correctly. | | `skyg remote status --all` | Check status of all remote hosts | | `skyg remote boot-all` | Boot all remote hosts | | `skyg remote reboot-pending` | Reboot every remote host with a pending reboot (`booted-system` != next-boot generation, so `boot` deploys count). Lists them, asks for confirmation, then reboots one at a time. Each host must prove it rebooted (boot id changed) and come back on its next-boot generation, otherwise it is reported as failed. Sends a desktop notification per host. | diff --git a/bin/lib/remote.nu b/bin/lib/remote.nu index 2d384aa..5e226d0 100644 --- a/bin/lib/remote.nu +++ b/bin/lib/remote.nu @@ -140,7 +140,7 @@ Starting parallel boots (concurrency: 2)..." let boot_results = ($online_hosts | par-each --threads 1 { |host| let target_host = $"root@($host).lab.internal" let profile = $"hl-($host)" - let runcmd = $"nh os boot --target-host ($target_host) .#($profile)" + let runcmd = $"nh os boot --no-nom --target-host ($target_host) .#($profile)" print $runcmd let ok = (try { bash -c $runcmd @@ -214,8 +214,13 @@ export def "skyg remote status-all" [] { let current = (if ($info.current | is-empty) { "-" } else { $info.current | path basename }) let next_boot = (if ($info.next_boot | is-empty) { "-" } else { $info.next_boot | path basename }) let reboot_line = if $info.reboot_pending { $"(ansi yellow)REBOOT PENDING(ansi reset)\n" } else { "" } - let build_line = if $info.needs_build { $"(ansi red)NEEDS NEW BUILD(ansi reset)" } else { $"(ansi green)UP TO DATE(ansi reset)" } - print $"($host) | Booted: ($booted) | Current: ($current) | Next boot: ($next_boot)" + # needs_build is true, false, or "unknown" (local eval failed) + let build_line = match $info.needs_build { + true => $"(ansi red)NEEDS NEW BUILD(ansi reset)", + false => $"(ansi green)UP TO DATE(ansi reset)", + _ => $"(ansi yellow)BUILD UNKNOWN (eval failed)(ansi reset)", + } + print $"($host) | Booted: ($booted) | Current: ($current) | Next boot: ($next_boot)\n ($reboot_line)($build_line)" $info }) diff --git a/flake.lock b/flake.lock index 56a5b2d..ee80c02 100644 --- a/flake.lock +++ b/flake.lock @@ -3155,11 +3155,11 @@ ] }, "locked": { - "lastModified": 1792553170, - "narHash": "sha256-wajyEzplVR8XWkjAZFPiGp+1aVcZjC14pspWy+pl5qw=", + "lastModified": 1792555264, + "narHash": "sha256-Mo6kcBIKiZ618SqwPjxlDU1PgNDfqF5Ai0YKLadXIsg=", "ref": "refs/heads/main", - "rev": "020fa4871fac5517490fd51e914d539cffbf8afd", - "revCount": 213, + "rev": "d8ad1c44c57a2abe5261656817ca25f72c062140", + "revCount": 216, "type": "git", "url": "https://gitea.skyg.ca/ari/skyg-linux-tracker.git" }, diff --git a/flake.nix b/flake.nix index 1f433da..676a215 100644 --- a/flake.nix +++ b/flake.nix @@ -179,6 +179,12 @@ buildbot-gitea-oauth-secret = ./secrets/buildbot-gitea-oauth-secret.age; immich-shared = ./secrets/immich-shared.age; grafana-secret-key = ./secrets/grafana-secret-key.age; + linux-tracker-peer-minipc1 = ./secrets/linux-tracker-peer-minipc1.age; + linux-tracker-peer-minipc2 = ./secrets/linux-tracker-peer-minipc2.age; + linux-tracker-peer-minipc3 = ./secrets/linux-tracker-peer-minipc3.age; + linux-tracker-peer-bigbox1 = ./secrets/linux-tracker-peer-bigbox1.age; + linux-tracker-peer-bigbox2 = ./secrets/linux-tracker-peer-bigbox2.age; + linux-tracker-peer-terra1 = ./secrets/linux-tracker-peer-terra1.age; }; inherit hyprlauncher diff --git a/hosts/hl-fwdesk/services/hermenix.nix b/hosts/hl-fwdesk/services/hermenix.nix index b010a8c..a4a086d 100644 --- a/hosts/hl-fwdesk/services/hermenix.nix +++ b/hosts/hl-fwdesk/services/hermenix.nix @@ -8,6 +8,7 @@ let hermesCache = 5001; ollama = 11434; ds4 = 8000; + tracker = 1984; }; in { @@ -30,6 +31,8 @@ in hostAccess = [ { port = ports.ollama; } { port = ports.ds4; } + # Primary skyg-linux-tracker on this host (it consolidates the lab). + { port = ports.tracker; } ]; lanAccess = [ { address = "10.0.12.1"; port = 80; } @@ -39,8 +42,7 @@ in { address = "10.0.101.3"; port = 8080; } { address = "10.0.101.3"; port = 9102; } { address = "10.0.101.8"; port = 3000; } - # skyg-linux-tracker (REST + MCP) on every lab host. - { address = "10.0.0.0/16"; port = 1984; } + ]; publish = [ { hostPort = ports.hermes; guestPort = 8642; } diff --git a/hosts/hl-fwdesk/services/linux-tracker.nix b/hosts/hl-fwdesk/services/linux-tracker.nix index da2b6c8..4a05e5a 100644 --- a/hosts/hl-fwdesk/services/linux-tracker.nix +++ b/hosts/hl-fwdesk/services/linux-tracker.nix @@ -1,12 +1,33 @@ -{ config, ... }: +{ config, lib, skyg-secrets, ... }: +let + # Hosts this primary tracker consolidates. + peers = [ "minipc1" "minipc2" "minipc3" "bigbox1" "bigbox2" "terra1" ]; + secretName = peer: "linux-tracker-peer-${peer}"; +in { + age.secrets = lib.genAttrs (map secretName peers) (name: { + file = skyg-secrets.${name}; + owner = "root"; + mode = "0400"; + }); + services.skyg-linux-tracker = { enable = true; bind = "0.0.0.0"; openFirewall = true; docker.enable = config.virtualisation.docker.enable; - # Reachable only by its lab FQDN; the short name does not resolve. - allowedHosts = [ "fwdesk.lab.internal" ]; + # The Hermes VM reaches this host at 10.77.0.1. + allowedHosts = [ "fwdesk.lab.internal" "10.77.0.1" ]; + # Primary: one endpoint for the whole lab (Hermes only talks to this one). + peers = { + enable = true; + peers = lib.genAttrs peers (peer: { + url = "http://${peer}.lab.internal:1984"; + tokenFile = config.age.secrets.${secretName peer}.path; + # Trackers have no TLS; token travels in the clear on the lab LAN. + allowPlainHttp = true; + }); + }; # OTLP out to the collector on minipc1. settings.otel = { enabled = true; diff --git a/secrets.nix b/secrets.nix index 8073d45..66cbabb 100644 --- a/secrets.nix +++ b/secrets.nix @@ -59,4 +59,13 @@ in # Grafana signing key (settings.security.secret_key) on hl-minipc1 "secrets/grafana-secret-key.age".publicKeys = [ ari minipc1 ]; + + # skyg-linux-tracker peer tokens. fwdesk is the primary and replays them; + # each one is a `consolidator-fwdesk` token minted on that peer. + "secrets/linux-tracker-peer-minipc1.age".publicKeys = [ ari fwdesk ]; + "secrets/linux-tracker-peer-minipc2.age".publicKeys = [ ari fwdesk ]; + "secrets/linux-tracker-peer-minipc3.age".publicKeys = [ ari fwdesk ]; + "secrets/linux-tracker-peer-bigbox1.age".publicKeys = [ ari fwdesk ]; + "secrets/linux-tracker-peer-bigbox2.age".publicKeys = [ ari fwdesk ]; + "secrets/linux-tracker-peer-terra1.age".publicKeys = [ ari fwdesk ]; } diff --git a/secrets/ABOUTME.md b/secrets/ABOUTME.md index 14ca495..ccd6af6 100644 --- a/secrets/ABOUTME.md +++ b/secrets/ABOUTME.md @@ -4,22 +4,23 @@ Age-encrypted secret files managed with [agenix](https://github.com/ryantm/ageni ## Contents -| File | Description | -| -------------------------------------- | ------------------------------------------------- | -| `dns-addresses.conf.age` | DNS address overrides for the dnsmasq module | -| `glmain.json.age` | OpenWrt router credentials/config for `glmain` | -| `hermes-env.age` | Hermes agent env vars (fwbook + fwdesk) | -| `hermes-env` | Unencrypted working copy of `hermes-env.age` | -| `iu-project.age` | iu-project credentials (minipc1) | -| `portainer-agent-bigbox1.age` | Portainer agent env vars (bigbox1) | -| `stack1.age` | Compose definition for stack1 (minipc2) | -| `stack2.age` | Compose definition for stack2 (minipc2) | -| `buildbot-worker-password-minipc3.age` | buildbot-nix worker password (minipc3) | -| `buildbot-worker-password-fwdesk.age` | buildbot-nix worker password (fwdesk) | -| `buildbot-gitea-token.age` | Gitea access token for buildbot (minipc3) | -| `buildbot-gitea-webhook-secret.age` | Gitea webhook secret for buildbot (minipc3) | -| `buildbot-gitea-oauth-secret.age` | Gitea OAuth2 client secret for buildbot (minipc3) | -| `cloudflare-ddclient.age` | Cloudflare DNS API token for ddclient (minipc1) | +| File | Description | +| -------------------------------------- | ------------------------------------------------------------------------------------------ | +| `dns-addresses.conf.age` | DNS address overrides for the dnsmasq module | +| `glmain.json.age` | OpenWrt router credentials/config for `glmain` | +| `hermes-env.age` | Hermes agent env vars (fwbook + fwdesk) | +| `hermes-env` | Unencrypted working copy of `hermes-env.age` | +| `iu-project.age` | iu-project credentials (minipc1) | +| `portainer-agent-bigbox1.age` | Portainer agent env vars (bigbox1) | +| `stack1.age` | Compose definition for stack1 (minipc2) | +| `stack2.age` | Compose definition for stack2 (minipc2) | +| `buildbot-worker-password-minipc3.age` | buildbot-nix worker password (minipc3) | +| `buildbot-worker-password-fwdesk.age` | buildbot-nix worker password (fwdesk) | +| `buildbot-gitea-token.age` | Gitea access token for buildbot (minipc3) | +| `buildbot-gitea-webhook-secret.age` | Gitea webhook secret for buildbot (minipc3) | +| `buildbot-gitea-oauth-secret.age` | Gitea OAuth2 client secret for buildbot (minipc3) | +| `cloudflare-ddclient.age` | Cloudflare DNS API token for ddclient (minipc1) | +| `linux-tracker-peer-.age` | `consolidator-fwdesk` tracker token per peer (minipc1/2/3, bigbox1/2, terra1); fwdesk only | The authoritative list of names + authorized keys is `secrets.nix` in the repo root.