From e3f01e3c35f12fd11c4a47af7b8826f4271f6784 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Thu, 5 Nov 2026 15:21:01 -0500 Subject: [PATCH] cleaning up old bots --- bin/lib/buzz.nu | 1 - flake.lock | 274 +------------- flake.nix | 33 -- hosts/ABOUTME.md | 4 +- hosts/hl-fwdesk/services/default.nix | 2 - hosts/hl-fwdesk/services/nbot.nix | 1 - hosts/hl-fwdesk/services/robotnix.nix | 124 ------- hosts/hl-fwdesk/services/robotnix/buzz.nix | 77 ---- hosts/hl-fwdesk/services/robotnix/common.nix | 45 --- hosts/hl-fwdesk/services/robotnix/genix.nix | 13 - hosts/hl-fwdesk/services/robotnix/sibli.nix | 71 ---- hosts/hl-fwdesk/services/slackerbot.nix | 83 ----- hosts/hl-fws1/services/default.nix | 2 +- hosts/hl-fws1/services/nbot.nix | 4 +- hosts/hl-fws1/services/slackerbot.nix | 69 ---- modules/ABOUTME.md | 1 - modules/main.nix | 2 - modules/nixos/server/ABOUTME.md | 3 - modules/nixos/server/default.nix | 1 - modules/nixos/server/robotnix/ABOUTME.md | 32 -- modules/nixos/server/robotnix/default.nix | 35 -- modules/nixos/server/services/ABOUTME.md | 46 --- modules/nixos/server/services/bazel-build.nix | 345 ------------------ modules/nixos/server/services/default.nix | 1 - secrets.nix | 10 - 25 files changed, 12 insertions(+), 1267 deletions(-) delete mode 100644 hosts/hl-fwdesk/services/robotnix.nix delete mode 100644 hosts/hl-fwdesk/services/robotnix/buzz.nix delete mode 100644 hosts/hl-fwdesk/services/robotnix/common.nix delete mode 100644 hosts/hl-fwdesk/services/robotnix/genix.nix delete mode 100644 hosts/hl-fwdesk/services/robotnix/sibli.nix delete mode 100644 hosts/hl-fwdesk/services/slackerbot.nix delete mode 100644 hosts/hl-fws1/services/slackerbot.nix delete mode 100644 modules/nixos/server/robotnix/ABOUTME.md delete mode 100644 modules/nixos/server/robotnix/default.nix delete mode 100644 modules/nixos/server/services/bazel-build.nix diff --git a/bin/lib/buzz.nu b/bin/lib/buzz.nu index 1d0ab9f..2c1144f 100644 --- a/bin/lib/buzz.nu +++ b/bin/lib/buzz.nu @@ -103,7 +103,6 @@ export def "skyg buzz identity" [ print $"npub: ($npub)" print "Next:" print $" 1. On hl-minipc1: sudo buzz-manage add-member ($npub)" - print $" 2. Rebuild, then restart the robot: robotnix restart ($robot)" if $restricted and not $gated { print "⚠️ --restricted with no BUZZ_ALLOWED_USERS: the bot will ignore everyone. Re-run with --rotate --allowed-users or edit the secret." } diff --git a/flake.lock b/flake.lock index ccafc23..eee1c7e 100644 --- a/flake.lock +++ b/flake.lock @@ -279,21 +279,6 @@ "type": "github" } }, - "crane_2": { - "locked": { - "lastModified": 1780029330, - "narHash": "sha256-fWFKEKB5CP+NO8/3uOaEZIVbd03mvDJ//VsnKkXty08=", - "owner": "ipetkov", - "repo": "crane", - "rev": "6823b493a0bc2142082075576efa6633b537197e", - "type": "github" - }, - "original": { - "owner": "ipetkov", - "repo": "crane", - "type": "github" - } - }, "crate2nix": { "flake": false, "locked": { @@ -629,22 +614,6 @@ "type": "github" } }, - "flake-compat_10": { - "flake": false, - "locked": { - "lastModified": 1767039857, - "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", - "owner": "NixOS", - "repo": "flake-compat", - "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "flake-compat", - "type": "github" - } - }, "flake-compat_2": { "flake": false, "locked": { @@ -860,24 +829,6 @@ } }, "flake-parts_5": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib" - }, - "locked": { - "lastModified": 1769996383, - "narHash": "sha256-AnYjnFWgS49RlqX7LrC4uA+sCCDBj0Ry/WOJ5XWAsa0=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "57928607ea566b5db3ad13af0e57e921e6b12381", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "flake-parts_6": { "inputs": { "nixpkgs-lib": [ "stylix", @@ -898,21 +849,6 @@ "type": "github" } }, - "flake-root": { - "locked": { - "lastModified": 1723604017, - "narHash": "sha256-rBtQ8gg+Dn4Sx/s+pvjdq3CB2wQNzx9XGFq/JVGCB6k=", - "owner": "srid", - "repo": "flake-root", - "rev": "b759a56851e10cb13f6b8e5698af7b59c44be26e", - "type": "github" - }, - "original": { - "owner": "srid", - "repo": "flake-root", - "type": "github" - } - }, "flox": { "inputs": { "crane": "crane", @@ -1083,29 +1019,6 @@ "type": "github" } }, - "git-hooks_4": { - "inputs": { - "flake-compat": "flake-compat_9", - "gitignore": "gitignore_2", - "nixpkgs": [ - "nativelink", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1772024342, - "narHash": "sha256-+eXlIc4/7dE6EcPs9a2DaSY3fTA9AE526hGqkNID3Wg=", - "owner": "cachix", - "repo": "git-hooks.nix", - "rev": "6e34e97ed9788b17796ee43ccdbaf871a5c2b476", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "git-hooks.nix", - "type": "github" - } - }, "gitignore": { "inputs": { "nixpkgs": [ @@ -1128,28 +1041,6 @@ "type": "github" } }, - "gitignore_2": { - "inputs": { - "nixpkgs": [ - "nativelink", - "git-hooks", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1709087332, - "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, "gnome-shell": { "flake": false, "locked": { @@ -1688,7 +1579,7 @@ "hyprutils": "hyprutils_3", "hyprwayland-scanner": "hyprwayland-scanner_3", "hyprwire": "hyprwire_3", - "nixpkgs": "nixpkgs_12", + "nixpkgs": "nixpkgs_11", "pre-commit-hooks": "pre-commit-hooks_3", "systems": "systems_3", "xdph": "xdph_2" @@ -2306,31 +2197,6 @@ "type": "github" } }, - "nativelink": { - "inputs": { - "crane": "crane_2", - "flake-parts": "flake-parts_5", - "flake-root": "flake-root", - "git-hooks": "git-hooks_4", - "nix2container": "nix2container", - "nixpkgs": "nixpkgs_11", - "rust-overlay": "rust-overlay_3" - }, - "locked": { - "lastModified": 1793212176, - "narHash": "sha256-k9Vzp5nOq2D/JnYfnG1vlIZrca8dt7F6qz6SwirXLec=", - "owner": "TraceMachina", - "repo": "nativelink", - "rev": "bd336137d794a6a4e7ac32e561ab55e2d7e70ff3", - "type": "github" - }, - "original": { - "owner": "TraceMachina", - "ref": "v1.8.3", - "repo": "nativelink", - "type": "github" - } - }, "nbot": { "inputs": { "nixpkgs": [ @@ -2467,28 +2333,6 @@ "type": "github" } }, - "nix2container": { - "inputs": { - "nixpkgs": [ - "nativelink", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1775487831, - "narHash": "sha256-2lguQpLPQaxpQCJjXhmEEAfabwsAhkP29Z7fgLzHARA=", - "owner": "nlewo", - "repo": "nix2container", - "rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e", - "type": "github" - }, - "original": { - "owner": "nlewo", - "repo": "nix2container", - "rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e", - "type": "github" - } - }, "nix_2": { "inputs": { "flake-parts": "flake-parts_2", @@ -2707,7 +2551,7 @@ }, "nixos-hardware_2": { "inputs": { - "nixpkgs": "nixpkgs_13" + "nixpkgs": "nixpkgs_12" }, "locked": { "lastModified": 1792491056, @@ -2772,21 +2616,6 @@ "type": "github" } }, - "nixpkgs-lib": { - "locked": { - "lastModified": 1769909678, - "narHash": "sha256-cBEymOf4/o3FD5AZnzC3J9hLbiZ+QDT/KDuyHXVJOpM=", - "owner": "nix-community", - "repo": "nixpkgs.lib", - "rev": "72716169fe93074c333e8d0173151350670b824c", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nixpkgs.lib", - "type": "github" - } - }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -2866,22 +2695,6 @@ } }, "nixpkgs_11": { - "locked": { - "lastModified": 1787498568, - "narHash": "sha256-9i/VTdusq/+NM/tz+J1Re+ojkMB8MBf0QshnYfzHz30=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "56c02bc00adcf003215cc4bd996d6efaf4cff188", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_12": { "locked": { "lastModified": 1788404924, "narHash": "sha256-lhEhY8X5EgkQ/eg6IFz4cc8jRuSYSvnxx1al7d1dvZ0=", @@ -2897,7 +2710,7 @@ "type": "github" } }, - "nixpkgs_13": { + "nixpkgs_12": { "locked": { "lastModified": 1789546076, "narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=", @@ -2910,7 +2723,7 @@ "url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz" } }, - "nixpkgs_14": { + "nixpkgs_13": { "locked": { "lastModified": 1792503283, "narHash": "sha256-vhe2jyvRCgufWPqDuDq1S3t3ypE+ka46IFutV02xKso=", @@ -3191,7 +3004,7 @@ }, "pre-commit-hooks_3": { "inputs": { - "flake-compat": "flake-compat_10", + "flake-compat": "flake-compat_9", "nixpkgs": [ "nixarchy", "hyprland", @@ -3262,35 +3075,6 @@ "type": "github" } }, - "robotnix": { - "inputs": { - "devenv": [], - "git-hooks": [ - "git-hooks" - ], - "hermes-agent": [ - "hermes-agent" - ], - "nixpkgs": [ - "nixpkgs" - ], - "nutest": [], - "treefmt-nix": [] - }, - "locked": { - "lastModified": 1793232093, - "narHash": "sha256-lH8MNrXEyWzkM+iA0ILRH3qoqSrE0ZJ76FeFoEHWgac=", - "ref": "refs/heads/main", - "rev": "c8a494361d085a355f63f98ec92c5aee58f3c51c", - "revCount": 4, - "type": "git", - "url": "ssh://gitea@gitea.app.internal:22/ari/robotnix.git" - }, - "original": { - "type": "git", - "url": "ssh://gitea@gitea.app.internal:22/ari/robotnix.git" - } - }, "root": { "inputs": { "agenix": "agenix", @@ -3309,20 +3093,17 @@ "hyprland": "hyprland", "hyprlauncher": "hyprlauncher", "my-nixpkgs": "my-nixpkgs", - "nativelink": "nativelink", "nbot": "nbot", "nix-darwin": "nix-darwin", "nix-flatpak": "nix-flatpak", "nix-index-database": "nix-index-database", "nixarchy": "nixarchy", "nixos-hardware": "nixos-hardware_2", - "nixpkgs": "nixpkgs_14", + "nixpkgs": "nixpkgs_13", "nixpkgs-unstable": "nixpkgs-unstable", "noctalia": "noctalia", - "robotnix": "robotnix", "skyg-linux-tracker": "skyg-linux-tracker", "skygqts": "skygqts", - "slackerbot": "slackerbot", "stylix": "stylix", "systems": "systems_6" } @@ -3387,27 +3168,6 @@ "type": "github" } }, - "rust-overlay_3": { - "inputs": { - "nixpkgs": [ - "nativelink", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1787713867, - "narHash": "sha256-ZRYTEvDb8QZLCHRoMyIZle9V3Bvw905m0teeo1RvI7M=", - "owner": "oxalica", - "repo": "rust-overlay", - "rev": "494680f9ffa2b4bf9f2df0f442c7096e44adfa00", - "type": "github" - }, - "original": { - "owner": "oxalica", - "repo": "rust-overlay", - "type": "github" - } - }, "skyg-linux-tracker": { "inputs": { "nixpkgs": [ @@ -3448,26 +3208,6 @@ "url": "ssh://gitea@gitea.app.internal:22/ari/skygqts.git" } }, - "slackerbot": { - "inputs": { - "nixpkgs": [ - "nixpkgs-unstable" - ] - }, - "locked": { - "lastModified": 1793590815, - "narHash": "sha256-Z2HRkYW0547L309qm4e0w++xVKIJFpKbqBUDLEoWG4c=", - "ref": "refs/heads/main", - "rev": "4db99eaa2a0e1529194983962a07109fdb716bcd", - "revCount": 22, - "type": "git", - "url": "https://gitea.skyg.ca/ari/slacker-bot.git" - }, - "original": { - "type": "git", - "url": "https://gitea.skyg.ca/ari/slacker-bot.git" - } - }, "sops-nix": { "inputs": { "nixpkgs": [ @@ -3513,7 +3253,7 @@ "base16-helix": "base16-helix", "base16-vim": "base16-vim", "firefox-gnome-theme": "firefox-gnome-theme", - "flake-parts": "flake-parts_6", + "flake-parts": "flake-parts_5", "gnome-shell": "gnome-shell", "nixpkgs": [ "nixpkgs" diff --git a/flake.nix b/flake.nix index c6d72d2..304609a 100644 --- a/flake.nix +++ b/flake.nix @@ -4,17 +4,6 @@ skygqts.url = "git+ssh://gitea@gitea.app.internal:22/ari/skygqts.git"; skygqts.inputs.nixpkgs.follows = "nixpkgs"; hermenix.url = "git+ssh://gitea@gitea.app.internal:22/ari/hermenix.git"; - # Self-managed nspawn robots. Robots pin the same hermes-agent rev. - robotnix = { - url = "git+ssh://gitea@gitea.app.internal:22/ari/robotnix.git"; - inputs.nixpkgs.follows = "nixpkgs"; - inputs.hermes-agent.follows = "hermes-agent"; - # Dev/test-only inputs of robotnix: keep them out of this lock. - inputs.git-hooks.follows = "git-hooks"; - inputs.devenv.follows = ""; - inputs.treefmt-nix.follows = ""; - inputs.nutest.follows = ""; - }; # Read-only host monitor (systemd, journal, Docker) over REST + MCP. skyg-linux-tracker = { url = "git+https://gitea.skyg.ca/ari/skyg-linux-tracker.git"; @@ -25,14 +14,6 @@ url = "git+ssh://git@github.com/asosnovsky/nbot.git"; inputs.nixpkgs.follows = "nixpkgs-unstable"; }; - # Off. - slackerbot = { - url = "git+https://gitea.skyg.ca/ari/slacker-bot.git"; - inputs.nixpkgs.follows = "nixpkgs-unstable"; - }; - # Bazel remote build server for slackerbot containers. Pinned: its config - # format changes between releases (see bazel-build.nix). - nativelink.url = "github:TraceMachina/nativelink/v1.8.3"; # Headless Hyprland session streamed over a Unix socket. hhstream = { url = "git+ssh://gitea@gitea.app.internal:22/Hermenix/hyprland-headless-stream-plugin.git"; @@ -131,11 +112,8 @@ { self , skygqts , hermenix - , robotnix , skyg-linux-tracker , nbot - , slackerbot - , nativelink , hhstream , nixpkgs-unstable , flox @@ -212,12 +190,6 @@ linux-tracker-peer-bigbox2 = ./secrets/linux-tracker-peer-bigbox2.age; linux-tracker-peer-terra1 = ./secrets/linux-tracker-peer-terra1.age; scrypted-envs = ./secrets/scrypted-envs.age; - robots-shared = ./secrets/robots-shared.age; - robots-sibli = ./secrets/robots-sibli.age; - robots-genix = ./secrets/robots-genix.age; - slackerbot-slack = ./secrets/slackerbot-slack.age; - slackerbot-goose = ./secrets/slackerbot-goose.age; - slackerbot-cluster = ./secrets/slackerbot-cluster.age; nbot-slack = ./secrets/nbot-slack.age; nbot-goose = ./secrets/nbot-goose.age; nbot-cluster = ./secrets/nbot-cluster.age; @@ -236,11 +208,8 @@ hermes-agent skygqts hermenix - robotnix skyg-linux-tracker nbot - slackerbot - nativelink hhstream flox determinate @@ -399,7 +368,6 @@ ./hosts/hl-fws1.disko.nix nixos-hardware.nixosModules.framework-11th-gen-intel disko.nixosModules.disko - slackerbot.nixosModules.host nbot.nixosModules.host ]; }; @@ -411,7 +379,6 @@ nixos-hardware.nixosModules.framework-desktop-amd-ai-max-300-series hermenix.nixosModules.default buildbot-nix.nixosModules.buildbot-worker - slackerbot.nixosModules.host nbot.nixosModules.host ]; }; diff --git a/hosts/ABOUTME.md b/hosts/ABOUTME.md index c9ddd8d..27997b0 100644 --- a/hosts/ABOUTME.md +++ b/hosts/ABOUTME.md @@ -31,8 +31,8 @@ hosts/ | Folder | Host | Role | | ------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | | `fwbook/` | fwbook | Framework 13 AMD — personal laptop / daily driver | -| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); nbot master (slackerbot off) | -| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; nbot worker (slackerbot off) | +| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); nbot master | +| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; nbot worker | | `hl-bigbox1/` | hl-bigbox1 | AI/compute server (NVIDIA + AMD); Immich machine-learning (CUDA) | | `hl-bigbox2/` | hl-bigbox2 | NFS storage server; Immich server/db (docker, `immich.home.sosnovsky.ca`) | | `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI), monitoring (Grafana, Prometheus, Loki, OTel collector) | diff --git a/hosts/hl-fwdesk/services/default.nix b/hosts/hl-fwdesk/services/default.nix index 42293e0..c444d7d 100644 --- a/hosts/hl-fwdesk/services/default.nix +++ b/hosts/hl-fwdesk/services/default.nix @@ -8,8 +8,6 @@ ./steam.nix ./buildbot.nix ./linux-tracker.nix - ./robotnix.nix - ./slackerbot.nix ./nbot.nix ]; } diff --git a/hosts/hl-fwdesk/services/nbot.nix b/hosts/hl-fwdesk/services/nbot.nix index 6d6c265..a50ac96 100644 --- a/hosts/hl-fwdesk/services/nbot.nix +++ b/hosts/hl-fwdesk/services/nbot.nix @@ -28,7 +28,6 @@ let OPENROUTER_API_KEY = "$OPENROUTER_API_KEY"; CLAUDE_CODE_OAUTH_TOKEN = "$CLAUDE_CODE_OAUTH_TOKEN"; COMMAND_CODE_API_KEY = "$COMMAND_CODE_API_KEY"; - # Every agent can use this PAT, so anyone who can prompt one can too. GH_TOKEN = "$GH_TOKEN"; }; diff --git a/hosts/hl-fwdesk/services/robotnix.nix b/hosts/hl-fwdesk/services/robotnix.nix deleted file mode 100644 index 4c051a6..0000000 --- a/hosts/hl-fwdesk/services/robotnix.nix +++ /dev/null @@ -1,124 +0,0 @@ -{ config, lib, skyg-secrets, ... }: -let - apps = config.skyg.internalNetworkingMap.apps; - robotSecrets = [ "robots-shared" "robots-sibli" "robots-genix" ]; - secret = name: config.age.secrets.${name}.path; - ports = { - ollama = 11434; - ds4 = 8000; - tracker = 1984; - }; - # A robot's seed: its config plus the files it shares with the others. - # Each robot gets copies under /etc/nixos/seed (same layout as this repo), - # so a robot that rebuilds itself (genix) keeps working. - seedOf = file: { - root = ../../..; - configuration = file; - extraFiles = [ - ./robotnix/common.nix - ./robotnix/buzz.nix - ../../../pkgs/buzz-cli - ]; - }; -in -{ - age.secrets = lib.genAttrs robotSecrets (name: { - file = skyg-secrets.${name}; - owner = "root"; - mode = "0400"; - }); - - # robotnix flake input; options and docs in its repo. - services.robotnix = { - enable = true; - externalInterface = "enp191s0"; - - # Every robot gets this tier. Same holes as the Hermes VM. - access.shared = { - internet = "all"; - hostAccess = [ - { port = ports.ollama; } - ]; - lanAccess = [ - { address = "10.0.101.3"; port = 443; } - { address = "10.0.101.3"; port = 3000; } - { address = "10.0.101.3"; port = 8080; } - { address = "10.0.101.3"; port = 9102; } - ]; - }; - - # A Hermes agent in every robot. - hermes = { - enable = true; - environmentFiles = [ (secret "robots-shared") ]; - }; - - robots.sibli = { - id = 2; - address = apps.sibli.ip; - cpus = { min = 1; max = 4; }; - memory = { min = "2G"; max = "16G"; }; - disk = { rootSize = "100G"; homeSize = "60G"; }; - seed = seedOf ./robotnix/sibli.nix; - hermes = { - name = "Siblix"; - environmentFiles = [ (secret "robots-sibli") ]; - settings.model = { - provider = "anthropic"; - default = "claude-haiku-5-5"; - }; - }; - allowDocker = true; - allowTun = true; - ephemeral = true; - persist = [ - "/var/lib/tailscale" - "/var/lib/docker" - ]; - access = { }; - git.enable = false; - }; - - # Persistent and self-managing: genix changes and applies its own system - # (robotnix-apply), pushes its flake to gitea, and pulls it back after a - # reseed. Add /var/lib/robotnix-git/id_ed25519.pub (inside genix) as a - # deploy key with write access on ari/genix. - robots.genix = { - id = 3; - address = apps.genix.ip; - cpus = { min = 1; max = 6; }; - memory = { min = "4G"; max = "24G"; }; - disk = { rootSize = "150G"; homeSize = "100G"; }; - seed = seedOf ./robotnix/genix.nix; - selfManage.enable = true; - git = { - remoteUrl = "gitea@gitea.app.internal:Hermenix/genix.git"; - interval = "hourly"; - }; - hermes = { - name = "Genix"; - environmentFiles = [ (secret "robots-genix") ]; - settings = { - model = { - provider = "commandcode"; - default = "deepseek/deepseek-v4.1-flash"; - }; - fallback_providers = [ - { provider = "anthropic"; model = "claude-haiku-5-5"; } - { provider = "openrouter"; model = "deepseek/deepseek-v4.1-flash"; } - ]; - }; - }; - allowDocker = true; - # Survive a reseed (root is rebuilt from the git remote). - persist = [ - "/var/lib/hermes" - "/var/lib/docker" - ]; - # Push/pull its flake over gitea's ssh. - access.lanAccess = [ - { address = apps.gitea.ip; port = 22; } - ]; - }; - }; -} diff --git a/hosts/hl-fwdesk/services/robotnix/buzz.nix b/hosts/hl-fwdesk/services/robotnix/buzz.nix deleted file mode 100644 index 8c7b6ec..0000000 --- a/hosts/hl-fwdesk/services/robotnix/buzz.nix +++ /dev/null @@ -1,77 +0,0 @@ -# Buzz for a robot's Hermes: the `buzz` CLI and a profile for its key. -# Shared by this host's robots (copied into each under /etc/nixos/seed). -# Needs a Buzz identity in the robot's secret: `skyg buzz identity `. -{ config, lib, pkgs, ... }: -let - cfg = config.skyg.robot.buzz; - # Built from this repo; the robot gets a copy of pkgs/buzz-cli with this file. - buzz-cli = pkgs.callPackage ../../../../pkgs/buzz-cli { }; -in -{ - options.skyg.robot.buzz = { - enable = lib.mkEnableOption "Buzz for this robot's Hermes"; - homeChannel = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - description = "BUZZ_HOME_CHANNEL (not secret, so it lives here, not in the secret)."; - }; - profileName = lib.mkOption { - type = lib.types.str; - default = config.services.hermes-agent.environment.API_SERVER_MODEL_NAME or config.networking.hostName; - defaultText = lib.literalExpression "the Hermes gateway model name, else the host name"; - description = "Name published for a new key that has no Buzz profile yet."; - }; - }; - - config = lib.mkIf cfg.enable { - environment.systemPackages = [ buzz-cli ]; - - # The Hermes service PATH has no system packages. - services.hermes-agent.environment = { - BUZZ_CLI_PATH = "${buzz-cli}/bin/buzz"; - } // lib.optionalAttrs (cfg.homeChannel != null) { - BUZZ_HOME_CHANNEL = cfg.homeChannel; - }; - - # A new Buzz key has no profile, and Hermes won't connect without one. - # Publish one at start if missing; the relay keeps it. - systemd.services.buzz-profile = { - description = "Publish the Buzz profile if missing"; - wantedBy = [ "multi-user.target" ]; - before = [ "hermes-agent.service" ]; - wants = [ "network-online.target" ]; - after = [ "network-online.target" ]; - path = [ buzz-cli pkgs.jq ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - User = "hermes"; - EnvironmentFile = config.systemd.services.hermes-agent.serviceConfig.EnvironmentFile; - }; - script = '' - if [ -z "''${BUZZ_PRIVATE_KEY:-}" ] || [ -z "''${BUZZ_RELAY_URL:-}" ]; then - echo "No Buzz creds, skipping." - exit 0 - fi - # Retry: the relay may not be reachable yet. Failure is not fatal. - for _ in 1 2 3 4 5; do - n="" - if out=$(buzz users get); then - n=$(jq 'length' <<<"$out") || n="" - fi - if [ "$n" = "0" ]; then - if buzz users set-profile --name ${lib.escapeShellArg cfg.profileName}; then - echo "Published Buzz profile." - exit 0 - fi - elif [ -n "$n" ]; then - echo "Buzz profile exists." - exit 0 - fi - sleep 5 - done - echo "Could not check or publish the Buzz profile." >&2 - ''; - }; - }; -} diff --git a/hosts/hl-fwdesk/services/robotnix/common.nix b/hosts/hl-fwdesk/services/robotnix/common.nix deleted file mode 100644 index d28c0b6..0000000 --- a/hosts/hl-fwdesk/services/robotnix/common.nix +++ /dev/null @@ -1,45 +0,0 @@ -# Shared base for this host's robots (sibli, genix). Robots get a copy under -# /etc/nixos/seed (see seedOf in ../robotnix.nix), so a robot that rebuilds -# itself keeps working; edit it here for new installs and ephemeral robots. -{ config, lib, pkgs, ... }: -let - user = config.robotnix.user; -in -{ - # The user has no password; ssh keys only. - security.sudo.wheelNeedsPassword = false; - - environment.systemPackages = with pkgs; [ - git - vim - curl - htop - claude-code - devenv - direnv - (python3.withPackages (ps: [ ps.pip ])) - nodejs - docker-compose - zellij - ]; - nixpkgs.config.allowUnfree = true; - - # direnv: hook into bash so the repo's devenv shims work. - programs.bash = { - enable = true; - interactiveShellInit = '' - eval "$(direnv hook bash)" - ''; - }; - - # Docker + compose. Needs allowDocker = true on the host. - virtualisation.docker.enable = true; - users.users.${user}.extraGroups = [ "docker" ]; - users.users.hermes.extraGroups = lib.mkIf config.robotnix.hermes.enable [ "docker" ]; - - nix.gc = { - automatic = true; - dates = "weekly"; - options = "--delete-older-than 14d"; - }; -} diff --git a/hosts/hl-fwdesk/services/robotnix/genix.nix b/hosts/hl-fwdesk/services/robotnix/genix.nix deleted file mode 100644 index 66a4206..0000000 --- a/hosts/hl-fwdesk/services/robotnix/genix.nix +++ /dev/null @@ -1,13 +0,0 @@ -# genix's starting config. It is persistent and manages itself: after the -# first install this (copied to /etc/nixos/seed/...) belongs to genix, which -# changes and applies its own system with robotnix-apply. Edits here only -# matter for a fresh install with nothing on its git remote. -{ ... }: -{ - imports = [ - ./common.nix - ./buzz.nix - ]; - - skyg.robot.buzz.enable = true; -} diff --git a/hosts/hl-fwdesk/services/robotnix/sibli.nix b/hosts/hl-fwdesk/services/robotnix/sibli.nix deleted file mode 100644 index c466769..0000000 --- a/hosts/hl-fwdesk/services/robotnix/sibli.nix +++ /dev/null @@ -1,71 +0,0 @@ -# sibli is ephemeral: root resets to this config on every start, so change -# things here (then rebuild the host), not inside the robot. -{ config, pkgs, ... }: -let - robotoName = config.services.hermes-agent.environment.API_SERVER_MODEL_NAME; -in -{ - imports = [ - ./common.nix - ./buzz.nix - ]; - - skyg.robot.buzz = { - enable = true; - homeChannel = "a8969cdc-7ba2-4e25-ae18-b898ab59003d"; - }; - - services.hermes-agent = { - extraPackages = [ pkgs.nodejs ]; - mcpServers.shortcut = { - command = "${pkgs.nodejs}/bin/npx"; - args = [ "-y" "@shortcut/mcp@latest" ]; - env.SHORTCUT_API_TOKEN = "\${SHORTCUT_API_KEY}"; - }; - }; - - services.tailscale.enable = true; - services.tailscale.openFirewall = true; - networking.firewall.trustedInterfaces = [ "tailscale0" ]; - - systemd.services.tailscale-login = { - description = "Log in to Tailscale if needed"; - wantedBy = [ "multi-user.target" ]; - wants = [ "network-online.target" ]; - after = [ "network-online.target" "tailscaled.service" ]; - requires = [ "tailscaled.service" ]; - path = [ config.services.tailscale.package pkgs.jq ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - EnvironmentFile = config.systemd.services.hermes-agent.serviceConfig.EnvironmentFile; - }; - script = '' - if [ -z "''${TS_AUTHKEY:-}" ]; then - echo "No TS_AUTHKEY, skipping." - exit 0 - fi - # Retry: the daemon may not be ready yet. Failure is not fatal. - for _ in 1 2 3 4 5 6; do - state="" - if out=$(tailscale status --json); then - state=$(jq -r '.BackendState' <<<"$out") || state="" - fi - case "$state" in - Running) - echo "Already logged in." - exit 0 - ;; - NeedsLogin|Stopped) - if tailscale up --auth-key="$TS_AUTHKEY" --hostname=ari-fwdesk-${robotoName}-bot; then - echo "Logged in." - exit 0 - fi - ;; - esac - sleep 5 - done - echo "Could not log in to Tailscale." >&2 - ''; - }; -} diff --git a/hosts/hl-fwdesk/services/slackerbot.nix b/hosts/hl-fwdesk/services/slackerbot.nix deleted file mode 100644 index eb06df3..0000000 --- a/hosts/hl-fwdesk/services/slackerbot.nix +++ /dev/null @@ -1,83 +0,0 @@ -{ config, lib, pkgs, skyg-secrets, ... }: -let - ports = { cluster = 7946; }; - homes = "/data/slackerbot"; - secrets = [ "slackerbot-slack" "slackerbot-goose" "slackerbot-cluster" ]; - bazel = config.skyg.nixos.server.services.bazel-build; - settings = { - # The system each container boots (slackerbot's agent + bazel/rbuild); - # updated on every rebuild. - system = "${bazel.agentSystem}"; - # Shared Bazel download cache (/repo-cache) for the containers. - rw_binds = bazel.binds.rw; - ro_binds = bazel.binds.ro; - machines_dir = "/var/lib/machines"; - state_dir = "/var/lib/slackerbot"; - agent_cwd = "/root"; - - idle_timeout = "6h"; - max_idle_timeout = "0s"; - allow_indefinite = true; - boot_timeout = "2m"; - - # Goose settings sent to every container (also on fws1). - # Values come from the slackerbot-goose secret. - agent_env = { - GOOSE_PROVIDER = "$GOOSE_PROVIDER"; - GOOSE_MODEL = "$GOOSE_MODEL"; - OPENROUTER_API_KEY = "$OPENROUTER_API_KEY"; - }; - - container = { memory_mb = 4096; cpus = 1.0; }; - max_container = { memory_mb = 8192; cpus = 2.0; }; - # Leave room for ollama, ds4, comfyui, hermes and the robots. - host_reserve = { memory_mb = 32768; cpus = 4.0; }; - storage = { min = "2G"; max = "5G"; limit = "20G"; }; - drives = [{ path = homes; reserve = "40G"; }]; - - cluster = { - secret = "$CLUSTER_SECRET"; - listen = ":${toString ports.cluster}"; - }; - - # Only read on first start; after that, manage permissions from Slack. - permissions = { - managers = [ "U0CDGSJADMM" ]; - default.U0CDGSJADMM = [ "all" ]; - }; - }; -in -{ - # Off. - config = lib.mkIf false { - age.secrets = lib.genAttrs secrets (name: { - file = skyg-secrets.${name}; - owner = "root"; - mode = "0400"; - }); - - # slackerbot flake input; options and docs in its repo. - services.slackerbot = { - enable = true; - mode = "master"; - clusterPort = ports.cluster; - configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; - }; - # Bazel build server for the containers (16 cores / 32 threads). - skyg.nixos.server.services.bazel-build = { - enable = true; - cpuCount = 32; - cacheGB = 200; - }; - - # The module takes one env file; these add to it. - systemd.services.slackerbot.serviceConfig.EnvironmentFile = - map (name: config.age.secrets.${name}.path) secrets; - - # Container links belong to systemd-networkd. - networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; - networking.dhcpcd.denyInterfaces = [ "ve-*" ]; - - systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; - }; -} diff --git a/hosts/hl-fws1/services/default.nix b/hosts/hl-fws1/services/default.nix index 72b9008..fbe9ff7 100644 --- a/hosts/hl-fws1/services/default.nix +++ b/hosts/hl-fws1/services/default.nix @@ -1,6 +1,6 @@ { ... }: { - imports = [ ./slackerbot.nix ./nbot.nix ]; + imports = [ ./nbot.nix ]; services.pipewire = { extraConfig.pipewire-pulse."10-raop-discover" = { diff --git a/hosts/hl-fws1/services/nbot.nix b/hosts/hl-fws1/services/nbot.nix index 5eb1d2b..3aaee4a 100644 --- a/hosts/hl-fws1/services/nbot.nix +++ b/hosts/hl-fws1/services/nbot.nix @@ -16,8 +16,8 @@ let container = { memory_mb = 4096; cpus = 1.0; }; max_container = { memory_mb = 51200; cpus = 6; }; host_reserve = { memory_mb = 8192; cpus = 2.0; }; - storage = { min = "2G"; max = "5G"; limit = "20G"; }; - drives = [{ path = homes; reserve = "100G"; }]; + storage = { min = "2G"; max = "10G"; limit = "50G"; }; + drives = [{ path = homes; reserve = "200G"; }]; # Worker of the master on fwdesk. Slack tokens and goose settings come # from the master. Trusts the first certificate it sees (saved in diff --git a/hosts/hl-fws1/services/slackerbot.nix b/hosts/hl-fws1/services/slackerbot.nix deleted file mode 100644 index c0595f4..0000000 --- a/hosts/hl-fws1/services/slackerbot.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ config, lib, pkgs, skyg-secrets, ... }: -let - homes = "/var/lib/slackerbot/homes"; - bazel = config.skyg.nixos.server.services.bazel-build; - settings = { - # The system each container boots (slackerbot's agent + bazel/rbuild); - # updated on every rebuild. - system = "${bazel.agentSystem}"; - # Shared Bazel download cache (/repo-cache) for the containers. - rw_binds = bazel.binds.rw; - ro_binds = bazel.binds.ro; - machines_dir = "/var/lib/machines"; - state_dir = "/var/lib/slackerbot"; - agent_cwd = "/root"; - - idle_timeout = "6h"; - max_idle_timeout = "0s"; - allow_indefinite = true; - boot_timeout = "2m"; - - container = { memory_mb = 4096; cpus = 1.0; }; - max_container = { memory_mb = 8192; cpus = 2.0; }; - host_reserve = { memory_mb = 8192; cpus = 2.0; }; - storage = { min = "2G"; max = "5G"; limit = "20G"; }; - drives = [{ path = homes; reserve = "100G"; }]; - - # Worker of the master on fwdesk. Slack tokens and goose settings come - # from the master. Trusts the first certificate it sees (saved in - # state_dir/master.fingerprint); pin it in master_fingerprint if wanted. - cluster = { - secret = "$CLUSTER_SECRET"; - master = "wss://fwdesk.lab.internal:7946"; - }; - }; -in -{ - # Off. - config = lib.mkIf false { - age.secrets.slackerbot-cluster = { - file = skyg-secrets.slackerbot-cluster; - owner = "root"; - mode = "0400"; - }; - - # slackerbot flake input; options and docs in its repo. - services.slackerbot = { - enable = true; - mode = "slave"; - configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; - environmentFile = config.age.secrets.slackerbot-cluster.path; - }; - - # Bazel build server for the containers (4 cores / 8 threads). - skyg.nixos.server.services.bazel-build = { - enable = true; - cpuCount = 8; - cacheGB = 50; - }; - - # Container links belong to systemd-networkd. - networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; - networking.dhcpcd.denyInterfaces = [ "ve-*" ]; - # networkd only has the container links here (NetworkManager owns wlan), - # so wait-online would always time out. - systemd.network.wait-online.enable = false; - - systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; - }; -} diff --git a/modules/ABOUTME.md b/modules/ABOUTME.md index 07ba593..baeda3e 100644 --- a/modules/ABOUTME.md +++ b/modules/ABOUTME.md @@ -38,7 +38,6 @@ skyg.rootDir → core/default.nix (set by the flake; repo root stor skyg.nixos.common.* → nixos/common/ skyg.nixos.desktop.* → nixos/desktop/ skyg.nixos.server.* → nixos/server/ -services.robotnix.* → nixos/server/robotnix/ (adapter for the robotnix flake) skyg.core.* → core/ ``` diff --git a/modules/main.nix b/modules/main.nix index d87e1d2..4bcd688 100644 --- a/modules/main.nix +++ b/modules/main.nix @@ -112,8 +112,6 @@ in immich = { ip = "10.0.101.10"; }; immich-ml = { ip = "10.0.101.11"; }; gilix = { ip = "10.0.101.13"; }; - sibli = { ip = "10.0.101.14"; }; - genix = { ip = "10.0.101.12"; }; jellyfin = { ip = "10.0.101.5"; }; nvr = { ip = "10.0.101.6"; }; portainer = { ip = "10.0.101.241"; }; diff --git a/modules/nixos/server/ABOUTME.md b/modules/nixos/server/ABOUTME.md index 54e7842..eb70402 100644 --- a/modules/nixos/server/ABOUTME.md +++ b/modules/nixos/server/ABOUTME.md @@ -18,7 +18,6 @@ server/ ├── k3s/ # Single-binary K3s cluster role ├── k8s/ # Full upstream Kubernetes (master/node) ├── portainer.nix # skyg.nixos.server.portainer — Portainer server or Edge agent -├── robotnix/ # adapter: imports the robotnix flake (services.robotnix) └── services/ # Self-hosted apps (jellyfin, audiobookshelf, comfyui, ds4, colibri, ai…) ``` @@ -36,7 +35,6 @@ server/ | Structure | Namespace | Notes | | ------------ | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `robotnix/` | `services.robotnix` | Adapter for the robotnix flake (own repo): self-managed NixOS nspawn robots. Sets skyg defaults; the one sanctioned nspawn exception to the oci-containers rule | | `timers.nix` | `skyg.server.timers` | Generic attrset → timer + oneshot service | ## Option Namespace @@ -50,7 +48,6 @@ skyg.server.arrs.* → arrs/ skyg.server.dns.* → dns/ skyg.nixos.server.k3s.* → k3s/ skyg.nixos.server.k8s.* → k8s/ -services.robotnix.* → robotnix/ (adapter for the robotnix flake) skyg.nixos.server.services.* → services/ skyg.nixos.server.portainer.* → portainer.nix ``` diff --git a/modules/nixos/server/default.nix b/modules/nixos/server/default.nix index df84263..502aa38 100644 --- a/modules/nixos/server/default.nix +++ b/modules/nixos/server/default.nix @@ -11,6 +11,5 @@ ./timers.nix ./dns ./portainer.nix - ./robotnix ]; } diff --git a/modules/nixos/server/robotnix/ABOUTME.md b/modules/nixos/server/robotnix/ABOUTME.md deleted file mode 100644 index 8c1a8dd..0000000 --- a/modules/nixos/server/robotnix/ABOUTME.md +++ /dev/null @@ -1,32 +0,0 @@ -# robotnix — adapter for the robotnix flake - -robotnix (self-managed NixOS `systemd-nspawn` robots) lives in its own repo: -`git+ssh://gitea@gitea.app.internal:22/ari/robotnix.git`, a flake input of -this repo. Its README and `docs/` describe the design, options, CLI and -tests. Hosts configure it under `services.robotnix` (like `services.hermenix`). - -This folder only holds `default.nix`, which: - -- imports `robotnix.nixosModules.default` (passed in via `specialArgs`); -- defaults `user`, `authorizedKeys`, `dnsUpstreams` and `localDomains` from - the `skyg` user, the SSH master keys and the internal networking map; -- asserts no robot address clashes with an app in - `skyg.internalNetworkingMap.apps`. - -## Decisions - -- **Own repo** (2026-10): robotnix moved out of `modules/nixos/server` so it - can grow (storage backends, Genix, child robots) with its own tests. -- **Namespace**: `services.robotnix`, not `skyg.*`, same exception as - hermenix: the options belong to an external flake. -- **hermes-agent pin**: the input follows this repo's `hermes-agent`, so - robots pin the same revision the host locked. -- **Buzz stays here**: robotnix knows nothing about Buzz. The robots' shared - `hosts/hl-fwdesk/services/robotnix/buzz.nix` builds `pkgs/buzz-cli` from - this repo and sets `BUZZ_CLI_PATH`. Robots get copies of it, `common.nix` - and `pkgs/buzz-cli` under `/etc/nixos/seed` (robotnix `seed.extraFiles`, - `seedOf` in the host file), so a robot that rebuilds itself (genix) has - its own copy. -- **nspawn, not `oci-containers`**: the oci-containers rule targets - Docker/Podman. nspawn robots need their own store for self-rebuilds, which - the `containers.*` module can't give. diff --git a/modules/nixos/server/robotnix/default.nix b/modules/nixos/server/robotnix/default.nix deleted file mode 100644 index 34ece56..0000000 --- a/modules/nixos/server/robotnix/default.nix +++ /dev/null @@ -1,35 +0,0 @@ -# robotnix lives in its own repo (flake input `robotnix`). This adapter -# imports its host module and fills its defaults from skyg values. -# Hosts configure it under services.robotnix. See ABOUTME.md. -{ config, lib, robotnix, ... }: -let - cfg = config.services.robotnix; - apps = config.skyg.internalNetworkingMap.apps; -in -{ - imports = [ robotnix.nixosModules.default ]; - - config = lib.mkIf cfg.enable { - services.robotnix = { - user = lib.mkDefault config.skyg.user.name; - authorizedKeys = lib.mkDefault config.skyg.nixos.common.ssh-server.masterPubKeys; - dnsUpstreams = lib.mkDefault [ config.skyg.internalNetworkingMap.appNetwork.gateway ]; - localDomains = lib.mkDefault [ - "app.internal" - "lab.internal" - ]; - }; - - # A robot's address must not clash with an app from the network map. - assertions = lib.mapAttrsToList - (name: r: - let - clash = lib.attrNames (lib.filterAttrs (an: a: a.ip == r.address && an != name) apps); - in - { - assertion = clash == [ ]; - message = "robotnix: ${name}'s address ${r.address} is already used by app(s): ${toString clash}."; - }) - cfg.robots; - }; -} diff --git a/modules/nixos/server/services/ABOUTME.md b/modules/nixos/server/services/ABOUTME.md index aff5a83..500417e 100644 --- a/modules/nixos/server/services/ABOUTME.md +++ b/modules/nixos/server/services/ABOUTME.md @@ -11,7 +11,6 @@ independent `skyg.nixos.server.services..enable` flag. Most run as OCI con services/ ├── default.nix # imports the services below ├── ai-services.nix # skyg.nixos.server.services.ai — Wyoming (whisper/piper/wake) -├── bazel-build.nix # skyg.nixos.server.services.bazel-build — NativeLink Bazel build server for slackerbot containers ├── audiobookshelf.nix # skyg.nixos.server.services.audiobookshelf — audiobook/podcast server ├── jellyfin.nix # skyg.nixos.server.services.jellyfin — media server (uid/gid 7777) ├── nbot-agent.nix # skyg.nixos.server.services.nbot-agent — nbot's agent system + gh, for nbot's `system` @@ -24,50 +23,6 @@ services/ - **`ai-services.nix`** openwakeword as containers and enables NixOS-native Wyoming faster-whisper + piper. Opens 11434/10200/10300/10400. -- **`bazel-build.nix`** runs NativeLink (from the `nativelink` flake input, pinned - to v1.8.3 because its config format changes) as a remote cache and executor for - slackerbot containers. Off on every host: only the disabled slackerbot configs on - hl-fws1 and hl-fwdesk turn it on, and nbot doesn't use it. Goal: as little as possible - runs in a container. Measured on slacker-bot (1 CPU container, 2026-11-01): local - 458 s, remote with no cache 65 s, remote with warm cache 17 s. - - NativeLink uses `DynamicUser`, whose state folder systemd mounts `noexec`; - `ExecPaths` re-allows it, or every remote action fails with "Permission denied". - - Containers get their own `bazel` (`agentBazel`), with no plain bazel/bazelisk on - PATH. At boot `bazel-remote-rc` writes the gateway into `/run/bazel-remote.bazelrc`, - which `/etc/bazel.bazelrc` try-imports. Only remote actions are allowed - (`--spawn_strategy=remote`, no local fallback), so a missing file or unreachable - host fails the build instead of building in the container. The wrapper refuses - command-line flags that change this (`--spawn_strategy`, `--strategy*`, - `--remote_executor`, `--bazelrc`, ...). Flags in a repo's own `.bazelrc` are not - checked. - - Final outputs are downloaded into `bazel-bin` (`--remote_download_toplevel`). - `rbuild` is just `bazel build`. A `/handoff` → `/outputs` path that hardlinked - outputs from the cache was tried and removed: outputs are a few MB over a local - link, so it saved nothing (download ~20 s vs handoff ~344 s). Most of that was a - lock bug: Bazel's server inherits open file handles, so a lock held on an fd in - the wrapper stays held until the server exits. Close such fds for Bazel (`9>&-`). - - Still in the container: goose, the Bazel server (512M heap, exits after 5 idle - minutes) and repository rules, which Bazel can't run remotely. - - Downloads share one cache per host (`/var/lib/bazel-repo-cache` → `/repo-cache`, - sha256-checked, 30-day cleanup). Fetched repo folders stay per container - (`--repo_contents_cache`), since their contents are not checked. - - Writable binds (`/repo-cache`) use `idmap`, or the container can't write - to them. Container root isn't root on the host, but its files there are - owned by host root. TODO: own the cache by a dedicated host user and map - container root to it. - - Global goose hints (`/root/.config/goose/.goosehints` → `/etc/goose/goosehints`): - a local build means something is wrong (stop and report); build Go through Bazel. - - Exposes `agentSystem` (slackerbot's agent + bazel, go, bazelrc, rbuild) and - `binds` (slackerbot `ro_binds`/`rw_binds`). Slackerbot creates missing bind - source folders, or containers fail to start. - `bazel` is a bazelisk wrapper on the host and in containers: a repo's - `.bazelversion` wins, otherwise 8.7.0 (`USE_BAZEL_FALLBACK_VERSION`). Bazel 8.7.0 - is fetched once per node by Nix (official release, sha256-checked) and the wrapper - enters it into each user's bazelisk cache as a link, so containers share that one - read-only copy and never download it. Other versions download into each container's - own `/root/.cache/bazelisk`. Upstream Bazel only runs on NixOS through - `programs.nix-ld`, enabled on the host and in containers. Bump `bazelVersion` and - `bazelSha256` together. - **`nbot-agent.nix`** adds `gh` to nbot's agent system from this repo, leaving the nbot repo unchanged. `gh` reads `GH_TOKEN` from `agent_env`, and git over HTTPS to github.com uses it through `gh`. Every container gets that token (chosen on purpose, 2026-11-03), @@ -82,7 +37,6 @@ services/ ``` skyg.nixos.server.services.ai.enable skyg.nixos.server.services.audiobookshelf.{enable,package,configDir,dataDir} -skyg.nixos.server.services.bazel-build.{enable,cpuCount,cacheGB,port,agentSystem,binds} skyg.nixos.server.services.jellyfin.enable skyg.nixos.server.services.nbot-agent.system skyg.nixos.server.services.signal-cli.{enable,host,port,configDir,account,environmentFile,openFirewall} diff --git a/modules/nixos/server/services/bazel-build.nix b/modules/nixos/server/services/bazel-build.nix deleted file mode 100644 index bab8c56..0000000 --- a/modules/nixos/server/services/bazel-build.nix +++ /dev/null @@ -1,345 +0,0 @@ -# Bazel build server for slackerbot containers on the same host. -# One NativeLink (cache + scheduler + worker) runs builds on the host CPUs. -# -# container: bazel build //app ──grpc──▶ NativeLink (host) ──▶ host CPUs -# ◀── final outputs downloaded (small, same machine) -# -# slackerbot mounts the shared download cache through `binds`. Container root -# isn't root on the host, but with `idmap` its files in /repo-cache are owned -# by host root. -# TODO: own the cache by a dedicated host user and map container root to it. -{ config, lib, pkgs, nativelink, slackerbot, ... }: - -with lib; - -let - cfg = config.skyg.nixos.server.services.bazel-build; - system = pkgs.stdenv.hostPlatform.system; - # Official Bazel release, checked against its sha256. Containers see - # /nix/store read-only, so one copy per node serves all of them. Upstream - # builds only run on NixOS through nix-ld. To bump: change both values. - bazelVersion = "8.7.0"; - bazelSha256 = "d7606e679b78067c811096fb3d6cf135225b528835ca396e3a4dddf957859544"; - # Copied as-is: patching would break the archive appended to the binary. - bazelRelease = pkgs.runCommand "bazel-${bazelVersion}-release" { } '' - install -Dm755 ${pkgs.fetchurl { - url = "https://releases.bazel.build/${bazelVersion}/release/bazel-${bazelVersion}-linux-x86_64"; - sha256 = bazelSha256; - }} $out/bin/bazel - ''; - - # `bazel` runs bazelisk on the host and in containers: a repo's - # .bazelversion wins, otherwise bazelVersion. bazelVersion is pre-entered in - # bazelisk's cache (format of bazelisk 1.28) as a link to bazelRelease, so it - # is never downloaded. Other versions download into each user's own cache. - bazel = pkgs.writeShellScriptBin "bazel" '' - export USE_BAZEL_FALLBACK_VERSION=''${USE_BAZEL_FALLBACK_VERSION:-silent:${bazelVersion}} - dl=''${BAZELISK_HOME_LINUX:-''${BAZELISK_HOME:-''${XDG_CACHE_HOME:-$HOME/.cache}/bazelisk}}/downloads - meta=$dl/metadata/bazelbuild/bazel-${bazelVersion}-linux-x86_64 - if [ ! -e "$dl/sha256/${bazelSha256}/bin/bazel" ] || [ "$(cat "$meta" 2>/dev/null)" != ${bazelSha256} ]; then - mkdir -p "$dl/sha256" "''${meta%/*}" - ln -sfn ${bazelRelease} "$dl/sha256/${bazelSha256}" - printf %s ${bazelSha256} > "$meta" - fi - exec ${pkgs.bazelisk}/bin/bazelisk "$@" - ''; - - stateDir = "/var/lib/nativelink"; - # Bazel download cache shared by all containers (files are sha256-checked). - repoCacheRoot = "/var/lib/bazel-repo-cache"; - workerPort = 50061; # host only - - # Tools build actions run with. Same store paths on the host and in containers. - toolchain = pkgs.buildEnv { - name = "bazel-action-toolchain"; - paths = with pkgs; [ gcc binutils coreutils bash gnused gnugrep gawk findutils diffutils python3 ]; - }; - - # Matches NativeLink v1.8.3 (basic_cas example). Field names change between - # releases; check them when bumping the flake input. - instances = f: map (instance_name: { inherit instance_name; } // f) [ "" "main" ]; - nativelinkConfig = pkgs.writeText "nativelink.json" (builtins.toJSON { - stores = [ - { - # If any output of a cached result was evicted, report a miss so the - # action is rebuilt instead of returning a result with missing files. - name = "AC_MAIN_STORE"; - completeness_checking = { - backend.filesystem = { - content_path = "${stateDir}/ac/content"; - temp_path = "${stateDir}/ac/tmp"; - eviction_policy.max_bytes = 10 * 1000 * 1000 * 1000; - }; - cas_store.ref_store.name = "WORKER_FAST_SLOW_STORE"; - }; - } - { - name = "WORKER_FAST_SLOW_STORE"; - fast_slow = { - fast.filesystem = { - content_path = "${stateDir}/cas/content"; - temp_path = "${stateDir}/cas/tmp"; - eviction_policy.max_bytes = cfg.cacheGB * 1000 * 1000 * 1000; - }; - slow.noop = { }; - }; - } - ]; - - schedulers = [{ - name = "MAIN_SCHEDULER"; - simple.supported_platform_properties = { - cpu_count = "minimum"; - OSFamily = "priority"; - "container-image" = "priority"; - }; - }]; - - workers = [{ - local = { - worker_api_endpoint.uri = "grpc://127.0.0.1:${toString workerPort}"; - cas_fast_slow_store = "WORKER_FAST_SLOW_STORE"; - upload_action_result.ac_store = "AC_MAIN_STORE"; - work_directory = "${stateDir}/work"; - platform_properties = { - cpu_count.values = [ (toString cfg.cpuCount) ]; - OSFamily.values = [ "" ]; - "container-image".values = [ "" ]; - }; - }; - }]; - - servers = [ - { - name = "public"; - # Only container links (ve-*) are trusted; the LAN port stays closed. - listener.http.socket_address = "0.0.0.0:${toString cfg.port}"; - services = { - cas = instances { cas_store = "WORKER_FAST_SLOW_STORE"; }; - ac = instances { ac_store = "AC_MAIN_STORE"; }; - execution = instances { cas_store = "WORKER_FAST_SLOW_STORE"; scheduler = "MAIN_SCHEDULER"; }; - capabilities = instances { remote_execution.scheduler = "MAIN_SCHEDULER"; }; - bytestream = instances { cas_store = "WORKER_FAST_SLOW_STORE"; }; - }; - } - { - name = "private_workers_servers"; - listener.http.socket_address = "127.0.0.1:${toString workerPort}"; - services = { - worker_api.scheduler = "MAIN_SCHEDULER"; - admin = { }; - health = { }; - }; - } - ]; - - global.max_open_files = 65536; - }); - - # Container side `bazel`. Refuses flags that would move work into the - # container and forces remote actions. The host address comes from - # /run/bazel-remote.bazelrc (see bazelrc). - agentBazel = pkgs.writeShellScriptBin "bazel" '' - set -u - real=${bazel}/bin/bazel - enforce=(--spawn_strategy=remote --noremote_local_fallback) - - # Find the command (first word that isn't a startup option) and check flags. - # Arguments after `--` belong to the program being run, so they're skipped. - cmd="" at=0 i=0 - for a in "$@"; do - [ "$a" = "--" ] && break - case "$a" in - --spawn_strategy*|--strategy*|--test_strategy*|--remote_executor*|\ - --remote_local_fallback*|--bazelrc*|--ignore_all_rc_files|--nosystem_rc) - echo "bazel: $a isn't allowed here. Builds must run on the host's build server;" \ - "tell the user instead of working around it." >&2 - exit 2 ;; - esac - if [ -z "$cmd" ] && [ "''${a#-}" = "$a" ]; then cmd=$a at=$i; fi - i=$((i + 1)) - done - [ -z "$cmd" ] && exec "$real" "$@" - pre=("''${@:1:at}") - post=("''${@:at+2}") - - case "$cmd" in - build|test|run|coverage) exec "$real" "''${pre[@]}" "$cmd" "''${enforce[@]}" "''${post[@]}" ;; - *) exec "$real" "$@" ;; - esac - ''; - - # Kept for old habits and docs: same as `bazel build`. - rbuild = pkgs.writeShellScriptBin "rbuild" ''exec ${agentBazel}/bin/bazel build "$@"''; - - remoteRc = "/run/bazel-remote.bazelrc"; - - bazelrc = '' - # Builds run on the host, so the Bazel server in a container stays small, - # and exits after 5 idle minutes instead of 3 hours. - startup --host_jvm_args=-Xmx512m - startup --host_jvm_args=-XX:+UseSerialGC - startup --max_idle_secs=300 - - # Every action runs on the host; one that can't fails instead of running here. - build --spawn_strategy=remote - build --noremote_local_fallback - - # Downloads are shared by all containers on this host. Fetched repo folders - # are not (unchecked, so one container could tamper with another's). - common --repository_cache=/repo-cache - common --repo_contents_cache=/root/.cache/bazel/repo-contents - - build --remote_instance_name=main - # Final outputs are downloaded into bazel-bin (a few MB, same machine). - build --remote_download_toplevel - build --jobs=${toString (cfg.cpuCount * 4)} - - # Rebuild instead of failing when the cache evicts something Bazel needs. - build --experimental_remote_cache_eviction_retries=5 - - build --action_env=PATH=${toolchain}/bin - build --host_action_env=PATH=${toolchain}/bin - build --repo_env=CC=${toolchain}/bin/gcc - - # Points plain `bazel` at the host (written at boot). Missing = builds fail. - try-import ${remoteRc} - ''; - - # Global goose instructions in each container. - goosehints = '' - # Bazel builds run on the host, never in this container - - This container is small (a few GB RAM, 1 CPU). `bazel build/test/run` runs - every action on the host's build server. - - Outputs are downloaded into bazel-bin as usual. - Flags that change where actions run (--spawn_strategy, --strategy, - --remote_executor, --bazelrc, ...) are refused on purpose. - - A normal build ends with a line like - `INFO: 120 processes: 110 remote, 10 internal.` (or "remote cache hit"). - If instead you see `linux-sandbox`, `local` or `processwrapper-sandbox` - doing the work, or /run/bazel-remote.bazelrc is missing, or a build can't - reach the remote executor, something is wrong: - - stop the build (`bazel shutdown`), don't retry it locally, - - don't remove or override the --remote_* or --spawn_strategy flags, - - tell the user what you saw, with the error lines. - - Build and test Go code through Bazel (`bazel build`, `bazel test`), not - `go build` / `go test`: those run here, in this small container. `go mod` - and other quick `go` commands are fine. - ''; - - # Added to slackerbot's agent system. Uses host packages so bazel and the - # toolchain match the host exactly. - agentModule = { - # No plain bazel/bazelisk on PATH, so the checks in agentBazel can't be skipped by accident. - environment.systemPackages = [ agentBazel pkgs.go pkgs.git rbuild ]; - environment.etc."bazel.bazelrc".text = bazelrc; - environment.etc."goose/goosehints".text = goosehints; - programs.nix-ld.enable = true; - - systemd.tmpfiles.rules = [ - "d /root/.config/goose 0755 root root -" - "L+ /root/.config/goose/.goosehints - - - - /etc/goose/goosehints" - ]; - - # The host is the default gateway, whose address changes per container, - # so write it for bazel.bazelrc once the route is up. Doesn't hold up boot. - systemd.services.bazel-remote-rc = { - description = "Point Bazel at the host build server"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - path = [ pkgs.iproute2 pkgs.gawk pkgs.coreutils ]; - script = '' - for _ in $(seq 60); do - gw=$(ip -4 route show default | awk '{print $3; exit}') - if [ -n "$gw" ]; then - echo "build --remote_executor=grpc://$gw:${toString cfg.port}" > ${remoteRc} - exit 0 - fi - sleep 1 - done - echo "no default route after 60s; bazel will build locally" >&2 - exit 1 - ''; - }; - }; -in -{ - options = { - skyg.nixos.server.services.bazel-build = { - enable = mkEnableOption "Bazel build server (NativeLink) for slackerbot containers"; - - cpuCount = mkOption { - description = "CPU threads the host worker offers to builds."; - type = types.ints.positive; - }; - - cacheGB = mkOption { - description = "Size of the build cache in GB."; - default = 100; - type = types.ints.positive; - }; - - port = mkOption { - description = "Port containers reach NativeLink on (container links only)."; - default = 50051; - type = types.port; - }; - - agentSystem = mkOption { - description = "slackerbot's agent system with bazel, go, bazelrc and rbuild added. Use for slackerbot's `system`."; - readOnly = true; - type = types.package; - default = (slackerbot.nixosConfigurations.agent.extendModules { - modules = [ agentModule ]; - }).config.system.build.toplevel; - # Keeps the manual from building it on every host. - defaultText = literalExpression "slackerbot agent + bazel, go, bazelrc, rbuild"; - }; - - binds = mkOption { - description = "slackerbot ro_binds / rw_binds that give each container the shared /repo-cache."; - readOnly = true; - type = types.attrsOf (types.listOf types.str); - default = { - # idmap: every container writes as the same host user, so all can share it. - rw = [ "${repoCacheRoot}:/repo-cache:idmap" ]; - ro = [ ]; - }; - }; - }; - }; - - config = mkIf cfg.enable { - environment.systemPackages = [ bazel pkgs.bazelisk ]; - programs.nix-ld.enable = true; # for the Bazel bazelisk downloads - - systemd.services.nativelink = { - description = "NativeLink remote cache and build service"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - path = [ toolchain ]; - serviceConfig = { - ExecStart = "${nativelink.packages.${system}.nativelink}/bin/nativelink ${nativelinkConfig}"; - DynamicUser = true; - StateDirectory = "nativelink"; - # DynamicUser's state dir is mounted noexec; build actions run from work/. - ExecPaths = [ "/var/lib/private/nativelink" ]; - UMask = "0022"; - LimitNOFILE = 65536; - Restart = "on-failure"; - }; - }; - - systemd.tmpfiles.rules = [ - # Downloads unused for 30 days are removed (Bazel never cleans this). - "d ${repoCacheRoot} 0755 root root 30d" - ]; - - # Keep the toolchain from garbage collection. - system.extraDependencies = [ toolchain ]; - }; -} diff --git a/modules/nixos/server/services/default.nix b/modules/nixos/server/services/default.nix index 7fea907..9374053 100644 --- a/modules/nixos/server/services/default.nix +++ b/modules/nixos/server/services/default.nix @@ -3,7 +3,6 @@ imports = [ ./audiobookshelf.nix ./ai-services.nix - ./bazel-build.nix ./colibri.nix ./comfyui ./jellyfin.nix diff --git a/secrets.nix b/secrets.nix index 3df9ee7..b059746 100644 --- a/secrets.nix +++ b/secrets.nix @@ -28,16 +28,6 @@ in "secrets/hermes-env.age".publicKeys = [ ari fwdesk ]; "secrets/iu-project.age".publicKeys = [ ari minipc1 ]; - # Robots - "secrets/robots-shared.age".publicKeys = [ ari fwdesk ]; - "secrets/robots-genix.age".publicKeys = [ ari fwdesk ]; - "secrets/robots-sibli.age".publicKeys = [ ari fwdesk ]; - - # Slackerbot (master on fwdesk, worker on fws1) - "secrets/slackerbot-slack.age".publicKeys = [ ari fwdesk ]; - "secrets/slackerbot-goose.age".publicKeys = [ ari fwdesk ]; - "secrets/slackerbot-cluster.age".publicKeys = [ ari fwdesk fws1 ]; - # nbot (master on fwdesk, worker on fws1) "secrets/nbot-slack.age".publicKeys = [ ari fwdesk ]; "secrets/nbot-goose.age".publicKeys = [ ari fwdesk ];