cleaning up old bots

This commit is contained in:
ari committed 2026-11-05 15:21:01 -05:00
1 parent 880c85facb
commit e3f01e3c35
25 files changed
+12 -1267

No files matched your search

-1
View File
@@ -103,7 +103,6 @@ export def "skyg buzz identity" [
print $"npub: ($npub)"
print "Next:"
print $" 1. On hl-minipc1: sudo buzz-manage add-member ($npub)"
print $" 2. Rebuild, then restart the robot: robotnix restart ($robot)"
if $restricted and not $gated {
print "⚠️ --restricted with no BUZZ_ALLOWED_USERS: the bot will ignore everyone. Re-run with --rotate --allowed-users <npub,...> or edit the secret."
}
Generated
+7 -267
View File
@@ -279,21 +279,6 @@
"type": "github"
}
},
"crane_2": {
"locked": {
"lastModified": 1780029330,
"narHash": "sha256-fWFKEKB5CP+NO8/3uOaEZIVbd03mvDJ//VsnKkXty08=",
"owner": "ipetkov",
"repo": "crane",
"rev": "6823b493a0bc2142082075576efa6633b537197e",
"type": "github"
},
"original": {
"owner": "ipetkov",
"repo": "crane",
"type": "github"
}
},
"crate2nix": {
"flake": false,
"locked": {
@@ -629,22 +614,6 @@
"type": "github"
}
},
"flake-compat_10": {
"flake": false,
"locked": {
"lastModified": 1767039857,
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
"owner": "NixOS",
"repo": "flake-compat",
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_2": {
"flake": false,
"locked": {
@@ -860,24 +829,6 @@
}
},
"flake-parts_5": {
"inputs": {
"nixpkgs-lib": "nixpkgs-lib"
},
"locked": {
"lastModified": 1769996383,
"narHash": "sha256-AnYjnFWgS49RlqX7LrC4uA+sCCDBj0Ry/WOJ5XWAsa0=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "57928607ea566b5db3ad13af0e57e921e6b12381",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "flake-parts",
"type": "github"
}
},
"flake-parts_6": {
"inputs": {
"nixpkgs-lib": [
"stylix",
@@ -898,21 +849,6 @@
"type": "github"
}
},
"flake-root": {
"locked": {
"lastModified": 1723604017,
"narHash": "sha256-rBtQ8gg+Dn4Sx/s+pvjdq3CB2wQNzx9XGFq/JVGCB6k=",
"owner": "srid",
"repo": "flake-root",
"rev": "b759a56851e10cb13f6b8e5698af7b59c44be26e",
"type": "github"
},
"original": {
"owner": "srid",
"repo": "flake-root",
"type": "github"
}
},
"flox": {
"inputs": {
"crane": "crane",
@@ -1083,29 +1019,6 @@
"type": "github"
}
},
"git-hooks_4": {
"inputs": {
"flake-compat": "flake-compat_9",
"gitignore": "gitignore_2",
"nixpkgs": [
"nativelink",
"nixpkgs"
]
},
"locked": {
"lastModified": 1772024342,
"narHash": "sha256-+eXlIc4/7dE6EcPs9a2DaSY3fTA9AE526hGqkNID3Wg=",
"owner": "cachix",
"repo": "git-hooks.nix",
"rev": "6e34e97ed9788b17796ee43ccdbaf871a5c2b476",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "git-hooks.nix",
"type": "github"
}
},
"gitignore": {
"inputs": {
"nixpkgs": [
@@ -1128,28 +1041,6 @@
"type": "github"
}
},
"gitignore_2": {
"inputs": {
"nixpkgs": [
"nativelink",
"git-hooks",
"nixpkgs"
]
},
"locked": {
"lastModified": 1709087332,
"narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"gnome-shell": {
"flake": false,
"locked": {
@@ -1688,7 +1579,7 @@
"hyprutils": "hyprutils_3",
"hyprwayland-scanner": "hyprwayland-scanner_3",
"hyprwire": "hyprwire_3",
"nixpkgs": "nixpkgs_12",
"nixpkgs": "nixpkgs_11",
"pre-commit-hooks": "pre-commit-hooks_3",
"systems": "systems_3",
"xdph": "xdph_2"
@@ -2306,31 +2197,6 @@
"type": "github"
}
},
"nativelink": {
"inputs": {
"crane": "crane_2",
"flake-parts": "flake-parts_5",
"flake-root": "flake-root",
"git-hooks": "git-hooks_4",
"nix2container": "nix2container",
"nixpkgs": "nixpkgs_11",
"rust-overlay": "rust-overlay_3"
},
"locked": {
"lastModified": 1793212176,
"narHash": "sha256-k9Vzp5nOq2D/JnYfnG1vlIZrca8dt7F6qz6SwirXLec=",
"owner": "TraceMachina",
"repo": "nativelink",
"rev": "bd336137d794a6a4e7ac32e561ab55e2d7e70ff3",
"type": "github"
},
"original": {
"owner": "TraceMachina",
"ref": "v1.8.3",
"repo": "nativelink",
"type": "github"
}
},
"nbot": {
"inputs": {
"nixpkgs": [
@@ -2467,28 +2333,6 @@
"type": "github"
}
},
"nix2container": {
"inputs": {
"nixpkgs": [
"nativelink",
"nixpkgs"
]
},
"locked": {
"lastModified": 1775487831,
"narHash": "sha256-2lguQpLPQaxpQCJjXhmEEAfabwsAhkP29Z7fgLzHARA=",
"owner": "nlewo",
"repo": "nix2container",
"rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e",
"type": "github"
},
"original": {
"owner": "nlewo",
"repo": "nix2container",
"rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e",
"type": "github"
}
},
"nix_2": {
"inputs": {
"flake-parts": "flake-parts_2",
@@ -2707,7 +2551,7 @@
},
"nixos-hardware_2": {
"inputs": {
"nixpkgs": "nixpkgs_13"
"nixpkgs": "nixpkgs_12"
},
"locked": {
"lastModified": 1792491056,
@@ -2772,21 +2616,6 @@
"type": "github"
}
},
"nixpkgs-lib": {
"locked": {
"lastModified": 1769909678,
"narHash": "sha256-cBEymOf4/o3FD5AZnzC3J9hLbiZ+QDT/KDuyHXVJOpM=",
"owner": "nix-community",
"repo": "nixpkgs.lib",
"rev": "72716169fe93074c333e8d0173151350670b824c",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nixpkgs.lib",
"type": "github"
}
},
"nixpkgs-regression": {
"locked": {
"lastModified": 1643052045,
@@ -2866,22 +2695,6 @@
}
},
"nixpkgs_11": {
"locked": {
"lastModified": 1787498568,
"narHash": "sha256-9i/VTdusq/+NM/tz+J1Re+ojkMB8MBf0QshnYfzHz30=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "56c02bc00adcf003215cc4bd996d6efaf4cff188",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_12": {
"locked": {
"lastModified": 1788404924,
"narHash": "sha256-lhEhY8X5EgkQ/eg6IFz4cc8jRuSYSvnxx1al7d1dvZ0=",
@@ -2897,7 +2710,7 @@
"type": "github"
}
},
"nixpkgs_13": {
"nixpkgs_12": {
"locked": {
"lastModified": 1789546076,
"narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=",
@@ -2910,7 +2723,7 @@
"url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz"
}
},
"nixpkgs_14": {
"nixpkgs_13": {
"locked": {
"lastModified": 1792503283,
"narHash": "sha256-vhe2jyvRCgufWPqDuDq1S3t3ypE+ka46IFutV02xKso=",
@@ -3191,7 +3004,7 @@
},
"pre-commit-hooks_3": {
"inputs": {
"flake-compat": "flake-compat_10",
"flake-compat": "flake-compat_9",
"nixpkgs": [
"nixarchy",
"hyprland",
@@ -3262,35 +3075,6 @@
"type": "github"
}
},
"robotnix": {
"inputs": {
"devenv": [],
"git-hooks": [
"git-hooks"
],
"hermes-agent": [
"hermes-agent"
],
"nixpkgs": [
"nixpkgs"
],
"nutest": [],
"treefmt-nix": []
},
"locked": {
"lastModified": 1793232093,
"narHash": "sha256-lH8MNrXEyWzkM+iA0ILRH3qoqSrE0ZJ76FeFoEHWgac=",
"ref": "refs/heads/main",
"rev": "c8a494361d085a355f63f98ec92c5aee58f3c51c",
"revCount": 4,
"type": "git",
"url": "ssh://gitea@gitea.app.internal:22/ari/robotnix.git"
},
"original": {
"type": "git",
"url": "ssh://gitea@gitea.app.internal:22/ari/robotnix.git"
}
},
"root": {
"inputs": {
"agenix": "agenix",
@@ -3309,20 +3093,17 @@
"hyprland": "hyprland",
"hyprlauncher": "hyprlauncher",
"my-nixpkgs": "my-nixpkgs",
"nativelink": "nativelink",
"nbot": "nbot",
"nix-darwin": "nix-darwin",
"nix-flatpak": "nix-flatpak",
"nix-index-database": "nix-index-database",
"nixarchy": "nixarchy",
"nixos-hardware": "nixos-hardware_2",
"nixpkgs": "nixpkgs_14",
"nixpkgs": "nixpkgs_13",
"nixpkgs-unstable": "nixpkgs-unstable",
"noctalia": "noctalia",
"robotnix": "robotnix",
"skyg-linux-tracker": "skyg-linux-tracker",
"skygqts": "skygqts",
"slackerbot": "slackerbot",
"stylix": "stylix",
"systems": "systems_6"
}
@@ -3387,27 +3168,6 @@
"type": "github"
}
},
"rust-overlay_3": {
"inputs": {
"nixpkgs": [
"nativelink",
"nixpkgs"
]
},
"locked": {
"lastModified": 1787713867,
"narHash": "sha256-ZRYTEvDb8QZLCHRoMyIZle9V3Bvw905m0teeo1RvI7M=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "494680f9ffa2b4bf9f2df0f442c7096e44adfa00",
"type": "github"
},
"original": {
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
}
},
"skyg-linux-tracker": {
"inputs": {
"nixpkgs": [
@@ -3448,26 +3208,6 @@
"url": "ssh://gitea@gitea.app.internal:22/ari/skygqts.git"
}
},
"slackerbot": {
"inputs": {
"nixpkgs": [
"nixpkgs-unstable"
]
},
"locked": {
"lastModified": 1793590815,
"narHash": "sha256-Z2HRkYW0547L309qm4e0w++xVKIJFpKbqBUDLEoWG4c=",
"ref": "refs/heads/main",
"rev": "4db99eaa2a0e1529194983962a07109fdb716bcd",
"revCount": 22,
"type": "git",
"url": "https://gitea.skyg.ca/ari/slacker-bot.git"
},
"original": {
"type": "git",
"url": "https://gitea.skyg.ca/ari/slacker-bot.git"
}
},
"sops-nix": {
"inputs": {
"nixpkgs": [
@@ -3513,7 +3253,7 @@
"base16-helix": "base16-helix",
"base16-vim": "base16-vim",
"firefox-gnome-theme": "firefox-gnome-theme",
"flake-parts": "flake-parts_6",
"flake-parts": "flake-parts_5",
"gnome-shell": "gnome-shell",
"nixpkgs": [
"nixpkgs"
-33
View File
@@ -4,17 +4,6 @@
skygqts.url = "git+ssh://gitea@gitea.app.internal:22/ari/skygqts.git";
skygqts.inputs.nixpkgs.follows = "nixpkgs";
hermenix.url = "git+ssh://gitea@gitea.app.internal:22/ari/hermenix.git";
# Self-managed nspawn robots. Robots pin the same hermes-agent rev.
robotnix = {
url = "git+ssh://gitea@gitea.app.internal:22/ari/robotnix.git";
inputs.nixpkgs.follows = "nixpkgs";
inputs.hermes-agent.follows = "hermes-agent";
# Dev/test-only inputs of robotnix: keep them out of this lock.
inputs.git-hooks.follows = "git-hooks";
inputs.devenv.follows = "";
inputs.treefmt-nix.follows = "";
inputs.nutest.follows = "";
};
# Read-only host monitor (systemd, journal, Docker) over REST + MCP.
skyg-linux-tracker = {
url = "git+https://gitea.skyg.ca/ari/skyg-linux-tracker.git";
@@ -25,14 +14,6 @@
url = "git+ssh://git@github.com/asosnovsky/nbot.git";
inputs.nixpkgs.follows = "nixpkgs-unstable";
};
# Off.
slackerbot = {
url = "git+https://gitea.skyg.ca/ari/slacker-bot.git";
inputs.nixpkgs.follows = "nixpkgs-unstable";
};
# Bazel remote build server for slackerbot containers. Pinned: its config
# format changes between releases (see bazel-build.nix).
nativelink.url = "github:TraceMachina/nativelink/v1.8.3";
# Headless Hyprland session streamed over a Unix socket.
hhstream = {
url = "git+ssh://gitea@gitea.app.internal:22/Hermenix/hyprland-headless-stream-plugin.git";
@@ -131,11 +112,8 @@
{ self
, skygqts
, hermenix
, robotnix
, skyg-linux-tracker
, nbot
, slackerbot
, nativelink
, hhstream
, nixpkgs-unstable
, flox
@@ -212,12 +190,6 @@
linux-tracker-peer-bigbox2 = ./secrets/linux-tracker-peer-bigbox2.age;
linux-tracker-peer-terra1 = ./secrets/linux-tracker-peer-terra1.age;
scrypted-envs = ./secrets/scrypted-envs.age;
robots-shared = ./secrets/robots-shared.age;
robots-sibli = ./secrets/robots-sibli.age;
robots-genix = ./secrets/robots-genix.age;
slackerbot-slack = ./secrets/slackerbot-slack.age;
slackerbot-goose = ./secrets/slackerbot-goose.age;
slackerbot-cluster = ./secrets/slackerbot-cluster.age;
nbot-slack = ./secrets/nbot-slack.age;
nbot-goose = ./secrets/nbot-goose.age;
nbot-cluster = ./secrets/nbot-cluster.age;
@@ -236,11 +208,8 @@
hermes-agent
skygqts
hermenix
robotnix
skyg-linux-tracker
nbot
slackerbot
nativelink
hhstream
flox
determinate
@@ -399,7 +368,6 @@
./hosts/hl-fws1.disko.nix
nixos-hardware.nixosModules.framework-11th-gen-intel
disko.nixosModules.disko
slackerbot.nixosModules.host
nbot.nixosModules.host
];
};
@@ -411,7 +379,6 @@
nixos-hardware.nixosModules.framework-desktop-amd-ai-max-300-series
hermenix.nixosModules.default
buildbot-nix.nixosModules.buildbot-worker
slackerbot.nixosModules.host
nbot.nixosModules.host
];
};
+2 -2
View File
@@ -31,8 +31,8 @@ hosts/
| Folder | Host | Role |
| ------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
| `fwbook/` | fwbook | Framework 13 AMD — personal laptop / daily driver |
| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); nbot master (slackerbot off) |
| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; nbot worker (slackerbot off) |
| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); nbot master |
| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; nbot worker |
| `hl-bigbox1/` | hl-bigbox1 | AI/compute server (NVIDIA + AMD); Immich machine-learning (CUDA) |
| `hl-bigbox2/` | hl-bigbox2 | NFS storage server; Immich server/db (docker, `immich.home.sosnovsky.ca`) |
| `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI), monitoring (Grafana, Prometheus, Loki, OTel collector) |
-2
View File
@@ -8,8 +8,6 @@
./steam.nix
./buildbot.nix
./linux-tracker.nix
./robotnix.nix
./slackerbot.nix
./nbot.nix
];
}
-1
View File
@@ -28,7 +28,6 @@ let
OPENROUTER_API_KEY = "$OPENROUTER_API_KEY";
CLAUDE_CODE_OAUTH_TOKEN = "$CLAUDE_CODE_OAUTH_TOKEN";
COMMAND_CODE_API_KEY = "$COMMAND_CODE_API_KEY";
# Every agent can use this PAT, so anyone who can prompt one can too.
GH_TOKEN = "$GH_TOKEN";
};
-124
View File
@@ -1,124 +0,0 @@
{ config, lib, skyg-secrets, ... }:
let
apps = config.skyg.internalNetworkingMap.apps;
robotSecrets = [ "robots-shared" "robots-sibli" "robots-genix" ];
secret = name: config.age.secrets.${name}.path;
ports = {
ollama = 11434;
ds4 = 8000;
tracker = 1984;
};
# A robot's seed: its config plus the files it shares with the others.
# Each robot gets copies under /etc/nixos/seed (same layout as this repo),
# so a robot that rebuilds itself (genix) keeps working.
seedOf = file: {
root = ../../..;
configuration = file;
extraFiles = [
./robotnix/common.nix
./robotnix/buzz.nix
../../../pkgs/buzz-cli
];
};
in
{
age.secrets = lib.genAttrs robotSecrets (name: {
file = skyg-secrets.${name};
owner = "root";
mode = "0400";
});
# robotnix flake input; options and docs in its repo.
services.robotnix = {
enable = true;
externalInterface = "enp191s0";
# Every robot gets this tier. Same holes as the Hermes VM.
access.shared = {
internet = "all";
hostAccess = [
{ port = ports.ollama; }
];
lanAccess = [
{ address = "10.0.101.3"; port = 443; }
{ address = "10.0.101.3"; port = 3000; }
{ address = "10.0.101.3"; port = 8080; }
{ address = "10.0.101.3"; port = 9102; }
];
};
# A Hermes agent in every robot.
hermes = {
enable = true;
environmentFiles = [ (secret "robots-shared") ];
};
robots.sibli = {
id = 2;
address = apps.sibli.ip;
cpus = { min = 1; max = 4; };
memory = { min = "2G"; max = "16G"; };
disk = { rootSize = "100G"; homeSize = "60G"; };
seed = seedOf ./robotnix/sibli.nix;
hermes = {
name = "Siblix";
environmentFiles = [ (secret "robots-sibli") ];
settings.model = {
provider = "anthropic";
default = "claude-haiku-5-5";
};
};
allowDocker = true;
allowTun = true;
ephemeral = true;
persist = [
"/var/lib/tailscale"
"/var/lib/docker"
];
access = { };
git.enable = false;
};
# Persistent and self-managing: genix changes and applies its own system
# (robotnix-apply), pushes its flake to gitea, and pulls it back after a
# reseed. Add /var/lib/robotnix-git/id_ed25519.pub (inside genix) as a
# deploy key with write access on ari/genix.
robots.genix = {
id = 3;
address = apps.genix.ip;
cpus = { min = 1; max = 6; };
memory = { min = "4G"; max = "24G"; };
disk = { rootSize = "150G"; homeSize = "100G"; };
seed = seedOf ./robotnix/genix.nix;
selfManage.enable = true;
git = {
remoteUrl = "gitea@gitea.app.internal:Hermenix/genix.git";
interval = "hourly";
};
hermes = {
name = "Genix";
environmentFiles = [ (secret "robots-genix") ];
settings = {
model = {
provider = "commandcode";
default = "deepseek/deepseek-v4.1-flash";
};
fallback_providers = [
{ provider = "anthropic"; model = "claude-haiku-5-5"; }
{ provider = "openrouter"; model = "deepseek/deepseek-v4.1-flash"; }
];
};
};
allowDocker = true;
# Survive a reseed (root is rebuilt from the git remote).
persist = [
"/var/lib/hermes"
"/var/lib/docker"
];
# Push/pull its flake over gitea's ssh.
access.lanAccess = [
{ address = apps.gitea.ip; port = 22; }
];
};
};
}
@@ -1,77 +0,0 @@
# Buzz for a robot's Hermes: the `buzz` CLI and a profile for its key.
# Shared by this host's robots (copied into each under /etc/nixos/seed).
# Needs a Buzz identity in the robot's secret: `skyg buzz identity <robot>`.
{ config, lib, pkgs, ... }:
let
cfg = config.skyg.robot.buzz;
# Built from this repo; the robot gets a copy of pkgs/buzz-cli with this file.
buzz-cli = pkgs.callPackage ../../../../pkgs/buzz-cli { };
in
{
options.skyg.robot.buzz = {
enable = lib.mkEnableOption "Buzz for this robot's Hermes";
homeChannel = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "BUZZ_HOME_CHANNEL (not secret, so it lives here, not in the secret).";
};
profileName = lib.mkOption {
type = lib.types.str;
default = config.services.hermes-agent.environment.API_SERVER_MODEL_NAME or config.networking.hostName;
defaultText = lib.literalExpression "the Hermes gateway model name, else the host name";
description = "Name published for a new key that has no Buzz profile yet.";
};
};
config = lib.mkIf cfg.enable {
environment.systemPackages = [ buzz-cli ];
# The Hermes service PATH has no system packages.
services.hermes-agent.environment = {
BUZZ_CLI_PATH = "${buzz-cli}/bin/buzz";
} // lib.optionalAttrs (cfg.homeChannel != null) {
BUZZ_HOME_CHANNEL = cfg.homeChannel;
};
# A new Buzz key has no profile, and Hermes won't connect without one.
# Publish one at start if missing; the relay keeps it.
systemd.services.buzz-profile = {
description = "Publish the Buzz profile if missing";
wantedBy = [ "multi-user.target" ];
before = [ "hermes-agent.service" ];
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
path = [ buzz-cli pkgs.jq ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
User = "hermes";
EnvironmentFile = config.systemd.services.hermes-agent.serviceConfig.EnvironmentFile;
};
script = ''
if [ -z "''${BUZZ_PRIVATE_KEY:-}" ] || [ -z "''${BUZZ_RELAY_URL:-}" ]; then
echo "No Buzz creds, skipping."
exit 0
fi
# Retry: the relay may not be reachable yet. Failure is not fatal.
for _ in 1 2 3 4 5; do
n=""
if out=$(buzz users get); then
n=$(jq 'length' <<<"$out") || n=""
fi
if [ "$n" = "0" ]; then
if buzz users set-profile --name ${lib.escapeShellArg cfg.profileName}; then
echo "Published Buzz profile."
exit 0
fi
elif [ -n "$n" ]; then
echo "Buzz profile exists."
exit 0
fi
sleep 5
done
echo "Could not check or publish the Buzz profile." >&2
'';
};
};
}
@@ -1,45 +0,0 @@
# Shared base for this host's robots (sibli, genix). Robots get a copy under
# /etc/nixos/seed (see seedOf in ../robotnix.nix), so a robot that rebuilds
# itself keeps working; edit it here for new installs and ephemeral robots.
{ config, lib, pkgs, ... }:
let
user = config.robotnix.user;
in
{
# The user has no password; ssh keys only.
security.sudo.wheelNeedsPassword = false;
environment.systemPackages = with pkgs; [
git
vim
curl
htop
claude-code
devenv
direnv
(python3.withPackages (ps: [ ps.pip ]))
nodejs
docker-compose
zellij
];
nixpkgs.config.allowUnfree = true;
# direnv: hook into bash so the repo's devenv shims work.
programs.bash = {
enable = true;
interactiveShellInit = ''
eval "$(direnv hook bash)"
'';
};
# Docker + compose. Needs allowDocker = true on the host.
virtualisation.docker.enable = true;
users.users.${user}.extraGroups = [ "docker" ];
users.users.hermes.extraGroups = lib.mkIf config.robotnix.hermes.enable [ "docker" ];
nix.gc = {
automatic = true;
dates = "weekly";
options = "--delete-older-than 14d";
};
}
@@ -1,13 +0,0 @@
# genix's starting config. It is persistent and manages itself: after the
# first install this (copied to /etc/nixos/seed/...) belongs to genix, which
# changes and applies its own system with robotnix-apply. Edits here only
# matter for a fresh install with nothing on its git remote.
{ ... }:
{
imports = [
./common.nix
./buzz.nix
];
skyg.robot.buzz.enable = true;
}
@@ -1,71 +0,0 @@
# sibli is ephemeral: root resets to this config on every start, so change
# things here (then rebuild the host), not inside the robot.
{ config, pkgs, ... }:
let
robotoName = config.services.hermes-agent.environment.API_SERVER_MODEL_NAME;
in
{
imports = [
./common.nix
./buzz.nix
];
skyg.robot.buzz = {
enable = true;
homeChannel = "a8969cdc-7ba2-4e25-ae18-b898ab59003d";
};
services.hermes-agent = {
extraPackages = [ pkgs.nodejs ];
mcpServers.shortcut = {
command = "${pkgs.nodejs}/bin/npx";
args = [ "-y" "@shortcut/mcp@latest" ];
env.SHORTCUT_API_TOKEN = "\${SHORTCUT_API_KEY}";
};
};
services.tailscale.enable = true;
services.tailscale.openFirewall = true;
networking.firewall.trustedInterfaces = [ "tailscale0" ];
systemd.services.tailscale-login = {
description = "Log in to Tailscale if needed";
wantedBy = [ "multi-user.target" ];
wants = [ "network-online.target" ];
after = [ "network-online.target" "tailscaled.service" ];
requires = [ "tailscaled.service" ];
path = [ config.services.tailscale.package pkgs.jq ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
EnvironmentFile = config.systemd.services.hermes-agent.serviceConfig.EnvironmentFile;
};
script = ''
if [ -z "''${TS_AUTHKEY:-}" ]; then
echo "No TS_AUTHKEY, skipping."
exit 0
fi
# Retry: the daemon may not be ready yet. Failure is not fatal.
for _ in 1 2 3 4 5 6; do
state=""
if out=$(tailscale status --json); then
state=$(jq -r '.BackendState' <<<"$out") || state=""
fi
case "$state" in
Running)
echo "Already logged in."
exit 0
;;
NeedsLogin|Stopped)
if tailscale up --auth-key="$TS_AUTHKEY" --hostname=ari-fwdesk-${robotoName}-bot; then
echo "Logged in."
exit 0
fi
;;
esac
sleep 5
done
echo "Could not log in to Tailscale." >&2
'';
};
}
-83
View File
@@ -1,83 +0,0 @@
{ config, lib, pkgs, skyg-secrets, ... }:
let
ports = { cluster = 7946; };
homes = "/data/slackerbot";
secrets = [ "slackerbot-slack" "slackerbot-goose" "slackerbot-cluster" ];
bazel = config.skyg.nixos.server.services.bazel-build;
settings = {
# The system each container boots (slackerbot's agent + bazel/rbuild);
# updated on every rebuild.
system = "${bazel.agentSystem}";
# Shared Bazel download cache (/repo-cache) for the containers.
rw_binds = bazel.binds.rw;
ro_binds = bazel.binds.ro;
machines_dir = "/var/lib/machines";
state_dir = "/var/lib/slackerbot";
agent_cwd = "/root";
idle_timeout = "6h";
max_idle_timeout = "0s";
allow_indefinite = true;
boot_timeout = "2m";
# Goose settings sent to every container (also on fws1).
# Values come from the slackerbot-goose secret.
agent_env = {
GOOSE_PROVIDER = "$GOOSE_PROVIDER";
GOOSE_MODEL = "$GOOSE_MODEL";
OPENROUTER_API_KEY = "$OPENROUTER_API_KEY";
};
container = { memory_mb = 4096; cpus = 1.0; };
max_container = { memory_mb = 8192; cpus = 2.0; };
# Leave room for ollama, ds4, comfyui, hermes and the robots.
host_reserve = { memory_mb = 32768; cpus = 4.0; };
storage = { min = "2G"; max = "5G"; limit = "20G"; };
drives = [{ path = homes; reserve = "40G"; }];
cluster = {
secret = "$CLUSTER_SECRET";
listen = ":${toString ports.cluster}";
};
# Only read on first start; after that, manage permissions from Slack.
permissions = {
managers = [ "U0CDGSJADMM" ];
default.U0CDGSJADMM = [ "all" ];
};
};
in
{
# Off.
config = lib.mkIf false {
age.secrets = lib.genAttrs secrets (name: {
file = skyg-secrets.${name};
owner = "root";
mode = "0400";
});
# slackerbot flake input; options and docs in its repo.
services.slackerbot = {
enable = true;
mode = "master";
clusterPort = ports.cluster;
configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings;
};
# Bazel build server for the containers (16 cores / 32 threads).
skyg.nixos.server.services.bazel-build = {
enable = true;
cpuCount = 32;
cacheGB = 200;
};
# The module takes one env file; these add to it.
systemd.services.slackerbot.serviceConfig.EnvironmentFile =
map (name: config.age.secrets.${name}.path) secrets;
# Container links belong to systemd-networkd.
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ];
};
}
+1 -1
View File
@@ -1,6 +1,6 @@
{ ... }:
{
imports = [ ./slackerbot.nix ./nbot.nix ];
imports = [ ./nbot.nix ];
services.pipewire = {
extraConfig.pipewire-pulse."10-raop-discover" = {
+2 -2
View File
@@ -16,8 +16,8 @@ let
container = { memory_mb = 4096; cpus = 1.0; };
max_container = { memory_mb = 51200; cpus = 6; };
host_reserve = { memory_mb = 8192; cpus = 2.0; };
storage = { min = "2G"; max = "5G"; limit = "20G"; };
drives = [{ path = homes; reserve = "100G"; }];
storage = { min = "2G"; max = "10G"; limit = "50G"; };
drives = [{ path = homes; reserve = "200G"; }];
# Worker of the master on fwdesk. Slack tokens and goose settings come
# from the master. Trusts the first certificate it sees (saved in
-69
View File
@@ -1,69 +0,0 @@
{ config, lib, pkgs, skyg-secrets, ... }:
let
homes = "/var/lib/slackerbot/homes";
bazel = config.skyg.nixos.server.services.bazel-build;
settings = {
# The system each container boots (slackerbot's agent + bazel/rbuild);
# updated on every rebuild.
system = "${bazel.agentSystem}";
# Shared Bazel download cache (/repo-cache) for the containers.
rw_binds = bazel.binds.rw;
ro_binds = bazel.binds.ro;
machines_dir = "/var/lib/machines";
state_dir = "/var/lib/slackerbot";
agent_cwd = "/root";
idle_timeout = "6h";
max_idle_timeout = "0s";
allow_indefinite = true;
boot_timeout = "2m";
container = { memory_mb = 4096; cpus = 1.0; };
max_container = { memory_mb = 8192; cpus = 2.0; };
host_reserve = { memory_mb = 8192; cpus = 2.0; };
storage = { min = "2G"; max = "5G"; limit = "20G"; };
drives = [{ path = homes; reserve = "100G"; }];
# Worker of the master on fwdesk. Slack tokens and goose settings come
# from the master. Trusts the first certificate it sees (saved in
# state_dir/master.fingerprint); pin it in master_fingerprint if wanted.
cluster = {
secret = "$CLUSTER_SECRET";
master = "wss://fwdesk.lab.internal:7946";
};
};
in
{
# Off.
config = lib.mkIf false {
age.secrets.slackerbot-cluster = {
file = skyg-secrets.slackerbot-cluster;
owner = "root";
mode = "0400";
};
# slackerbot flake input; options and docs in its repo.
services.slackerbot = {
enable = true;
mode = "slave";
configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings;
environmentFile = config.age.secrets.slackerbot-cluster.path;
};
# Bazel build server for the containers (4 cores / 8 threads).
skyg.nixos.server.services.bazel-build = {
enable = true;
cpuCount = 8;
cacheGB = 50;
};
# Container links belong to systemd-networkd.
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
# networkd only has the container links here (NetworkManager owns wlan),
# so wait-online would always time out.
systemd.network.wait-online.enable = false;
systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ];
};
}
-1
View File
@@ -38,7 +38,6 @@ skyg.rootDir → core/default.nix (set by the flake; repo root stor
skyg.nixos.common.* → nixos/common/
skyg.nixos.desktop.* → nixos/desktop/
skyg.nixos.server.* → nixos/server/
services.robotnix.* → nixos/server/robotnix/ (adapter for the robotnix flake)
skyg.core.* → core/
```
-2
View File
@@ -112,8 +112,6 @@ in
immich = { ip = "10.0.101.10"; };
immich-ml = { ip = "10.0.101.11"; };
gilix = { ip = "10.0.101.13"; };
sibli = { ip = "10.0.101.14"; };
genix = { ip = "10.0.101.12"; };
jellyfin = { ip = "10.0.101.5"; };
nvr = { ip = "10.0.101.6"; };
portainer = { ip = "10.0.101.241"; };
-3
View File
@@ -18,7 +18,6 @@ server/
├── k3s/ # Single-binary K3s cluster role
├── k8s/ # Full upstream Kubernetes (master/node)
├── portainer.nix # skyg.nixos.server.portainer — Portainer server or Edge agent
├── robotnix/ # adapter: imports the robotnix flake (services.robotnix)
└── services/ # Self-hosted apps (jellyfin, audiobookshelf, comfyui, ds4, colibri, ai…)
```
@@ -36,7 +35,6 @@ server/
| Structure | Namespace | Notes |
| ------------ | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `robotnix/` | `services.robotnix` | Adapter for the robotnix flake (own repo): self-managed NixOS nspawn robots. Sets skyg defaults; the one sanctioned nspawn exception to the oci-containers rule |
| `timers.nix` | `skyg.server.timers` | Generic attrset → timer + oneshot service |
## Option Namespace
@@ -50,7 +48,6 @@ skyg.server.arrs.* → arrs/
skyg.server.dns.* → dns/
skyg.nixos.server.k3s.* → k3s/
skyg.nixos.server.k8s.* → k8s/
services.robotnix.* → robotnix/ (adapter for the robotnix flake)
skyg.nixos.server.services.* → services/
skyg.nixos.server.portainer.* → portainer.nix
```
-1
View File
@@ -11,6 +11,5 @@
./timers.nix
./dns
./portainer.nix
./robotnix
];
}
-32
View File
@@ -1,32 +0,0 @@
# robotnix — adapter for the robotnix flake
robotnix (self-managed NixOS `systemd-nspawn` robots) lives in its own repo:
`git+ssh://gitea@gitea.app.internal:22/ari/robotnix.git`, a flake input of
this repo. Its README and `docs/` describe the design, options, CLI and
tests. Hosts configure it under `services.robotnix` (like `services.hermenix`).
This folder only holds `default.nix`, which:
- imports `robotnix.nixosModules.default` (passed in via `specialArgs`);
- defaults `user`, `authorizedKeys`, `dnsUpstreams` and `localDomains` from
the `skyg` user, the SSH master keys and the internal networking map;
- asserts no robot address clashes with an app in
`skyg.internalNetworkingMap.apps`.
## Decisions
- **Own repo** (2026-10): robotnix moved out of `modules/nixos/server` so it
can grow (storage backends, Genix, child robots) with its own tests.
- **Namespace**: `services.robotnix`, not `skyg.*`, same exception as
hermenix: the options belong to an external flake.
- **hermes-agent pin**: the input follows this repo's `hermes-agent`, so
robots pin the same revision the host locked.
- **Buzz stays here**: robotnix knows nothing about Buzz. The robots' shared
`hosts/hl-fwdesk/services/robotnix/buzz.nix` builds `pkgs/buzz-cli` from
this repo and sets `BUZZ_CLI_PATH`. Robots get copies of it, `common.nix`
and `pkgs/buzz-cli` under `/etc/nixos/seed` (robotnix `seed.extraFiles`,
`seedOf` in the host file), so a robot that rebuilds itself (genix) has
its own copy.
- **nspawn, not `oci-containers`**: the oci-containers rule targets
Docker/Podman. nspawn robots need their own store for self-rebuilds, which
the `containers.*` module can't give.
-35
View File
@@ -1,35 +0,0 @@
# robotnix lives in its own repo (flake input `robotnix`). This adapter
# imports its host module and fills its defaults from skyg values.
# Hosts configure it under services.robotnix. See ABOUTME.md.
{ config, lib, robotnix, ... }:
let
cfg = config.services.robotnix;
apps = config.skyg.internalNetworkingMap.apps;
in
{
imports = [ robotnix.nixosModules.default ];
config = lib.mkIf cfg.enable {
services.robotnix = {
user = lib.mkDefault config.skyg.user.name;
authorizedKeys = lib.mkDefault config.skyg.nixos.common.ssh-server.masterPubKeys;
dnsUpstreams = lib.mkDefault [ config.skyg.internalNetworkingMap.appNetwork.gateway ];
localDomains = lib.mkDefault [
"app.internal"
"lab.internal"
];
};
# A robot's address must not clash with an app from the network map.
assertions = lib.mapAttrsToList
(name: r:
let
clash = lib.attrNames (lib.filterAttrs (an: a: a.ip == r.address && an != name) apps);
in
{
assertion = clash == [ ];
message = "robotnix: ${name}'s address ${r.address} is already used by app(s): ${toString clash}.";
})
cfg.robots;
};
}
-46
View File
@@ -11,7 +11,6 @@ independent `skyg.nixos.server.services.<name>.enable` flag. Most run as OCI con
services/
├── default.nix # imports the services below
├── ai-services.nix # skyg.nixos.server.services.ai — Wyoming (whisper/piper/wake)
├── bazel-build.nix # skyg.nixos.server.services.bazel-build — NativeLink Bazel build server for slackerbot containers
├── audiobookshelf.nix # skyg.nixos.server.services.audiobookshelf — audiobook/podcast server
├── jellyfin.nix # skyg.nixos.server.services.jellyfin — media server (uid/gid 7777)
├── nbot-agent.nix # skyg.nixos.server.services.nbot-agent — nbot's agent system + gh, for nbot's `system`
@@ -24,50 +23,6 @@ services/
- **`ai-services.nix`** openwakeword as containers and
enables NixOS-native Wyoming faster-whisper + piper. Opens 11434/10200/10300/10400.
- **`bazel-build.nix`** runs NativeLink (from the `nativelink` flake input, pinned
to v1.8.3 because its config format changes) as a remote cache and executor for
slackerbot containers. Off on every host: only the disabled slackerbot configs on
hl-fws1 and hl-fwdesk turn it on, and nbot doesn't use it. Goal: as little as possible
runs in a container. Measured on slacker-bot (1 CPU container, 2026-11-01): local
458 s, remote with no cache 65 s, remote with warm cache 17 s.
- NativeLink uses `DynamicUser`, whose state folder systemd mounts `noexec`;
`ExecPaths` re-allows it, or every remote action fails with "Permission denied".
- Containers get their own `bazel` (`agentBazel`), with no plain bazel/bazelisk on
PATH. At boot `bazel-remote-rc` writes the gateway into `/run/bazel-remote.bazelrc`,
which `/etc/bazel.bazelrc` try-imports. Only remote actions are allowed
(`--spawn_strategy=remote`, no local fallback), so a missing file or unreachable
host fails the build instead of building in the container. The wrapper refuses
command-line flags that change this (`--spawn_strategy`, `--strategy*`,
`--remote_executor`, `--bazelrc`, ...). Flags in a repo's own `.bazelrc` are not
checked.
- Final outputs are downloaded into `bazel-bin` (`--remote_download_toplevel`).
`rbuild` is just `bazel build`. A `/handoff` → `/outputs` path that hardlinked
outputs from the cache was tried and removed: outputs are a few MB over a local
link, so it saved nothing (download ~20 s vs handoff ~344 s). Most of that was a
lock bug: Bazel's server inherits open file handles, so a lock held on an fd in
the wrapper stays held until the server exits. Close such fds for Bazel (`9>&-`).
- Still in the container: goose, the Bazel server (512M heap, exits after 5 idle
minutes) and repository rules, which Bazel can't run remotely.
- Downloads share one cache per host (`/var/lib/bazel-repo-cache` → `/repo-cache`,
sha256-checked, 30-day cleanup). Fetched repo folders stay per container
(`--repo_contents_cache`), since their contents are not checked.
- Writable binds (`/repo-cache`) use `idmap`, or the container can't write
to them. Container root isn't root on the host, but its files there are
owned by host root. TODO: own the cache by a dedicated host user and map
container root to it.
- Global goose hints (`/root/.config/goose/.goosehints` → `/etc/goose/goosehints`):
a local build means something is wrong (stop and report); build Go through Bazel.
- Exposes `agentSystem` (slackerbot's agent + bazel, go, bazelrc, rbuild) and
`binds` (slackerbot `ro_binds`/`rw_binds`). Slackerbot creates missing bind
source folders, or containers fail to start.
`bazel` is a bazelisk wrapper on the host and in containers: a repo's
`.bazelversion` wins, otherwise 8.7.0 (`USE_BAZEL_FALLBACK_VERSION`). Bazel 8.7.0
is fetched once per node by Nix (official release, sha256-checked) and the wrapper
enters it into each user's bazelisk cache as a link, so containers share that one
read-only copy and never download it. Other versions download into each container's
own `/root/.cache/bazelisk`. Upstream Bazel only runs on NixOS through
`programs.nix-ld`, enabled on the host and in containers. Bump `bazelVersion` and
`bazelSha256` together.
- **`nbot-agent.nix`** adds `gh` to nbot's agent system from this repo, leaving the nbot
repo unchanged. `gh` reads `GH_TOKEN` from `agent_env`, and git over HTTPS to github.com
uses it through `gh`. Every container gets that token (chosen on purpose, 2026-11-03),
@@ -82,7 +37,6 @@ services/
```
skyg.nixos.server.services.ai.enable
skyg.nixos.server.services.audiobookshelf.{enable,package,configDir,dataDir}
skyg.nixos.server.services.bazel-build.{enable,cpuCount,cacheGB,port,agentSystem,binds}
skyg.nixos.server.services.jellyfin.enable
skyg.nixos.server.services.nbot-agent.system
skyg.nixos.server.services.signal-cli.{enable,host,port,configDir,account,environmentFile,openFirewall}
@@ -1,345 +0,0 @@
# Bazel build server for slackerbot containers on the same host.
# One NativeLink (cache + scheduler + worker) runs builds on the host CPUs.
#
# container: bazel build //app ──grpc──▶ NativeLink (host) ──▶ host CPUs
# ◀── final outputs downloaded (small, same machine)
#
# slackerbot mounts the shared download cache through `binds`. Container root
# isn't root on the host, but with `idmap` its files in /repo-cache are owned
# by host root.
# TODO: own the cache by a dedicated host user and map container root to it.
{ config, lib, pkgs, nativelink, slackerbot, ... }:
with lib;
let
cfg = config.skyg.nixos.server.services.bazel-build;
system = pkgs.stdenv.hostPlatform.system;
# Official Bazel release, checked against its sha256. Containers see
# /nix/store read-only, so one copy per node serves all of them. Upstream
# builds only run on NixOS through nix-ld. To bump: change both values.
bazelVersion = "8.7.0";
bazelSha256 = "d7606e679b78067c811096fb3d6cf135225b528835ca396e3a4dddf957859544";
# Copied as-is: patching would break the archive appended to the binary.
bazelRelease = pkgs.runCommand "bazel-${bazelVersion}-release" { } ''
install -Dm755 ${pkgs.fetchurl {
url = "https://releases.bazel.build/${bazelVersion}/release/bazel-${bazelVersion}-linux-x86_64";
sha256 = bazelSha256;
}} $out/bin/bazel
'';
# `bazel` runs bazelisk on the host and in containers: a repo's
# .bazelversion wins, otherwise bazelVersion. bazelVersion is pre-entered in
# bazelisk's cache (format of bazelisk 1.28) as a link to bazelRelease, so it
# is never downloaded. Other versions download into each user's own cache.
bazel = pkgs.writeShellScriptBin "bazel" ''
export USE_BAZEL_FALLBACK_VERSION=''${USE_BAZEL_FALLBACK_VERSION:-silent:${bazelVersion}}
dl=''${BAZELISK_HOME_LINUX:-''${BAZELISK_HOME:-''${XDG_CACHE_HOME:-$HOME/.cache}/bazelisk}}/downloads
meta=$dl/metadata/bazelbuild/bazel-${bazelVersion}-linux-x86_64
if [ ! -e "$dl/sha256/${bazelSha256}/bin/bazel" ] || [ "$(cat "$meta" 2>/dev/null)" != ${bazelSha256} ]; then
mkdir -p "$dl/sha256" "''${meta%/*}"
ln -sfn ${bazelRelease} "$dl/sha256/${bazelSha256}"
printf %s ${bazelSha256} > "$meta"
fi
exec ${pkgs.bazelisk}/bin/bazelisk "$@"
'';
stateDir = "/var/lib/nativelink";
# Bazel download cache shared by all containers (files are sha256-checked).
repoCacheRoot = "/var/lib/bazel-repo-cache";
workerPort = 50061; # host only
# Tools build actions run with. Same store paths on the host and in containers.
toolchain = pkgs.buildEnv {
name = "bazel-action-toolchain";
paths = with pkgs; [ gcc binutils coreutils bash gnused gnugrep gawk findutils diffutils python3 ];
};
# Matches NativeLink v1.8.3 (basic_cas example). Field names change between
# releases; check them when bumping the flake input.
instances = f: map (instance_name: { inherit instance_name; } // f) [ "" "main" ];
nativelinkConfig = pkgs.writeText "nativelink.json" (builtins.toJSON {
stores = [
{
# If any output of a cached result was evicted, report a miss so the
# action is rebuilt instead of returning a result with missing files.
name = "AC_MAIN_STORE";
completeness_checking = {
backend.filesystem = {
content_path = "${stateDir}/ac/content";
temp_path = "${stateDir}/ac/tmp";
eviction_policy.max_bytes = 10 * 1000 * 1000 * 1000;
};
cas_store.ref_store.name = "WORKER_FAST_SLOW_STORE";
};
}
{
name = "WORKER_FAST_SLOW_STORE";
fast_slow = {
fast.filesystem = {
content_path = "${stateDir}/cas/content";
temp_path = "${stateDir}/cas/tmp";
eviction_policy.max_bytes = cfg.cacheGB * 1000 * 1000 * 1000;
};
slow.noop = { };
};
}
];
schedulers = [{
name = "MAIN_SCHEDULER";
simple.supported_platform_properties = {
cpu_count = "minimum";
OSFamily = "priority";
"container-image" = "priority";
};
}];
workers = [{
local = {
worker_api_endpoint.uri = "grpc://127.0.0.1:${toString workerPort}";
cas_fast_slow_store = "WORKER_FAST_SLOW_STORE";
upload_action_result.ac_store = "AC_MAIN_STORE";
work_directory = "${stateDir}/work";
platform_properties = {
cpu_count.values = [ (toString cfg.cpuCount) ];
OSFamily.values = [ "" ];
"container-image".values = [ "" ];
};
};
}];
servers = [
{
name = "public";
# Only container links (ve-*) are trusted; the LAN port stays closed.
listener.http.socket_address = "0.0.0.0:${toString cfg.port}";
services = {
cas = instances { cas_store = "WORKER_FAST_SLOW_STORE"; };
ac = instances { ac_store = "AC_MAIN_STORE"; };
execution = instances { cas_store = "WORKER_FAST_SLOW_STORE"; scheduler = "MAIN_SCHEDULER"; };
capabilities = instances { remote_execution.scheduler = "MAIN_SCHEDULER"; };
bytestream = instances { cas_store = "WORKER_FAST_SLOW_STORE"; };
};
}
{
name = "private_workers_servers";
listener.http.socket_address = "127.0.0.1:${toString workerPort}";
services = {
worker_api.scheduler = "MAIN_SCHEDULER";
admin = { };
health = { };
};
}
];
global.max_open_files = 65536;
});
# Container side `bazel`. Refuses flags that would move work into the
# container and forces remote actions. The host address comes from
# /run/bazel-remote.bazelrc (see bazelrc).
agentBazel = pkgs.writeShellScriptBin "bazel" ''
set -u
real=${bazel}/bin/bazel
enforce=(--spawn_strategy=remote --noremote_local_fallback)
# Find the command (first word that isn't a startup option) and check flags.
# Arguments after `--` belong to the program being run, so they're skipped.
cmd="" at=0 i=0
for a in "$@"; do
[ "$a" = "--" ] && break
case "$a" in
--spawn_strategy*|--strategy*|--test_strategy*|--remote_executor*|\
--remote_local_fallback*|--bazelrc*|--ignore_all_rc_files|--nosystem_rc)
echo "bazel: $a isn't allowed here. Builds must run on the host's build server;" \
"tell the user instead of working around it." >&2
exit 2 ;;
esac
if [ -z "$cmd" ] && [ "''${a#-}" = "$a" ]; then cmd=$a at=$i; fi
i=$((i + 1))
done
[ -z "$cmd" ] && exec "$real" "$@"
pre=("''${@:1:at}")
post=("''${@:at+2}")
case "$cmd" in
build|test|run|coverage) exec "$real" "''${pre[@]}" "$cmd" "''${enforce[@]}" "''${post[@]}" ;;
*) exec "$real" "$@" ;;
esac
'';
# Kept for old habits and docs: same as `bazel build`.
rbuild = pkgs.writeShellScriptBin "rbuild" ''exec ${agentBazel}/bin/bazel build "$@"'';
remoteRc = "/run/bazel-remote.bazelrc";
bazelrc = ''
# Builds run on the host, so the Bazel server in a container stays small,
# and exits after 5 idle minutes instead of 3 hours.
startup --host_jvm_args=-Xmx512m
startup --host_jvm_args=-XX:+UseSerialGC
startup --max_idle_secs=300
# Every action runs on the host; one that can't fails instead of running here.
build --spawn_strategy=remote
build --noremote_local_fallback
# Downloads are shared by all containers on this host. Fetched repo folders
# are not (unchecked, so one container could tamper with another's).
common --repository_cache=/repo-cache
common --repo_contents_cache=/root/.cache/bazel/repo-contents
build --remote_instance_name=main
# Final outputs are downloaded into bazel-bin (a few MB, same machine).
build --remote_download_toplevel
build --jobs=${toString (cfg.cpuCount * 4)}
# Rebuild instead of failing when the cache evicts something Bazel needs.
build --experimental_remote_cache_eviction_retries=5
build --action_env=PATH=${toolchain}/bin
build --host_action_env=PATH=${toolchain}/bin
build --repo_env=CC=${toolchain}/bin/gcc
# Points plain `bazel` at the host (written at boot). Missing = builds fail.
try-import ${remoteRc}
'';
# Global goose instructions in each container.
goosehints = ''
# Bazel builds run on the host, never in this container
This container is small (a few GB RAM, 1 CPU). `bazel build/test/run` runs
every action on the host's build server.
Outputs are downloaded into bazel-bin as usual.
Flags that change where actions run (--spawn_strategy, --strategy,
--remote_executor, --bazelrc, ...) are refused on purpose.
A normal build ends with a line like
`INFO: 120 processes: 110 remote, 10 internal.` (or "remote cache hit").
If instead you see `linux-sandbox`, `local` or `processwrapper-sandbox`
doing the work, or /run/bazel-remote.bazelrc is missing, or a build can't
reach the remote executor, something is wrong:
- stop the build (`bazel shutdown`), don't retry it locally,
- don't remove or override the --remote_* or --spawn_strategy flags,
- tell the user what you saw, with the error lines.
Build and test Go code through Bazel (`bazel build`, `bazel test`), not
`go build` / `go test`: those run here, in this small container. `go mod`
and other quick `go` commands are fine.
'';
# Added to slackerbot's agent system. Uses host packages so bazel and the
# toolchain match the host exactly.
agentModule = {
# No plain bazel/bazelisk on PATH, so the checks in agentBazel can't be skipped by accident.
environment.systemPackages = [ agentBazel pkgs.go pkgs.git rbuild ];
environment.etc."bazel.bazelrc".text = bazelrc;
environment.etc."goose/goosehints".text = goosehints;
programs.nix-ld.enable = true;
systemd.tmpfiles.rules = [
"d /root/.config/goose 0755 root root -"
"L+ /root/.config/goose/.goosehints - - - - /etc/goose/goosehints"
];
# The host is the default gateway, whose address changes per container,
# so write it for bazel.bazelrc once the route is up. Doesn't hold up boot.
systemd.services.bazel-remote-rc = {
description = "Point Bazel at the host build server";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
path = [ pkgs.iproute2 pkgs.gawk pkgs.coreutils ];
script = ''
for _ in $(seq 60); do
gw=$(ip -4 route show default | awk '{print $3; exit}')
if [ -n "$gw" ]; then
echo "build --remote_executor=grpc://$gw:${toString cfg.port}" > ${remoteRc}
exit 0
fi
sleep 1
done
echo "no default route after 60s; bazel will build locally" >&2
exit 1
'';
};
};
in
{
options = {
skyg.nixos.server.services.bazel-build = {
enable = mkEnableOption "Bazel build server (NativeLink) for slackerbot containers";
cpuCount = mkOption {
description = "CPU threads the host worker offers to builds.";
type = types.ints.positive;
};
cacheGB = mkOption {
description = "Size of the build cache in GB.";
default = 100;
type = types.ints.positive;
};
port = mkOption {
description = "Port containers reach NativeLink on (container links only).";
default = 50051;
type = types.port;
};
agentSystem = mkOption {
description = "slackerbot's agent system with bazel, go, bazelrc and rbuild added. Use for slackerbot's `system`.";
readOnly = true;
type = types.package;
default = (slackerbot.nixosConfigurations.agent.extendModules {
modules = [ agentModule ];
}).config.system.build.toplevel;
# Keeps the manual from building it on every host.
defaultText = literalExpression "slackerbot agent + bazel, go, bazelrc, rbuild";
};
binds = mkOption {
description = "slackerbot ro_binds / rw_binds that give each container the shared /repo-cache.";
readOnly = true;
type = types.attrsOf (types.listOf types.str);
default = {
# idmap: every container writes as the same host user, so all can share it.
rw = [ "${repoCacheRoot}:/repo-cache:idmap" ];
ro = [ ];
};
};
};
};
config = mkIf cfg.enable {
environment.systemPackages = [ bazel pkgs.bazelisk ];
programs.nix-ld.enable = true; # for the Bazel bazelisk downloads
systemd.services.nativelink = {
description = "NativeLink remote cache and build service";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
path = [ toolchain ];
serviceConfig = {
ExecStart = "${nativelink.packages.${system}.nativelink}/bin/nativelink ${nativelinkConfig}";
DynamicUser = true;
StateDirectory = "nativelink";
# DynamicUser's state dir is mounted noexec; build actions run from work/.
ExecPaths = [ "/var/lib/private/nativelink" ];
UMask = "0022";
LimitNOFILE = 65536;
Restart = "on-failure";
};
};
systemd.tmpfiles.rules = [
# Downloads unused for 30 days are removed (Bazel never cleans this).
"d ${repoCacheRoot} 0755 root root 30d"
];
# Keep the toolchain from garbage collection.
system.extraDependencies = [ toolchain ];
};
}
@@ -3,7 +3,6 @@
imports = [
./audiobookshelf.nix
./ai-services.nix
./bazel-build.nix
./colibri.nix
./comfyui
./jellyfin.nix
-10
View File
@@ -28,16 +28,6 @@ in
"secrets/hermes-env.age".publicKeys = [ ari fwdesk ];
"secrets/iu-project.age".publicKeys = [ ari minipc1 ];
# Robots
"secrets/robots-shared.age".publicKeys = [ ari fwdesk ];
"secrets/robots-genix.age".publicKeys = [ ari fwdesk ];
"secrets/robots-sibli.age".publicKeys = [ ari fwdesk ];
# Slackerbot (master on fwdesk, worker on fws1)
"secrets/slackerbot-slack.age".publicKeys = [ ari fwdesk ];
"secrets/slackerbot-goose.age".publicKeys = [ ari fwdesk ];
"secrets/slackerbot-cluster.age".publicKeys = [ ari fwdesk fws1 ];
# nbot (master on fwdesk, worker on fws1)
"secrets/nbot-slack.age".publicKeys = [ ari fwdesk ];
"secrets/nbot-goose.age".publicKeys = [ ari fwdesk ];