From 2b83e9c606373a619a242201194c5f3d1eec285d Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Tue, 3 Nov 2026 10:22:40 -0500 Subject: [PATCH] nbots --- flake.nix | 12 ++++ hosts/ABOUTME.md | 4 +- hosts/fwbook/networking.nix | 1 + hosts/hl-fwdesk/services/default.nix | 1 + hosts/hl-fwdesk/services/nbot.nix | 69 +++++++++++++++++++++++ hosts/hl-fwdesk/services/slackerbot.nix | 57 ++++++++++--------- hosts/hl-fws1/services/default.nix | 2 +- hosts/hl-fws1/services/nbot.nix | 54 ++++++++++++++++++ hosts/hl-fws1/services/slackerbot.nix | 61 ++++++++++---------- modules/nixos/server/services/ABOUTME.md | 3 +- secrets.nix | 5 ++ secrets/nbot-cluster.age | Bin 0 -> 512 bytes secrets/nbot-goose.age | 9 +++ secrets/nbot-slack.age | Bin 0 -> 513 bytes secrets/slackerbot-slack.age | Bin 513 -> 513 bytes 15 files changed, 218 insertions(+), 60 deletions(-) create mode 100644 hosts/hl-fwdesk/services/nbot.nix create mode 100644 hosts/hl-fws1/services/nbot.nix create mode 100644 secrets/nbot-cluster.age create mode 100644 secrets/nbot-goose.age create mode 100644 secrets/nbot-slack.age diff --git a/flake.nix b/flake.nix index cc41a32..ad7a6f4 100644 --- a/flake.nix +++ b/flake.nix @@ -21,6 +21,11 @@ inputs.nixpkgs.follows = "nixpkgs"; }; # Slack bot that runs goose agents in nspawn containers. + nbot = { + url = "git+ssh://git@github.com/asosnovsky/nbot.git"; + inputs.nixpkgs.follows = "nixpkgs-unstable"; + }; + # Off. slackerbot = { url = "git+https://gitea.skyg.ca/ari/slacker-bot.git"; inputs.nixpkgs.follows = "nixpkgs-unstable"; @@ -128,6 +133,7 @@ , hermenix , robotnix , skyg-linux-tracker + , nbot , slackerbot , nativelink , hhstream @@ -212,6 +218,9 @@ slackerbot-slack = ./secrets/slackerbot-slack.age; slackerbot-goose = ./secrets/slackerbot-goose.age; slackerbot-cluster = ./secrets/slackerbot-cluster.age; + nbot-slack = ./secrets/nbot-slack.age; + nbot-goose = ./secrets/nbot-goose.age; + nbot-cluster = ./secrets/nbot-cluster.age; }; inherit hyprlauncher @@ -228,6 +237,7 @@ hermenix robotnix skyg-linux-tracker + nbot slackerbot nativelink hhstream @@ -389,6 +399,7 @@ nixos-hardware.nixosModules.framework-11th-gen-intel disko.nixosModules.disko slackerbot.nixosModules.host + nbot.nixosModules.host ]; }; hl-fwdesk = lib.makeNixOs { @@ -400,6 +411,7 @@ hermenix.nixosModules.default buildbot-nix.nixosModules.buildbot-worker slackerbot.nixosModules.host + nbot.nixosModules.host ]; }; hl-bigbox1 = lib.makeNixOs { diff --git a/hosts/ABOUTME.md b/hosts/ABOUTME.md index 76a5c88..c9ddd8d 100644 --- a/hosts/ABOUTME.md +++ b/hosts/ABOUTME.md @@ -31,8 +31,8 @@ hosts/ | Folder | Host | Role | | ------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | | `fwbook/` | fwbook | Framework 13 AMD — personal laptop / daily driver | -| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); slackerbot master | -| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; slackerbot worker | +| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); nbot master (slackerbot off) | +| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; nbot worker (slackerbot off) | | `hl-bigbox1/` | hl-bigbox1 | AI/compute server (NVIDIA + AMD); Immich machine-learning (CUDA) | | `hl-bigbox2/` | hl-bigbox2 | NFS storage server; Immich server/db (docker, `immich.home.sosnovsky.ca`) | | `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI), monitoring (Grafana, Prometheus, Loki, OTel collector) | diff --git a/hosts/fwbook/networking.nix b/hosts/fwbook/networking.nix index 3d81265..91da42a 100644 --- a/hosts/fwbook/networking.nix +++ b/hosts/fwbook/networking.nix @@ -13,6 +13,7 @@ in }; networking.firewall.allowedUDPPorts = openPorts; networking.firewall.allowedTCPPorts = openPorts; + programs.mosh.enable = true; networking.hosts = { "0.0.0.0" = [ "auth.me.internal" diff --git a/hosts/hl-fwdesk/services/default.nix b/hosts/hl-fwdesk/services/default.nix index 848748c..42293e0 100644 --- a/hosts/hl-fwdesk/services/default.nix +++ b/hosts/hl-fwdesk/services/default.nix @@ -10,5 +10,6 @@ ./linux-tracker.nix ./robotnix.nix ./slackerbot.nix + ./nbot.nix ]; } diff --git a/hosts/hl-fwdesk/services/nbot.nix b/hosts/hl-fwdesk/services/nbot.nix new file mode 100644 index 0000000..a379c60 --- /dev/null +++ b/hosts/hl-fwdesk/services/nbot.nix @@ -0,0 +1,69 @@ +{ config, lib, pkgs, nbot, skyg-secrets, ... }: +let + ports = { cluster = 7946; }; + homes = "/data/nbot"; + secrets = [ "nbot-slack" "nbot-goose" "nbot-cluster" ]; + settings = { + # The system each container boots; updated on every rebuild. + system = "${nbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}"; + machines_dir = "/var/lib/machines"; + state_dir = "/var/lib/nbot"; + agent_cwd = "/root"; + + idle_timeout = "6h"; + max_idle_timeout = "0s"; + allow_indefinite = true; + boot_timeout = "2m"; + + # Goose settings sent to every container (also on fws1). + # Values come from the nbot-goose secret. + agent_env = { + GOOSE_PROVIDER = "$GOOSE_PROVIDER"; + GOOSE_MODEL = "$GOOSE_MODEL"; + OPENROUTER_API_KEY = "$OPENROUTER_API_KEY"; + }; + + container = { memory_mb = 4096; cpus = 1.0; }; + max_container = { memory_mb = 8192; cpus = 2.0; }; + # Leave room for ollama, ds4, comfyui, hermes and the robots. + host_reserve = { memory_mb = 32768; cpus = 4.0; }; + storage = { min = "2G"; max = "5G"; limit = "20G"; }; + drives = [{ path = homes; reserve = "40G"; }]; + + cluster = { + secret = "$CLUSTER_SECRET"; + listen = ":${toString ports.cluster}"; + }; + + # Only read on first start; after that, manage permissions from Slack. + permissions = { + managers = [ "U0CDGSJADMM" ]; + default.U0CDGSJADMM = [ "all" ]; + }; + }; +in +{ + age.secrets = lib.genAttrs secrets (name: { + file = skyg-secrets.${name}; + owner = "root"; + mode = "0400"; + }); + + # nbot flake input; options and docs in its repo. + services.nbot = { + enable = true; + mode = "master"; + clusterPort = ports.cluster; + configFile = (pkgs.formats.toml { }).generate "nbot.toml" settings; + }; + + # The module takes one env file; these add to it. + systemd.services.nbot.serviceConfig.EnvironmentFile = + map (name: config.age.secrets.${name}.path) secrets; + + # Container links belong to systemd-networkd. + networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; + networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + + systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; +} diff --git a/hosts/hl-fwdesk/services/slackerbot.nix b/hosts/hl-fwdesk/services/slackerbot.nix index d7c7b67..eb06df3 100644 --- a/hosts/hl-fwdesk/services/slackerbot.nix +++ b/hosts/hl-fwdesk/services/slackerbot.nix @@ -48,33 +48,36 @@ let }; in { - age.secrets = lib.genAttrs secrets (name: { - file = skyg-secrets.${name}; - owner = "root"; - mode = "0400"; - }); + # Off. + config = lib.mkIf false { + age.secrets = lib.genAttrs secrets (name: { + file = skyg-secrets.${name}; + owner = "root"; + mode = "0400"; + }); - # slackerbot flake input; options and docs in its repo. - services.slackerbot = { - enable = true; - mode = "master"; - clusterPort = ports.cluster; - configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; + # slackerbot flake input; options and docs in its repo. + services.slackerbot = { + enable = true; + mode = "master"; + clusterPort = ports.cluster; + configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; + }; + # Bazel build server for the containers (16 cores / 32 threads). + skyg.nixos.server.services.bazel-build = { + enable = true; + cpuCount = 32; + cacheGB = 200; + }; + + # The module takes one env file; these add to it. + systemd.services.slackerbot.serviceConfig.EnvironmentFile = + map (name: config.age.secrets.${name}.path) secrets; + + # Container links belong to systemd-networkd. + networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; + networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + + systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; }; - # Bazel build server for the containers (16 cores / 32 threads). - skyg.nixos.server.services.bazel-build = { - enable = true; - cpuCount = 32; - cacheGB = 200; - }; - - # The module takes one env file; these add to it. - systemd.services.slackerbot.serviceConfig.EnvironmentFile = - map (name: config.age.secrets.${name}.path) secrets; - - # Container links belong to systemd-networkd. - networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; - networking.dhcpcd.denyInterfaces = [ "ve-*" ]; - - systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; } diff --git a/hosts/hl-fws1/services/default.nix b/hosts/hl-fws1/services/default.nix index 71d5668..72b9008 100644 --- a/hosts/hl-fws1/services/default.nix +++ b/hosts/hl-fws1/services/default.nix @@ -1,6 +1,6 @@ { ... }: { - imports = [ ./slackerbot.nix ]; + imports = [ ./slackerbot.nix ./nbot.nix ]; services.pipewire = { extraConfig.pipewire-pulse."10-raop-discover" = { diff --git a/hosts/hl-fws1/services/nbot.nix b/hosts/hl-fws1/services/nbot.nix new file mode 100644 index 0000000..e2359e2 --- /dev/null +++ b/hosts/hl-fws1/services/nbot.nix @@ -0,0 +1,54 @@ +{ config, pkgs, nbot, skyg-secrets, ... }: +let + homes = "/var/lib/nbot/homes"; + settings = { + # The system each container boots; updated on every rebuild. + system = "${nbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}"; + machines_dir = "/var/lib/machines"; + state_dir = "/var/lib/nbot"; + agent_cwd = "/root"; + + idle_timeout = "6h"; + max_idle_timeout = "0s"; + allow_indefinite = true; + boot_timeout = "2m"; + + container = { memory_mb = 4096; cpus = 1.0; }; + max_container = { memory_mb = 8192; cpus = 2.0; }; + host_reserve = { memory_mb = 8192; cpus = 2.0; }; + storage = { min = "2G"; max = "5G"; limit = "20G"; }; + drives = [{ path = homes; reserve = "100G"; }]; + + # Worker of the master on fwdesk. Slack tokens and goose settings come + # from the master. Trusts the first certificate it sees (saved in + # state_dir/master.fingerprint); pin it in master_fingerprint if wanted. + cluster = { + secret = "$CLUSTER_SECRET"; + master = "wss://fwdesk.lab.internal:7946"; + }; + }; +in +{ + age.secrets.nbot-cluster = { + file = skyg-secrets.nbot-cluster; + owner = "root"; + mode = "0400"; + }; + + # nbot flake input; options and docs in its repo. + services.nbot = { + enable = true; + mode = "slave"; + configFile = (pkgs.formats.toml { }).generate "nbot.toml" settings; + environmentFile = config.age.secrets.nbot-cluster.path; + }; + + # Container links belong to systemd-networkd. + networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; + networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + # networkd only has the container links here (NetworkManager owns wlan), + # so wait-online would always time out. + systemd.network.wait-online.enable = false; + + systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; +} diff --git a/hosts/hl-fws1/services/slackerbot.nix b/hosts/hl-fws1/services/slackerbot.nix index ff46e6c..c0595f4 100644 --- a/hosts/hl-fws1/services/slackerbot.nix +++ b/hosts/hl-fws1/services/slackerbot.nix @@ -1,4 +1,4 @@ -{ config, pkgs, skyg-secrets, ... }: +{ config, lib, pkgs, skyg-secrets, ... }: let homes = "/var/lib/slackerbot/homes"; bazel = config.skyg.nixos.server.services.bazel-build; @@ -34,33 +34,36 @@ let }; in { - age.secrets.slackerbot-cluster = { - file = skyg-secrets.slackerbot-cluster; - owner = "root"; - mode = "0400"; + # Off. + config = lib.mkIf false { + age.secrets.slackerbot-cluster = { + file = skyg-secrets.slackerbot-cluster; + owner = "root"; + mode = "0400"; + }; + + # slackerbot flake input; options and docs in its repo. + services.slackerbot = { + enable = true; + mode = "slave"; + configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; + environmentFile = config.age.secrets.slackerbot-cluster.path; + }; + + # Bazel build server for the containers (4 cores / 8 threads). + skyg.nixos.server.services.bazel-build = { + enable = true; + cpuCount = 8; + cacheGB = 50; + }; + + # Container links belong to systemd-networkd. + networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; + networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + # networkd only has the container links here (NetworkManager owns wlan), + # so wait-online would always time out. + systemd.network.wait-online.enable = false; + + systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; }; - - # slackerbot flake input; options and docs in its repo. - services.slackerbot = { - enable = true; - mode = "slave"; - configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; - environmentFile = config.age.secrets.slackerbot-cluster.path; - }; - - # Bazel build server for the containers (4 cores / 8 threads). - skyg.nixos.server.services.bazel-build = { - enable = true; - cpuCount = 8; - cacheGB = 50; - }; - - # Container links belong to systemd-networkd. - networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; - networking.dhcpcd.denyInterfaces = [ "ve-*" ]; - # networkd only has the container links here (NetworkManager owns wlan), - # so wait-online would always time out. - systemd.network.wait-online.enable = false; - - systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; } diff --git a/modules/nixos/server/services/ABOUTME.md b/modules/nixos/server/services/ABOUTME.md index f1951fc..2c7cfc5 100644 --- a/modules/nixos/server/services/ABOUTME.md +++ b/modules/nixos/server/services/ABOUTME.md @@ -25,7 +25,8 @@ services/ enables NixOS-native Wyoming faster-whisper + piper. Opens 11434/10200/10300/10400. - **`bazel-build.nix`** runs NativeLink (from the `nativelink` flake input, pinned to v1.8.3 because its config format changes) as a remote cache and executor for - slackerbot containers. Used on hl-fws1 and hl-fwdesk. Goal: as little as possible + slackerbot containers. Off on every host: only the disabled slackerbot configs on + hl-fws1 and hl-fwdesk turn it on, and nbot doesn't use it. Goal: as little as possible runs in a container. Measured on slacker-bot (1 CPU container, 2026-11-01): local 458 s, remote with no cache 65 s, remote with warm cache 17 s. - NativeLink uses `DynamicUser`, whose state folder systemd mounts `noexec`; diff --git a/secrets.nix b/secrets.nix index 00f9bec..0e1acf3 100644 --- a/secrets.nix +++ b/secrets.nix @@ -38,6 +38,11 @@ in "secrets/slackerbot-goose.age".publicKeys = [ ari fwdesk ]; "secrets/slackerbot-cluster.age".publicKeys = [ ari fwdesk fws1 ]; + # nbot (master on fwdesk, worker on fws1) + "secrets/nbot-slack.age".publicKeys = [ ari fwdesk ]; + "secrets/nbot-goose.age".publicKeys = [ ari fwdesk ]; + "secrets/nbot-cluster.age".publicKeys = [ ari fwdesk fws1 ]; + # Buzz stack env (minipc1) "secrets/buzz-env.age".publicKeys = [ ari minipc1 ]; diff --git a/secrets/nbot-cluster.age b/secrets/nbot-cluster.age new file mode 100644 index 0000000000000000000000000000000000000000..2c17f5ac11a60db1fe036819f21ba048230e17f9 GIT binary patch literal 512 zcmZ9_J&V&|003Ye&EzZ!IZ-$bxh5q^8xSG)Ch14g^eb)B3UX z(LvAAB6e^&aB^}}+z%IpLl4wR;SjtIBD&x62ObMLq!Zesc|7h#;Z`()sVR+-H}NF# zNT-`-S(?W|p+*G)9@;umb*oSxDH>u3K9~1}q1LthHp_)#oQ9&8$pUHC??X&gcs^ff zI+8a@g+}W}s*Vtz0IOQ0JMAnd)*?#?64Pch zBd3+KI^P%psn+pI&8pd4coH@%Dp50&ni`>&?%8aZk@c{l_NaDIHY(DfC7N<$0wlBC zsh-{xc{^4SK7qE8jb%`=fJ4Z+#5en#$IQ)&pGaPjBckzwL%z`#4FOa7jNJw%A=9oN z(}-f}lsnDqe1-5MW;;;ii3x{nA=45NMv^364El8_a;V;;=bWvgT%(nb@`g(Zf$NrM z)?|>9P+3VF7Gj@1LjCD`Car;^gR>u;z4N2wX!rMn7l*5VpRC?F$bF;PSCK z-+_+~zM$`=8=HUM>|cBZ>HD|3tIaFN&o}Oz`5OF6R_rDDuy*~~6NcV8C!Z|$f4l^D UudcuCyypMBd%L{n?>%<^138qrHvj+t literal 0 HcmV?d00001 diff --git a/secrets/nbot-goose.age b/secrets/nbot-goose.age new file mode 100644 index 0000000..1435c5a --- /dev/null +++ b/secrets/nbot-goose.age @@ -0,0 +1,9 @@ +age-encryption.org/v1 +-> ssh-ed25519 CIR0mA 5rbuNoaaPYjl/B4MQW2jufxjCKC5gf05MRXBQbl6+AA +dB6sf5Q2Bnk/hwtWSblio1GEoM3OwwFCEC4sn6Nla3U +-> ssh-ed25519 gzLTww tI/mZILMxoFNQg4jL2gm7T6ZL/IGFASrbEIAqiOOoCA +5AUdlQm5Bg4JkVT4TyuW4rPaJ+5yHuPEXe81cdMyxZ8 +--- lj92JFLzMvqXZP18XBdXAkhhvPdOHxzfqN14oBrPCFw + +B æZö¯yéš*£à£[¸¯(O%�Ö2[c¬] ƒþÖ‘\4üÕ󉘦Óv†Éxqqí6ųá@æ�îß Â•3 áù +œaþ-»‚¬Ó.•¥{Þ™,åÂ)éâQÖä=ïìå¯LO9¹®ñD–Ð�û(piÃôq4°\£Ærê°](Û¥—ÊÆÂÖ™é)ò%|~Ö9É¢ ä̾û?NFˆåÏ{~ÄW`@³+Üñ2<âÒÆÓ#[ÆÊ„QÌb˜ \ No newline at end of file diff --git a/secrets/nbot-slack.age b/secrets/nbot-slack.age new file mode 100644 index 0000000000000000000000000000000000000000..f202f2eae5c6b0bb90216b72e603594fb1590f77 GIT binary patch literal 513 zcmYdHPt{G$OD?J`D9Oyv)5|YP*Do{V(zR14F3!+RO))YxHMCT4_6#z}byTRx@`&;; zh)ONakMawN^a#q0D5!9)@bb(LD9$wWU;J1sD`q$JYIu&~rOEg&tsw7|$D!;#C!Te~t%-^@7KE6XF>vDC*hIkeC- zq_QBlAR{FrExRn-)T_WCz%(K!r<_YyS63l0yC}fVxYXS}HLoHgEGMuiE!kf`J;bNj zJjkOYEF|5tq9VB{C(KB@Adu_e+bV{6Gv9AlkDYNhe{+$!*%xionPRHKx7xE-pSZ~G zzBB7Mf9$-?S^+;=E2m4opW0^Ma93fcLUGf|S>_D~u1%W}Dm-D2RPKiR>VMZrc4riE zt>xsnbEKZlpsM81S&oyt7XQCrC?$?)2??rwoKcod#KRp;kJKP{DL2wjW%w*wYV+ny!`v6AL1sl9NMV9)8_vd zylD!q0Y3a{T820KB5o A-T(jq literal 0 HcmV?d00001 diff --git a/secrets/slackerbot-slack.age b/secrets/slackerbot-slack.age index b83e0bf44f005b37349bf2bb26b5f6cae74d8240..f202f2eae5c6b0bb90216b72e603594fb1590f77 100644 GIT binary patch delta 479 zcmV<50U-W?1c3yQEPr@vNLo*DT4i%@T2541NK$QBaCk*{N=a`}cWiWWSz$GBM`%h{ zcXUoNX9{aEMo&sbHdjk)Sb0`?ZbolqczH5bLs5BId2BOOaBoCeXm3wbHZM&vRSGRW zAaiqQEoEdfH8n9gAZL0^RCjkEN_kTNNYiLOgUp!aY^_P);*-M@MCDcxYB^QF3NuPcLUwOmjCAq0G^rew|^&HnB8x)ayK^gD>a!GB@^0*YOBc62S>ST#}8eevno*dg?X15@0Er> zfZZUuAajDLnm2&J)|Qx66Og_XmSdz2?*T8e+Dpmbi&OD$-1Ne+{LTd zv6WXf4l8x5L}BPg$GPdvvicSk9Gn;}L4GK_ESsoga9ELd0=n+Fw3e|g!f`U;w*EQ~ z5%GgEv9;QxhFZ@b@1^iwk_5uBC%HWT^fOoSO?XpX%aH(#n$ws{S>u0E?rI0wg=rt` V;x>Iut<>L3-9zDV^FHKkPn@Fn$gKbX delta 479 zcmV<50U-W?1c3yQEPq6KMsY=GHB3l(bY*oxQCDYiPex5_Ze~_=Sa&&ULU43!a7Z*} zGH!5IcM42vNoaaTD`hKDY(-2lQdn_nQ&2%SXEH@cSYb0bPgP@DZ%#}(S7dQbcM2^& zAaiqQEoEdfH8n9gAZL0^RCjkEVN^9MM|n;*Y)~^dbWeG7Vt;dXVpA|`OKC7_P-S*u zZDMz7NoqDPNNzAm3R-wFX>CP#W@<=UX-Q~xP;6vxF)=|$enbcTrZ^3F)iAf$b5`6gBaY>X{rn!Wvn?O5pe`u`T4{CQTU%kiP#Htgkq zTKTlA=#Up4Z36vcL|(*U)}`RTe8!+Yw^iA>b+`iN Vu&)^M_V2@s%@rq9sf70g>ZCe))Di#y