diff --git a/flake.nix b/flake.nix index cc41a32..ad7a6f4 100644 --- a/flake.nix +++ b/flake.nix @@ -21,6 +21,11 @@ inputs.nixpkgs.follows = "nixpkgs"; }; # Slack bot that runs goose agents in nspawn containers. + nbot = { + url = "git+ssh://git@github.com/asosnovsky/nbot.git"; + inputs.nixpkgs.follows = "nixpkgs-unstable"; + }; + # Off. slackerbot = { url = "git+https://gitea.skyg.ca/ari/slacker-bot.git"; inputs.nixpkgs.follows = "nixpkgs-unstable"; @@ -128,6 +133,7 @@ , hermenix , robotnix , skyg-linux-tracker + , nbot , slackerbot , nativelink , hhstream @@ -212,6 +218,9 @@ slackerbot-slack = ./secrets/slackerbot-slack.age; slackerbot-goose = ./secrets/slackerbot-goose.age; slackerbot-cluster = ./secrets/slackerbot-cluster.age; + nbot-slack = ./secrets/nbot-slack.age; + nbot-goose = ./secrets/nbot-goose.age; + nbot-cluster = ./secrets/nbot-cluster.age; }; inherit hyprlauncher @@ -228,6 +237,7 @@ hermenix robotnix skyg-linux-tracker + nbot slackerbot nativelink hhstream @@ -389,6 +399,7 @@ nixos-hardware.nixosModules.framework-11th-gen-intel disko.nixosModules.disko slackerbot.nixosModules.host + nbot.nixosModules.host ]; }; hl-fwdesk = lib.makeNixOs { @@ -400,6 +411,7 @@ hermenix.nixosModules.default buildbot-nix.nixosModules.buildbot-worker slackerbot.nixosModules.host + nbot.nixosModules.host ]; }; hl-bigbox1 = lib.makeNixOs { diff --git a/hosts/ABOUTME.md b/hosts/ABOUTME.md index 76a5c88..c9ddd8d 100644 --- a/hosts/ABOUTME.md +++ b/hosts/ABOUTME.md @@ -31,8 +31,8 @@ hosts/ | Folder | Host | Role | | ------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | | `fwbook/` | fwbook | Framework 13 AMD — personal laptop / daily driver | -| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); slackerbot master | -| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; slackerbot worker | +| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); nbot master (slackerbot off) | +| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; nbot worker (slackerbot off) | | `hl-bigbox1/` | hl-bigbox1 | AI/compute server (NVIDIA + AMD); Immich machine-learning (CUDA) | | `hl-bigbox2/` | hl-bigbox2 | NFS storage server; Immich server/db (docker, `immich.home.sosnovsky.ca`) | | `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI), monitoring (Grafana, Prometheus, Loki, OTel collector) | diff --git a/hosts/fwbook/networking.nix b/hosts/fwbook/networking.nix index 3d81265..91da42a 100644 --- a/hosts/fwbook/networking.nix +++ b/hosts/fwbook/networking.nix @@ -13,6 +13,7 @@ in }; networking.firewall.allowedUDPPorts = openPorts; networking.firewall.allowedTCPPorts = openPorts; + programs.mosh.enable = true; networking.hosts = { "0.0.0.0" = [ "auth.me.internal" diff --git a/hosts/hl-fwdesk/services/default.nix b/hosts/hl-fwdesk/services/default.nix index 848748c..42293e0 100644 --- a/hosts/hl-fwdesk/services/default.nix +++ b/hosts/hl-fwdesk/services/default.nix @@ -10,5 +10,6 @@ ./linux-tracker.nix ./robotnix.nix ./slackerbot.nix + ./nbot.nix ]; } diff --git a/hosts/hl-fwdesk/services/nbot.nix b/hosts/hl-fwdesk/services/nbot.nix new file mode 100644 index 0000000..a379c60 --- /dev/null +++ b/hosts/hl-fwdesk/services/nbot.nix @@ -0,0 +1,69 @@ +{ config, lib, pkgs, nbot, skyg-secrets, ... }: +let + ports = { cluster = 7946; }; + homes = "/data/nbot"; + secrets = [ "nbot-slack" "nbot-goose" "nbot-cluster" ]; + settings = { + # The system each container boots; updated on every rebuild. + system = "${nbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}"; + machines_dir = "/var/lib/machines"; + state_dir = "/var/lib/nbot"; + agent_cwd = "/root"; + + idle_timeout = "6h"; + max_idle_timeout = "0s"; + allow_indefinite = true; + boot_timeout = "2m"; + + # Goose settings sent to every container (also on fws1). + # Values come from the nbot-goose secret. + agent_env = { + GOOSE_PROVIDER = "$GOOSE_PROVIDER"; + GOOSE_MODEL = "$GOOSE_MODEL"; + OPENROUTER_API_KEY = "$OPENROUTER_API_KEY"; + }; + + container = { memory_mb = 4096; cpus = 1.0; }; + max_container = { memory_mb = 8192; cpus = 2.0; }; + # Leave room for ollama, ds4, comfyui, hermes and the robots. + host_reserve = { memory_mb = 32768; cpus = 4.0; }; + storage = { min = "2G"; max = "5G"; limit = "20G"; }; + drives = [{ path = homes; reserve = "40G"; }]; + + cluster = { + secret = "$CLUSTER_SECRET"; + listen = ":${toString ports.cluster}"; + }; + + # Only read on first start; after that, manage permissions from Slack. + permissions = { + managers = [ "U0CDGSJADMM" ]; + default.U0CDGSJADMM = [ "all" ]; + }; + }; +in +{ + age.secrets = lib.genAttrs secrets (name: { + file = skyg-secrets.${name}; + owner = "root"; + mode = "0400"; + }); + + # nbot flake input; options and docs in its repo. + services.nbot = { + enable = true; + mode = "master"; + clusterPort = ports.cluster; + configFile = (pkgs.formats.toml { }).generate "nbot.toml" settings; + }; + + # The module takes one env file; these add to it. + systemd.services.nbot.serviceConfig.EnvironmentFile = + map (name: config.age.secrets.${name}.path) secrets; + + # Container links belong to systemd-networkd. + networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; + networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + + systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; +} diff --git a/hosts/hl-fwdesk/services/slackerbot.nix b/hosts/hl-fwdesk/services/slackerbot.nix index d7c7b67..eb06df3 100644 --- a/hosts/hl-fwdesk/services/slackerbot.nix +++ b/hosts/hl-fwdesk/services/slackerbot.nix @@ -48,33 +48,36 @@ let }; in { - age.secrets = lib.genAttrs secrets (name: { - file = skyg-secrets.${name}; - owner = "root"; - mode = "0400"; - }); + # Off. + config = lib.mkIf false { + age.secrets = lib.genAttrs secrets (name: { + file = skyg-secrets.${name}; + owner = "root"; + mode = "0400"; + }); - # slackerbot flake input; options and docs in its repo. - services.slackerbot = { - enable = true; - mode = "master"; - clusterPort = ports.cluster; - configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; + # slackerbot flake input; options and docs in its repo. + services.slackerbot = { + enable = true; + mode = "master"; + clusterPort = ports.cluster; + configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; + }; + # Bazel build server for the containers (16 cores / 32 threads). + skyg.nixos.server.services.bazel-build = { + enable = true; + cpuCount = 32; + cacheGB = 200; + }; + + # The module takes one env file; these add to it. + systemd.services.slackerbot.serviceConfig.EnvironmentFile = + map (name: config.age.secrets.${name}.path) secrets; + + # Container links belong to systemd-networkd. + networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; + networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + + systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; }; - # Bazel build server for the containers (16 cores / 32 threads). - skyg.nixos.server.services.bazel-build = { - enable = true; - cpuCount = 32; - cacheGB = 200; - }; - - # The module takes one env file; these add to it. - systemd.services.slackerbot.serviceConfig.EnvironmentFile = - map (name: config.age.secrets.${name}.path) secrets; - - # Container links belong to systemd-networkd. - networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; - networking.dhcpcd.denyInterfaces = [ "ve-*" ]; - - systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; } diff --git a/hosts/hl-fws1/services/default.nix b/hosts/hl-fws1/services/default.nix index 71d5668..72b9008 100644 --- a/hosts/hl-fws1/services/default.nix +++ b/hosts/hl-fws1/services/default.nix @@ -1,6 +1,6 @@ { ... }: { - imports = [ ./slackerbot.nix ]; + imports = [ ./slackerbot.nix ./nbot.nix ]; services.pipewire = { extraConfig.pipewire-pulse."10-raop-discover" = { diff --git a/hosts/hl-fws1/services/nbot.nix b/hosts/hl-fws1/services/nbot.nix new file mode 100644 index 0000000..e2359e2 --- /dev/null +++ b/hosts/hl-fws1/services/nbot.nix @@ -0,0 +1,54 @@ +{ config, pkgs, nbot, skyg-secrets, ... }: +let + homes = "/var/lib/nbot/homes"; + settings = { + # The system each container boots; updated on every rebuild. + system = "${nbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}"; + machines_dir = "/var/lib/machines"; + state_dir = "/var/lib/nbot"; + agent_cwd = "/root"; + + idle_timeout = "6h"; + max_idle_timeout = "0s"; + allow_indefinite = true; + boot_timeout = "2m"; + + container = { memory_mb = 4096; cpus = 1.0; }; + max_container = { memory_mb = 8192; cpus = 2.0; }; + host_reserve = { memory_mb = 8192; cpus = 2.0; }; + storage = { min = "2G"; max = "5G"; limit = "20G"; }; + drives = [{ path = homes; reserve = "100G"; }]; + + # Worker of the master on fwdesk. Slack tokens and goose settings come + # from the master. Trusts the first certificate it sees (saved in + # state_dir/master.fingerprint); pin it in master_fingerprint if wanted. + cluster = { + secret = "$CLUSTER_SECRET"; + master = "wss://fwdesk.lab.internal:7946"; + }; + }; +in +{ + age.secrets.nbot-cluster = { + file = skyg-secrets.nbot-cluster; + owner = "root"; + mode = "0400"; + }; + + # nbot flake input; options and docs in its repo. + services.nbot = { + enable = true; + mode = "slave"; + configFile = (pkgs.formats.toml { }).generate "nbot.toml" settings; + environmentFile = config.age.secrets.nbot-cluster.path; + }; + + # Container links belong to systemd-networkd. + networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; + networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + # networkd only has the container links here (NetworkManager owns wlan), + # so wait-online would always time out. + systemd.network.wait-online.enable = false; + + systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; +} diff --git a/hosts/hl-fws1/services/slackerbot.nix b/hosts/hl-fws1/services/slackerbot.nix index ff46e6c..c0595f4 100644 --- a/hosts/hl-fws1/services/slackerbot.nix +++ b/hosts/hl-fws1/services/slackerbot.nix @@ -1,4 +1,4 @@ -{ config, pkgs, skyg-secrets, ... }: +{ config, lib, pkgs, skyg-secrets, ... }: let homes = "/var/lib/slackerbot/homes"; bazel = config.skyg.nixos.server.services.bazel-build; @@ -34,33 +34,36 @@ let }; in { - age.secrets.slackerbot-cluster = { - file = skyg-secrets.slackerbot-cluster; - owner = "root"; - mode = "0400"; + # Off. + config = lib.mkIf false { + age.secrets.slackerbot-cluster = { + file = skyg-secrets.slackerbot-cluster; + owner = "root"; + mode = "0400"; + }; + + # slackerbot flake input; options and docs in its repo. + services.slackerbot = { + enable = true; + mode = "slave"; + configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; + environmentFile = config.age.secrets.slackerbot-cluster.path; + }; + + # Bazel build server for the containers (4 cores / 8 threads). + skyg.nixos.server.services.bazel-build = { + enable = true; + cpuCount = 8; + cacheGB = 50; + }; + + # Container links belong to systemd-networkd. + networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; + networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + # networkd only has the container links here (NetworkManager owns wlan), + # so wait-online would always time out. + systemd.network.wait-online.enable = false; + + systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; }; - - # slackerbot flake input; options and docs in its repo. - services.slackerbot = { - enable = true; - mode = "slave"; - configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; - environmentFile = config.age.secrets.slackerbot-cluster.path; - }; - - # Bazel build server for the containers (4 cores / 8 threads). - skyg.nixos.server.services.bazel-build = { - enable = true; - cpuCount = 8; - cacheGB = 50; - }; - - # Container links belong to systemd-networkd. - networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; - networking.dhcpcd.denyInterfaces = [ "ve-*" ]; - # networkd only has the container links here (NetworkManager owns wlan), - # so wait-online would always time out. - systemd.network.wait-online.enable = false; - - systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; } diff --git a/modules/nixos/server/services/ABOUTME.md b/modules/nixos/server/services/ABOUTME.md index f1951fc..2c7cfc5 100644 --- a/modules/nixos/server/services/ABOUTME.md +++ b/modules/nixos/server/services/ABOUTME.md @@ -25,7 +25,8 @@ services/ enables NixOS-native Wyoming faster-whisper + piper. Opens 11434/10200/10300/10400. - **`bazel-build.nix`** runs NativeLink (from the `nativelink` flake input, pinned to v1.8.3 because its config format changes) as a remote cache and executor for - slackerbot containers. Used on hl-fws1 and hl-fwdesk. Goal: as little as possible + slackerbot containers. Off on every host: only the disabled slackerbot configs on + hl-fws1 and hl-fwdesk turn it on, and nbot doesn't use it. Goal: as little as possible runs in a container. Measured on slacker-bot (1 CPU container, 2026-11-01): local 458 s, remote with no cache 65 s, remote with warm cache 17 s. - NativeLink uses `DynamicUser`, whose state folder systemd mounts `noexec`; diff --git a/secrets.nix b/secrets.nix index 00f9bec..0e1acf3 100644 --- a/secrets.nix +++ b/secrets.nix @@ -38,6 +38,11 @@ in "secrets/slackerbot-goose.age".publicKeys = [ ari fwdesk ]; "secrets/slackerbot-cluster.age".publicKeys = [ ari fwdesk fws1 ]; + # nbot (master on fwdesk, worker on fws1) + "secrets/nbot-slack.age".publicKeys = [ ari fwdesk ]; + "secrets/nbot-goose.age".publicKeys = [ ari fwdesk ]; + "secrets/nbot-cluster.age".publicKeys = [ ari fwdesk fws1 ]; + # Buzz stack env (minipc1) "secrets/buzz-env.age".publicKeys = [ ari minipc1 ]; diff --git a/secrets/nbot-cluster.age b/secrets/nbot-cluster.age new file mode 100644 index 0000000..2c17f5a Binary files /dev/null and b/secrets/nbot-cluster.age differ diff --git a/secrets/nbot-goose.age b/secrets/nbot-goose.age new file mode 100644 index 0000000..1435c5a --- /dev/null +++ b/secrets/nbot-goose.age @@ -0,0 +1,9 @@ +age-encryption.org/v1 +-> ssh-ed25519 CIR0mA 5rbuNoaaPYjl/B4MQW2jufxjCKC5gf05MRXBQbl6+AA +dB6sf5Q2Bnk/hwtWSblio1GEoM3OwwFCEC4sn6Nla3U +-> ssh-ed25519 gzLTww tI/mZILMxoFNQg4jL2gm7T6ZL/IGFASrbEIAqiOOoCA +5AUdlQm5Bg4JkVT4TyuW4rPaJ+5yHuPEXe81cdMyxZ8 +--- lj92JFLzMvqXZP18XBdXAkhhvPdOHxzfqN14oBrPCFw + +B æZö¯yéš*£à£[¸¯(O%�Ö2[c¬] ƒþÖ‘\4üÕ󉘦Óv†Éxqqí6ųá@æ�îß Â•3 áù +œaþ-»‚¬Ó.•¥{Þ™,åÂ)éâQÖä=ïìå¯LO9¹®ñD–Ð�û(piÃôq4°\£Ærê°](Û¥—ÊÆÂÖ™é)ò%|~Ö9É¢ ä̾û?NFˆåÏ{~ÄW`@³+Üñ2<âÒÆÓ#[ÆÊ„QÌb˜ \ No newline at end of file diff --git a/secrets/nbot-slack.age b/secrets/nbot-slack.age new file mode 100644 index 0000000..f202f2e Binary files /dev/null and b/secrets/nbot-slack.age differ diff --git a/secrets/slackerbot-slack.age b/secrets/slackerbot-slack.age index b83e0bf..f202f2e 100644 Binary files a/secrets/slackerbot-slack.age and b/secrets/slackerbot-slack.age differ