diff --git a/flake.lock b/flake.lock index cbdc664..8b0103a 100644 --- a/flake.lock +++ b/flake.lock @@ -3113,6 +3113,7 @@ "robotnix": "robotnix", "skyg-linux-tracker": "skyg-linux-tracker", "skygqts": "skygqts", + "slackerbot": "slackerbot", "stylix": "stylix", "systems": "systems_6" } @@ -3217,6 +3218,26 @@ "url": "ssh://gitea@gitea.app.internal:22/ari/skygqts.git" } }, + "slackerbot": { + "inputs": { + "nixpkgs": [ + "nixpkgs-unstable" + ] + }, + "locked": { + "lastModified": 1793556114, + "narHash": "sha256-TxDFlMNXyd9GXHa1FUCfZjauf5W//m73aMlhArE4vAQ=", + "ref": "refs/heads/main", + "rev": "0ac61b4eceb017ccf94f86c8cde30d1fb6db3d3e", + "revCount": 2, + "type": "git", + "url": "https://gitea.skyg.ca/ari/slacker-bot.git" + }, + "original": { + "type": "git", + "url": "https://gitea.skyg.ca/ari/slacker-bot.git" + } + }, "sops-nix": { "inputs": { "nixpkgs": [ diff --git a/flake.nix b/flake.nix index 7d5be49..cd35f3a 100644 --- a/flake.nix +++ b/flake.nix @@ -20,6 +20,11 @@ url = "git+https://gitea.skyg.ca/ari/skyg-linux-tracker.git"; inputs.nixpkgs.follows = "nixpkgs"; }; + # Slack bot that runs goose agents in nspawn containers. + slackerbot = { + url = "git+https://gitea.skyg.ca/ari/slacker-bot.git"; + inputs.nixpkgs.follows = "nixpkgs-unstable"; + }; # Headless Hyprland session streamed over a Unix socket. hhstream = { url = "git+ssh://gitea@gitea.app.internal:22/Hermenix/hyprland-headless-stream-plugin.git"; @@ -120,6 +125,7 @@ , hermenix , robotnix , skyg-linux-tracker + , slackerbot , hhstream , nixpkgs-unstable , flox @@ -199,6 +205,9 @@ robots-shared = ./secrets/robots-shared.age; robots-sibli = ./secrets/robots-sibli.age; robots-genix = ./secrets/robots-genix.age; + slackerbot-slack = ./secrets/slackerbot-slack.age; + slackerbot-goose = ./secrets/slackerbot-goose.age; + slackerbot-cluster = ./secrets/slackerbot-cluster.age; }; inherit hyprlauncher @@ -215,6 +224,7 @@ hermenix robotnix skyg-linux-tracker + slackerbot hhstream flox determinate @@ -373,6 +383,7 @@ ./hosts/hl-fws1.disko.nix nixos-hardware.nixosModules.framework-11th-gen-intel disko.nixosModules.disko + slackerbot.nixosModules.host ]; }; hl-fwdesk = lib.makeNixOs { @@ -383,6 +394,7 @@ nixos-hardware.nixosModules.framework-desktop-amd-ai-max-300-series hermenix.nixosModules.default buildbot-nix.nixosModules.buildbot-worker + slackerbot.nixosModules.host ]; }; hl-bigbox1 = lib.makeNixOs { diff --git a/hosts/ABOUTME.md b/hosts/ABOUTME.md index 91c058e..76a5c88 100644 --- a/hosts/ABOUTME.md +++ b/hosts/ABOUTME.md @@ -31,8 +31,8 @@ hosts/ | Folder | Host | Role | | ------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | | `fwbook/` | fwbook | Framework 13 AMD — personal laptop / daily driver | -| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores) | -| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node | +| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); slackerbot master | +| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; slackerbot worker | | `hl-bigbox1/` | hl-bigbox1 | AI/compute server (NVIDIA + AMD); Immich machine-learning (CUDA) | | `hl-bigbox2/` | hl-bigbox2 | NFS storage server; Immich server/db (docker, `immich.home.sosnovsky.ca`) | | `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI), monitoring (Grafana, Prometheus, Loki, OTel collector) | diff --git a/hosts/hl-fwdesk/services/default.nix b/hosts/hl-fwdesk/services/default.nix index dd946e8..848748c 100644 --- a/hosts/hl-fwdesk/services/default.nix +++ b/hosts/hl-fwdesk/services/default.nix @@ -9,5 +9,6 @@ ./buildbot.nix ./linux-tracker.nix ./robotnix.nix + ./slackerbot.nix ]; } diff --git a/hosts/hl-fwdesk/services/slackerbot.nix b/hosts/hl-fwdesk/services/slackerbot.nix new file mode 100644 index 0000000..1a0c4c1 --- /dev/null +++ b/hosts/hl-fwdesk/services/slackerbot.nix @@ -0,0 +1,68 @@ +{ config, lib, pkgs, slackerbot, skyg-secrets, ... }: +let + ports = { cluster = 7946; }; + homes = "/data/slackerbot"; + secrets = [ "slackerbot-slack" "slackerbot-goose" "slackerbot-cluster" ]; + settings = { + # The system each container boots; updated on every rebuild. + system = "${slackerbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}"; + machines_dir = "/var/lib/machines"; + state_dir = "/var/lib/slackerbot"; + agent_cwd = "/root"; + + idle_timeout = "6h"; + max_idle_timeout = "0s"; + allow_indefinite = true; + boot_timeout = "2m"; + + # Goose settings sent to every container (also on fws1). + # Values come from the slackerbot-goose secret. + agent_env = { + GOOSE_PROVIDER = "$GOOSE_PROVIDER"; + GOOSE_MODEL = "$GOOSE_MODEL"; + OPENROUTER_API_KEY = "$OPENROUTER_API_KEY"; + }; + + container = { memory_mb = 1024; cpus = 1.0; }; + max_container = { memory_mb = 4096; cpus = 2.0; }; + # Leave room for ollama, ds4, comfyui, hermes and the robots. + host_reserve = { memory_mb = 32768; cpus = 4.0; }; + storage = { min = "2G"; max = "5G"; limit = "20G"; }; + drives = [{ path = homes; reserve = "40G"; }]; + + cluster = { + secret = "$CLUSTER_SECRET"; + listen = ":${toString ports.cluster}"; + }; + + # Only read on first start; after that, manage permissions from Slack. + permissions = { + managers = [ "U0CDGSJADMM" ]; + default.U0CDGSJADMM = [ "all" ]; + }; + }; +in +{ + age.secrets = lib.genAttrs secrets (name: { + file = skyg-secrets.${name}; + owner = "root"; + mode = "0400"; + }); + + # slackerbot flake input; options and docs in its repo. + services.slackerbot = { + enable = true; + mode = "master"; + clusterPort = ports.cluster; + configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; + }; + # The module takes one env file; these add to it. + systemd.services.slackerbot.serviceConfig.EnvironmentFile = + map (name: config.age.secrets.${name}.path) secrets; + + # Container links belong to systemd-networkd. + networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; + networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + + systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; +} diff --git a/hosts/hl-fws1/default.nix b/hosts/hl-fws1/default.nix index ad824fe..73ba748 100644 --- a/hosts/hl-fws1/default.nix +++ b/hosts/hl-fws1/default.nix @@ -1,7 +1,7 @@ { pkgs, config, ... }: { imports = [ - ./services.nix + ./services ./hardware-configuration ./hardware.nix ./systemd.nix diff --git a/hosts/hl-fws1/services.nix b/hosts/hl-fws1/services/default.nix similarity index 97% rename from hosts/hl-fws1/services.nix rename to hosts/hl-fws1/services/default.nix index 82b9d83..71d5668 100644 --- a/hosts/hl-fws1/services.nix +++ b/hosts/hl-fws1/services/default.nix @@ -1,5 +1,7 @@ { ... }: { + imports = [ ./slackerbot.nix ]; + services.pipewire = { extraConfig.pipewire-pulse."10-raop-discover" = { context.modules = [ diff --git a/hosts/hl-fws1/services/slackerbot.nix b/hosts/hl-fws1/services/slackerbot.nix new file mode 100644 index 0000000..c5af7de --- /dev/null +++ b/hosts/hl-fws1/services/slackerbot.nix @@ -0,0 +1,51 @@ +{ config, pkgs, slackerbot, skyg-secrets, ... }: +let + homes = "/var/lib/slackerbot/homes"; + settings = { + # The system each container boots; updated on every rebuild. + system = "${slackerbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}"; + machines_dir = "/var/lib/machines"; + state_dir = "/var/lib/slackerbot"; + agent_cwd = "/root"; + + idle_timeout = "6h"; + max_idle_timeout = "0s"; + allow_indefinite = true; + boot_timeout = "2m"; + + container = { memory_mb = 1024; cpus = 1.0; }; + max_container = { memory_mb = 4096; cpus = 2.0; }; + host_reserve = { memory_mb = 8192; cpus = 2.0; }; + storage = { min = "2G"; max = "5G"; limit = "20G"; }; + drives = [{ path = homes; reserve = "100G"; }]; + + # Worker of the master on fwdesk. Slack tokens and goose settings come + # from the master. Trusts the first certificate it sees (saved in + # state_dir/master.fingerprint); pin it in master_fingerprint if wanted. + cluster = { + secret = "$CLUSTER_SECRET"; + master = "wss://fwdesk.lab.internal:7946"; + }; + }; +in +{ + age.secrets.slackerbot-cluster = { + file = skyg-secrets.slackerbot-cluster; + owner = "root"; + mode = "0400"; + }; + + # slackerbot flake input; options and docs in its repo. + services.slackerbot = { + enable = true; + mode = "slave"; + configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; + environmentFile = config.age.secrets.slackerbot-cluster.path; + }; + + # Container links belong to systemd-networkd. + networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; + networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + + systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; +} diff --git a/secrets.nix b/secrets.nix index 3532eda..00f9bec 100644 --- a/secrets.nix +++ b/secrets.nix @@ -8,6 +8,7 @@ let bigbox1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFl5xiH4aFlmDWdoU+KmERLYOeOQK2aKP7jenY2G3rFb root@nixos"; bigbox2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6l6VqQ23PkGkWVABBoLp1KA6PzkX5oKTy5RBHbQXet root@hl-bigbox2"; hermenix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPfMrT6dDPw3+J22k3X5aR9ERQ8RMRLeBGgqQjmuL7EI hermenix@lab"; + fws1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM6r0AIJyTlE+U6K97GijzAP4jNaAueVlwXbeMpEMcpe root@hl-fws1"; in { "secrets/dns-addresses.conf.age".publicKeys = [ @@ -32,6 +33,11 @@ in "secrets/robots-genix.age".publicKeys = [ ari fwdesk ]; "secrets/robots-sibli.age".publicKeys = [ ari fwdesk ]; + # Slackerbot (master on fwdesk, worker on fws1) + "secrets/slackerbot-slack.age".publicKeys = [ ari fwdesk ]; + "secrets/slackerbot-goose.age".publicKeys = [ ari fwdesk ]; + "secrets/slackerbot-cluster.age".publicKeys = [ ari fwdesk fws1 ]; + # Buzz stack env (minipc1) "secrets/buzz-env.age".publicKeys = [ ari minipc1 ]; diff --git a/secrets/slackerbot-cluster.age b/secrets/slackerbot-cluster.age new file mode 100644 index 0000000..2c17f5a Binary files /dev/null and b/secrets/slackerbot-cluster.age differ diff --git a/secrets/slackerbot-goose.age b/secrets/slackerbot-goose.age new file mode 100644 index 0000000..1435c5a --- /dev/null +++ b/secrets/slackerbot-goose.age @@ -0,0 +1,9 @@ +age-encryption.org/v1 +-> ssh-ed25519 CIR0mA 5rbuNoaaPYjl/B4MQW2jufxjCKC5gf05MRXBQbl6+AA +dB6sf5Q2Bnk/hwtWSblio1GEoM3OwwFCEC4sn6Nla3U +-> ssh-ed25519 gzLTww tI/mZILMxoFNQg4jL2gm7T6ZL/IGFASrbEIAqiOOoCA +5AUdlQm5Bg4JkVT4TyuW4rPaJ+5yHuPEXe81cdMyxZ8 +--- lj92JFLzMvqXZP18XBdXAkhhvPdOHxzfqN14oBrPCFw + +B æZö¯yéš*£à£[¸¯(O%�Ö2[c¬] ƒþÖ‘\4üÕ󉘦Óv†Éxqqí6ųá@æ�îß Â•3 áù +œaþ-»‚¬Ó.•¥{Þ™,åÂ)éâQÖä=ïìå¯LO9¹®ñD–Ð�û(piÃôq4°\£Ærê°](Û¥—ÊÆÂÖ™é)ò%|~Ö9É¢ ä̾û?NFˆåÏ{~ÄW`@³+Üñ2<âÒÆÓ#[ÆÊ„QÌb˜ \ No newline at end of file diff --git a/secrets/slackerbot-slack.age b/secrets/slackerbot-slack.age new file mode 100644 index 0000000..b83e0bf --- /dev/null +++ b/secrets/slackerbot-slack.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> ssh-ed25519 CIR0mA DyFqEh5LHyteuAQWgqOFMlnfVtXw9jBptlpH4f2npVw +LkIhzF+e+QlEL1RXqjSPA7g2EGXa38OUcZoNL9WdqMw +-> ssh-ed25519 gzLTww aT5+GyN6lP37tOytbsvbS0jKi0jPevambwjIj6/Hn0I +Zx2imExfjHZiIhuPldo11APa2pVEBWzgNMZBdu3FqQ8 +--- 29nZzi1W+D7ZXsslpj3aOZ5UUL6ZO1CJHrp4mfEdvMI +’»¼#L=5É×ÿû,áüÚ³ Ô¼D�ZY°´¸,¦ãt(gZ¿¸"…Í2dÔsÙ8KöwAÇô¡Ù(8šùÇ{SäWKr#tÐÝîÿ)ü´‚C�ðœZ4w½Õ½‡»¿³Ëü÷z˜Eÿ–ÞÚâË<ðt†ˆôòÎHÌ ¤}mù&VlŒ-dš½ý…íèøúÿ�üyV¦Ëñ¡ß6ìå€Zù´«èí¶ìcXÎï”èÕ\0ÄnMJ 묛ÏmýcD^ÄaÖ¥à¿|Æ >·UÙ¹u¸æ°¯òöïËÍ'T©„÷ê¤ \ No newline at end of file