nbots
This commit is contained in:
1 parent
8fec1dd4bb
commit
2b83e9c606
15 files changed
+218
-60
No files matched your search
@@ -21,6 +21,11 @@
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
# Slack bot that runs goose agents in nspawn containers.
|
||||
nbot = {
|
||||
url = "git+ssh://git@github.com/asosnovsky/nbot.git";
|
||||
inputs.nixpkgs.follows = "nixpkgs-unstable";
|
||||
};
|
||||
# Off.
|
||||
slackerbot = {
|
||||
url = "git+https://gitea.skyg.ca/ari/slacker-bot.git";
|
||||
inputs.nixpkgs.follows = "nixpkgs-unstable";
|
||||
@@ -128,6 +133,7 @@
|
||||
, hermenix
|
||||
, robotnix
|
||||
, skyg-linux-tracker
|
||||
, nbot
|
||||
, slackerbot
|
||||
, nativelink
|
||||
, hhstream
|
||||
@@ -212,6 +218,9 @@
|
||||
slackerbot-slack = ./secrets/slackerbot-slack.age;
|
||||
slackerbot-goose = ./secrets/slackerbot-goose.age;
|
||||
slackerbot-cluster = ./secrets/slackerbot-cluster.age;
|
||||
nbot-slack = ./secrets/nbot-slack.age;
|
||||
nbot-goose = ./secrets/nbot-goose.age;
|
||||
nbot-cluster = ./secrets/nbot-cluster.age;
|
||||
};
|
||||
inherit
|
||||
hyprlauncher
|
||||
@@ -228,6 +237,7 @@
|
||||
hermenix
|
||||
robotnix
|
||||
skyg-linux-tracker
|
||||
nbot
|
||||
slackerbot
|
||||
nativelink
|
||||
hhstream
|
||||
@@ -389,6 +399,7 @@
|
||||
nixos-hardware.nixosModules.framework-11th-gen-intel
|
||||
disko.nixosModules.disko
|
||||
slackerbot.nixosModules.host
|
||||
nbot.nixosModules.host
|
||||
];
|
||||
};
|
||||
hl-fwdesk = lib.makeNixOs {
|
||||
@@ -400,6 +411,7 @@
|
||||
hermenix.nixosModules.default
|
||||
buildbot-nix.nixosModules.buildbot-worker
|
||||
slackerbot.nixosModules.host
|
||||
nbot.nixosModules.host
|
||||
];
|
||||
};
|
||||
hl-bigbox1 = lib.makeNixOs {
|
||||
|
||||
+2
-2
@@ -31,8 +31,8 @@ hosts/
|
||||
| Folder | Host | Role |
|
||||
| ------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `fwbook/` | fwbook | Framework 13 AMD — personal laptop / daily driver |
|
||||
| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); slackerbot master |
|
||||
| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; slackerbot worker |
|
||||
| `hl-fwdesk/` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation; buildbot-nix CI worker (4 cores); nbot master (slackerbot off) |
|
||||
| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node; nbot worker (slackerbot off) |
|
||||
| `hl-bigbox1/` | hl-bigbox1 | AI/compute server (NVIDIA + AMD); Immich machine-learning (CUDA) |
|
||||
| `hl-bigbox2/` | hl-bigbox2 | NFS storage server; Immich server/db (docker, `immich.home.sosnovsky.ca`) |
|
||||
| `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI), monitoring (Grafana, Prometheus, Loki, OTel collector) |
|
||||
|
||||
@@ -13,6 +13,7 @@ in
|
||||
};
|
||||
networking.firewall.allowedUDPPorts = openPorts;
|
||||
networking.firewall.allowedTCPPorts = openPorts;
|
||||
programs.mosh.enable = true;
|
||||
networking.hosts = {
|
||||
"0.0.0.0" = [
|
||||
"auth.me.internal"
|
||||
|
||||
@@ -10,5 +10,6 @@
|
||||
./linux-tracker.nix
|
||||
./robotnix.nix
|
||||
./slackerbot.nix
|
||||
./nbot.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
{ config, lib, pkgs, nbot, skyg-secrets, ... }:
|
||||
let
|
||||
ports = { cluster = 7946; };
|
||||
homes = "/data/nbot";
|
||||
secrets = [ "nbot-slack" "nbot-goose" "nbot-cluster" ];
|
||||
settings = {
|
||||
# The system each container boots; updated on every rebuild.
|
||||
system = "${nbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}";
|
||||
machines_dir = "/var/lib/machines";
|
||||
state_dir = "/var/lib/nbot";
|
||||
agent_cwd = "/root";
|
||||
|
||||
idle_timeout = "6h";
|
||||
max_idle_timeout = "0s";
|
||||
allow_indefinite = true;
|
||||
boot_timeout = "2m";
|
||||
|
||||
# Goose settings sent to every container (also on fws1).
|
||||
# Values come from the nbot-goose secret.
|
||||
agent_env = {
|
||||
GOOSE_PROVIDER = "$GOOSE_PROVIDER";
|
||||
GOOSE_MODEL = "$GOOSE_MODEL";
|
||||
OPENROUTER_API_KEY = "$OPENROUTER_API_KEY";
|
||||
};
|
||||
|
||||
container = { memory_mb = 4096; cpus = 1.0; };
|
||||
max_container = { memory_mb = 8192; cpus = 2.0; };
|
||||
# Leave room for ollama, ds4, comfyui, hermes and the robots.
|
||||
host_reserve = { memory_mb = 32768; cpus = 4.0; };
|
||||
storage = { min = "2G"; max = "5G"; limit = "20G"; };
|
||||
drives = [{ path = homes; reserve = "40G"; }];
|
||||
|
||||
cluster = {
|
||||
secret = "$CLUSTER_SECRET";
|
||||
listen = ":${toString ports.cluster}";
|
||||
};
|
||||
|
||||
# Only read on first start; after that, manage permissions from Slack.
|
||||
permissions = {
|
||||
managers = [ "U0CDGSJADMM" ];
|
||||
default.U0CDGSJADMM = [ "all" ];
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
age.secrets = lib.genAttrs secrets (name: {
|
||||
file = skyg-secrets.${name};
|
||||
owner = "root";
|
||||
mode = "0400";
|
||||
});
|
||||
|
||||
# nbot flake input; options and docs in its repo.
|
||||
services.nbot = {
|
||||
enable = true;
|
||||
mode = "master";
|
||||
clusterPort = ports.cluster;
|
||||
configFile = (pkgs.formats.toml { }).generate "nbot.toml" settings;
|
||||
};
|
||||
|
||||
# The module takes one env file; these add to it.
|
||||
systemd.services.nbot.serviceConfig.EnvironmentFile =
|
||||
map (name: config.age.secrets.${name}.path) secrets;
|
||||
|
||||
# Container links belong to systemd-networkd.
|
||||
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
|
||||
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
|
||||
|
||||
systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ];
|
||||
}
|
||||
@@ -48,33 +48,36 @@ let
|
||||
};
|
||||
in
|
||||
{
|
||||
age.secrets = lib.genAttrs secrets (name: {
|
||||
file = skyg-secrets.${name};
|
||||
owner = "root";
|
||||
mode = "0400";
|
||||
});
|
||||
# Off.
|
||||
config = lib.mkIf false {
|
||||
age.secrets = lib.genAttrs secrets (name: {
|
||||
file = skyg-secrets.${name};
|
||||
owner = "root";
|
||||
mode = "0400";
|
||||
});
|
||||
|
||||
# slackerbot flake input; options and docs in its repo.
|
||||
services.slackerbot = {
|
||||
enable = true;
|
||||
mode = "master";
|
||||
clusterPort = ports.cluster;
|
||||
configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings;
|
||||
# slackerbot flake input; options and docs in its repo.
|
||||
services.slackerbot = {
|
||||
enable = true;
|
||||
mode = "master";
|
||||
clusterPort = ports.cluster;
|
||||
configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings;
|
||||
};
|
||||
# Bazel build server for the containers (16 cores / 32 threads).
|
||||
skyg.nixos.server.services.bazel-build = {
|
||||
enable = true;
|
||||
cpuCount = 32;
|
||||
cacheGB = 200;
|
||||
};
|
||||
|
||||
# The module takes one env file; these add to it.
|
||||
systemd.services.slackerbot.serviceConfig.EnvironmentFile =
|
||||
map (name: config.age.secrets.${name}.path) secrets;
|
||||
|
||||
# Container links belong to systemd-networkd.
|
||||
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
|
||||
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
|
||||
|
||||
systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ];
|
||||
};
|
||||
# Bazel build server for the containers (16 cores / 32 threads).
|
||||
skyg.nixos.server.services.bazel-build = {
|
||||
enable = true;
|
||||
cpuCount = 32;
|
||||
cacheGB = 200;
|
||||
};
|
||||
|
||||
# The module takes one env file; these add to it.
|
||||
systemd.services.slackerbot.serviceConfig.EnvironmentFile =
|
||||
map (name: config.age.secrets.${name}.path) secrets;
|
||||
|
||||
# Container links belong to systemd-networkd.
|
||||
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
|
||||
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
|
||||
|
||||
systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ];
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{ ... }:
|
||||
{
|
||||
imports = [ ./slackerbot.nix ];
|
||||
imports = [ ./slackerbot.nix ./nbot.nix ];
|
||||
|
||||
services.pipewire = {
|
||||
extraConfig.pipewire-pulse."10-raop-discover" = {
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
{ config, pkgs, nbot, skyg-secrets, ... }:
|
||||
let
|
||||
homes = "/var/lib/nbot/homes";
|
||||
settings = {
|
||||
# The system each container boots; updated on every rebuild.
|
||||
system = "${nbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}";
|
||||
machines_dir = "/var/lib/machines";
|
||||
state_dir = "/var/lib/nbot";
|
||||
agent_cwd = "/root";
|
||||
|
||||
idle_timeout = "6h";
|
||||
max_idle_timeout = "0s";
|
||||
allow_indefinite = true;
|
||||
boot_timeout = "2m";
|
||||
|
||||
container = { memory_mb = 4096; cpus = 1.0; };
|
||||
max_container = { memory_mb = 8192; cpus = 2.0; };
|
||||
host_reserve = { memory_mb = 8192; cpus = 2.0; };
|
||||
storage = { min = "2G"; max = "5G"; limit = "20G"; };
|
||||
drives = [{ path = homes; reserve = "100G"; }];
|
||||
|
||||
# Worker of the master on fwdesk. Slack tokens and goose settings come
|
||||
# from the master. Trusts the first certificate it sees (saved in
|
||||
# state_dir/master.fingerprint); pin it in master_fingerprint if wanted.
|
||||
cluster = {
|
||||
secret = "$CLUSTER_SECRET";
|
||||
master = "wss://fwdesk.lab.internal:7946";
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
age.secrets.nbot-cluster = {
|
||||
file = skyg-secrets.nbot-cluster;
|
||||
owner = "root";
|
||||
mode = "0400";
|
||||
};
|
||||
|
||||
# nbot flake input; options and docs in its repo.
|
||||
services.nbot = {
|
||||
enable = true;
|
||||
mode = "slave";
|
||||
configFile = (pkgs.formats.toml { }).generate "nbot.toml" settings;
|
||||
environmentFile = config.age.secrets.nbot-cluster.path;
|
||||
};
|
||||
|
||||
# Container links belong to systemd-networkd.
|
||||
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
|
||||
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
|
||||
# networkd only has the container links here (NetworkManager owns wlan),
|
||||
# so wait-online would always time out.
|
||||
systemd.network.wait-online.enable = false;
|
||||
|
||||
systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ];
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
{ config, pkgs, skyg-secrets, ... }:
|
||||
{ config, lib, pkgs, skyg-secrets, ... }:
|
||||
let
|
||||
homes = "/var/lib/slackerbot/homes";
|
||||
bazel = config.skyg.nixos.server.services.bazel-build;
|
||||
@@ -34,33 +34,36 @@ let
|
||||
};
|
||||
in
|
||||
{
|
||||
age.secrets.slackerbot-cluster = {
|
||||
file = skyg-secrets.slackerbot-cluster;
|
||||
owner = "root";
|
||||
mode = "0400";
|
||||
# Off.
|
||||
config = lib.mkIf false {
|
||||
age.secrets.slackerbot-cluster = {
|
||||
file = skyg-secrets.slackerbot-cluster;
|
||||
owner = "root";
|
||||
mode = "0400";
|
||||
};
|
||||
|
||||
# slackerbot flake input; options and docs in its repo.
|
||||
services.slackerbot = {
|
||||
enable = true;
|
||||
mode = "slave";
|
||||
configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings;
|
||||
environmentFile = config.age.secrets.slackerbot-cluster.path;
|
||||
};
|
||||
|
||||
# Bazel build server for the containers (4 cores / 8 threads).
|
||||
skyg.nixos.server.services.bazel-build = {
|
||||
enable = true;
|
||||
cpuCount = 8;
|
||||
cacheGB = 50;
|
||||
};
|
||||
|
||||
# Container links belong to systemd-networkd.
|
||||
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
|
||||
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
|
||||
# networkd only has the container links here (NetworkManager owns wlan),
|
||||
# so wait-online would always time out.
|
||||
systemd.network.wait-online.enable = false;
|
||||
|
||||
systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ];
|
||||
};
|
||||
|
||||
# slackerbot flake input; options and docs in its repo.
|
||||
services.slackerbot = {
|
||||
enable = true;
|
||||
mode = "slave";
|
||||
configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings;
|
||||
environmentFile = config.age.secrets.slackerbot-cluster.path;
|
||||
};
|
||||
|
||||
# Bazel build server for the containers (4 cores / 8 threads).
|
||||
skyg.nixos.server.services.bazel-build = {
|
||||
enable = true;
|
||||
cpuCount = 8;
|
||||
cacheGB = 50;
|
||||
};
|
||||
|
||||
# Container links belong to systemd-networkd.
|
||||
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
|
||||
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
|
||||
# networkd only has the container links here (NetworkManager owns wlan),
|
||||
# so wait-online would always time out.
|
||||
systemd.network.wait-online.enable = false;
|
||||
|
||||
systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ];
|
||||
}
|
||||
@@ -25,7 +25,8 @@ services/
|
||||
enables NixOS-native Wyoming faster-whisper + piper. Opens 11434/10200/10300/10400.
|
||||
- **`bazel-build.nix`** runs NativeLink (from the `nativelink` flake input, pinned
|
||||
to v1.8.3 because its config format changes) as a remote cache and executor for
|
||||
slackerbot containers. Used on hl-fws1 and hl-fwdesk. Goal: as little as possible
|
||||
slackerbot containers. Off on every host: only the disabled slackerbot configs on
|
||||
hl-fws1 and hl-fwdesk turn it on, and nbot doesn't use it. Goal: as little as possible
|
||||
runs in a container. Measured on slacker-bot (1 CPU container, 2026-11-01): local
|
||||
458 s, remote with no cache 65 s, remote with warm cache 17 s.
|
||||
- NativeLink uses `DynamicUser`, whose state folder systemd mounts `noexec`;
|
||||
|
||||
@@ -38,6 +38,11 @@ in
|
||||
"secrets/slackerbot-goose.age".publicKeys = [ ari fwdesk ];
|
||||
"secrets/slackerbot-cluster.age".publicKeys = [ ari fwdesk fws1 ];
|
||||
|
||||
# nbot (master on fwdesk, worker on fws1)
|
||||
"secrets/nbot-slack.age".publicKeys = [ ari fwdesk ];
|
||||
"secrets/nbot-goose.age".publicKeys = [ ari fwdesk ];
|
||||
"secrets/nbot-cluster.age".publicKeys = [ ari fwdesk fws1 ];
|
||||
|
||||
# Buzz stack env (minipc1)
|
||||
"secrets/buzz-env.age".publicKeys = [ ari minipc1 ];
|
||||
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,9 @@
|
||||
age-encryption.org/v1
|
||||
-> ssh-ed25519 CIR0mA 5rbuNoaaPYjl/B4MQW2jufxjCKC5gf05MRXBQbl6+AA
|
||||
dB6sf5Q2Bnk/hwtWSblio1GEoM3OwwFCEC4sn6Nla3U
|
||||
-> ssh-ed25519 gzLTww tI/mZILMxoFNQg4jL2gm7T6ZL/IGFASrbEIAqiOOoCA
|
||||
5AUdlQm5Bg4JkVT4TyuW4rPaJ+5yHuPEXe81cdMyxZ8
|
||||
--- lj92JFLzMvqXZP18XBdXAkhhvPdOHxzfqN14oBrPCFw
|
||||
|
||||
B
æZö¯yéš*£à£[¸¯(O%�Ö2[c¬] ƒþÖ‘\4üÕ󉘦Óv†Éxqqí6ųá@æ�îß Â•3
áù
|
||||
œaþ-»‚¬Ó.•¥{Þ™,åÂ)éâQÖä=ïìå¯LO9¹®ñD–Ð�û(piÃôq4°\£Ærê°](Û¥—ÊÆÂÖ™é)ò%|~Ö9É¢ ä̾û?NFˆåÏ{~ÄW`@³+Üñ2<âÒÆÓ#[ÆÊ„QÌb˜
|
||||
Binary file not shown.
Binary file not shown.
Reference in new issue
Block a user