diff --git a/flake.lock b/flake.lock index 9917839..6d6bdb9 100644 --- a/flake.lock +++ b/flake.lock @@ -2204,11 +2204,11 @@ ] }, "locked": { - "lastModified": 1794413151, - "narHash": "sha256-GWrOXpDvbMOocsS331/GpJ9W660OadAcAkW3FjpU5LQ=", + "lastModified": 1794432179, + "narHash": "sha256-473TO/GoK1dbQR/RgYeniCXDC+hVHzBUYEU2Ig6Ta24=", "ref": "refs/heads/main", - "rev": "bec7fea9638423c8ed27cb0534c9d09e596a2e22", - "revCount": 1285, + "rev": "f23eb0352c48ec5b279b8dcb7280c19b0da58cbe", + "revCount": 1304, "type": "git", "url": "ssh://git@github.com/asosnovsky/nbot.git" }, diff --git a/hosts/hl-bigbox1/default.nix b/hosts/hl-bigbox1/default.nix index 899debe..ba32a07 100644 --- a/hosts/hl-bigbox1/default.nix +++ b/hosts/hl-bigbox1/default.nix @@ -17,6 +17,7 @@ gnome.enable = false; }; common.ssh-server.enable = true; + common.diagnix.enable = true; common.hardware = { nvidia.enable = true; diff --git a/hosts/hl-bigbox2/default.nix b/hosts/hl-bigbox2/default.nix index 10d4410..8e8d68a 100644 --- a/hosts/hl-bigbox2/default.nix +++ b/hosts/hl-bigbox2/default.nix @@ -13,6 +13,7 @@ user.enable = true; nixos.common = { ssh-server.enable = true; + diagnix.enable = true; containers.networks.ipvlanLab = { enable = true; diff --git a/hosts/hl-fwdesk/default.nix b/hosts/hl-fwdesk/default.nix index 643cf94..05ab63a 100644 --- a/hosts/hl-fwdesk/default.nix +++ b/hosts/hl-fwdesk/default.nix @@ -50,6 +50,7 @@ in core.qemu.enable = true; nixos = { common.ssh-server.enable = true; + common.diagnix.enable = true; common.cachePush.enable = true; common.hardware = { pipewire.enable = true; diff --git a/hosts/hl-fwdesk/services/nbot.nix b/hosts/hl-fwdesk/services/nbot.nix index 3288a9f..bd5cc99 100644 --- a/hosts/hl-fwdesk/services/nbot.nix +++ b/hosts/hl-fwdesk/services/nbot.nix @@ -28,6 +28,7 @@ let CLAUDE_CODE_OAUTH_TOKEN = "$CLAUDE_CODE_OAUTH_TOKEN"; COMMAND_CODE_API_KEY = "$COMMAND_CODE_API_KEY"; GH_TOKEN = "$GH_TOKEN"; + DIAGNIX_KEY = "$DIAGNIX_KEY"; # base64 private key for lab-ssh (diagnix) }; model_providers = [ diff --git a/hosts/hl-fws1/default.nix b/hosts/hl-fws1/default.nix index d5bc29e..3359c4c 100644 --- a/hosts/hl-fws1/default.nix +++ b/hosts/hl-fws1/default.nix @@ -11,6 +11,7 @@ user.enable = true; nixos = { common.ssh-server.enable = true; + common.diagnix.enable = true; common.hardware.pipewire.enable = true; desktop = { enable = true; diff --git a/hosts/hl-minipc1/default.nix b/hosts/hl-minipc1/default.nix index 2364be6..d913c98 100644 --- a/hosts/hl-minipc1/default.nix +++ b/hosts/hl-minipc1/default.nix @@ -37,6 +37,7 @@ in skyg.user.enable = true; skyg.nixos.common.ssh-server.enable = true; + skyg.nixos.common.diagnix.enable = true; skyg.server.dns.routing = { enable = false; openFirewall = true; diff --git a/hosts/hl-minipc2/default.nix b/hosts/hl-minipc2/default.nix index aa302bb..494e994 100644 --- a/hosts/hl-minipc2/default.nix +++ b/hosts/hl-minipc2/default.nix @@ -8,6 +8,7 @@ skyg.user.enable = true; skyg.nixos.common.ssh-server.enable = true; + skyg.nixos.common.diagnix.enable = true; skyg.nixos.common.containers.networks.ipvlanLab = { enable = true; parent = "enp2s0"; diff --git a/hosts/hl-minipc3/default.nix b/hosts/hl-minipc3/default.nix index 781cb98..7c01f22 100644 --- a/hosts/hl-minipc3/default.nix +++ b/hosts/hl-minipc3/default.nix @@ -8,6 +8,7 @@ skyg.user.enable = true; skyg.nixos.common.ssh-server.enable = true; + skyg.nixos.common.diagnix.enable = true; skyg.server.dns.routing = { enable = false; openFirewall = true; diff --git a/hosts/hl-pi1/default.nix b/hosts/hl-pi1/default.nix index 4e6308e..33e87b0 100644 --- a/hosts/hl-pi1/default.nix +++ b/hosts/hl-pi1/default.nix @@ -13,6 +13,7 @@ minimal = true; fonts.minimal = true; ssh-server.enable = true; + diagnix.enable = true; containers.enable = false; networking.nfsServer.enable = false; hardware.pipewire.enable = true; diff --git a/hosts/hl-pi2/default.nix b/hosts/hl-pi2/default.nix index 7b8fd1f..f2c18fb 100644 --- a/hosts/hl-pi2/default.nix +++ b/hosts/hl-pi2/default.nix @@ -10,6 +10,7 @@ user.enable = true; nixos = { common.ssh-server.enable = true; + common.diagnix.enable = true; common.hardware.pipewire.enable = true; desktop = { enable = true; diff --git a/hosts/hl-terra1/default.nix b/hosts/hl-terra1/default.nix index 2665307..cc513a4 100644 --- a/hosts/hl-terra1/default.nix +++ b/hosts/hl-terra1/default.nix @@ -20,6 +20,7 @@ in skyg.user.enable = true; skyg.nixos.common.ssh-server.enable = true; + skyg.nixos.common.diagnix.enable = true; skyg.server.dns.routing = { enable = false; openFirewall = true; diff --git a/modules/nixos/common/ABOUTME.md b/modules/nixos/common/ABOUTME.md index f3d87bd..0067792 100644 --- a/modules/nixos/common/ABOUTME.md +++ b/modules/nixos/common/ABOUTME.md @@ -15,6 +15,7 @@ common/ ├── user.nix # skyg.user.createSystemUser — creates the system user/groups + zsh shell ├── fonts.nix # System font packages + fontconfig defaults (Fira Code, Noto, …); skyg.nixos.common.fonts.minimal ├── ssh-server.nix # skyg.nixos.common.ssh-server — hardened sshd + master public keys (list) +├── diagnix/ # skyg.nixos.common.diagnix — read-only user nbot uses to diagnose hosts ├── qemu.nix # skyg.core.qemu — QEMU/quickemu + SPICE guest tooling ├── binary-cache.nix # skyg.nixos.common.cachePush — push finished builds to the bigbox2 cache @@ -38,6 +39,18 @@ common/ - **`ssh-server.nix`** ships the master public keys (`masterPubKeys`, a list so a key can be rotated by adding the new one before dropping the old) and a hardened sshd config (key-only auth, long keepalives). Disabled by default. +- **`diagnix/`** adds a read-only `diagnix` user (uid 2950) that nbot uses over SSH, via + `lab-ssh` in the agent system (`server/services/nbot-agent.nix`). It reads logs + (`systemd-journal` group), units, `/proc` and the store, and can't change anything. It + can't read root-only files. Through sudo it can run only the fixed `diag-*` scripts + (docker ps/logs, smart, ports, firewall, btrfs, io). The key is `restrict,pty` and only + works from the nbot hosts (`hosts.nix`). Hosts running nbot also let their own containers + reach port 22 and accept them from private IPs outside the LAN. Their IPs are random, so + they can't be listed. Resources are capped (1G memory, 1 CPU, 256 tasks). The Nix daemon + is left open on purpose: an allow-list would break nix-serve/buildbot. + The private key (base64) is `DIAGNIX_KEY` in the `nbot-goose` secret; fwdesk passes it to + every container via `agent_env`. + `hosts.nix` also holds each host's address and host key; the agent pins those keys. ## Option Namespace @@ -46,6 +59,7 @@ skyg.core.hostName → networking.nix skyg.user.createSystemUser → user.nix skyg.core.qemu.* → qemu.nix skyg.nixos.common.ssh-server.* → ssh-server.nix +skyg.nixos.common.diagnix.* → diagnix/ skyg.nixos.common.cachePush.* → binary-cache.nix skyg.nixos.common.minimal → core.nix (skip heavy always-on system packages) diff --git a/modules/nixos/common/default.nix b/modules/nixos/common/default.nix index db9123c..ac7687b 100644 --- a/modules/nixos/common/default.nix +++ b/modules/nixos/common/default.nix @@ -8,6 +8,7 @@ ./user.nix ./fonts.nix ./ssh-server.nix + ./diagnix ./qemu.nix ./binary-cache.nix ./pritunl diff --git a/modules/nixos/common/diagnix/default.nix b/modules/nixos/common/diagnix/default.nix new file mode 100644 index 0000000..58b31e0 --- /dev/null +++ b/modules/nixos/common/diagnix/default.nix @@ -0,0 +1,116 @@ +# diagnix: a read-only user nbot logs in as to diagnose a host. +# Sees logs, units, /proc and /nix/store; can't change the system. +{ config, lib, pkgs, options, ... }: +let + cfg = config.skyg.nixos.common.diagnix; + data = import ./hosts.nix; + uid = 2950; + + # This host runs nbot, so its own containers connect from random private IPs. + hasNbot = options.services ? nbot && config.services.nbot.enable; + + keyLine = from: ''restrict,pty,from="${from}" ${cfg.publicKey}''; + + runtime = + if config.virtualisation.docker.enable then "${config.virtualisation.docker.package}/bin/docker" + else if config.virtualisation.podman.enable then "${config.virtualisation.podman.package}/bin/podman" + else null; + rt = toString runtime; + + # Fixed root scripts diagnix may run with sudo. Anything taking input checks it. + script = name: inputs: text: pkgs.writeShellApplication { + inherit name text; + runtimeInputs = inputs; + }; + scripts = [ + (script "diag-smart" [ pkgs.smartmontools pkgs.nvme-cli ] '' + smartctl --scan | while read -r dev _; do + echo "=== $dev"; smartctl -a "$dev" || true + done + for dev in /dev/nvme[0-9]; do + [ -e "$dev" ] || continue + echo "=== $dev"; nvme smart-log "$dev" || true + done + '') + (script "diag-ports" [ pkgs.iproute2 ] "ss -tulpn") + (script "diag-firewall" [ pkgs.iptables pkgs.nftables ] '' + iptables -S || true; iptables -t nat -S || true; ip6tables -S || true + nft list ruleset || true + '') + (script "diag-btrfs" [ pkgs.btrfs-progs pkgs.util-linux ] '' + findmnt -rn -t btrfs -o TARGET | sort -u | while read -r mnt; do + echo "=== $mnt"; btrfs filesystem usage "$mnt" || true; btrfs device stats "$mnt" || true + done + '') + (script "diag-io" [ pkgs.iotop ] "iotop -b -o -n 3 -d 1") + ] ++ lib.optionals (runtime != null) [ + (script "diag-docker-ps" [ ] '' + ${rt} ps -a; ${rt} stats --no-stream; ${rt} system df + '') + (script "diag-docker-logs" [ ] '' + if [ $# -ne 1 ] || ! [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then + echo "usage: diag-docker-logs " >&2; exit 2 + fi + ${rt} logs --tail 500 "$1" 2>&1 + '') + ]; +in +{ + options.skyg.nixos.common.diagnix = { + enable = lib.mkEnableOption "the read-only diagnix user for nbot"; + publicKey = lib.mkOption { + type = lib.types.str; + default = data.publicKey; + description = "nbot's client public key for diagnix."; + }; + }; + + config = lib.mkIf cfg.enable { + users.groups.diagnix.gid = uid; + users.users.diagnix = { + inherit uid; + isNormalUser = true; + group = "diagnix"; + description = "nbot read-only diagnostics"; + shell = pkgs.bash; + extraGroups = [ "systemd-journal" ]; + openssh.authorizedKeys.keys = + [ (keyLine (lib.concatStringsSep "," data.nbotHosts)) ] + # Containers on this host: any private IP outside the LAN. + ++ lib.optional hasNbot (keyLine "!${data.lanSubnet},10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"); + }; + + # A runaway command can't starve the host. + systemd.slices."user-${toString uid}" = { + overrideStrategy = "asDropin"; + sliceConfig = { MemoryMax = "1G"; CPUQuota = "100%"; TasksMax = 256; }; + }; + + security.sudo.extraRules = [{ + users = [ "diagnix" ]; + commands = map (s: { command = lib.getExe s; options = [ "NOPASSWD" ]; }) scripts; + }]; + + programs.mtr.enable = true; + environment.systemPackages = scripts ++ (with pkgs; [ + lsof + pciutils + usbutils + dnsutils + sysstat + ethtool + lm_sensors + htop + ]); + + # nbot drops all container traffic to the host except a few ports; let SSH in too. + networking.firewall = lib.mkIf (hasNbot && !config.networking.nftables.enable) { + extraCommands = lib.mkAfter '' + iptables -C INPUT -i ve-+ -p tcp --dport 22 -j ACCEPT 2>/dev/null || iptables -I INPUT 1 -i ve-+ -p tcp --dport 22 -j ACCEPT + ''; + extraStopCommands = '' + iptables -D INPUT -i ve-+ -p tcp --dport 22 -j ACCEPT 2>/dev/null || true + ''; + }; + }; +} diff --git a/modules/nixos/common/diagnix/hosts.nix b/modules/nixos/common/diagnix/hosts.nix new file mode 100644 index 0000000..906f00d --- /dev/null +++ b/modules/nixos/common/diagnix/hosts.nix @@ -0,0 +1,25 @@ +# Lab hosts diagnix lives on, shared by the host module and nbot's agent system. +# hostKey is the host's ssh_host_ed25519 public key (from secrets.nix); +# null means the agent saves it on first connect. +{ + # LAN addresses of the hosts that run nbot. The key only works from these. + nbotHosts = [ "10.0.10.11" "10.0.10.13" ]; # hl-fwdesk, hl-fws1 + lanSubnet = "10.0.0.0/16"; + + # The nbot client public key. Private half (base64) is DIAGNIX_KEY in the + # nbot-goose secret, sent to containers via agent_env on fwdesk. + publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPKsgkysVa3BiJQkUjiyESwjoi9ZMmUiWEB66JlbgIrO nbot-diagnix"; + + hosts = { + hl-bigbox1 = { address = "bigbox1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFl5xiH4aFlmDWdoU+KmERLYOeOQK2aKP7jenY2G3rFb"; }; + hl-bigbox2 = { address = "bigbox2.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6l6VqQ23PkGkWVABBoLp1KA6PzkX5oKTy5RBHbQXet"; }; + hl-fwdesk = { address = "fwdesk.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHbBcdMpNPaELUzHRIoRkTUQde+mBQ87Ra2R+974FuUy"; }; + hl-fws1 = { address = "fws1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM6r0AIJyTlE+U6K97GijzAP4jNaAueVlwXbeMpEMcpe"; }; + hl-minipc1 = { address = "minipc1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP42vtV9Qo7q5F9M/rCGD/TIZG3d0mm1uP6lMvy3zF/a"; }; + hl-minipc2 = { address = "minipc2.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDqi2WT0pa8m+kZNN2Rrg1F01zwTvkCwD3wjSOBNvSjz"; }; + hl-minipc3 = { address = "minipc3.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEbvoe/UvhpRSWVXMCyj/qzuR7WnJdNF7CYRmmJE3yYu"; }; + hl-pi1 = { address = "pi1.lab.internal"; hostKey = null; }; + hl-pi2 = { address = "pi2.lab.internal"; hostKey = null; }; + hl-terra1 = { address = "terra1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHBKI8gqd7DDubShle4D3JGq61BP1NBKCZ81V3C9uyym"; }; + }; +} diff --git a/modules/nixos/server/services/nbot-agent.nix b/modules/nixos/server/services/nbot-agent.nix index 0e3d5c7..d9519d4 100644 --- a/modules/nixos/server/services/nbot-agent.nix +++ b/modules/nixos/server/services/nbot-agent.nix @@ -1,11 +1,38 @@ -# nbot's agent system with our additions. Use it for nbot's `system`. +# nbot's agent system with our additions (gh, lab-ssh). Use it for nbot's `system`. { lib, nbot, ... }: with lib; let + lab = import ../../common/diagnix/hosts.nix; + + # lab-ssh [cmd...]: log in to a lab host as the read-only diagnix user. + # The key comes base64-encoded in $DIAGNIX_KEY (agent_env, from the nbot-goose secret). + labSsh = pkgs: pkgs.writeShellApplication { + name = "lab-ssh"; + runtimeInputs = [ pkgs.openssh pkgs.coreutils ]; + text = '' + host="''${1:-}" + case "$host" in + ${concatStrings (mapAttrsToList (name: h: '' + ${name}|${removePrefix "hl-" name}) addr=${h.address}; check=${if h.hostKey == null then "accept-new" else "yes"} ;; + '') lab.hosts)} + *) echo "usage: lab-ssh [command...]; hosts: ${concatStringsSep " " (attrNames lab.hosts)}" >&2; exit 2 ;; + esac + shift + : "''${DIAGNIX_KEY:?DIAGNIX_KEY missing from agent_env}" + key="$HOME/.ssh/diagnix" + mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh" + (umask 077; base64 -d <<<"$DIAGNIX_KEY" > "$key") + exec ssh -i "$key" -o IdentitiesOnly=yes -o StrictHostKeyChecking="$check" "diagnix@$addr" "$@" + ''; + }; + agentModule = { pkgs, ... }: { - environment.systemPackages = [ pkgs.gh ]; + environment.systemPackages = [ pkgs.gh (labSsh pkgs) ]; + programs.ssh.knownHosts = mapAttrs + (name: h: { hostNames = [ name h.address ]; publicKey = h.hostKey; }) + (filterAttrs (_: h: h.hostKey != null) lab.hosts); # gh reads GH_TOKEN (from agent_env); git over HTTPS uses it through gh. programs.git = { enable = true; @@ -15,13 +42,13 @@ let in { options.skyg.nixos.server.services.nbot-agent.system = mkOption { - description = "nbot's agent system with gh added. Use for nbot's `system`."; + description = "nbot's agent system with gh and lab-ssh added. Use for nbot's `system`."; readOnly = true; type = types.package; default = (nbot.nixosConfigurations.agent.extendModules { modules = [ agentModule ]; }).config.system.build.toplevel; # Keeps the manual from building it on every host. - defaultText = literalExpression "nbot agent + gh"; + defaultText = literalExpression "nbot agent + gh + lab-ssh"; }; } diff --git a/secrets.nix b/secrets.nix index b059746..a58f687 100644 --- a/secrets.nix +++ b/secrets.nix @@ -9,6 +9,7 @@ let bigbox2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6l6VqQ23PkGkWVABBoLp1KA6PzkX5oKTy5RBHbQXet root@hl-bigbox2"; hermenix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPfMrT6dDPw3+J22k3X5aR9ERQ8RMRLeBGgqQjmuL7EI hermenix@lab"; fws1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM6r0AIJyTlE+U6K97GijzAP4jNaAueVlwXbeMpEMcpe root@hl-fws1"; + diagnix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPKsgkysVa3BiJQkUjiyESwjoi9ZMmUiWEB66JlbgIrO nbot-diagnix"; in { "secrets/dns-addresses.conf.age".publicKeys = [ diff --git a/secrets/nbot-goose.age b/secrets/nbot-goose.age index c37f15b..c522c8e 100644 Binary files a/secrets/nbot-goose.age and b/secrets/nbot-goose.age differ