debug agent

This commit is contained in:
ari committed 2026-11-11 17:02:36 -05:00
1 parent 2547ec45b4
commit 09c6fe2235
19 files changed
+203 -8

No files matched your search

+14
View File
@@ -15,6 +15,7 @@ common/
├── user.nix # skyg.user.createSystemUser — creates the system user/groups + zsh shell
├── fonts.nix # System font packages + fontconfig defaults (Fira Code, Noto, …); skyg.nixos.common.fonts.minimal
├── ssh-server.nix # skyg.nixos.common.ssh-server — hardened sshd + master public keys (list)
├── diagnix/ # skyg.nixos.common.diagnix — read-only user nbot uses to diagnose hosts
├── qemu.nix # skyg.core.qemu — QEMU/quickemu + SPICE guest tooling
├── binary-cache.nix # skyg.nixos.common.cachePush — push finished builds to the bigbox2 cache
@@ -38,6 +39,18 @@ common/
- **`ssh-server.nix`** ships the master public keys (`masterPubKeys`, a list so a key can be
rotated by adding the new one before dropping the old) and a hardened sshd config
(key-only auth, long keepalives). Disabled by default.
- **`diagnix/`** adds a read-only `diagnix` user (uid 2950) that nbot uses over SSH, via
`lab-ssh` in the agent system (`server/services/nbot-agent.nix`). It reads logs
(`systemd-journal` group), units, `/proc` and the store, and can't change anything. It
can't read root-only files. Through sudo it can run only the fixed `diag-*` scripts
(docker ps/logs, smart, ports, firewall, btrfs, io). The key is `restrict,pty` and only
works from the nbot hosts (`hosts.nix`). Hosts running nbot also let their own containers
reach port 22 and accept them from private IPs outside the LAN. Their IPs are random, so
they can't be listed. Resources are capped (1G memory, 1 CPU, 256 tasks). The Nix daemon
is left open on purpose: an allow-list would break nix-serve/buildbot.
The private key (base64) is `DIAGNIX_KEY` in the `nbot-goose` secret; fwdesk passes it to
every container via `agent_env`.
`hosts.nix` also holds each host's address and host key; the agent pins those keys.
## Option Namespace
@@ -46,6 +59,7 @@ skyg.core.hostName → networking.nix
skyg.user.createSystemUser → user.nix
skyg.core.qemu.* → qemu.nix
skyg.nixos.common.ssh-server.* → ssh-server.nix
skyg.nixos.common.diagnix.* → diagnix/
skyg.nixos.common.cachePush.* → binary-cache.nix
skyg.nixos.common.minimal → core.nix (skip heavy always-on system packages)
+1
View File
@@ -8,6 +8,7 @@
./user.nix
./fonts.nix
./ssh-server.nix
./diagnix
./qemu.nix
./binary-cache.nix
./pritunl
+116
View File
@@ -0,0 +1,116 @@
# diagnix: a read-only user nbot logs in as to diagnose a host.
# Sees logs, units, /proc and /nix/store; can't change the system.
{ config, lib, pkgs, options, ... }:
let
cfg = config.skyg.nixos.common.diagnix;
data = import ./hosts.nix;
uid = 2950;
# This host runs nbot, so its own containers connect from random private IPs.
hasNbot = options.services ? nbot && config.services.nbot.enable;
keyLine = from: ''restrict,pty,from="${from}" ${cfg.publicKey}'';
runtime =
if config.virtualisation.docker.enable then "${config.virtualisation.docker.package}/bin/docker"
else if config.virtualisation.podman.enable then "${config.virtualisation.podman.package}/bin/podman"
else null;
rt = toString runtime;
# Fixed root scripts diagnix may run with sudo. Anything taking input checks it.
script = name: inputs: text: pkgs.writeShellApplication {
inherit name text;
runtimeInputs = inputs;
};
scripts = [
(script "diag-smart" [ pkgs.smartmontools pkgs.nvme-cli ] ''
smartctl --scan | while read -r dev _; do
echo "=== $dev"; smartctl -a "$dev" || true
done
for dev in /dev/nvme[0-9]; do
[ -e "$dev" ] || continue
echo "=== $dev"; nvme smart-log "$dev" || true
done
'')
(script "diag-ports" [ pkgs.iproute2 ] "ss -tulpn")
(script "diag-firewall" [ pkgs.iptables pkgs.nftables ] ''
iptables -S || true; iptables -t nat -S || true; ip6tables -S || true
nft list ruleset || true
'')
(script "diag-btrfs" [ pkgs.btrfs-progs pkgs.util-linux ] ''
findmnt -rn -t btrfs -o TARGET | sort -u | while read -r mnt; do
echo "=== $mnt"; btrfs filesystem usage "$mnt" || true; btrfs device stats "$mnt" || true
done
'')
(script "diag-io" [ pkgs.iotop ] "iotop -b -o -n 3 -d 1")
] ++ lib.optionals (runtime != null) [
(script "diag-docker-ps" [ ] ''
${rt} ps -a; ${rt} stats --no-stream; ${rt} system df
'')
(script "diag-docker-logs" [ ] ''
if [ $# -ne 1 ] || ! [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then
echo "usage: diag-docker-logs <container-name>" >&2; exit 2
fi
${rt} logs --tail 500 "$1" 2>&1
'')
];
in
{
options.skyg.nixos.common.diagnix = {
enable = lib.mkEnableOption "the read-only diagnix user for nbot";
publicKey = lib.mkOption {
type = lib.types.str;
default = data.publicKey;
description = "nbot's client public key for diagnix.";
};
};
config = lib.mkIf cfg.enable {
users.groups.diagnix.gid = uid;
users.users.diagnix = {
inherit uid;
isNormalUser = true;
group = "diagnix";
description = "nbot read-only diagnostics";
shell = pkgs.bash;
extraGroups = [ "systemd-journal" ];
openssh.authorizedKeys.keys =
[ (keyLine (lib.concatStringsSep "," data.nbotHosts)) ]
# Containers on this host: any private IP outside the LAN.
++ lib.optional hasNbot (keyLine "!${data.lanSubnet},10.0.0.0/8,172.16.0.0/12,192.168.0.0/16");
};
# A runaway command can't starve the host.
systemd.slices."user-${toString uid}" = {
overrideStrategy = "asDropin";
sliceConfig = { MemoryMax = "1G"; CPUQuota = "100%"; TasksMax = 256; };
};
security.sudo.extraRules = [{
users = [ "diagnix" ];
commands = map (s: { command = lib.getExe s; options = [ "NOPASSWD" ]; }) scripts;
}];
programs.mtr.enable = true;
environment.systemPackages = scripts ++ (with pkgs; [
lsof
pciutils
usbutils
dnsutils
sysstat
ethtool
lm_sensors
htop
]);
# nbot drops all container traffic to the host except a few ports; let SSH in too.
networking.firewall = lib.mkIf (hasNbot && !config.networking.nftables.enable) {
extraCommands = lib.mkAfter ''
iptables -C INPUT -i ve-+ -p tcp --dport 22 -j ACCEPT 2>/dev/null || iptables -I INPUT 1 -i ve-+ -p tcp --dport 22 -j ACCEPT
'';
extraStopCommands = ''
iptables -D INPUT -i ve-+ -p tcp --dport 22 -j ACCEPT 2>/dev/null || true
'';
};
};
}
+25
View File
@@ -0,0 +1,25 @@
# Lab hosts diagnix lives on, shared by the host module and nbot's agent system.
# hostKey is the host's ssh_host_ed25519 public key (from secrets.nix);
# null means the agent saves it on first connect.
{
# LAN addresses of the hosts that run nbot. The key only works from these.
nbotHosts = [ "10.0.10.11" "10.0.10.13" ]; # hl-fwdesk, hl-fws1
lanSubnet = "10.0.0.0/16";
# The nbot client public key. Private half (base64) is DIAGNIX_KEY in the
# nbot-goose secret, sent to containers via agent_env on fwdesk.
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPKsgkysVa3BiJQkUjiyESwjoi9ZMmUiWEB66JlbgIrO nbot-diagnix";
hosts = {
hl-bigbox1 = { address = "bigbox1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFl5xiH4aFlmDWdoU+KmERLYOeOQK2aKP7jenY2G3rFb"; };
hl-bigbox2 = { address = "bigbox2.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6l6VqQ23PkGkWVABBoLp1KA6PzkX5oKTy5RBHbQXet"; };
hl-fwdesk = { address = "fwdesk.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHbBcdMpNPaELUzHRIoRkTUQde+mBQ87Ra2R+974FuUy"; };
hl-fws1 = { address = "fws1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM6r0AIJyTlE+U6K97GijzAP4jNaAueVlwXbeMpEMcpe"; };
hl-minipc1 = { address = "minipc1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP42vtV9Qo7q5F9M/rCGD/TIZG3d0mm1uP6lMvy3zF/a"; };
hl-minipc2 = { address = "minipc2.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDqi2WT0pa8m+kZNN2Rrg1F01zwTvkCwD3wjSOBNvSjz"; };
hl-minipc3 = { address = "minipc3.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEbvoe/UvhpRSWVXMCyj/qzuR7WnJdNF7CYRmmJE3yYu"; };
hl-pi1 = { address = "pi1.lab.internal"; hostKey = null; };
hl-pi2 = { address = "pi2.lab.internal"; hostKey = null; };
hl-terra1 = { address = "terra1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHBKI8gqd7DDubShle4D3JGq61BP1NBKCZ81V3C9uyym"; };
};
}
+31 -4
View File
@@ -1,11 +1,38 @@
# nbot's agent system with our additions. Use it for nbot's `system`.
# nbot's agent system with our additions (gh, lab-ssh). Use it for nbot's `system`.
{ lib, nbot, ... }:
with lib;
let
lab = import ../../common/diagnix/hosts.nix;
# lab-ssh <host> [cmd...]: log in to a lab host as the read-only diagnix user.
# The key comes base64-encoded in $DIAGNIX_KEY (agent_env, from the nbot-goose secret).
labSsh = pkgs: pkgs.writeShellApplication {
name = "lab-ssh";
runtimeInputs = [ pkgs.openssh pkgs.coreutils ];
text = ''
host="''${1:-}"
case "$host" in
${concatStrings (mapAttrsToList (name: h: ''
${name}|${removePrefix "hl-" name}) addr=${h.address}; check=${if h.hostKey == null then "accept-new" else "yes"} ;;
'') lab.hosts)}
*) echo "usage: lab-ssh <host> [command...]; hosts: ${concatStringsSep " " (attrNames lab.hosts)}" >&2; exit 2 ;;
esac
shift
: "''${DIAGNIX_KEY:?DIAGNIX_KEY missing from agent_env}"
key="$HOME/.ssh/diagnix"
mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
(umask 077; base64 -d <<<"$DIAGNIX_KEY" > "$key")
exec ssh -i "$key" -o IdentitiesOnly=yes -o StrictHostKeyChecking="$check" "diagnix@$addr" "$@"
'';
};
agentModule = { pkgs, ... }: {
environment.systemPackages = [ pkgs.gh ];
environment.systemPackages = [ pkgs.gh (labSsh pkgs) ];
programs.ssh.knownHosts = mapAttrs
(name: h: { hostNames = [ name h.address ]; publicKey = h.hostKey; })
(filterAttrs (_: h: h.hostKey != null) lab.hosts);
# gh reads GH_TOKEN (from agent_env); git over HTTPS uses it through gh.
programs.git = {
enable = true;
@@ -15,13 +42,13 @@ let
in
{
options.skyg.nixos.server.services.nbot-agent.system = mkOption {
description = "nbot's agent system with gh added. Use for nbot's `system`.";
description = "nbot's agent system with gh and lab-ssh added. Use for nbot's `system`.";
readOnly = true;
type = types.package;
default = (nbot.nixosConfigurations.agent.extendModules {
modules = [ agentModule ];
}).config.system.build.toplevel;
# Keeps the manual from building it on every host.
defaultText = literalExpression "nbot agent + gh";
defaultText = literalExpression "nbot agent + gh + lab-ssh";
};
}