debug agent
This commit is contained in:
1 parent
2547ec45b4
commit
09c6fe2235
19 files changed
+203
-8
No files matched your search
@@ -15,6 +15,7 @@ common/
|
||||
├── user.nix # skyg.user.createSystemUser — creates the system user/groups + zsh shell
|
||||
├── fonts.nix # System font packages + fontconfig defaults (Fira Code, Noto, …); skyg.nixos.common.fonts.minimal
|
||||
├── ssh-server.nix # skyg.nixos.common.ssh-server — hardened sshd + master public keys (list)
|
||||
├── diagnix/ # skyg.nixos.common.diagnix — read-only user nbot uses to diagnose hosts
|
||||
|
||||
├── qemu.nix # skyg.core.qemu — QEMU/quickemu + SPICE guest tooling
|
||||
├── binary-cache.nix # skyg.nixos.common.cachePush — push finished builds to the bigbox2 cache
|
||||
@@ -38,6 +39,18 @@ common/
|
||||
- **`ssh-server.nix`** ships the master public keys (`masterPubKeys`, a list so a key can be
|
||||
rotated by adding the new one before dropping the old) and a hardened sshd config
|
||||
(key-only auth, long keepalives). Disabled by default.
|
||||
- **`diagnix/`** adds a read-only `diagnix` user (uid 2950) that nbot uses over SSH, via
|
||||
`lab-ssh` in the agent system (`server/services/nbot-agent.nix`). It reads logs
|
||||
(`systemd-journal` group), units, `/proc` and the store, and can't change anything. It
|
||||
can't read root-only files. Through sudo it can run only the fixed `diag-*` scripts
|
||||
(docker ps/logs, smart, ports, firewall, btrfs, io). The key is `restrict,pty` and only
|
||||
works from the nbot hosts (`hosts.nix`). Hosts running nbot also let their own containers
|
||||
reach port 22 and accept them from private IPs outside the LAN. Their IPs are random, so
|
||||
they can't be listed. Resources are capped (1G memory, 1 CPU, 256 tasks). The Nix daemon
|
||||
is left open on purpose: an allow-list would break nix-serve/buildbot.
|
||||
The private key (base64) is `DIAGNIX_KEY` in the `nbot-goose` secret; fwdesk passes it to
|
||||
every container via `agent_env`.
|
||||
`hosts.nix` also holds each host's address and host key; the agent pins those keys.
|
||||
|
||||
## Option Namespace
|
||||
|
||||
@@ -46,6 +59,7 @@ skyg.core.hostName → networking.nix
|
||||
skyg.user.createSystemUser → user.nix
|
||||
skyg.core.qemu.* → qemu.nix
|
||||
skyg.nixos.common.ssh-server.* → ssh-server.nix
|
||||
skyg.nixos.common.diagnix.* → diagnix/
|
||||
|
||||
skyg.nixos.common.cachePush.* → binary-cache.nix
|
||||
skyg.nixos.common.minimal → core.nix (skip heavy always-on system packages)
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
./user.nix
|
||||
./fonts.nix
|
||||
./ssh-server.nix
|
||||
./diagnix
|
||||
./qemu.nix
|
||||
./binary-cache.nix
|
||||
./pritunl
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
# diagnix: a read-only user nbot logs in as to diagnose a host.
|
||||
# Sees logs, units, /proc and /nix/store; can't change the system.
|
||||
{ config, lib, pkgs, options, ... }:
|
||||
let
|
||||
cfg = config.skyg.nixos.common.diagnix;
|
||||
data = import ./hosts.nix;
|
||||
uid = 2950;
|
||||
|
||||
# This host runs nbot, so its own containers connect from random private IPs.
|
||||
hasNbot = options.services ? nbot && config.services.nbot.enable;
|
||||
|
||||
keyLine = from: ''restrict,pty,from="${from}" ${cfg.publicKey}'';
|
||||
|
||||
runtime =
|
||||
if config.virtualisation.docker.enable then "${config.virtualisation.docker.package}/bin/docker"
|
||||
else if config.virtualisation.podman.enable then "${config.virtualisation.podman.package}/bin/podman"
|
||||
else null;
|
||||
rt = toString runtime;
|
||||
|
||||
# Fixed root scripts diagnix may run with sudo. Anything taking input checks it.
|
||||
script = name: inputs: text: pkgs.writeShellApplication {
|
||||
inherit name text;
|
||||
runtimeInputs = inputs;
|
||||
};
|
||||
scripts = [
|
||||
(script "diag-smart" [ pkgs.smartmontools pkgs.nvme-cli ] ''
|
||||
smartctl --scan | while read -r dev _; do
|
||||
echo "=== $dev"; smartctl -a "$dev" || true
|
||||
done
|
||||
for dev in /dev/nvme[0-9]; do
|
||||
[ -e "$dev" ] || continue
|
||||
echo "=== $dev"; nvme smart-log "$dev" || true
|
||||
done
|
||||
'')
|
||||
(script "diag-ports" [ pkgs.iproute2 ] "ss -tulpn")
|
||||
(script "diag-firewall" [ pkgs.iptables pkgs.nftables ] ''
|
||||
iptables -S || true; iptables -t nat -S || true; ip6tables -S || true
|
||||
nft list ruleset || true
|
||||
'')
|
||||
(script "diag-btrfs" [ pkgs.btrfs-progs pkgs.util-linux ] ''
|
||||
findmnt -rn -t btrfs -o TARGET | sort -u | while read -r mnt; do
|
||||
echo "=== $mnt"; btrfs filesystem usage "$mnt" || true; btrfs device stats "$mnt" || true
|
||||
done
|
||||
'')
|
||||
(script "diag-io" [ pkgs.iotop ] "iotop -b -o -n 3 -d 1")
|
||||
] ++ lib.optionals (runtime != null) [
|
||||
(script "diag-docker-ps" [ ] ''
|
||||
${rt} ps -a; ${rt} stats --no-stream; ${rt} system df
|
||||
'')
|
||||
(script "diag-docker-logs" [ ] ''
|
||||
if [ $# -ne 1 ] || ! [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then
|
||||
echo "usage: diag-docker-logs <container-name>" >&2; exit 2
|
||||
fi
|
||||
${rt} logs --tail 500 "$1" 2>&1
|
||||
'')
|
||||
];
|
||||
in
|
||||
{
|
||||
options.skyg.nixos.common.diagnix = {
|
||||
enable = lib.mkEnableOption "the read-only diagnix user for nbot";
|
||||
publicKey = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = data.publicKey;
|
||||
description = "nbot's client public key for diagnix.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
users.groups.diagnix.gid = uid;
|
||||
users.users.diagnix = {
|
||||
inherit uid;
|
||||
isNormalUser = true;
|
||||
group = "diagnix";
|
||||
description = "nbot read-only diagnostics";
|
||||
shell = pkgs.bash;
|
||||
extraGroups = [ "systemd-journal" ];
|
||||
openssh.authorizedKeys.keys =
|
||||
[ (keyLine (lib.concatStringsSep "," data.nbotHosts)) ]
|
||||
# Containers on this host: any private IP outside the LAN.
|
||||
++ lib.optional hasNbot (keyLine "!${data.lanSubnet},10.0.0.0/8,172.16.0.0/12,192.168.0.0/16");
|
||||
};
|
||||
|
||||
# A runaway command can't starve the host.
|
||||
systemd.slices."user-${toString uid}" = {
|
||||
overrideStrategy = "asDropin";
|
||||
sliceConfig = { MemoryMax = "1G"; CPUQuota = "100%"; TasksMax = 256; };
|
||||
};
|
||||
|
||||
security.sudo.extraRules = [{
|
||||
users = [ "diagnix" ];
|
||||
commands = map (s: { command = lib.getExe s; options = [ "NOPASSWD" ]; }) scripts;
|
||||
}];
|
||||
|
||||
programs.mtr.enable = true;
|
||||
environment.systemPackages = scripts ++ (with pkgs; [
|
||||
lsof
|
||||
pciutils
|
||||
usbutils
|
||||
dnsutils
|
||||
sysstat
|
||||
ethtool
|
||||
lm_sensors
|
||||
htop
|
||||
]);
|
||||
|
||||
# nbot drops all container traffic to the host except a few ports; let SSH in too.
|
||||
networking.firewall = lib.mkIf (hasNbot && !config.networking.nftables.enable) {
|
||||
extraCommands = lib.mkAfter ''
|
||||
iptables -C INPUT -i ve-+ -p tcp --dport 22 -j ACCEPT 2>/dev/null || iptables -I INPUT 1 -i ve-+ -p tcp --dport 22 -j ACCEPT
|
||||
'';
|
||||
extraStopCommands = ''
|
||||
iptables -D INPUT -i ve-+ -p tcp --dport 22 -j ACCEPT 2>/dev/null || true
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Lab hosts diagnix lives on, shared by the host module and nbot's agent system.
|
||||
# hostKey is the host's ssh_host_ed25519 public key (from secrets.nix);
|
||||
# null means the agent saves it on first connect.
|
||||
{
|
||||
# LAN addresses of the hosts that run nbot. The key only works from these.
|
||||
nbotHosts = [ "10.0.10.11" "10.0.10.13" ]; # hl-fwdesk, hl-fws1
|
||||
lanSubnet = "10.0.0.0/16";
|
||||
|
||||
# The nbot client public key. Private half (base64) is DIAGNIX_KEY in the
|
||||
# nbot-goose secret, sent to containers via agent_env on fwdesk.
|
||||
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPKsgkysVa3BiJQkUjiyESwjoi9ZMmUiWEB66JlbgIrO nbot-diagnix";
|
||||
|
||||
hosts = {
|
||||
hl-bigbox1 = { address = "bigbox1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFl5xiH4aFlmDWdoU+KmERLYOeOQK2aKP7jenY2G3rFb"; };
|
||||
hl-bigbox2 = { address = "bigbox2.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6l6VqQ23PkGkWVABBoLp1KA6PzkX5oKTy5RBHbQXet"; };
|
||||
hl-fwdesk = { address = "fwdesk.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHbBcdMpNPaELUzHRIoRkTUQde+mBQ87Ra2R+974FuUy"; };
|
||||
hl-fws1 = { address = "fws1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM6r0AIJyTlE+U6K97GijzAP4jNaAueVlwXbeMpEMcpe"; };
|
||||
hl-minipc1 = { address = "minipc1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP42vtV9Qo7q5F9M/rCGD/TIZG3d0mm1uP6lMvy3zF/a"; };
|
||||
hl-minipc2 = { address = "minipc2.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDqi2WT0pa8m+kZNN2Rrg1F01zwTvkCwD3wjSOBNvSjz"; };
|
||||
hl-minipc3 = { address = "minipc3.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEbvoe/UvhpRSWVXMCyj/qzuR7WnJdNF7CYRmmJE3yYu"; };
|
||||
hl-pi1 = { address = "pi1.lab.internal"; hostKey = null; };
|
||||
hl-pi2 = { address = "pi2.lab.internal"; hostKey = null; };
|
||||
hl-terra1 = { address = "terra1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHBKI8gqd7DDubShle4D3JGq61BP1NBKCZ81V3C9uyym"; };
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user