debug agent
This commit is contained in:
1 parent
2547ec45b4
commit
09c6fe2235
19 files changed
+203
-8
No files matched your search
@@ -15,6 +15,7 @@ common/
|
||||
├── user.nix # skyg.user.createSystemUser — creates the system user/groups + zsh shell
|
||||
├── fonts.nix # System font packages + fontconfig defaults (Fira Code, Noto, …); skyg.nixos.common.fonts.minimal
|
||||
├── ssh-server.nix # skyg.nixos.common.ssh-server — hardened sshd + master public keys (list)
|
||||
├── diagnix/ # skyg.nixos.common.diagnix — read-only user nbot uses to diagnose hosts
|
||||
|
||||
├── qemu.nix # skyg.core.qemu — QEMU/quickemu + SPICE guest tooling
|
||||
├── binary-cache.nix # skyg.nixos.common.cachePush — push finished builds to the bigbox2 cache
|
||||
@@ -38,6 +39,18 @@ common/
|
||||
- **`ssh-server.nix`** ships the master public keys (`masterPubKeys`, a list so a key can be
|
||||
rotated by adding the new one before dropping the old) and a hardened sshd config
|
||||
(key-only auth, long keepalives). Disabled by default.
|
||||
- **`diagnix/`** adds a read-only `diagnix` user (uid 2950) that nbot uses over SSH, via
|
||||
`lab-ssh` in the agent system (`server/services/nbot-agent.nix`). It reads logs
|
||||
(`systemd-journal` group), units, `/proc` and the store, and can't change anything. It
|
||||
can't read root-only files. Through sudo it can run only the fixed `diag-*` scripts
|
||||
(docker ps/logs, smart, ports, firewall, btrfs, io). The key is `restrict,pty` and only
|
||||
works from the nbot hosts (`hosts.nix`). Hosts running nbot also let their own containers
|
||||
reach port 22 and accept them from private IPs outside the LAN. Their IPs are random, so
|
||||
they can't be listed. Resources are capped (1G memory, 1 CPU, 256 tasks). The Nix daemon
|
||||
is left open on purpose: an allow-list would break nix-serve/buildbot.
|
||||
The private key (base64) is `DIAGNIX_KEY` in the `nbot-goose` secret; fwdesk passes it to
|
||||
every container via `agent_env`.
|
||||
`hosts.nix` also holds each host's address and host key; the agent pins those keys.
|
||||
|
||||
## Option Namespace
|
||||
|
||||
@@ -46,6 +59,7 @@ skyg.core.hostName → networking.nix
|
||||
skyg.user.createSystemUser → user.nix
|
||||
skyg.core.qemu.* → qemu.nix
|
||||
skyg.nixos.common.ssh-server.* → ssh-server.nix
|
||||
skyg.nixos.common.diagnix.* → diagnix/
|
||||
|
||||
skyg.nixos.common.cachePush.* → binary-cache.nix
|
||||
skyg.nixos.common.minimal → core.nix (skip heavy always-on system packages)
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
./user.nix
|
||||
./fonts.nix
|
||||
./ssh-server.nix
|
||||
./diagnix
|
||||
./qemu.nix
|
||||
./binary-cache.nix
|
||||
./pritunl
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
# diagnix: a read-only user nbot logs in as to diagnose a host.
|
||||
# Sees logs, units, /proc and /nix/store; can't change the system.
|
||||
{ config, lib, pkgs, options, ... }:
|
||||
let
|
||||
cfg = config.skyg.nixos.common.diagnix;
|
||||
data = import ./hosts.nix;
|
||||
uid = 2950;
|
||||
|
||||
# This host runs nbot, so its own containers connect from random private IPs.
|
||||
hasNbot = options.services ? nbot && config.services.nbot.enable;
|
||||
|
||||
keyLine = from: ''restrict,pty,from="${from}" ${cfg.publicKey}'';
|
||||
|
||||
runtime =
|
||||
if config.virtualisation.docker.enable then "${config.virtualisation.docker.package}/bin/docker"
|
||||
else if config.virtualisation.podman.enable then "${config.virtualisation.podman.package}/bin/podman"
|
||||
else null;
|
||||
rt = toString runtime;
|
||||
|
||||
# Fixed root scripts diagnix may run with sudo. Anything taking input checks it.
|
||||
script = name: inputs: text: pkgs.writeShellApplication {
|
||||
inherit name text;
|
||||
runtimeInputs = inputs;
|
||||
};
|
||||
scripts = [
|
||||
(script "diag-smart" [ pkgs.smartmontools pkgs.nvme-cli ] ''
|
||||
smartctl --scan | while read -r dev _; do
|
||||
echo "=== $dev"; smartctl -a "$dev" || true
|
||||
done
|
||||
for dev in /dev/nvme[0-9]; do
|
||||
[ -e "$dev" ] || continue
|
||||
echo "=== $dev"; nvme smart-log "$dev" || true
|
||||
done
|
||||
'')
|
||||
(script "diag-ports" [ pkgs.iproute2 ] "ss -tulpn")
|
||||
(script "diag-firewall" [ pkgs.iptables pkgs.nftables ] ''
|
||||
iptables -S || true; iptables -t nat -S || true; ip6tables -S || true
|
||||
nft list ruleset || true
|
||||
'')
|
||||
(script "diag-btrfs" [ pkgs.btrfs-progs pkgs.util-linux ] ''
|
||||
findmnt -rn -t btrfs -o TARGET | sort -u | while read -r mnt; do
|
||||
echo "=== $mnt"; btrfs filesystem usage "$mnt" || true; btrfs device stats "$mnt" || true
|
||||
done
|
||||
'')
|
||||
(script "diag-io" [ pkgs.iotop ] "iotop -b -o -n 3 -d 1")
|
||||
] ++ lib.optionals (runtime != null) [
|
||||
(script "diag-docker-ps" [ ] ''
|
||||
${rt} ps -a; ${rt} stats --no-stream; ${rt} system df
|
||||
'')
|
||||
(script "diag-docker-logs" [ ] ''
|
||||
if [ $# -ne 1 ] || ! [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then
|
||||
echo "usage: diag-docker-logs <container-name>" >&2; exit 2
|
||||
fi
|
||||
${rt} logs --tail 500 "$1" 2>&1
|
||||
'')
|
||||
];
|
||||
in
|
||||
{
|
||||
options.skyg.nixos.common.diagnix = {
|
||||
enable = lib.mkEnableOption "the read-only diagnix user for nbot";
|
||||
publicKey = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = data.publicKey;
|
||||
description = "nbot's client public key for diagnix.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
users.groups.diagnix.gid = uid;
|
||||
users.users.diagnix = {
|
||||
inherit uid;
|
||||
isNormalUser = true;
|
||||
group = "diagnix";
|
||||
description = "nbot read-only diagnostics";
|
||||
shell = pkgs.bash;
|
||||
extraGroups = [ "systemd-journal" ];
|
||||
openssh.authorizedKeys.keys =
|
||||
[ (keyLine (lib.concatStringsSep "," data.nbotHosts)) ]
|
||||
# Containers on this host: any private IP outside the LAN.
|
||||
++ lib.optional hasNbot (keyLine "!${data.lanSubnet},10.0.0.0/8,172.16.0.0/12,192.168.0.0/16");
|
||||
};
|
||||
|
||||
# A runaway command can't starve the host.
|
||||
systemd.slices."user-${toString uid}" = {
|
||||
overrideStrategy = "asDropin";
|
||||
sliceConfig = { MemoryMax = "1G"; CPUQuota = "100%"; TasksMax = 256; };
|
||||
};
|
||||
|
||||
security.sudo.extraRules = [{
|
||||
users = [ "diagnix" ];
|
||||
commands = map (s: { command = lib.getExe s; options = [ "NOPASSWD" ]; }) scripts;
|
||||
}];
|
||||
|
||||
programs.mtr.enable = true;
|
||||
environment.systemPackages = scripts ++ (with pkgs; [
|
||||
lsof
|
||||
pciutils
|
||||
usbutils
|
||||
dnsutils
|
||||
sysstat
|
||||
ethtool
|
||||
lm_sensors
|
||||
htop
|
||||
]);
|
||||
|
||||
# nbot drops all container traffic to the host except a few ports; let SSH in too.
|
||||
networking.firewall = lib.mkIf (hasNbot && !config.networking.nftables.enable) {
|
||||
extraCommands = lib.mkAfter ''
|
||||
iptables -C INPUT -i ve-+ -p tcp --dport 22 -j ACCEPT 2>/dev/null || iptables -I INPUT 1 -i ve-+ -p tcp --dport 22 -j ACCEPT
|
||||
'';
|
||||
extraStopCommands = ''
|
||||
iptables -D INPUT -i ve-+ -p tcp --dport 22 -j ACCEPT 2>/dev/null || true
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Lab hosts diagnix lives on, shared by the host module and nbot's agent system.
|
||||
# hostKey is the host's ssh_host_ed25519 public key (from secrets.nix);
|
||||
# null means the agent saves it on first connect.
|
||||
{
|
||||
# LAN addresses of the hosts that run nbot. The key only works from these.
|
||||
nbotHosts = [ "10.0.10.11" "10.0.10.13" ]; # hl-fwdesk, hl-fws1
|
||||
lanSubnet = "10.0.0.0/16";
|
||||
|
||||
# The nbot client public key. Private half (base64) is DIAGNIX_KEY in the
|
||||
# nbot-goose secret, sent to containers via agent_env on fwdesk.
|
||||
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPKsgkysVa3BiJQkUjiyESwjoi9ZMmUiWEB66JlbgIrO nbot-diagnix";
|
||||
|
||||
hosts = {
|
||||
hl-bigbox1 = { address = "bigbox1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFl5xiH4aFlmDWdoU+KmERLYOeOQK2aKP7jenY2G3rFb"; };
|
||||
hl-bigbox2 = { address = "bigbox2.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6l6VqQ23PkGkWVABBoLp1KA6PzkX5oKTy5RBHbQXet"; };
|
||||
hl-fwdesk = { address = "fwdesk.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHbBcdMpNPaELUzHRIoRkTUQde+mBQ87Ra2R+974FuUy"; };
|
||||
hl-fws1 = { address = "fws1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM6r0AIJyTlE+U6K97GijzAP4jNaAueVlwXbeMpEMcpe"; };
|
||||
hl-minipc1 = { address = "minipc1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP42vtV9Qo7q5F9M/rCGD/TIZG3d0mm1uP6lMvy3zF/a"; };
|
||||
hl-minipc2 = { address = "minipc2.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDqi2WT0pa8m+kZNN2Rrg1F01zwTvkCwD3wjSOBNvSjz"; };
|
||||
hl-minipc3 = { address = "minipc3.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEbvoe/UvhpRSWVXMCyj/qzuR7WnJdNF7CYRmmJE3yYu"; };
|
||||
hl-pi1 = { address = "pi1.lab.internal"; hostKey = null; };
|
||||
hl-pi2 = { address = "pi2.lab.internal"; hostKey = null; };
|
||||
hl-terra1 = { address = "terra1.lab.internal"; hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHBKI8gqd7DDubShle4D3JGq61BP1NBKCZ81V3C9uyym"; };
|
||||
};
|
||||
}
|
||||
@@ -1,11 +1,38 @@
|
||||
# nbot's agent system with our additions. Use it for nbot's `system`.
|
||||
# nbot's agent system with our additions (gh, lab-ssh). Use it for nbot's `system`.
|
||||
{ lib, nbot, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
lab = import ../../common/diagnix/hosts.nix;
|
||||
|
||||
# lab-ssh <host> [cmd...]: log in to a lab host as the read-only diagnix user.
|
||||
# The key comes base64-encoded in $DIAGNIX_KEY (agent_env, from the nbot-goose secret).
|
||||
labSsh = pkgs: pkgs.writeShellApplication {
|
||||
name = "lab-ssh";
|
||||
runtimeInputs = [ pkgs.openssh pkgs.coreutils ];
|
||||
text = ''
|
||||
host="''${1:-}"
|
||||
case "$host" in
|
||||
${concatStrings (mapAttrsToList (name: h: ''
|
||||
${name}|${removePrefix "hl-" name}) addr=${h.address}; check=${if h.hostKey == null then "accept-new" else "yes"} ;;
|
||||
'') lab.hosts)}
|
||||
*) echo "usage: lab-ssh <host> [command...]; hosts: ${concatStringsSep " " (attrNames lab.hosts)}" >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
: "''${DIAGNIX_KEY:?DIAGNIX_KEY missing from agent_env}"
|
||||
key="$HOME/.ssh/diagnix"
|
||||
mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
|
||||
(umask 077; base64 -d <<<"$DIAGNIX_KEY" > "$key")
|
||||
exec ssh -i "$key" -o IdentitiesOnly=yes -o StrictHostKeyChecking="$check" "diagnix@$addr" "$@"
|
||||
'';
|
||||
};
|
||||
|
||||
agentModule = { pkgs, ... }: {
|
||||
environment.systemPackages = [ pkgs.gh ];
|
||||
environment.systemPackages = [ pkgs.gh (labSsh pkgs) ];
|
||||
programs.ssh.knownHosts = mapAttrs
|
||||
(name: h: { hostNames = [ name h.address ]; publicKey = h.hostKey; })
|
||||
(filterAttrs (_: h: h.hostKey != null) lab.hosts);
|
||||
# gh reads GH_TOKEN (from agent_env); git over HTTPS uses it through gh.
|
||||
programs.git = {
|
||||
enable = true;
|
||||
@@ -15,13 +42,13 @@ let
|
||||
in
|
||||
{
|
||||
options.skyg.nixos.server.services.nbot-agent.system = mkOption {
|
||||
description = "nbot's agent system with gh added. Use for nbot's `system`.";
|
||||
description = "nbot's agent system with gh and lab-ssh added. Use for nbot's `system`.";
|
||||
readOnly = true;
|
||||
type = types.package;
|
||||
default = (nbot.nixosConfigurations.agent.extendModules {
|
||||
modules = [ agentModule ];
|
||||
}).config.system.build.toplevel;
|
||||
# Keeps the manual from building it on every host.
|
||||
defaultText = literalExpression "nbot agent + gh";
|
||||
defaultText = literalExpression "nbot agent + gh + lab-ssh";
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user