Files
nixos-setup/modules/nixos/server/robotnix/robots.nix
T
2026-10-26 14:36:03 -04:00

623 lines
22 KiB
Nix

# robotnix robots: storage, first install, nspawn units, snapshots, CLI.
#
# Layout per robot (all btrfs subvolumes):
# <baseDir>/<name>/root/root.img whole OS incl. /nix (hard size limit)
# <baseDir>/<name>/home/home.img /home (hard size limit)
# <baseDir>/<name>/snapshots/<id>/{root,home}
# The host mounts the images under /run/robotnix/<name>/ so it can freeze
# them for consistent snapshots while the robot runs.
{ config
, lib
, pkgs
, hermes-agent
, ...
}:
let
cfg = config.skyg.nixos.server.robotnix;
h = import ./helpers.nix { inherit config lib; };
rs = h.robotList;
sw = cfg.switch.address;
br = cfg.switch.name;
ext = cfg.externalInterface;
hostSystem = pkgs.stdenv.hostPlatform.system;
# The robot's flake pins the same nixpkgs the host used to build its seed.
nixpkgsRef =
let
rev = config.system.nixos.revision;
in
if rev != null then builtins.unsafeDiscardStringContext rev else "nixos-unstable";
keys = r: lib.concatMapStringsSep " " h.nixStr r.authorizedKeys;
# Ephemeral robots: kept folders live here, inside the home image.
persistDir = "/home/.persist";
# Robots pin the same hermes-agent revision the host has locked.
hermesRef = hermes-agent.rev or "main";
# Fixed ids so kept Hermes files stay owned right across reinstalls.
hermesId = 870;
# Secrets reach Hermes as container credentials, one per file.
hermesCreds = r: lib.imap0 (i: f: { id = "robotnix-hermes-${toString i}"; file = f; }) r.hermes.environmentFiles;
# Stock Hermes identity, with the robot's own name.
soul =
name:
"You are ${name}, a Hermes-based agent. Be direct: match the length of your reply to the weight of "
+ "the ask — a one-line question gets a one-line answer, and finished work gets a short report of what "
+ "changed, what's verified, and what's left, never a replay of the process. No filler (\"Great question,\" "
+ "\"I'd be happy to\"), no restating the request back, no re-summarizing what you already said, no narrating "
+ "tool calls the user can see. Plain claims over adjectives; when unsure, say so plainly. Agree because it's "
+ "right, not because the user said it. Depth is earned — give it when the user asks for detail, teaches, or "
+ "the stakes demand it, not by default.\n";
tz = lib.optionalString
(
config.time.timeZone != null
) "time.timeZone = ${h.nixStr config.time.timeZone};";
# The host-owned part of the robot's config -> /etc/nixos/robotnix.nix.
robotnixNix =
r:
let
gw = r.hermes.gateway;
db = r.hermes.dashboard;
gwOn = r.hermes.enable && gw.enable;
dbOn = r.hermes.enable && db.enable;
openPorts = lib.optional (gwOn && gw.openFirewall) gw.port ++ lib.optional (dbOn && db.openFirewall) db.port;
envFiles = "[ ${lib.concatMapStringsSep " " (c: h.nixStr "/run/credentials/@system/${c.id}") (hermesCreds r)} ]";
gwEnv = {
API_SERVER_ENABLED = "true";
API_SERVER_HOST = "0.0.0.0";
API_SERVER_PORT = toString gw.port;
API_SERVER_MODEL_NAME = if r.hermes.name != null then r.hermes.name else r.name;
};
in
builtins.toFile "robotnix-${r.name}.nix" ''
# Generated by robotnix on the host (first install / reseed, or a host
# rebuild if you changed the robot's host-side settings).
# The network and git-sync values must match the host's robotnix
# settings; change them and this robot loses its network or its
# auto-commits stop. Everything else is yours.
{ lib, pkgs, ... }:
{
boot.isContainer = true;
nixpkgs.hostPlatform = ${h.nixStr hostSystem};
networking.hostName = ${h.nixStr r.name};
# Address and route handed out by the host. IPv4 only.
networking.useDHCP = false;
networking.useNetworkd = true;
networking.enableIPv6 = false;
networking.useHostResolvConf = lib.mkForce false;
networking.nameservers = [ ${h.nixStr sw} ];
systemd.network.networks."10-host0" = {
matchConfig.Name = "host0";
address = [ ${h.nixStr "${r.address}/32"} ];
routes = [ { Gateway = ${h.nixStr sw}; GatewayOnLink = true; } ];
networkConfig.LinkLocalAddressing = "no";
};
nix.settings.experimental-features = [ "nix-command" "flakes" ];
services.openssh = {
enable = true;
settings.PasswordAuthentication = false;
settings.KbdInteractiveAuthentication = false;
};
users.users.${h.nixStr r.user} = {
isNormalUser = true;
extraGroups = [ "wheel" ${lib.optionalString r.hermes.enable ''"hermes" ''}];
openssh.authorizedKeys.keys = [ ${keys r} ];
};
users.users.root.openssh.authorizedKeys.keys = [ ${keys r} ];
${tz}
system.stateVersion = ${h.nixStr r.seed.stateVersion};
${lib.optionalString r.hermes.enable ''
# Hermes agent, managed by the host (robotnix hermes options).
services.hermes-agent = {
enable = true;
addToSystemPackages = true;
settings = builtins.fromJSON ${h.nixStr (builtins.toJSON r.hermes.settings)};
${lib.optionalString (r.hermes.name != null) ''
hermesHomeFiles."SOUL.md" = ${h.nixStr (soul r.hermes.name)};
''}
${lib.optionalString gwOn ''environment = { ${lib.concatStringsSep " " (lib.mapAttrsToList (k: v: "${k} = ${h.nixStr v};") gwEnv)} }; # gateway API, key from the secrets''}
${lib.optionalString dbOn ''backend = { mode = "dashboard"; host = "0.0.0.0"; port = ${toString db.port}; }; # login from the secrets''}
};
users.users.hermes.uid = ${toString hermesId};
users.groups.hermes.gid = ${toString hermesId};
# The module doesn't make skills/ (kept on ephemeral robots).
systemd.tmpfiles.rules = [ "d ${h.hermesHome}/skills 2770 hermes hermes - -" ];
# Secrets: container credentials from the host, read at start.
systemd.services.hermes-agent.serviceConfig.EnvironmentFile = ${envFiles};
${lib.optionalString dbOn ''systemd.services.hermes-backend.serviceConfig.EnvironmentFile = ${envFiles};''}
${lib.optionalString (dbOn && db.port < 1024) ''
# Ports below 1024 need this as the hermes user.
systemd.services.hermes-backend.serviceConfig.AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
''}
${lib.optionalString (openPorts != [ ]) ''networking.firewall.allowedTCPPorts = [ ${lib.concatMapStringsSep " " toString openPorts} ];''}
''}
${lib.optionalString r.ephemeral ''
# Ephemeral robot: root resets on every start. Keep the ssh host
# keys in /home so they don't change each boot.
services.openssh.hostKeys = [
{ type = "ed25519"; path = "${persistDir}/ssh/ssh_host_ed25519_key"; }
{ type = "rsa"; bits = 4096; path = "${persistDir}/ssh/ssh_host_rsa_key"; }
];
''}
${lib.optionalString r.git.enable ''
# git-tracking your flake: auto-commit, and push when a remote is set.
environment.systemPackages = [ pkgs.git ];
systemd.services."robotnix-flake-gitsync" = {
path = [ pkgs.git ];
script = '''
cd /etc/nixos
git add -A
git diff --cached --quiet || git commit -m "robotnix auto-sync $(date +'%Y-%m-%d %H:%M:%S')"
if git remote get-url origin >/dev/null 2>&1; then
git push -q
fi
''';
serviceConfig = {
Type = "oneshot";
User = "root";
};
};
systemd.timers."robotnix-flake-gitsync" = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = ${h.nixStr r.git.interval};
Persistent = true;
Unit = "robotnix-flake-gitsync.service";
};
};
''}
}
'';
flakeNix =
r:
builtins.toFile "robotnix-${r.name}-flake.nix" ''
# ${r.name}'s own flake. Apply changes with:
# sudo nixos-rebuild switch --flake /etc/nixos
{
inputs.nixpkgs.url = "github:NixOS/nixpkgs/${nixpkgsRef}";
${lib.optionalString r.hermes.enable ''inputs.hermes-agent.url = "github:NousResearch/hermes-agent/${hermesRef}";''}
outputs = { nixpkgs, ... }@inputs: {
nixosConfigurations.${h.nixStr r.name} = nixpkgs.lib.nixosSystem {
modules = [
./robotnix.nix
./configuration.nix
${lib.optionalString r.hermes.enable "inputs.hermes-agent.nixosModules.default"}
];
};
};
}
'';
# The first system, built on the host exactly as the robot's own flake
# builds it later: lib.nixosSystem is eval-config with system = null and
# nixpkgs' lib, so this mirrors flake.nix's `nixpkgs.lib.nixosSystem` call.
seedSystem =
r:
(import "${pkgs.path}/nixos/lib/eval-config.nix" {
system = null;
inherit lib;
modules = [
(robotnixNix r)
r.seed.configuration
]
++ lib.optional r.hermes.enable hermes-agent.nixosModules.default;
}).config.system.build.toplevel;
# Ephemeral robots: fingerprint of everything the seed installs. When it
# changes, the clean root is rebuilt.
seedId =
r:
builtins.hashString "sha256" (
lib.concatStringsSep " " [
"${seedSystem r}"
"${flakeNix r}"
"${robotnixNix r}"
"${r.seed.configuration}"
]
);
storageStart =
r:
pkgs.writeShellScript "${r.unit}-storage-start" (
''
set -euo pipefail
base=${r.base}
mkdir -p ${cfg.baseDir}
if [ "$(stat -f -c %T ${cfg.baseDir})" != btrfs ]; then
echo "robotnix: ${cfg.baseDir} must be on btrfs (needed for snapshots)" >&2
exit 1
fi
for sub in "$base" "$base/root" "$base/home" "$base/snapshots"; do
[ -d "$sub" ] || btrfs subvolume create "$sub"
done
# No copy-on-write for the images (only affects new files).
chattr +C "$base/root" "$base/home"
''
+ lib.optionalString r.ephemeral ''
# Ephemeral: seed changed (or first run) -> wipe root so the seed step
# reinstalls and saves a new clean root.
if ! mountpoint -q ${r.run}/root \
&& [ "$(cat "$base/pristine.id" 2>/dev/null)" != ${seedId r} ]; then
echo "robotnix: seed for ${r.name} changed, reinstalling"
rm -f "$base/pristine.id"
if [ -d "$base/pristine" ]; then btrfs subvolume delete "$base/pristine"; fi
btrfs subvolume delete "$base/root"
btrfs subvolume create "$base/root"
chattr +C "$base/root"
fi
''
+ ''
image() { # <img> <size> <mountpoint>
img=$1 size=$2 dir=$3
if [ ! -e "$img" ]; then
truncate -s "$size" "$img"
mkfs.ext4 -q -F "$img"
fi
mkdir -p -m 0755 "$dir"
if ! mountpoint -q "$dir"; then
rc=0
e2fsck -p "$img" || rc=$?
if [ "$rc" -ge 4 ]; then
echo "robotnix: $img needs a manual e2fsck" >&2
exit 1
fi
mount -o loop,noatime "$img" "$dir"
fi
# Grow when the configured size goes up. Shrinking is not supported.
want=$(numfmt --from=iec "$size")
have=$(stat -c %s "$img")
if [ "$want" -gt "$have" ]; then
truncate -s "$size" "$img"
dev=$(findmnt -n -o SOURCE "$dir")
losetup -c "$dev"
resize2fs "$dev"
elif [ "$want" -lt "$have" ]; then
echo "robotnix: $img is bigger than $size; shrinking is not supported" >&2
fi
}
mkdir -p /run/robotnix ${r.run}
chmod 0755 /run/robotnix ${r.run}
image "$base/root/root.img" ${r.disk.rootSize} ${r.run}/root
image "$base/home/home.img" ${r.disk.homeSize} ${r.run}/home
chmod 0755 ${r.run}/root
''
+ lib.optionalString (r.ephemeral || r.persist != [ ]) ''
install -d -m 0700 ${r.run}${persistDir}
''
+ lib.concatMapStrings
(p: ''
mkdir -p ${lib.escapeShellArg "${r.run}${persistDir}${p}"}
'')
r.persist
);
storageStop =
r:
pkgs.writeShellScript "${r.unit}-storage-stop" ''
for d in ${r.run}/home ${r.run}/root; do
if mountpoint -q "$d"; then umount "$d"; fi
done
'';
# Installs the seed only when the root image has no system yet (first
# start, or after `robotnix reseed`). Never touches an installed robot.
# Ephemeral robots also save the fresh root as their clean copy.
seedScript =
r:
pkgs.writeShellScript "${r.unit}-seed" (
''
set -euo pipefail
root=${r.run}/root
if [ ! -e "$root/nix/var/nix/profiles/system" ]; then
echo "robotnix: installing the seed system into ${r.name}"
nixos-install --root "$root" --system ${seedSystem r} \
--no-root-passwd --no-bootloader --no-channel-copy
install -d -m 0755 "$root/etc/nixos"
install -m 0644 ${flakeNix r} "$root/etc/nixos/flake.nix"
install -m 0644 ${robotnixNix r} "$root/etc/nixos/robotnix.nix"
install -m 0644 ${r.seed.configuration} "$root/etc/nixos/configuration.nix"
${lib.optionalString r.git.enable ''
echo "robotnix: setting up the git repo for ${r.name}'s flake"
git -C "$root/etc/nixos" init -q
git -C "$root/etc/nixos" config user.name ${h.nixStr r.git.userName}
git -C "$root/etc/nixos" config user.email ${h.nixStr r.git.userEmail}
${lib.optionalString (r.git.remoteUrl != null) ''
git -C "$root/etc/nixos" remote add origin ${h.nixStr r.git.remoteUrl}
''}
git -C "$root/etc/nixos" add -A
git -C "$root/etc/nixos" commit -q -m "robotnix initial seed" || true
''}
fi
# What nspawn and NixOS's init expect (same as nixos-containers).
mkdir -p "$root/etc" "$root/var/lib" "$root/root" "$root/home"
mkdir -p -m 0755 "$root/usr/bin"
chmod 0700 "$root/root"
[ -e "$root/etc/os-release" ] || [ -L "$root/etc/os-release" ] || touch "$root/etc/os-release"
[ -e "$root/etc/machine-id" ] || touch "$root/etc/machine-id"
''
+ lib.optionalString r.ephemeral ''
# Ephemeral: save the clean root once, before the robot ever boots.
if [ ! -e ${r.base}/pristine.id ]; then
echo "robotnix: saving ${r.name}'s clean root"
if [ -d ${r.base}/pristine ]; then btrfs subvolume delete ${r.base}/pristine; fi
sync -f "$root"
fsfreeze -f "$root"
rc=0
btrfs subvolume snapshot -r ${r.base}/root ${r.base}/pristine || rc=$?
fsfreeze -u "$root"
[ "$rc" -eq 0 ]
echo ${seedId r} > ${r.base}/pristine.id
fi
''
);
# Ephemeral: swap root for a fresh copy of the clean root. Runs (as its
# own unit) before every start, including reboots from inside the robot.
resetRoot =
r:
pkgs.writeShellScript "${r.unit}-reset" ''
set -euo pipefail
base=${r.base}
root=${r.run}/root
if [ "$(cat "$base/pristine.id" 2>/dev/null)" != ${seedId r} ] || [ ! -d "$base/pristine" ]; then
echo "robotnix: no clean root for ${r.name}; restart its storage to reinstall" >&2
exit 1
fi
if mountpoint -q "$root"; then umount "$root"; fi
# Wait for the loop device to let go of the old image.
for _ in $(seq 1 20); do
[ -z "$(losetup -j "$base/root/root.img")" ] && break
sleep 0.5
done
btrfs subvolume delete "$base/root"
btrfs subvolume snapshot "$base/pristine" "$base/root"
mount -o loop,noatime "$base/root/root.img" "$root"
'';
preStart =
r:
pkgs.writeShellScript "${r.unit}-pre" ''
set -euo pipefail
for _ in $(seq 1 30); do
[ -e /sys/class/net/${br} ] && [ -e /sys/class/net/${ext} ] && break
sleep 1
done
[ -e /sys/class/net/${br} ] || { echo "robotnix: ${br} is missing" >&2; exit 1; }
# Answer ARP for the robot's LAN IP on the uplink (this IP only).
ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true
ip neigh add proxy ${r.address} dev ${ext}
'';
postStop =
r:
pkgs.writeShellScript "${r.unit}-post" ''
ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true
'';
idmap = r: lib.optionalString r.privateUsers ":idmap";
nspawnArgs =
r:
lib.escapeShellArgs (
[
"--keep-unit"
"--machine=${r.name}"
"--directory=${r.run}/root"
"--notify-ready=yes"
"--kill-signal=SIGRTMIN+3"
"--network-bridge=${br}"
"--link-journal=try-guest"
"--resolv-conf=off"
"--timezone=off"
"--bind=${r.run}/home:/home${idmap r}"
]
# Kept folders of an ephemeral robot, stored inside the home image.
++ map (p: "--bind=${r.run}${persistDir}${p}:${p}${idmap r}") r.persist
++ lib.optionals r.privateUsers [
"--private-users=pick"
"--private-users-ownership=auto"
]
# Docker needs the kernel keyring and bpf, which nspawn blocks by default.
++ lib.optional r.allowDocker "--system-call-filter=add_key keyctl bpf"
++ lib.optional r.allowTun "--bind=/dev/net/tun"
++ map (c: "--load-credential=${c.id}:${c.file}") (hermesCreds r)
++ [ "/nix/var/nix/profiles/system/init" ]
);
robotUnits = r: {
"${r.unit}-storage" = {
description = "robotnix storage for ${r.name}";
path = with pkgs; [
btrfs-progs
coreutils
e2fsprogs
util-linux
];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = storageStart r;
ExecStop = storageStop r;
};
};
"${r.unit}-seed" = {
description = "robotnix first install for ${r.name}";
requires = [ "${r.unit}-storage.service" ];
bindsTo = [ "${r.unit}-storage.service" ];
after = [ "${r.unit}-storage.service" ];
path = [
config.system.build.nixos-install
config.nix.package
pkgs.coreutils
pkgs.util-linux
pkgs.git
pkgs.btrfs-progs
];
environment.HOME = "/root";
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = seedScript r;
TimeoutStartSec = "30min";
};
};
# Ephemeral: fresh root before every robot start. Its own unit because
# the robot unit blocks devices, and mounting the image needs loop
# devices. No RemainAfterExit, so each robot start runs it again.
"${r.unit}-reset" = lib.mkIf r.ephemeral {
description = "robotnix fresh root for ${r.name}";
requires = [
"${r.unit}-storage.service"
"${r.unit}-seed.service"
];
bindsTo = [ "${r.unit}-storage.service" ];
after = [
"${r.unit}-storage.service"
"${r.unit}-seed.service"
];
path = with pkgs; [
btrfs-progs
coreutils
util-linux
];
serviceConfig = {
Type = "oneshot";
ExecStart = resetRoot r;
};
};
${r.unit} =
let
mem = h.toBytes r.memory.max;
deps = [
"${r.unit}-storage.service"
"${r.unit}-seed.service"
"${r.unit}-dns.service"
]
++ lib.optional r.ephemeral "${r.unit}-reset.service";
in
{
description = "robotnix robot ${r.name}";
wantedBy = lib.optional r.autoStart "machines.target";
requires = deps;
bindsTo = [ "${r.unit}-storage.service" ];
after = deps ++ [
"network.target"
"systemd-networkd.service"
"nftables.service"
];
path = [
pkgs.iproute2
pkgs.coreutils
];
environment.SYSTEMD_NSPAWN_UNIFIED_HIERARCHY = "1";
serviceConfig = {
ExecStartPre = preStart r;
ExecStart = "${config.systemd.package}/bin/systemd-nspawn ${nspawnArgs r}";
# Best effort: refresh LAN ARP caches.
ExecStartPost = "-${pkgs.iputils}/bin/arping -U -c 2 -I ${ext} ${r.address}";
ExecStopPost = postStop r;
# Same as nixos-containers.
Type = "notify";
RestartForceExitStatus = "133";
SuccessExitStatus = "133";
Restart = "on-failure";
TimeoutStartSec = "5min";
Slice = "machine.slice";
Delegate = true;
KillMode = "mixed";
KillSignal = "TERM";
DevicePolicy = "closed";
DeviceAllow = lib.mkIf r.allowTun [ "/dev/net/tun rwm" ];
# Resources.
CPUQuota = "${toString (r.cpus.max * 100)}%";
CPUWeight = toString (lib.min 10000 (100 * r.cpus.min));
MemoryLow = r.memory.min;
MemoryHigh = toString (mem - mem / 16);
MemoryMax = r.memory.max;
MemorySwapMax = r.memory.swapMax;
TasksMax = 16384;
};
};
};
cliConfig = pkgs.writeText "robotnix.json" (
builtins.toJSON {
inherit (cfg) baseDir;
robots = lib.mapAttrs
(_: r: {
inherit (r)
name
address
port
dnsPort
base
run
unit
user
nft
access
ephemeral
;
snapshots = { inherit (r.snapshots) enable interval keep; };
})
h.robots;
}
);
cliPath = lib.makeBinPath [
pkgs.btrfs-progs
pkgs.util-linux
pkgs.coreutils
pkgs.nftables
config.systemd.package
];
cli = pkgs.writers.writeNuBin "robotnix" (
''
const CONFIG = "${cliConfig}"
$env.PATH = ("${cliPath}" | split row ":" | append ($env.PATH | split row (char esep)))
''
+ builtins.readFile ./robotnix.nu
);
in
lib.mkIf cfg.enable {
systemd.services = lib.mkMerge (map robotUnits rs);
systemd.targets.multi-user.wants = [ "machines.target" ];
# Memory protection only works if the parent slice is protected too.
systemd.slices.machine.sliceConfig.MemoryLow = lib.mkIf (rs != [ ]) (
lib.mkDefault (toString (lib.foldl' (acc: r: acc + h.toBytes r.memory.min) 0 rs))
);
skyg.server.timers = lib.listToAttrs (
map
(
r:
lib.nameValuePair "${r.unit}-snapshot" {
script = "${cli}/bin/robotnix snapshot ${r.name} --auto";
OnCalendar = r.snapshots.interval;
}
)
(lib.filter (r: r.snapshots.enable) rs)
);
environment.systemPackages = [ cli ];
}