# robotnix robots: storage, first install, nspawn units, snapshots, CLI. # # Layout per robot (all btrfs subvolumes): # //root/root.img whole OS incl. /nix (hard size limit) # //home/home.img /home (hard size limit) # //snapshots//{root,home} # The host mounts the images under /run/robotnix// so it can freeze # them for consistent snapshots while the robot runs. { config , lib , pkgs , hermes-agent , ... }: let cfg = config.skyg.nixos.server.robotnix; h = import ./helpers.nix { inherit config lib; }; rs = h.robotList; sw = cfg.switch.address; br = cfg.switch.name; ext = cfg.externalInterface; hostSystem = pkgs.stdenv.hostPlatform.system; # The robot's flake pins the same nixpkgs the host used to build its seed. nixpkgsRef = let rev = config.system.nixos.revision; in if rev != null then builtins.unsafeDiscardStringContext rev else "nixos-unstable"; keys = r: lib.concatMapStringsSep " " h.nixStr r.authorizedKeys; # Ephemeral robots: kept folders live here, inside the home image. persistDir = "/home/.persist"; # Robots pin the same hermes-agent revision the host has locked. hermesRef = hermes-agent.rev or "main"; # Fixed ids so kept Hermes files stay owned right across reinstalls. hermesId = 870; # Secrets reach Hermes as container credentials, one per file. hermesCreds = r: lib.imap0 (i: f: { id = "robotnix-hermes-${toString i}"; file = f; }) r.hermes.environmentFiles; # Stock Hermes identity, with the robot's own name. soul = name: "You are ${name}, a Hermes-based agent. Be direct: match the length of your reply to the weight of " + "the ask — a one-line question gets a one-line answer, and finished work gets a short report of what " + "changed, what's verified, and what's left, never a replay of the process. No filler (\"Great question,\" " + "\"I'd be happy to\"), no restating the request back, no re-summarizing what you already said, no narrating " + "tool calls the user can see. Plain claims over adjectives; when unsure, say so plainly. Agree because it's " + "right, not because the user said it. Depth is earned — give it when the user asks for detail, teaches, or " + "the stakes demand it, not by default.\n"; tz = lib.optionalString ( config.time.timeZone != null ) "time.timeZone = ${h.nixStr config.time.timeZone};"; # The host-owned part of the robot's config -> /etc/nixos/robotnix.nix. robotnixNix = r: let gw = r.hermes.gateway; db = r.hermes.dashboard; gwOn = r.hermes.enable && gw.enable; dbOn = r.hermes.enable && db.enable; openPorts = lib.optional (gwOn && gw.openFirewall) gw.port ++ lib.optional (dbOn && db.openFirewall) db.port; envFiles = "[ ${lib.concatMapStringsSep " " (c: h.nixStr "/run/credentials/@system/${c.id}") (hermesCreds r)} ]"; gwEnv = { API_SERVER_ENABLED = "true"; API_SERVER_HOST = "0.0.0.0"; API_SERVER_PORT = toString gw.port; API_SERVER_MODEL_NAME = if r.hermes.name != null then r.hermes.name else r.name; }; in builtins.toFile "robotnix-${r.name}.nix" '' # Generated by robotnix on the host (first install / reseed, or a host # rebuild if you changed the robot's host-side settings). # The network and git-sync values must match the host's robotnix # settings; change them and this robot loses its network or its # auto-commits stop. Everything else is yours. { lib, pkgs, ... }: { boot.isContainer = true; nixpkgs.hostPlatform = ${h.nixStr hostSystem}; networking.hostName = ${h.nixStr r.name}; # Address and route handed out by the host. IPv4 only. networking.useDHCP = false; networking.useNetworkd = true; networking.enableIPv6 = false; networking.useHostResolvConf = lib.mkForce false; networking.nameservers = [ ${h.nixStr sw} ]; systemd.network.networks."10-host0" = { matchConfig.Name = "host0"; address = [ ${h.nixStr "${r.address}/32"} ]; routes = [ { Gateway = ${h.nixStr sw}; GatewayOnLink = true; } ]; networkConfig.LinkLocalAddressing = "no"; }; nix.settings.experimental-features = [ "nix-command" "flakes" ]; services.openssh = { enable = true; settings.PasswordAuthentication = false; settings.KbdInteractiveAuthentication = false; }; users.users.${h.nixStr r.user} = { isNormalUser = true; extraGroups = [ "wheel" ${lib.optionalString r.hermes.enable ''"hermes" ''}]; openssh.authorizedKeys.keys = [ ${keys r} ]; }; users.users.root.openssh.authorizedKeys.keys = [ ${keys r} ]; ${tz} system.stateVersion = ${h.nixStr r.seed.stateVersion}; ${lib.optionalString r.hermes.enable '' # Hermes agent, managed by the host (robotnix hermes options). services.hermes-agent = { enable = true; addToSystemPackages = true; settings = builtins.fromJSON ${h.nixStr (builtins.toJSON r.hermes.settings)}; ${lib.optionalString (r.hermes.name != null) '' hermesHomeFiles."SOUL.md" = ${h.nixStr (soul r.hermes.name)}; ''} ${lib.optionalString gwOn ''environment = { ${lib.concatStringsSep " " (lib.mapAttrsToList (k: v: "${k} = ${h.nixStr v};") gwEnv)} }; # gateway API, key from the secrets''} ${lib.optionalString dbOn ''backend = { mode = "dashboard"; host = "0.0.0.0"; port = ${toString db.port}; }; # login from the secrets''} }; users.users.hermes.uid = ${toString hermesId}; users.groups.hermes.gid = ${toString hermesId}; # The module doesn't make skills/ (kept on ephemeral robots). systemd.tmpfiles.rules = [ "d ${h.hermesHome}/skills 2770 hermes hermes - -" ]; # Secrets: container credentials from the host, read at start. systemd.services.hermes-agent.serviceConfig.EnvironmentFile = ${envFiles}; ${lib.optionalString dbOn ''systemd.services.hermes-backend.serviceConfig.EnvironmentFile = ${envFiles};''} ${lib.optionalString (dbOn && db.port < 1024) '' # Ports below 1024 need this as the hermes user. systemd.services.hermes-backend.serviceConfig.AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; ''} ${lib.optionalString (openPorts != [ ]) ''networking.firewall.allowedTCPPorts = [ ${lib.concatMapStringsSep " " toString openPorts} ];''} ''} ${lib.optionalString r.ephemeral '' # Ephemeral robot: root resets on every start. Keep the ssh host # keys in /home so they don't change each boot. services.openssh.hostKeys = [ { type = "ed25519"; path = "${persistDir}/ssh/ssh_host_ed25519_key"; } { type = "rsa"; bits = 4096; path = "${persistDir}/ssh/ssh_host_rsa_key"; } ]; ''} ${lib.optionalString r.git.enable '' # git-tracking your flake: auto-commit, and push when a remote is set. environment.systemPackages = [ pkgs.git ]; systemd.services."robotnix-flake-gitsync" = { path = [ pkgs.git ]; script = ''' cd /etc/nixos git add -A git diff --cached --quiet || git commit -m "robotnix auto-sync $(date +'%Y-%m-%d %H:%M:%S')" if git remote get-url origin >/dev/null 2>&1; then git push -q fi '''; serviceConfig = { Type = "oneshot"; User = "root"; }; }; systemd.timers."robotnix-flake-gitsync" = { wantedBy = [ "timers.target" ]; timerConfig = { OnCalendar = ${h.nixStr r.git.interval}; Persistent = true; Unit = "robotnix-flake-gitsync.service"; }; }; ''} } ''; flakeNix = r: builtins.toFile "robotnix-${r.name}-flake.nix" '' # ${r.name}'s own flake. Apply changes with: # sudo nixos-rebuild switch --flake /etc/nixos { inputs.nixpkgs.url = "github:NixOS/nixpkgs/${nixpkgsRef}"; ${lib.optionalString r.hermes.enable ''inputs.hermes-agent.url = "github:NousResearch/hermes-agent/${hermesRef}";''} outputs = { nixpkgs, ... }@inputs: { nixosConfigurations.${h.nixStr r.name} = nixpkgs.lib.nixosSystem { modules = [ ./robotnix.nix ./configuration.nix ${lib.optionalString r.hermes.enable "inputs.hermes-agent.nixosModules.default"} ]; }; }; } ''; # The first system, built on the host exactly as the robot's own flake # builds it later: lib.nixosSystem is eval-config with system = null and # nixpkgs' lib, so this mirrors flake.nix's `nixpkgs.lib.nixosSystem` call. seedSystem = r: (import "${pkgs.path}/nixos/lib/eval-config.nix" { system = null; inherit lib; modules = [ (robotnixNix r) r.seed.configuration ] ++ lib.optional r.hermes.enable hermes-agent.nixosModules.default; }).config.system.build.toplevel; # Ephemeral robots: fingerprint of everything the seed installs. When it # changes, the clean root is rebuilt. seedId = r: builtins.hashString "sha256" ( lib.concatStringsSep " " [ "${seedSystem r}" "${flakeNix r}" "${robotnixNix r}" "${r.seed.configuration}" ] ); storageStart = r: pkgs.writeShellScript "${r.unit}-storage-start" ( '' set -euo pipefail base=${r.base} mkdir -p ${cfg.baseDir} if [ "$(stat -f -c %T ${cfg.baseDir})" != btrfs ]; then echo "robotnix: ${cfg.baseDir} must be on btrfs (needed for snapshots)" >&2 exit 1 fi for sub in "$base" "$base/root" "$base/home" "$base/snapshots"; do [ -d "$sub" ] || btrfs subvolume create "$sub" done # No copy-on-write for the images (only affects new files). chattr +C "$base/root" "$base/home" '' + lib.optionalString r.ephemeral '' # Ephemeral: seed changed (or first run) -> wipe root so the seed step # reinstalls and saves a new clean root. if ! mountpoint -q ${r.run}/root \ && [ "$(cat "$base/pristine.id" 2>/dev/null)" != ${seedId r} ]; then echo "robotnix: seed for ${r.name} changed, reinstalling" rm -f "$base/pristine.id" if [ -d "$base/pristine" ]; then btrfs subvolume delete "$base/pristine"; fi btrfs subvolume delete "$base/root" btrfs subvolume create "$base/root" chattr +C "$base/root" fi '' + '' image() { # img=$1 size=$2 dir=$3 if [ ! -e "$img" ]; then truncate -s "$size" "$img" mkfs.ext4 -q -F "$img" fi mkdir -p -m 0755 "$dir" if ! mountpoint -q "$dir"; then rc=0 e2fsck -p "$img" || rc=$? if [ "$rc" -ge 4 ]; then echo "robotnix: $img needs a manual e2fsck" >&2 exit 1 fi mount -o loop,noatime "$img" "$dir" fi # Grow when the configured size goes up. Shrinking is not supported. want=$(numfmt --from=iec "$size") have=$(stat -c %s "$img") if [ "$want" -gt "$have" ]; then truncate -s "$size" "$img" dev=$(findmnt -n -o SOURCE "$dir") losetup -c "$dev" resize2fs "$dev" elif [ "$want" -lt "$have" ]; then echo "robotnix: $img is bigger than $size; shrinking is not supported" >&2 fi } mkdir -p /run/robotnix ${r.run} chmod 0755 /run/robotnix ${r.run} image "$base/root/root.img" ${r.disk.rootSize} ${r.run}/root image "$base/home/home.img" ${r.disk.homeSize} ${r.run}/home chmod 0755 ${r.run}/root '' + lib.optionalString (r.ephemeral || r.persist != [ ]) '' install -d -m 0700 ${r.run}${persistDir} '' + lib.concatMapStrings (p: '' mkdir -p ${lib.escapeShellArg "${r.run}${persistDir}${p}"} '') r.persist ); storageStop = r: pkgs.writeShellScript "${r.unit}-storage-stop" '' for d in ${r.run}/home ${r.run}/root; do if mountpoint -q "$d"; then umount "$d"; fi done ''; # Installs the seed only when the root image has no system yet (first # start, or after `robotnix reseed`). Never touches an installed robot. # Ephemeral robots also save the fresh root as their clean copy. seedScript = r: pkgs.writeShellScript "${r.unit}-seed" ( '' set -euo pipefail root=${r.run}/root if [ ! -e "$root/nix/var/nix/profiles/system" ]; then echo "robotnix: installing the seed system into ${r.name}" nixos-install --root "$root" --system ${seedSystem r} \ --no-root-passwd --no-bootloader --no-channel-copy install -d -m 0755 "$root/etc/nixos" install -m 0644 ${flakeNix r} "$root/etc/nixos/flake.nix" install -m 0644 ${robotnixNix r} "$root/etc/nixos/robotnix.nix" install -m 0644 ${r.seed.configuration} "$root/etc/nixos/configuration.nix" ${lib.optionalString r.git.enable '' echo "robotnix: setting up the git repo for ${r.name}'s flake" git -C "$root/etc/nixos" init -q git -C "$root/etc/nixos" config user.name ${h.nixStr r.git.userName} git -C "$root/etc/nixos" config user.email ${h.nixStr r.git.userEmail} ${lib.optionalString (r.git.remoteUrl != null) '' git -C "$root/etc/nixos" remote add origin ${h.nixStr r.git.remoteUrl} ''} git -C "$root/etc/nixos" add -A git -C "$root/etc/nixos" commit -q -m "robotnix initial seed" || true ''} fi # What nspawn and NixOS's init expect (same as nixos-containers). mkdir -p "$root/etc" "$root/var/lib" "$root/root" "$root/home" mkdir -p -m 0755 "$root/usr/bin" chmod 0700 "$root/root" [ -e "$root/etc/os-release" ] || [ -L "$root/etc/os-release" ] || touch "$root/etc/os-release" [ -e "$root/etc/machine-id" ] || touch "$root/etc/machine-id" '' + lib.optionalString r.ephemeral '' # Ephemeral: save the clean root once, before the robot ever boots. if [ ! -e ${r.base}/pristine.id ]; then echo "robotnix: saving ${r.name}'s clean root" if [ -d ${r.base}/pristine ]; then btrfs subvolume delete ${r.base}/pristine; fi sync -f "$root" fsfreeze -f "$root" rc=0 btrfs subvolume snapshot -r ${r.base}/root ${r.base}/pristine || rc=$? fsfreeze -u "$root" [ "$rc" -eq 0 ] echo ${seedId r} > ${r.base}/pristine.id fi '' ); # Ephemeral: swap root for a fresh copy of the clean root. Runs (as its # own unit) before every start, including reboots from inside the robot. resetRoot = r: pkgs.writeShellScript "${r.unit}-reset" '' set -euo pipefail base=${r.base} root=${r.run}/root if [ "$(cat "$base/pristine.id" 2>/dev/null)" != ${seedId r} ] || [ ! -d "$base/pristine" ]; then echo "robotnix: no clean root for ${r.name}; restart its storage to reinstall" >&2 exit 1 fi if mountpoint -q "$root"; then umount "$root"; fi # Wait for the loop device to let go of the old image. for _ in $(seq 1 20); do [ -z "$(losetup -j "$base/root/root.img")" ] && break sleep 0.5 done btrfs subvolume delete "$base/root" btrfs subvolume snapshot "$base/pristine" "$base/root" mount -o loop,noatime "$base/root/root.img" "$root" ''; preStart = r: pkgs.writeShellScript "${r.unit}-pre" '' set -euo pipefail for _ in $(seq 1 30); do [ -e /sys/class/net/${br} ] && [ -e /sys/class/net/${ext} ] && break sleep 1 done [ -e /sys/class/net/${br} ] || { echo "robotnix: ${br} is missing" >&2; exit 1; } # Answer ARP for the robot's LAN IP on the uplink (this IP only). ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true ip neigh add proxy ${r.address} dev ${ext} ''; postStop = r: pkgs.writeShellScript "${r.unit}-post" '' ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true ''; idmap = r: lib.optionalString r.privateUsers ":idmap"; nspawnArgs = r: lib.escapeShellArgs ( [ "--keep-unit" "--machine=${r.name}" "--directory=${r.run}/root" "--notify-ready=yes" "--kill-signal=SIGRTMIN+3" "--network-bridge=${br}" "--link-journal=try-guest" "--resolv-conf=off" "--timezone=off" "--bind=${r.run}/home:/home${idmap r}" ] # Kept folders of an ephemeral robot, stored inside the home image. ++ map (p: "--bind=${r.run}${persistDir}${p}:${p}${idmap r}") r.persist ++ lib.optionals r.privateUsers [ "--private-users=pick" "--private-users-ownership=auto" ] # Docker needs the kernel keyring and bpf, which nspawn blocks by default. ++ lib.optional r.allowDocker "--system-call-filter=add_key keyctl bpf" ++ lib.optional r.allowTun "--bind=/dev/net/tun" ++ map (c: "--load-credential=${c.id}:${c.file}") (hermesCreds r) ++ [ "/nix/var/nix/profiles/system/init" ] ); robotUnits = r: { "${r.unit}-storage" = { description = "robotnix storage for ${r.name}"; path = with pkgs; [ btrfs-progs coreutils e2fsprogs util-linux ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = storageStart r; ExecStop = storageStop r; }; }; "${r.unit}-seed" = { description = "robotnix first install for ${r.name}"; requires = [ "${r.unit}-storage.service" ]; bindsTo = [ "${r.unit}-storage.service" ]; after = [ "${r.unit}-storage.service" ]; path = [ config.system.build.nixos-install config.nix.package pkgs.coreutils pkgs.util-linux pkgs.git pkgs.btrfs-progs ]; environment.HOME = "/root"; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = seedScript r; TimeoutStartSec = "30min"; }; }; # Ephemeral: fresh root before every robot start. Its own unit because # the robot unit blocks devices, and mounting the image needs loop # devices. No RemainAfterExit, so each robot start runs it again. "${r.unit}-reset" = lib.mkIf r.ephemeral { description = "robotnix fresh root for ${r.name}"; requires = [ "${r.unit}-storage.service" "${r.unit}-seed.service" ]; bindsTo = [ "${r.unit}-storage.service" ]; after = [ "${r.unit}-storage.service" "${r.unit}-seed.service" ]; path = with pkgs; [ btrfs-progs coreutils util-linux ]; serviceConfig = { Type = "oneshot"; ExecStart = resetRoot r; }; }; ${r.unit} = let mem = h.toBytes r.memory.max; deps = [ "${r.unit}-storage.service" "${r.unit}-seed.service" "${r.unit}-dns.service" ] ++ lib.optional r.ephemeral "${r.unit}-reset.service"; in { description = "robotnix robot ${r.name}"; wantedBy = lib.optional r.autoStart "machines.target"; requires = deps; bindsTo = [ "${r.unit}-storage.service" ]; after = deps ++ [ "network.target" "systemd-networkd.service" "nftables.service" ]; path = [ pkgs.iproute2 pkgs.coreutils ]; environment.SYSTEMD_NSPAWN_UNIFIED_HIERARCHY = "1"; serviceConfig = { ExecStartPre = preStart r; ExecStart = "${config.systemd.package}/bin/systemd-nspawn ${nspawnArgs r}"; # Best effort: refresh LAN ARP caches. ExecStartPost = "-${pkgs.iputils}/bin/arping -U -c 2 -I ${ext} ${r.address}"; ExecStopPost = postStop r; # Same as nixos-containers. Type = "notify"; RestartForceExitStatus = "133"; SuccessExitStatus = "133"; Restart = "on-failure"; TimeoutStartSec = "5min"; Slice = "machine.slice"; Delegate = true; KillMode = "mixed"; KillSignal = "TERM"; DevicePolicy = "closed"; DeviceAllow = lib.mkIf r.allowTun [ "/dev/net/tun rwm" ]; # Resources. CPUQuota = "${toString (r.cpus.max * 100)}%"; CPUWeight = toString (lib.min 10000 (100 * r.cpus.min)); MemoryLow = r.memory.min; MemoryHigh = toString (mem - mem / 16); MemoryMax = r.memory.max; MemorySwapMax = r.memory.swapMax; TasksMax = 16384; }; }; }; cliConfig = pkgs.writeText "robotnix.json" ( builtins.toJSON { inherit (cfg) baseDir; robots = lib.mapAttrs (_: r: { inherit (r) name address port dnsPort base run unit user nft access ephemeral ; snapshots = { inherit (r.snapshots) enable interval keep; }; }) h.robots; } ); cliPath = lib.makeBinPath [ pkgs.btrfs-progs pkgs.util-linux pkgs.coreutils pkgs.nftables config.systemd.package ]; cli = pkgs.writers.writeNuBin "robotnix" ( '' const CONFIG = "${cliConfig}" $env.PATH = ("${cliPath}" | split row ":" | append ($env.PATH | split row (char esep))) '' + builtins.readFile ./robotnix.nu ); in lib.mkIf cfg.enable { systemd.services = lib.mkMerge (map robotUnits rs); systemd.targets.multi-user.wants = [ "machines.target" ]; # Memory protection only works if the parent slice is protected too. systemd.slices.machine.sliceConfig.MemoryLow = lib.mkIf (rs != [ ]) ( lib.mkDefault (toString (lib.foldl' (acc: r: acc + h.toBytes r.memory.min) 0 rs)) ); skyg.server.timers = lib.listToAttrs ( map ( r: lib.nameValuePair "${r.unit}-snapshot" { script = "${cli}/bin/robotnix snapshot ${r.name} --auto"; OnCalendar = r.snapshots.interval; } ) (lib.filter (r: r.snapshots.enable) rs) ); environment.systemPackages = [ cli ]; }