Files
nixos-setup/modules/nixos/server/robotnix/robots.nix
T
Hermenix 86f9ae010e robotnix: give the robot root a /usr tree before nspawn starts
robotnix-genix.service on hub was in start-limit-hit with exec_main_status 1.
systemd-nspawn refused the root: "Directory /run/robotnix/genix/root doesnt
look like it has an OS tree (/usr/ directory is missing). Refusing." The storage
and seed units were healthy (image mounted, "installation finished!"), so the
tree existed and was populated - it was just missing /usr.

The seed script ends with a tail block that runs on every invocation, not only
when it installs: it creates etc, var/lib, root and home, and touches os-release
and machine-id. That list was copied from nixos-containers but dropped its first
line, mkdir -p "$root/usr/bin" (nixos-containers.nix:102). nspawn treats the
presence of /usr as the marker of an OS tree, so every start died before the
init ran, and because the unit restarts on failure it burned through the start
limit instead of recovering.

Add that mkdir with an explicit 0755 mode to the same tail block. Because the
block is outside the first-install branch, the directory is created on the next
storage+seed run for an already seeded root, so this needs no robotnix reseed
and loses no data. The module layout is unchanged.

ABOUTME.md gains a Notes/decisions bullet recording why the root gets /usr/bin
and that no reseed is required.
2026-10-26 00:45:42 +00:00

431 lines
14 KiB
Nix

# robotnix robots: storage, first install, nspawn units, snapshots, CLI.
#
# Layout per robot (all btrfs subvolumes):
# <baseDir>/<name>/root/root.img whole OS incl. /nix (hard size limit)
# <baseDir>/<name>/home/home.img /home (hard size limit)
# <baseDir>/<name>/snapshots/<id>/{root,home}
# The host mounts the images under /run/robotnix/<name>/ so it can freeze
# them for consistent snapshots while the robot runs.
{ config
, lib
, pkgs
, ...
}:
let
cfg = config.skyg.nixos.server.robotnix;
h = import ./helpers.nix { inherit config lib; };
rs = h.robotList;
sw = cfg.switch.address;
br = cfg.switch.name;
ext = cfg.externalInterface;
hostSystem = pkgs.stdenv.hostPlatform.system;
# The robot's flake pins the same nixpkgs the host used to build its seed.
nixpkgsRef =
let
rev = config.system.nixos.revision;
in
if rev != null then builtins.unsafeDiscardStringContext rev else "nixos-unstable";
keys = r: lib.concatMapStringsSep " " h.nixStr r.authorizedKeys;
tz = lib.optionalString
(
config.time.timeZone != null
) "time.timeZone = ${h.nixStr config.time.timeZone};";
# The host-owned part of the robot's config -> /etc/nixos/robotnix.nix.
robotnixNix =
r:
builtins.toFile "robotnix-${r.name}.nix" ''
# Generated by robotnix on the host (first install / reseed, or a host
# rebuild if you changed the robot's host-side settings).
# The network and git-sync values must match the host's robotnix
# settings; change them and this robot loses its network or its
# auto-commits stop. Everything else is yours.
{ lib, pkgs, ... }:
{
boot.isContainer = true;
nixpkgs.hostPlatform = ${h.nixStr hostSystem};
networking.hostName = ${h.nixStr r.name};
# Address and route handed out by the host. IPv4 only.
networking.useDHCP = false;
networking.useNetworkd = true;
networking.enableIPv6 = false;
networking.useHostResolvConf = lib.mkForce false;
networking.nameservers = [ ${h.nixStr sw} ];
systemd.network.networks."10-host0" = {
matchConfig.Name = "host0";
address = [ ${h.nixStr "${r.address}/32"} ];
routes = [ { Gateway = ${h.nixStr sw}; GatewayOnLink = true; } ];
networkConfig.LinkLocalAddressing = "no";
};
nix.settings.experimental-features = [ "nix-command" "flakes" ];
services.openssh = {
enable = true;
settings.PasswordAuthentication = false;
settings.KbdInteractiveAuthentication = false;
};
users.users.${h.nixStr r.user} = {
isNormalUser = true;
extraGroups = [ "wheel" ];
openssh.authorizedKeys.keys = [ ${keys r} ];
};
users.users.root.openssh.authorizedKeys.keys = [ ${keys r} ];
${tz}
system.stateVersion = ${h.nixStr r.seed.stateVersion};
${lib.optionalString r.git.enable ''
# git-tracking your flake: auto-commit, and push when a remote is set.
environment.systemPackages = [ pkgs.git ];
systemd.services."robotnix-flake-gitsync" = {
path = [ pkgs.git ];
script = '''
cd /etc/nixos
git add -A
git diff --cached --quiet || git commit -m "robotnix auto-sync $(date +'%Y-%m-%d %H:%M:%S')"
if git remote get-url origin >/dev/null 2>&1; then
git push -q
fi
''';
serviceConfig = {
Type = "oneshot";
User = "root";
};
};
systemd.timers."robotnix-flake-gitsync" = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = ${h.nixStr r.git.interval};
Persistent = true;
Unit = "robotnix-flake-gitsync.service";
};
};
''}
}
'';
flakeNix =
r:
builtins.toFile "robotnix-${r.name}-flake.nix" ''
# ${r.name}'s own flake. Apply changes with:
# sudo nixos-rebuild switch --flake /etc/nixos
{
inputs.nixpkgs.url = "github:NixOS/nixpkgs/${nixpkgsRef}";
outputs = { nixpkgs, ... }: {
nixosConfigurations.${h.nixStr r.name} = nixpkgs.lib.nixosSystem {
modules = [ ./robotnix.nix ./configuration.nix ];
};
};
}
'';
# The first system, built on the host exactly as the robot's own flake
# builds it later: lib.nixosSystem is eval-config with system = null and
# nixpkgs' lib, so this mirrors flake.nix's `nixpkgs.lib.nixosSystem` call.
seedSystem =
r:
(import "${pkgs.path}/nixos/lib/eval-config.nix" {
system = null;
inherit lib;
modules = [
(robotnixNix r)
r.seed.configuration
];
}).config.system.build.toplevel;
storageStart =
r:
pkgs.writeShellScript "${r.unit}-storage-start" ''
set -euo pipefail
base=${r.base}
mkdir -p ${cfg.baseDir}
if [ "$(stat -f -c %T ${cfg.baseDir})" != btrfs ]; then
echo "robotnix: ${cfg.baseDir} must be on btrfs (needed for snapshots)" >&2
exit 1
fi
for sub in "$base" "$base/root" "$base/home" "$base/snapshots"; do
[ -d "$sub" ] || btrfs subvolume create "$sub"
done
# No copy-on-write for the images (only affects new files).
chattr +C "$base/root" "$base/home"
image() { # <img> <size> <mountpoint>
img=$1 size=$2 dir=$3
if [ ! -e "$img" ]; then
truncate -s "$size" "$img"
mkfs.ext4 -q -F "$img"
fi
mkdir -p -m 0755 "$dir"
if ! mountpoint -q "$dir"; then
rc=0
e2fsck -p "$img" || rc=$?
if [ "$rc" -ge 4 ]; then
echo "robotnix: $img needs a manual e2fsck" >&2
exit 1
fi
mount -o loop,noatime "$img" "$dir"
fi
# Grow when the configured size goes up. Shrinking is not supported.
want=$(numfmt --from=iec "$size")
have=$(stat -c %s "$img")
if [ "$want" -gt "$have" ]; then
truncate -s "$size" "$img"
dev=$(findmnt -n -o SOURCE "$dir")
losetup -c "$dev"
resize2fs "$dev"
elif [ "$want" -lt "$have" ]; then
echo "robotnix: $img is bigger than $size; shrinking is not supported" >&2
fi
}
mkdir -p /run/robotnix ${r.run}
chmod 0755 /run/robotnix ${r.run}
image "$base/root/root.img" ${r.disk.rootSize} ${r.run}/root
image "$base/home/home.img" ${r.disk.homeSize} ${r.run}/home
chmod 0755 ${r.run}/root
'';
storageStop =
r:
pkgs.writeShellScript "${r.unit}-storage-stop" ''
for d in ${r.run}/home ${r.run}/root; do
if mountpoint -q "$d"; then umount "$d"; fi
done
'';
# Installs the seed only when the root image has no system yet (first
# start, or after `robotnix reseed`). Never touches an installed robot.
seedScript =
r:
pkgs.writeShellScript "${r.unit}-seed" ''
set -euo pipefail
root=${r.run}/root
if [ ! -e "$root/nix/var/nix/profiles/system" ]; then
echo "robotnix: installing the seed system into ${r.name}"
nixos-install --root "$root" --system ${seedSystem r} \
--no-root-passwd --no-bootloader --no-channel-copy
install -d -m 0755 "$root/etc/nixos"
install -m 0644 ${flakeNix r} "$root/etc/nixos/flake.nix"
install -m 0644 ${robotnixNix r} "$root/etc/nixos/robotnix.nix"
install -m 0644 ${r.seed.configuration} "$root/etc/nixos/configuration.nix"
${lib.optionalString r.git.enable ''
echo "robotnix: setting up the git repo for ${r.name}'s flake"
git -C "$root/etc/nixos" init -q
git -C "$root/etc/nixos" config user.name ${h.nixStr r.git.userName}
git -C "$root/etc/nixos" config user.email ${h.nixStr r.git.userEmail}
${lib.optionalString (r.git.remoteUrl != null) ''
git -C "$root/etc/nixos" remote add origin ${h.nixStr r.git.remoteUrl}
''}
git -C "$root/etc/nixos" add -A
git -C "$root/etc/nixos" commit -q -m "robotnix initial seed" || true
''}
fi
# What nspawn and NixOS's init expect (same as nixos-containers).
mkdir -p "$root/etc" "$root/var/lib" "$root/root" "$root/home"
mkdir -p -m 0755 "$root/usr/bin"
chmod 0700 "$root/root"
[ -e "$root/etc/os-release" ] || [ -L "$root/etc/os-release" ] || touch "$root/etc/os-release"
[ -e "$root/etc/machine-id" ] || touch "$root/etc/machine-id"
'';
preStart =
r:
pkgs.writeShellScript "${r.unit}-pre" ''
set -euo pipefail
for _ in $(seq 1 30); do
[ -e /sys/class/net/${br} ] && [ -e /sys/class/net/${ext} ] && break
sleep 1
done
[ -e /sys/class/net/${br} ] || { echo "robotnix: ${br} is missing" >&2; exit 1; }
# Answer ARP for the robot's LAN IP on the uplink (this IP only).
ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true
ip neigh add proxy ${r.address} dev ${ext}
'';
postStop =
r:
pkgs.writeShellScript "${r.unit}-post" ''
ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true
'';
nspawnArgs =
r:
lib.escapeShellArgs (
[
"--keep-unit"
"--machine=${r.name}"
"--directory=${r.run}/root"
"--notify-ready=yes"
"--kill-signal=SIGRTMIN+3"
"--network-bridge=${br}"
"--link-journal=try-guest"
"--resolv-conf=off"
"--timezone=off"
"--bind=${r.run}/home:/home${lib.optionalString r.privateUsers ":idmap"}"
]
++ lib.optionals r.privateUsers [
"--private-users=pick"
"--private-users-ownership=auto"
]
++ [ "/nix/var/nix/profiles/system/init" ]
);
robotUnits = r: {
"${r.unit}-storage" = {
description = "robotnix storage for ${r.name}";
path = with pkgs; [
btrfs-progs
coreutils
e2fsprogs
util-linux
];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = storageStart r;
ExecStop = storageStop r;
};
};
"${r.unit}-seed" = {
description = "robotnix first install for ${r.name}";
requires = [ "${r.unit}-storage.service" ];
bindsTo = [ "${r.unit}-storage.service" ];
after = [ "${r.unit}-storage.service" ];
path = [
config.system.build.nixos-install
config.nix.package
pkgs.coreutils
pkgs.util-linux
pkgs.git
];
environment.HOME = "/root";
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = seedScript r;
TimeoutStartSec = "30min";
};
};
${r.unit} =
let
mem = h.toBytes r.memory.max;
deps = [
"${r.unit}-storage.service"
"${r.unit}-seed.service"
"${r.unit}-dns.service"
];
in
{
description = "robotnix robot ${r.name}";
wantedBy = lib.optional r.autoStart "machines.target";
requires = deps;
bindsTo = [ "${r.unit}-storage.service" ];
after = deps ++ [
"network.target"
"systemd-networkd.service"
"nftables.service"
];
path = [
pkgs.iproute2
pkgs.coreutils
];
environment.SYSTEMD_NSPAWN_UNIFIED_HIERARCHY = "1";
serviceConfig = {
ExecStartPre = preStart r;
ExecStart = "${config.systemd.package}/bin/systemd-nspawn ${nspawnArgs r}";
# Best effort: refresh LAN ARP caches.
ExecStartPost = "-${pkgs.iputils}/bin/arping -U -c 2 -I ${ext} ${r.address}";
ExecStopPost = postStop r;
# Same as nixos-containers.
Type = "notify";
RestartForceExitStatus = "133";
SuccessExitStatus = "133";
Restart = "on-failure";
TimeoutStartSec = "5min";
Slice = "machine.slice";
Delegate = true;
KillMode = "mixed";
KillSignal = "TERM";
DevicePolicy = "closed";
# Resources.
CPUQuota = "${toString (r.cpus.max * 100)}%";
CPUWeight = toString (lib.min 10000 (100 * r.cpus.min));
MemoryLow = r.memory.min;
MemoryHigh = toString (mem - mem / 16);
MemoryMax = r.memory.max;
MemorySwapMax = r.memory.swapMax;
TasksMax = 16384;
};
};
};
cliConfig = pkgs.writeText "robotnix.json" (
builtins.toJSON {
inherit (cfg) baseDir;
robots = lib.mapAttrs
(_: r: {
inherit (r)
name
address
port
dnsPort
base
run
unit
user
nft
access
;
snapshots = { inherit (r.snapshots) enable interval keep; };
})
h.robots;
}
);
cliPath = lib.makeBinPath [
pkgs.btrfs-progs
pkgs.util-linux
pkgs.coreutils
pkgs.nftables
config.systemd.package
];
cli = pkgs.writers.writeNuBin "robotnix" (
''
const CONFIG = "${cliConfig}"
$env.PATH = ("${cliPath}" | split row ":" | append ($env.PATH | split row (char esep)))
''
+ builtins.readFile ./robotnix.nu
);
in
lib.mkIf cfg.enable {
systemd.services = lib.mkMerge (map robotUnits rs);
systemd.targets.multi-user.wants = [ "machines.target" ];
# Memory protection only works if the parent slice is protected too.
systemd.slices.machine.sliceConfig.MemoryLow = lib.mkIf (rs != [ ]) (
lib.mkDefault (toString (lib.foldl' (acc: r: acc + h.toBytes r.memory.min) 0 rs))
);
skyg.server.timers = lib.listToAttrs (
map
(
r:
lib.nameValuePair "${r.unit}-snapshot" {
script = "${cli}/bin/robotnix snapshot ${r.name} --auto";
OnCalendar = r.snapshots.interval;
}
)
(lib.filter (r: r.snapshots.enable) rs)
);
environment.systemPackages = [ cli ];
}