robotnix-genix.service on hub was in start-limit-hit with exec_main_status 1. systemd-nspawn refused the root: "Directory /run/robotnix/genix/root doesnt look like it has an OS tree (/usr/ directory is missing). Refusing." The storage and seed units were healthy (image mounted, "installation finished!"), so the tree existed and was populated - it was just missing /usr. The seed script ends with a tail block that runs on every invocation, not only when it installs: it creates etc, var/lib, root and home, and touches os-release and machine-id. That list was copied from nixos-containers but dropped its first line, mkdir -p "$root/usr/bin" (nixos-containers.nix:102). nspawn treats the presence of /usr as the marker of an OS tree, so every start died before the init ran, and because the unit restarts on failure it burned through the start limit instead of recovering. Add that mkdir with an explicit 0755 mode to the same tail block. Because the block is outside the first-install branch, the directory is created on the next storage+seed run for an already seeded root, so this needs no robotnix reseed and loses no data. The module layout is unchanged. ABOUTME.md gains a Notes/decisions bullet recording why the root gets /usr/bin and that no reseed is required.
431 lines
14 KiB
Nix
431 lines
14 KiB
Nix
# robotnix robots: storage, first install, nspawn units, snapshots, CLI.
|
|
#
|
|
# Layout per robot (all btrfs subvolumes):
|
|
# <baseDir>/<name>/root/root.img whole OS incl. /nix (hard size limit)
|
|
# <baseDir>/<name>/home/home.img /home (hard size limit)
|
|
# <baseDir>/<name>/snapshots/<id>/{root,home}
|
|
# The host mounts the images under /run/robotnix/<name>/ so it can freeze
|
|
# them for consistent snapshots while the robot runs.
|
|
{ config
|
|
, lib
|
|
, pkgs
|
|
, ...
|
|
}:
|
|
let
|
|
cfg = config.skyg.nixos.server.robotnix;
|
|
h = import ./helpers.nix { inherit config lib; };
|
|
rs = h.robotList;
|
|
sw = cfg.switch.address;
|
|
br = cfg.switch.name;
|
|
ext = cfg.externalInterface;
|
|
hostSystem = pkgs.stdenv.hostPlatform.system;
|
|
|
|
# The robot's flake pins the same nixpkgs the host used to build its seed.
|
|
nixpkgsRef =
|
|
let
|
|
rev = config.system.nixos.revision;
|
|
in
|
|
if rev != null then builtins.unsafeDiscardStringContext rev else "nixos-unstable";
|
|
|
|
keys = r: lib.concatMapStringsSep " " h.nixStr r.authorizedKeys;
|
|
tz = lib.optionalString
|
|
(
|
|
config.time.timeZone != null
|
|
) "time.timeZone = ${h.nixStr config.time.timeZone};";
|
|
|
|
# The host-owned part of the robot's config -> /etc/nixos/robotnix.nix.
|
|
robotnixNix =
|
|
r:
|
|
builtins.toFile "robotnix-${r.name}.nix" ''
|
|
# Generated by robotnix on the host (first install / reseed, or a host
|
|
# rebuild if you changed the robot's host-side settings).
|
|
# The network and git-sync values must match the host's robotnix
|
|
# settings; change them and this robot loses its network or its
|
|
# auto-commits stop. Everything else is yours.
|
|
{ lib, pkgs, ... }:
|
|
{
|
|
boot.isContainer = true;
|
|
nixpkgs.hostPlatform = ${h.nixStr hostSystem};
|
|
networking.hostName = ${h.nixStr r.name};
|
|
|
|
# Address and route handed out by the host. IPv4 only.
|
|
networking.useDHCP = false;
|
|
networking.useNetworkd = true;
|
|
networking.enableIPv6 = false;
|
|
networking.useHostResolvConf = lib.mkForce false;
|
|
networking.nameservers = [ ${h.nixStr sw} ];
|
|
systemd.network.networks."10-host0" = {
|
|
matchConfig.Name = "host0";
|
|
address = [ ${h.nixStr "${r.address}/32"} ];
|
|
routes = [ { Gateway = ${h.nixStr sw}; GatewayOnLink = true; } ];
|
|
networkConfig.LinkLocalAddressing = "no";
|
|
};
|
|
|
|
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
|
|
|
services.openssh = {
|
|
enable = true;
|
|
settings.PasswordAuthentication = false;
|
|
settings.KbdInteractiveAuthentication = false;
|
|
};
|
|
users.users.${h.nixStr r.user} = {
|
|
isNormalUser = true;
|
|
extraGroups = [ "wheel" ];
|
|
openssh.authorizedKeys.keys = [ ${keys r} ];
|
|
};
|
|
users.users.root.openssh.authorizedKeys.keys = [ ${keys r} ];
|
|
${tz}
|
|
system.stateVersion = ${h.nixStr r.seed.stateVersion};
|
|
${lib.optionalString r.git.enable ''
|
|
# git-tracking your flake: auto-commit, and push when a remote is set.
|
|
environment.systemPackages = [ pkgs.git ];
|
|
systemd.services."robotnix-flake-gitsync" = {
|
|
path = [ pkgs.git ];
|
|
script = '''
|
|
cd /etc/nixos
|
|
git add -A
|
|
git diff --cached --quiet || git commit -m "robotnix auto-sync $(date +'%Y-%m-%d %H:%M:%S')"
|
|
if git remote get-url origin >/dev/null 2>&1; then
|
|
git push -q
|
|
fi
|
|
''';
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
User = "root";
|
|
};
|
|
};
|
|
systemd.timers."robotnix-flake-gitsync" = {
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnCalendar = ${h.nixStr r.git.interval};
|
|
Persistent = true;
|
|
Unit = "robotnix-flake-gitsync.service";
|
|
};
|
|
};
|
|
''}
|
|
}
|
|
'';
|
|
|
|
flakeNix =
|
|
r:
|
|
builtins.toFile "robotnix-${r.name}-flake.nix" ''
|
|
# ${r.name}'s own flake. Apply changes with:
|
|
# sudo nixos-rebuild switch --flake /etc/nixos
|
|
{
|
|
inputs.nixpkgs.url = "github:NixOS/nixpkgs/${nixpkgsRef}";
|
|
outputs = { nixpkgs, ... }: {
|
|
nixosConfigurations.${h.nixStr r.name} = nixpkgs.lib.nixosSystem {
|
|
modules = [ ./robotnix.nix ./configuration.nix ];
|
|
};
|
|
};
|
|
}
|
|
'';
|
|
|
|
# The first system, built on the host exactly as the robot's own flake
|
|
# builds it later: lib.nixosSystem is eval-config with system = null and
|
|
# nixpkgs' lib, so this mirrors flake.nix's `nixpkgs.lib.nixosSystem` call.
|
|
seedSystem =
|
|
r:
|
|
(import "${pkgs.path}/nixos/lib/eval-config.nix" {
|
|
system = null;
|
|
inherit lib;
|
|
modules = [
|
|
(robotnixNix r)
|
|
r.seed.configuration
|
|
];
|
|
}).config.system.build.toplevel;
|
|
|
|
storageStart =
|
|
r:
|
|
pkgs.writeShellScript "${r.unit}-storage-start" ''
|
|
set -euo pipefail
|
|
base=${r.base}
|
|
mkdir -p ${cfg.baseDir}
|
|
if [ "$(stat -f -c %T ${cfg.baseDir})" != btrfs ]; then
|
|
echo "robotnix: ${cfg.baseDir} must be on btrfs (needed for snapshots)" >&2
|
|
exit 1
|
|
fi
|
|
for sub in "$base" "$base/root" "$base/home" "$base/snapshots"; do
|
|
[ -d "$sub" ] || btrfs subvolume create "$sub"
|
|
done
|
|
# No copy-on-write for the images (only affects new files).
|
|
chattr +C "$base/root" "$base/home"
|
|
|
|
image() { # <img> <size> <mountpoint>
|
|
img=$1 size=$2 dir=$3
|
|
if [ ! -e "$img" ]; then
|
|
truncate -s "$size" "$img"
|
|
mkfs.ext4 -q -F "$img"
|
|
fi
|
|
mkdir -p -m 0755 "$dir"
|
|
if ! mountpoint -q "$dir"; then
|
|
rc=0
|
|
e2fsck -p "$img" || rc=$?
|
|
if [ "$rc" -ge 4 ]; then
|
|
echo "robotnix: $img needs a manual e2fsck" >&2
|
|
exit 1
|
|
fi
|
|
mount -o loop,noatime "$img" "$dir"
|
|
fi
|
|
# Grow when the configured size goes up. Shrinking is not supported.
|
|
want=$(numfmt --from=iec "$size")
|
|
have=$(stat -c %s "$img")
|
|
if [ "$want" -gt "$have" ]; then
|
|
truncate -s "$size" "$img"
|
|
dev=$(findmnt -n -o SOURCE "$dir")
|
|
losetup -c "$dev"
|
|
resize2fs "$dev"
|
|
elif [ "$want" -lt "$have" ]; then
|
|
echo "robotnix: $img is bigger than $size; shrinking is not supported" >&2
|
|
fi
|
|
}
|
|
|
|
mkdir -p /run/robotnix ${r.run}
|
|
chmod 0755 /run/robotnix ${r.run}
|
|
image "$base/root/root.img" ${r.disk.rootSize} ${r.run}/root
|
|
image "$base/home/home.img" ${r.disk.homeSize} ${r.run}/home
|
|
chmod 0755 ${r.run}/root
|
|
'';
|
|
|
|
storageStop =
|
|
r:
|
|
pkgs.writeShellScript "${r.unit}-storage-stop" ''
|
|
for d in ${r.run}/home ${r.run}/root; do
|
|
if mountpoint -q "$d"; then umount "$d"; fi
|
|
done
|
|
'';
|
|
|
|
# Installs the seed only when the root image has no system yet (first
|
|
# start, or after `robotnix reseed`). Never touches an installed robot.
|
|
seedScript =
|
|
r:
|
|
pkgs.writeShellScript "${r.unit}-seed" ''
|
|
set -euo pipefail
|
|
root=${r.run}/root
|
|
if [ ! -e "$root/nix/var/nix/profiles/system" ]; then
|
|
echo "robotnix: installing the seed system into ${r.name}"
|
|
nixos-install --root "$root" --system ${seedSystem r} \
|
|
--no-root-passwd --no-bootloader --no-channel-copy
|
|
install -d -m 0755 "$root/etc/nixos"
|
|
install -m 0644 ${flakeNix r} "$root/etc/nixos/flake.nix"
|
|
install -m 0644 ${robotnixNix r} "$root/etc/nixos/robotnix.nix"
|
|
install -m 0644 ${r.seed.configuration} "$root/etc/nixos/configuration.nix"
|
|
${lib.optionalString r.git.enable ''
|
|
echo "robotnix: setting up the git repo for ${r.name}'s flake"
|
|
git -C "$root/etc/nixos" init -q
|
|
git -C "$root/etc/nixos" config user.name ${h.nixStr r.git.userName}
|
|
git -C "$root/etc/nixos" config user.email ${h.nixStr r.git.userEmail}
|
|
${lib.optionalString (r.git.remoteUrl != null) ''
|
|
git -C "$root/etc/nixos" remote add origin ${h.nixStr r.git.remoteUrl}
|
|
''}
|
|
git -C "$root/etc/nixos" add -A
|
|
git -C "$root/etc/nixos" commit -q -m "robotnix initial seed" || true
|
|
''}
|
|
fi
|
|
# What nspawn and NixOS's init expect (same as nixos-containers).
|
|
mkdir -p "$root/etc" "$root/var/lib" "$root/root" "$root/home"
|
|
mkdir -p -m 0755 "$root/usr/bin"
|
|
chmod 0700 "$root/root"
|
|
[ -e "$root/etc/os-release" ] || [ -L "$root/etc/os-release" ] || touch "$root/etc/os-release"
|
|
[ -e "$root/etc/machine-id" ] || touch "$root/etc/machine-id"
|
|
'';
|
|
|
|
preStart =
|
|
r:
|
|
pkgs.writeShellScript "${r.unit}-pre" ''
|
|
set -euo pipefail
|
|
for _ in $(seq 1 30); do
|
|
[ -e /sys/class/net/${br} ] && [ -e /sys/class/net/${ext} ] && break
|
|
sleep 1
|
|
done
|
|
[ -e /sys/class/net/${br} ] || { echo "robotnix: ${br} is missing" >&2; exit 1; }
|
|
# Answer ARP for the robot's LAN IP on the uplink (this IP only).
|
|
ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true
|
|
ip neigh add proxy ${r.address} dev ${ext}
|
|
'';
|
|
|
|
postStop =
|
|
r:
|
|
pkgs.writeShellScript "${r.unit}-post" ''
|
|
ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true
|
|
'';
|
|
|
|
nspawnArgs =
|
|
r:
|
|
lib.escapeShellArgs (
|
|
[
|
|
"--keep-unit"
|
|
"--machine=${r.name}"
|
|
"--directory=${r.run}/root"
|
|
"--notify-ready=yes"
|
|
"--kill-signal=SIGRTMIN+3"
|
|
"--network-bridge=${br}"
|
|
"--link-journal=try-guest"
|
|
"--resolv-conf=off"
|
|
"--timezone=off"
|
|
"--bind=${r.run}/home:/home${lib.optionalString r.privateUsers ":idmap"}"
|
|
]
|
|
++ lib.optionals r.privateUsers [
|
|
"--private-users=pick"
|
|
"--private-users-ownership=auto"
|
|
]
|
|
++ [ "/nix/var/nix/profiles/system/init" ]
|
|
);
|
|
|
|
robotUnits = r: {
|
|
"${r.unit}-storage" = {
|
|
description = "robotnix storage for ${r.name}";
|
|
path = with pkgs; [
|
|
btrfs-progs
|
|
coreutils
|
|
e2fsprogs
|
|
util-linux
|
|
];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = storageStart r;
|
|
ExecStop = storageStop r;
|
|
};
|
|
};
|
|
|
|
"${r.unit}-seed" = {
|
|
description = "robotnix first install for ${r.name}";
|
|
requires = [ "${r.unit}-storage.service" ];
|
|
bindsTo = [ "${r.unit}-storage.service" ];
|
|
after = [ "${r.unit}-storage.service" ];
|
|
path = [
|
|
config.system.build.nixos-install
|
|
config.nix.package
|
|
pkgs.coreutils
|
|
pkgs.util-linux
|
|
pkgs.git
|
|
];
|
|
environment.HOME = "/root";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = seedScript r;
|
|
TimeoutStartSec = "30min";
|
|
};
|
|
};
|
|
|
|
${r.unit} =
|
|
let
|
|
mem = h.toBytes r.memory.max;
|
|
deps = [
|
|
"${r.unit}-storage.service"
|
|
"${r.unit}-seed.service"
|
|
"${r.unit}-dns.service"
|
|
];
|
|
in
|
|
{
|
|
description = "robotnix robot ${r.name}";
|
|
wantedBy = lib.optional r.autoStart "machines.target";
|
|
requires = deps;
|
|
bindsTo = [ "${r.unit}-storage.service" ];
|
|
after = deps ++ [
|
|
"network.target"
|
|
"systemd-networkd.service"
|
|
"nftables.service"
|
|
];
|
|
path = [
|
|
pkgs.iproute2
|
|
pkgs.coreutils
|
|
];
|
|
environment.SYSTEMD_NSPAWN_UNIFIED_HIERARCHY = "1";
|
|
serviceConfig = {
|
|
ExecStartPre = preStart r;
|
|
ExecStart = "${config.systemd.package}/bin/systemd-nspawn ${nspawnArgs r}";
|
|
# Best effort: refresh LAN ARP caches.
|
|
ExecStartPost = "-${pkgs.iputils}/bin/arping -U -c 2 -I ${ext} ${r.address}";
|
|
ExecStopPost = postStop r;
|
|
|
|
# Same as nixos-containers.
|
|
Type = "notify";
|
|
RestartForceExitStatus = "133";
|
|
SuccessExitStatus = "133";
|
|
Restart = "on-failure";
|
|
TimeoutStartSec = "5min";
|
|
Slice = "machine.slice";
|
|
Delegate = true;
|
|
KillMode = "mixed";
|
|
KillSignal = "TERM";
|
|
DevicePolicy = "closed";
|
|
|
|
# Resources.
|
|
CPUQuota = "${toString (r.cpus.max * 100)}%";
|
|
CPUWeight = toString (lib.min 10000 (100 * r.cpus.min));
|
|
MemoryLow = r.memory.min;
|
|
MemoryHigh = toString (mem - mem / 16);
|
|
MemoryMax = r.memory.max;
|
|
MemorySwapMax = r.memory.swapMax;
|
|
TasksMax = 16384;
|
|
};
|
|
};
|
|
};
|
|
|
|
cliConfig = pkgs.writeText "robotnix.json" (
|
|
builtins.toJSON {
|
|
inherit (cfg) baseDir;
|
|
robots = lib.mapAttrs
|
|
(_: r: {
|
|
inherit (r)
|
|
name
|
|
address
|
|
port
|
|
dnsPort
|
|
base
|
|
run
|
|
unit
|
|
user
|
|
nft
|
|
access
|
|
;
|
|
snapshots = { inherit (r.snapshots) enable interval keep; };
|
|
})
|
|
h.robots;
|
|
}
|
|
);
|
|
|
|
cliPath = lib.makeBinPath [
|
|
pkgs.btrfs-progs
|
|
pkgs.util-linux
|
|
pkgs.coreutils
|
|
pkgs.nftables
|
|
config.systemd.package
|
|
];
|
|
|
|
cli = pkgs.writers.writeNuBin "robotnix" (
|
|
''
|
|
const CONFIG = "${cliConfig}"
|
|
$env.PATH = ("${cliPath}" | split row ":" | append ($env.PATH | split row (char esep)))
|
|
''
|
|
+ builtins.readFile ./robotnix.nu
|
|
);
|
|
in
|
|
lib.mkIf cfg.enable {
|
|
systemd.services = lib.mkMerge (map robotUnits rs);
|
|
|
|
systemd.targets.multi-user.wants = [ "machines.target" ];
|
|
|
|
# Memory protection only works if the parent slice is protected too.
|
|
systemd.slices.machine.sliceConfig.MemoryLow = lib.mkIf (rs != [ ]) (
|
|
lib.mkDefault (toString (lib.foldl' (acc: r: acc + h.toBytes r.memory.min) 0 rs))
|
|
);
|
|
|
|
skyg.server.timers = lib.listToAttrs (
|
|
map
|
|
(
|
|
r:
|
|
lib.nameValuePair "${r.unit}-snapshot" {
|
|
script = "${cli}/bin/robotnix snapshot ${r.name} --auto";
|
|
OnCalendar = r.snapshots.interval;
|
|
}
|
|
)
|
|
(lib.filter (r: r.snapshots.enable) rs)
|
|
);
|
|
|
|
environment.systemPackages = [ cli ];
|
|
}
|