430 lines
14 KiB
Nix
430 lines
14 KiB
Nix
# robotnix robots: storage, first install, nspawn units, snapshots, CLI.
|
|
#
|
|
# Layout per robot (all btrfs subvolumes):
|
|
# <baseDir>/<name>/root/root.img whole OS incl. /nix (hard size limit)
|
|
# <baseDir>/<name>/home/home.img /home (hard size limit)
|
|
# <baseDir>/<name>/snapshots/<id>/{root,home}
|
|
# The host mounts the images under /run/robotnix/<name>/ so it can freeze
|
|
# them for consistent snapshots while the robot runs.
|
|
{ config
|
|
, lib
|
|
, pkgs
|
|
, ...
|
|
}:
|
|
let
|
|
cfg = config.skyg.nixos.server.robotnix;
|
|
h = import ./helpers.nix { inherit config lib; };
|
|
rs = h.robotList;
|
|
sw = cfg.switch.address;
|
|
br = cfg.switch.name;
|
|
ext = cfg.externalInterface;
|
|
hostSystem = pkgs.stdenv.hostPlatform.system;
|
|
|
|
# The robot's flake pins the same nixpkgs the host used to build its seed.
|
|
nixpkgsRef =
|
|
let
|
|
rev = config.system.nixos.revision;
|
|
in
|
|
if rev != null then builtins.unsafeDiscardStringContext rev else "nixos-unstable";
|
|
|
|
keys = r: lib.concatMapStringsSep " " h.nixStr r.authorizedKeys;
|
|
tz = lib.optionalString
|
|
(
|
|
config.time.timeZone != null
|
|
) "time.timeZone = ${h.nixStr config.time.timeZone};";
|
|
|
|
# The host-owned part of the robot's config -> /etc/nixos/robotnix.nix.
|
|
robotnixNix =
|
|
r:
|
|
builtins.toFile "robotnix-${r.name}.nix" ''
|
|
# Generated by robotnix on the host (first install / reseed, or a host
|
|
# rebuild if you changed the robot's host-side settings).
|
|
# The network and git-sync values must match the host's robotnix
|
|
# settings; change them and this robot loses its network or its
|
|
# auto-commits stop. Everything else is yours.
|
|
{ lib, pkgs, ... }:
|
|
{
|
|
boot.isContainer = true;
|
|
nixpkgs.hostPlatform = ${h.nixStr hostSystem};
|
|
networking.hostName = ${h.nixStr r.name};
|
|
|
|
# Address and route handed out by the host. IPv4 only.
|
|
networking.useDHCP = false;
|
|
networking.useNetworkd = true;
|
|
networking.enableIPv6 = false;
|
|
networking.useHostResolvConf = lib.mkForce false;
|
|
networking.nameservers = [ ${h.nixStr sw} ];
|
|
systemd.network.networks."10-host0" = {
|
|
matchConfig.Name = "host0";
|
|
address = [ ${h.nixStr "${r.address}/32"} ];
|
|
routes = [ { Gateway = ${h.nixStr sw}; GatewayOnLink = true; } ];
|
|
networkConfig.LinkLocalAddressing = "no";
|
|
};
|
|
|
|
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
|
|
|
services.openssh = {
|
|
enable = true;
|
|
settings.PasswordAuthentication = false;
|
|
settings.KbdInteractiveAuthentication = false;
|
|
};
|
|
users.users.${h.nixStr r.user} = {
|
|
isNormalUser = true;
|
|
extraGroups = [ "wheel" ];
|
|
openssh.authorizedKeys.keys = [ ${keys r} ];
|
|
};
|
|
users.users.root.openssh.authorizedKeys.keys = [ ${keys r} ];
|
|
${tz}
|
|
system.stateVersion = ${h.nixStr r.seed.stateVersion};
|
|
${lib.optionalString r.git.enable ''
|
|
# git-tracking your flake: auto-commit, and push when a remote is set.
|
|
environment.systemPackages = [ pkgs.git ];
|
|
systemd.services."robotnix-flake-gitsync" = {
|
|
path = [ pkgs.git ];
|
|
script = '''
|
|
cd /etc/nixos
|
|
git add -A
|
|
git diff --cached --quiet || git commit -m "robotnix auto-sync $(date +'%Y-%m-%d %H:%M:%S')"
|
|
if git remote get-url origin >/dev/null 2>&1; then
|
|
git push -q
|
|
fi
|
|
''';
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
User = "root";
|
|
};
|
|
};
|
|
systemd.timers."robotnix-flake-gitsync" = {
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnCalendar = ${h.nixStr r.git.interval};
|
|
Persistent = true;
|
|
Unit = "robotnix-flake-gitsync.service";
|
|
};
|
|
};
|
|
''}
|
|
}
|
|
'';
|
|
|
|
flakeNix =
|
|
r:
|
|
builtins.toFile "robotnix-${r.name}-flake.nix" ''
|
|
# ${r.name}'s own flake. Apply changes with:
|
|
# sudo nixos-rebuild switch --flake /etc/nixos
|
|
{
|
|
inputs.nixpkgs.url = "github:NixOS/nixpkgs/${nixpkgsRef}";
|
|
outputs = { nixpkgs, ... }: {
|
|
nixosConfigurations.${h.nixStr r.name} = nixpkgs.lib.nixosSystem {
|
|
modules = [ ./robotnix.nix ./configuration.nix ];
|
|
};
|
|
};
|
|
}
|
|
'';
|
|
|
|
# The first system, built on the host exactly as the robot's own flake
|
|
# builds it later: lib.nixosSystem is eval-config with system = null and
|
|
# nixpkgs' lib, so this mirrors flake.nix's `nixpkgs.lib.nixosSystem` call.
|
|
seedSystem =
|
|
r:
|
|
(import "${pkgs.path}/nixos/lib/eval-config.nix" {
|
|
system = null;
|
|
inherit lib;
|
|
modules = [
|
|
(robotnixNix r)
|
|
r.seed.configuration
|
|
];
|
|
}).config.system.build.toplevel;
|
|
|
|
storageStart =
|
|
r:
|
|
pkgs.writeShellScript "${r.unit}-storage-start" ''
|
|
set -euo pipefail
|
|
base=${r.base}
|
|
mkdir -p ${cfg.baseDir}
|
|
if [ "$(stat -f -c %T ${cfg.baseDir})" != btrfs ]; then
|
|
echo "robotnix: ${cfg.baseDir} must be on btrfs (needed for snapshots)" >&2
|
|
exit 1
|
|
fi
|
|
for sub in "$base" "$base/root" "$base/home" "$base/snapshots"; do
|
|
[ -d "$sub" ] || btrfs subvolume create "$sub"
|
|
done
|
|
# No copy-on-write for the images (only affects new files).
|
|
chattr +C "$base/root" "$base/home"
|
|
|
|
image() { # <img> <size> <mountpoint>
|
|
img=$1 size=$2 dir=$3
|
|
if [ ! -e "$img" ]; then
|
|
truncate -s "$size" "$img"
|
|
mkfs.ext4 -q -F "$img"
|
|
fi
|
|
mkdir -p -m 0755 "$dir"
|
|
if ! mountpoint -q "$dir"; then
|
|
rc=0
|
|
e2fsck -p "$img" || rc=$?
|
|
if [ "$rc" -ge 4 ]; then
|
|
echo "robotnix: $img needs a manual e2fsck" >&2
|
|
exit 1
|
|
fi
|
|
mount -o loop,noatime "$img" "$dir"
|
|
fi
|
|
# Grow when the configured size goes up. Shrinking is not supported.
|
|
want=$(numfmt --from=iec "$size")
|
|
have=$(stat -c %s "$img")
|
|
if [ "$want" -gt "$have" ]; then
|
|
truncate -s "$size" "$img"
|
|
dev=$(findmnt -n -o SOURCE "$dir")
|
|
losetup -c "$dev"
|
|
resize2fs "$dev"
|
|
elif [ "$want" -lt "$have" ]; then
|
|
echo "robotnix: $img is bigger than $size; shrinking is not supported" >&2
|
|
fi
|
|
}
|
|
|
|
mkdir -p /run/robotnix ${r.run}
|
|
chmod 0755 /run/robotnix ${r.run}
|
|
image "$base/root/root.img" ${r.disk.rootSize} ${r.run}/root
|
|
image "$base/home/home.img" ${r.disk.homeSize} ${r.run}/home
|
|
chmod 0755 ${r.run}/root
|
|
'';
|
|
|
|
storageStop =
|
|
r:
|
|
pkgs.writeShellScript "${r.unit}-storage-stop" ''
|
|
for d in ${r.run}/home ${r.run}/root; do
|
|
if mountpoint -q "$d"; then umount "$d"; fi
|
|
done
|
|
'';
|
|
|
|
# Installs the seed only when the root image has no system yet (first
|
|
# start, or after `robotnix reseed`). Never touches an installed robot.
|
|
seedScript =
|
|
r:
|
|
pkgs.writeShellScript "${r.unit}-seed" ''
|
|
set -euo pipefail
|
|
root=${r.run}/root
|
|
if [ ! -e "$root/nix/var/nix/profiles/system" ]; then
|
|
echo "robotnix: installing the seed system into ${r.name}"
|
|
nixos-install --root "$root" --system ${seedSystem r} \
|
|
--no-root-passwd --no-bootloader --no-channel-copy
|
|
install -d -m 0755 "$root/etc/nixos"
|
|
install -m 0644 ${flakeNix r} "$root/etc/nixos/flake.nix"
|
|
install -m 0644 ${robotnixNix r} "$root/etc/nixos/robotnix.nix"
|
|
install -m 0644 ${r.seed.configuration} "$root/etc/nixos/configuration.nix"
|
|
${lib.optionalString r.git.enable ''
|
|
echo "robotnix: setting up the git repo for ${r.name}'s flake"
|
|
git -C "$root/etc/nixos" init -q
|
|
git -C "$root/etc/nixos" config user.name ${h.nixStr r.git.userName}
|
|
git -C "$root/etc/nixos" config user.email ${h.nixStr r.git.userEmail}
|
|
${lib.optionalString (r.git.remoteUrl != null) ''
|
|
git -C "$root/etc/nixos" remote add origin ${h.nixStr r.git.remoteUrl}
|
|
''}
|
|
git -C "$root/etc/nixos" add -A
|
|
git -C "$root/etc/nixos" commit -q -m "robotnix initial seed" || true
|
|
''}
|
|
fi
|
|
# What nspawn and NixOS's init expect (same as nixos-containers).
|
|
mkdir -p "$root/etc" "$root/var/lib" "$root/root" "$root/home"
|
|
chmod 0700 "$root/root"
|
|
[ -e "$root/etc/os-release" ] || [ -L "$root/etc/os-release" ] || touch "$root/etc/os-release"
|
|
[ -e "$root/etc/machine-id" ] || touch "$root/etc/machine-id"
|
|
'';
|
|
|
|
preStart =
|
|
r:
|
|
pkgs.writeShellScript "${r.unit}-pre" ''
|
|
set -euo pipefail
|
|
for _ in $(seq 1 30); do
|
|
[ -e /sys/class/net/${br} ] && [ -e /sys/class/net/${ext} ] && break
|
|
sleep 1
|
|
done
|
|
[ -e /sys/class/net/${br} ] || { echo "robotnix: ${br} is missing" >&2; exit 1; }
|
|
# Answer ARP for the robot's LAN IP on the uplink (this IP only).
|
|
ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true
|
|
ip neigh add proxy ${r.address} dev ${ext}
|
|
'';
|
|
|
|
postStop =
|
|
r:
|
|
pkgs.writeShellScript "${r.unit}-post" ''
|
|
ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true
|
|
'';
|
|
|
|
nspawnArgs =
|
|
r:
|
|
lib.escapeShellArgs (
|
|
[
|
|
"--keep-unit"
|
|
"--machine=${r.name}"
|
|
"--directory=${r.run}/root"
|
|
"--notify-ready=yes"
|
|
"--kill-signal=SIGRTMIN+3"
|
|
"--network-bridge=${br}"
|
|
"--link-journal=try-guest"
|
|
"--resolv-conf=off"
|
|
"--timezone=off"
|
|
"--bind=${r.run}/home:/home${lib.optionalString r.privateUsers ":idmap"}"
|
|
]
|
|
++ lib.optionals r.privateUsers [
|
|
"--private-users=pick"
|
|
"--private-users-ownership=auto"
|
|
]
|
|
++ [ "/nix/var/nix/profiles/system/init" ]
|
|
);
|
|
|
|
robotUnits = r: {
|
|
"${r.unit}-storage" = {
|
|
description = "robotnix storage for ${r.name}";
|
|
path = with pkgs; [
|
|
btrfs-progs
|
|
coreutils
|
|
e2fsprogs
|
|
util-linux
|
|
];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = storageStart r;
|
|
ExecStop = storageStop r;
|
|
};
|
|
};
|
|
|
|
"${r.unit}-seed" = {
|
|
description = "robotnix first install for ${r.name}";
|
|
requires = [ "${r.unit}-storage.service" ];
|
|
bindsTo = [ "${r.unit}-storage.service" ];
|
|
after = [ "${r.unit}-storage.service" ];
|
|
path = [
|
|
config.system.build.nixos-install
|
|
config.nix.package
|
|
pkgs.coreutils
|
|
pkgs.util-linux
|
|
pkgs.git
|
|
];
|
|
environment.HOME = "/root";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = seedScript r;
|
|
TimeoutStartSec = "30min";
|
|
};
|
|
};
|
|
|
|
${r.unit} =
|
|
let
|
|
mem = h.toBytes r.memory.max;
|
|
deps = [
|
|
"${r.unit}-storage.service"
|
|
"${r.unit}-seed.service"
|
|
"${r.unit}-dns.service"
|
|
];
|
|
in
|
|
{
|
|
description = "robotnix robot ${r.name}";
|
|
wantedBy = lib.optional r.autoStart "machines.target";
|
|
requires = deps;
|
|
bindsTo = [ "${r.unit}-storage.service" ];
|
|
after = deps ++ [
|
|
"network.target"
|
|
"systemd-networkd.service"
|
|
"nftables.service"
|
|
];
|
|
path = [
|
|
pkgs.iproute2
|
|
pkgs.coreutils
|
|
];
|
|
environment.SYSTEMD_NSPAWN_UNIFIED_HIERARCHY = "1";
|
|
serviceConfig = {
|
|
ExecStartPre = preStart r;
|
|
ExecStart = "${config.systemd.package}/bin/systemd-nspawn ${nspawnArgs r}";
|
|
# Best effort: refresh LAN ARP caches.
|
|
ExecStartPost = "-${pkgs.iputils}/bin/arping -U -c 2 -I ${ext} ${r.address}";
|
|
ExecStopPost = postStop r;
|
|
|
|
# Same as nixos-containers.
|
|
Type = "notify";
|
|
RestartForceExitStatus = "133";
|
|
SuccessExitStatus = "133";
|
|
Restart = "on-failure";
|
|
TimeoutStartSec = "5min";
|
|
Slice = "machine.slice";
|
|
Delegate = true;
|
|
KillMode = "mixed";
|
|
KillSignal = "TERM";
|
|
DevicePolicy = "closed";
|
|
|
|
# Resources.
|
|
CPUQuota = "${toString (r.cpus.max * 100)}%";
|
|
CPUWeight = toString (lib.min 10000 (100 * r.cpus.min));
|
|
MemoryLow = r.memory.min;
|
|
MemoryHigh = toString (mem - mem / 16);
|
|
MemoryMax = r.memory.max;
|
|
MemorySwapMax = r.memory.swapMax;
|
|
TasksMax = 16384;
|
|
};
|
|
};
|
|
};
|
|
|
|
cliConfig = pkgs.writeText "robotnix.json" (
|
|
builtins.toJSON {
|
|
inherit (cfg) baseDir;
|
|
robots = lib.mapAttrs
|
|
(_: r: {
|
|
inherit (r)
|
|
name
|
|
address
|
|
port
|
|
dnsPort
|
|
base
|
|
run
|
|
unit
|
|
user
|
|
nft
|
|
access
|
|
;
|
|
snapshots = { inherit (r.snapshots) enable interval keep; };
|
|
})
|
|
h.robots;
|
|
}
|
|
);
|
|
|
|
cliPath = lib.makeBinPath [
|
|
pkgs.btrfs-progs
|
|
pkgs.util-linux
|
|
pkgs.coreutils
|
|
pkgs.nftables
|
|
config.systemd.package
|
|
];
|
|
|
|
cli = pkgs.writers.writeNuBin "robotnix" (
|
|
''
|
|
const CONFIG = "${cliConfig}"
|
|
$env.PATH = ("${cliPath}" | split row ":" | append ($env.PATH | split row (char esep)))
|
|
''
|
|
+ builtins.readFile ./robotnix.nu
|
|
);
|
|
in
|
|
lib.mkIf cfg.enable {
|
|
systemd.services = lib.mkMerge (map robotUnits rs);
|
|
|
|
systemd.targets.multi-user.wants = [ "machines.target" ];
|
|
|
|
# Memory protection only works if the parent slice is protected too.
|
|
systemd.slices.machine.sliceConfig.MemoryLow = lib.mkIf (rs != [ ]) (
|
|
lib.mkDefault (toString (lib.foldl' (acc: r: acc + h.toBytes r.memory.min) 0 rs))
|
|
);
|
|
|
|
skyg.server.timers = lib.listToAttrs (
|
|
map
|
|
(
|
|
r:
|
|
lib.nameValuePair "${r.unit}-snapshot" {
|
|
script = "${cli}/bin/robotnix snapshot ${r.name} --auto";
|
|
OnCalendar = r.snapshots.interval;
|
|
}
|
|
)
|
|
(lib.filter (r: r.snapshots.enable) rs)
|
|
);
|
|
|
|
environment.systemPackages = [ cli ];
|
|
}
|