From eb6380ddfee9f6b4d2b34f8d7aac14722d0d1503 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Fri, 23 Oct 2026 15:26:04 -0400 Subject: [PATCH] Robotnix v1 --- hosts/hl-fwdesk/services/default.nix | 1 + hosts/hl-fwdesk/services/robotnix.nix | 39 ++ hosts/hl-fwdesk/services/robotnix/genix.nix | 24 + modules/ABOUTME.md | 3 +- modules/main.nix | 1 + modules/nixos/server/ABOUTME.md | 7 + modules/nixos/server/default.nix | 1 + modules/nixos/server/robotnix/ABOUTME.md | 118 +++++ modules/nixos/server/robotnix/default.nix | 358 +++++++++++++++ modules/nixos/server/robotnix/helpers.nix | 86 ++++ modules/nixos/server/robotnix/network.nix | 294 ++++++++++++ modules/nixos/server/robotnix/robotnix.nu | 258 +++++++++++ modules/nixos/server/robotnix/robots.nix | 429 ++++++++++++++++++ .../server/robotnix/seed/configuration.nix | 24 + 14 files changed, 1642 insertions(+), 1 deletion(-) create mode 100644 hosts/hl-fwdesk/services/robotnix.nix create mode 100644 hosts/hl-fwdesk/services/robotnix/genix.nix create mode 100644 modules/nixos/server/robotnix/ABOUTME.md create mode 100644 modules/nixos/server/robotnix/default.nix create mode 100644 modules/nixos/server/robotnix/helpers.nix create mode 100644 modules/nixos/server/robotnix/network.nix create mode 100644 modules/nixos/server/robotnix/robotnix.nu create mode 100644 modules/nixos/server/robotnix/robots.nix create mode 100644 modules/nixos/server/robotnix/seed/configuration.nix diff --git a/hosts/hl-fwdesk/services/default.nix b/hosts/hl-fwdesk/services/default.nix index 600baf9..dd946e8 100644 --- a/hosts/hl-fwdesk/services/default.nix +++ b/hosts/hl-fwdesk/services/default.nix @@ -8,5 +8,6 @@ ./steam.nix ./buildbot.nix ./linux-tracker.nix + ./robotnix.nix ]; } diff --git a/hosts/hl-fwdesk/services/robotnix.nix b/hosts/hl-fwdesk/services/robotnix.nix new file mode 100644 index 0000000..7f26001 --- /dev/null +++ b/hosts/hl-fwdesk/services/robotnix.nix @@ -0,0 +1,39 @@ +{ config, ... }: +let + apps = config.skyg.internalNetworkingMap.apps; + ports = { + ollama = 11434; + ds4 = 8000; + tracker = 1984; + }; +in +{ + skyg.nixos.server.robotnix = { + enable = true; + externalInterface = "enp191s0"; + + # Every robot gets this tier. Same holes as the Hermes VM. + access.shared = { + internet = "all"; + hostAccess = [ + { port = ports.ollama; } + { port = ports.tracker; } + ]; + lanAccess = [ ]; + }; + + robots.genix = { + id = 1; + address = apps.genix.ip; + cpus = { min = 1; max = 4; }; + memory = { min = "8G"; max = "16G"; }; + disk = { rootSize = "100G"; homeSize = "60G"; }; + seed.configuration = ./robotnix/genix.nix; + access = { }; + git = { + enable = true; + remoteUrl = "gitea@gitea.app.internal:Hermenix/genix.git"; + }; + }; + }; +} diff --git a/hosts/hl-fwdesk/services/robotnix/genix.nix b/hosts/hl-fwdesk/services/robotnix/genix.nix new file mode 100644 index 0000000..42878a7 --- /dev/null +++ b/hosts/hl-fwdesk/services/robotnix/genix.nix @@ -0,0 +1,24 @@ +# genix's starting NixOS config (a robotnix seed, not a host module). +# +# The host copies this into the robot's /etc/nixos on first install and on +# `robotnix reseed`. After that the robot owns it. Inside the robot: +# sudo nixos-rebuild switch --flake /etc/nixos +# Host-managed basics (network, ssh, user) live in /etc/nixos/robotnix.nix. +{ pkgs, ... }: +{ + # The user has no password; ssh keys only. + security.sudo.wheelNeedsPassword = false; + + environment.systemPackages = with pkgs; [ + git + vim + curl + htop + ]; + + nix.gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 14d"; + }; +} diff --git a/modules/ABOUTME.md b/modules/ABOUTME.md index ee32e34..cba97f2 100644 --- a/modules/ABOUTME.md +++ b/modules/ABOUTME.md @@ -23,7 +23,7 @@ modules/ │ ├── common/ # Core system settings, networking, SSH, QEMU, fonts, hardware │ ├── server/ # Server services: K3s/K8s, DNS, AI stack, ARRs, media… │ └── desktop/ # Desktop: tiler (niri/hyprland), packages, theming, crypto -└── openwrt/ # OpenWrt router config generator +├── openwrt/ # OpenWrt router config generator ``` ## Option Namespace @@ -38,6 +38,7 @@ skyg.rootDir → core/default.nix (set by the flake; repo root stor skyg.nixos.common.* → nixos/common/ skyg.nixos.desktop.* → nixos/desktop/ skyg.nixos.server.* → nixos/server/ +skyg.nixos.server.robotnix.* → nixos/server/robotnix/ (self-managed nspawn robots) skyg.core.* → core/ ``` diff --git a/modules/main.nix b/modules/main.nix index 52a9755..f9c6fb3 100644 --- a/modules/main.nix +++ b/modules/main.nix @@ -111,6 +111,7 @@ in public-gitea = { ip = "10.0.101.9"; }; immich = { ip = "10.0.101.10"; }; immich-ml = { ip = "10.0.101.11"; }; + genix = { ip = "10.0.101.12"; }; jellyfin = { ip = "10.0.101.5"; }; nvr = { ip = "10.0.101.6"; }; portainer = { ip = "10.0.101.241"; }; diff --git a/modules/nixos/server/ABOUTME.md b/modules/nixos/server/ABOUTME.md index cdd9f1b..a138ae6 100644 --- a/modules/nixos/server/ABOUTME.md +++ b/modules/nixos/server/ABOUTME.md @@ -18,6 +18,7 @@ server/ ├── k3s/ # Single-binary K3s cluster role ├── k8s/ # Full upstream Kubernetes (master/node) ├── portainer.nix # skyg.nixos.server.portainer — Portainer server or Edge agent +├── robotnix/ # skyg.nixos.server.robotnix — self-managed NixOS nspawn containers └── services/ # Self-hosted apps (jellyfin, audiobookshelf, comfyui, ds4, colibri, ai…) ``` @@ -33,6 +34,11 @@ server/ (admin/timers/arrs/dns) and `skyg.nixos.server.*` for the K3s/K8s roles and the service modules under `services/`. +| Structure | Namespace | Notes | +| ------------ | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `robotnix/` | `skyg.nixos.server.robotnix` | Self-managed NixOS nspawn containers (persistent system, own LAN IP, snapshots, host-controlled web access). The one sanctioned nspawn exception to the oci-containers rule | +| `timers.nix` | `skyg.server.timers` | Generic attrset → timer + oneshot service (robotnix snapshot timers use it) | + ## Option Namespace ``` @@ -44,6 +50,7 @@ skyg.server.arrs.* → arrs/ skyg.server.dns.* → dns/ skyg.nixos.server.k3s.* → k3s/ skyg.nixos.server.k8s.* → k8s/ +skyg.nixos.server.robotnix.* → robotnix/ (self-managed nspawn containers) skyg.nixos.server.services.* → services/ skyg.nixos.server.portainer.* → portainer.nix ``` diff --git a/modules/nixos/server/default.nix b/modules/nixos/server/default.nix index 502aa38..df84263 100644 --- a/modules/nixos/server/default.nix +++ b/modules/nixos/server/default.nix @@ -11,5 +11,6 @@ ./timers.nix ./dns ./portainer.nix + ./robotnix ]; } diff --git a/modules/nixos/server/robotnix/ABOUTME.md b/modules/nixos/server/robotnix/ABOUTME.md new file mode 100644 index 0000000..d472fd0 --- /dev/null +++ b/modules/nixos/server/robotnix/ABOUTME.md @@ -0,0 +1,118 @@ +# robotnix — self-managed NixOS nspawn robots + +`skyg.nixos.server.robotnix` runs persistent NixOS `systemd-nspawn` containers +("robots") that fully own their own system. Each robot has a real LAN IP, hard +disk limits, btrfs snapshots, and network access the host controls completely. + +## Model + +- **The robot owns its system.** On first install (`robotnix--seed.service`) + the host builds a starting system with the same entry point the robot's own + flake uses (`lib.nixosSystem` = eval-config with `system = null`), installs it + into `root.img` with `nixos-install --no-bootloader`, and writes + `/etc/nixos/{flake.nix,robotnix.nix,configuration.nix}`. After that the host + never touches the robot's root: it rebuilds with + `nixos-rebuild switch --flake /etc/nixos` against its own pinned nixpkgs and + its own store. Changes survive reboots. +- **`robotnix.nix` is host-owned** (network, user, ssh keys, git-sync). + `configuration.nix` is the robot's own config (`seed.configuration`, default + `./seed/configuration.nix`). Changing a seed config only matters at the next + `robotnix reseed`. +- **No host root.** Robots run with `--private-users=pick` unless + `privateUsers = false`. + +## Storage + +Per robot under `//` (btrfs subvolumes, `nodatacow` images): + +``` +root/root.img 100G (default) sparse ext4, whole OS incl. /nix +home/home.img 60G (default) sparse ext4, /home +snapshots//{root,home} read-only subvolume snapshots +``` + +The host mounts the images at `/run/robotnix//` and passes them in, so a +snapshot freezes both filesystems (`fsfreeze`) for a consistent point in time +while the robot runs. Sizes are hard limits; raise them and the images grow on +the next restart (shrinking is not supported). + +Snapshots: per-robot `snapshots.enable/interval/keep` via `skyg.server.timers` +(`robotnix--snapshot`), `robotnix snapshot|restore|forget|snapshots`. +`restore` saves a `pre-restore` snapshot first. Because images are `nodatacow`, +keep the automatic snapshot count modest — every live snapshot reintroduces +copy-on-write for the referenced blocks. Backups are out of scope for now. + +## Network + +One routed virtual switch: + +- `robotnix-br` bridge (networkd) on 10.78.0.1/24, unmanaged by + NetworkManager; each robot's `vb-` port is `Isolated` (no robot-to- + robot L2). +- Robots hold `/32` with a default route via the switch. The host routes + to them and answers ARP for that IP on `externalInterface` (proxy ARP per IP; + global proxy ARP stays off), so each packet passes the host firewall. +- nftables: a bridge table drops spoofed addresses; an `inet robotnix` table + does the per-robot filtering before `nixos-fw` (priority filter - 10). IPv6 + from robots is dropped (`host0` is IPv4-only). Inbound from the LAN to a + robot is open on every port — make your own decisions inside the robot. +- Per-robot dnsmasq on the host (port 5300+id) is the robot's only resolver: + all port 53 is DNAT'd to it, DoT is dropped. The robot cannot change its + resolv.conf meaningfully; it cannot touch the host's nftables/networkd. + +## Access tiers + +`access.shared` + `robots..access` merge per robot (`internet` from the +robot tier wins, all lists are unions). Rule order in the out-chain: + +1. established/related +2. port 53 served by the host / 853 dropped (DoT) +3. `blockedCidrs` and blocked-domain IPs (nftsets fed by dnsmasq `nftset=`) +4. `lanAccess` / `hostAccess` endpoints (host input is matched per-robot by + port-source rules) +5. RFC1918/CGNAT/link-local drop unless `allowLan` +6. internet: `all` (default) or, with `allowlist`, only `allowedCidrs` + + allowed-domain IPs + +Domain rules are IP-based: acceptable for host-controlled filtering, but a +CDN shared with an allowed domain also lets that IP's other sites through (and +dnsmasq learns current IPs). `blockedDomains` resolve to NXDOMAIN; in +allowlist mode everything not allowed is NXDOMAIN + dropped. + +## Git-sync of the robot's flake (default) + +On first install / reseed, the seed creates a git repo at `/etc/nixos` +(`git.enable`, default on) and sets `git.remoteUrl` as `origin` when given. A +timer inside the robot (`robotnix-flake-gitsync` = `git add -A`, commit, push +if origin exists) runs on `git.interval` (`daily` default). The robot needs the +remote reachable (e.g. through `lanAccess`) and its own credentials if the +remote is private. + +## The `robotnix` CLI + +Nushell script (`pkgs.writers.writeNuBin`), part of this module: +`list`, `status`, `start/stop/restart`, `shell`, `snapshot`, +`snapshots`, `forget`, `restore`, `reseed [--wipe-home]`, `access`. +Runtime state comes from `/etc/robotnix.json`, written by the module. + +## Notes / decisions + +- **nspawn, not `oci-containers`**: the server/GUIDELINES rule about + `oci-containers` targets Docker/Podman. nspawn containers need + `systemd-nspawn` and the NixOS container init; there is no OCI equivalent. +- This is why `systemd-nspawn` is launched by a hand-written unit rather than + the `containers.*` module: that module shares the host store, which would + break the robot's self-rebuilds. The unit mirrors `nixos-containers.nix` + (`Type=notify`, SIGRTMIN+3, exit status 133, `machine.slice`). +- `systemd-nspawn` runs in `machine.slice` (systemd's nspawn home), not + `system.slice`. +- Timers use `skyg.server.timers`; eval asserts unique robot ids/IPs, name + length ≤ 12 (interface-name limit), no clash with + `skyg.internalNetworkingMap.apps`, and no domain/CIDR both allowed and + blocked. +- `fsfreeze` fails on unfrozen ro/sparse quirks only when the images are not + mounted (robot stopped); the CLI snapshots only mounted filesystems' mounts, + otherwise it snapshots the subvolumes directly. +- flake inputs: only the host nixpkgs is pinned into the robot's flake + (`github:NixOS/nixpkgs/`). Extra inputs belong in the robot's own + `configuration.nix` edits. diff --git a/modules/nixos/server/robotnix/default.nix b/modules/nixos/server/robotnix/default.nix new file mode 100644 index 0000000..7e1893a --- /dev/null +++ b/modules/nixos/server/robotnix/default.nix @@ -0,0 +1,358 @@ +# robotnix: self-managed NixOS nspawn containers ("robots"). +# See ABOUTME.md in this folder for the design. +{ config, lib, ... }: +let + inherit (lib) mkOption mkEnableOption types; + cfg = config.skyg.nixos.server.robotnix; + h = import ./helpers.nix { inherit config lib; }; + + size = types.strMatching "[0-9]+[KMGT]"; + + proto = mkOption { + type = types.enum [ + "tcp" + "udp" + ]; + default = "tcp"; + }; + + # One access tier. Used for `access.shared` and each robot's `access`. + tier = types.submodule { + options = { + internet = mkOption { + type = types.nullOr ( + types.enum [ + "all" + "allowlist" + ] + ); + default = null; + description = '' + "all": the whole public internet (minus blocked entries). + "allowlist": only allowedDomains / allowedCidrs. + null: inherit (robot tier -> shared tier -> "all"). + ''; + }; + allowedDomains = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "github.com" ]; + description = "Domains (and their subdomains) reachable in allowlist mode."; + }; + allowedCidrs = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "140.82.112.0/20" ]; + description = "Public IPv4 ranges reachable in allowlist mode."; + }; + blockedDomains = mkOption { + type = types.listOf types.str; + default = [ ]; + description = "Domains that never resolve. Applies in both modes."; + }; + blockedCidrs = mkOption { + type = types.listOf types.str; + default = [ ]; + description = "Public IPv4 ranges that are always dropped. Applies in both modes."; + }; + lanAccess = mkOption { + default = [ ]; + example = [ + { + address = "10.0.101.3"; + port = 3000; + } + ]; + description = "LAN endpoints punched through the private-range drop."; + type = types.listOf ( + types.submodule { + options = { + address = mkOption { type = types.str; }; + port = mkOption { type = types.port; }; + inherit proto; + }; + } + ); + }; + hostAccess = mkOption { + default = [ ]; + example = [{ port = 11434; }]; + description = "Ports on this host the robot may reach."; + type = types.listOf ( + types.submodule { + options = { + port = mkOption { type = types.port; }; + inherit proto; + }; + } + ); + }; + allowLan = mkOption { + type = types.bool; + default = false; + description = "Open all private ranges. Almost always the wrong tool; use lanAccess."; + }; + }; + }; + + robot = types.submodule ( + { name, ... }: + { + options = { + id = mkOption { + type = types.ints.between 1 250; + description = "Unique number per robot. Picks its DNS port (5300 + id)."; + }; + address = mkOption { + type = types.str; + example = "10.0.101.12"; + description = "The robot's own LAN IPv4 address."; + }; + autoStart = mkOption { + type = types.bool; + default = true; + }; + privateUsers = mkOption { + type = types.bool; + default = true; + description = "Run in a user namespace (root inside != root on the host)."; + }; + user = mkOption { + type = types.str; + default = config.skyg.user.name; + description = "Login user created in the robot."; + }; + authorizedKeys = mkOption { + type = types.listOf types.str; + default = config.skyg.nixos.common.ssh-server.masterPubKeys; + description = "SSH keys for the user and root in the robot."; + }; + cpus = { + min = mkOption { + type = types.ints.positive; + default = 1; + }; + max = mkOption { + type = types.ints.positive; + default = 4; + }; + }; + memory = { + min = mkOption { + type = size; + default = "8G"; + description = "Protected memory (MemoryLow)."; + }; + max = mkOption { + type = size; + default = "16G"; + description = "Hard limit (MemoryMax)."; + }; + swapMax = mkOption { + type = size; + default = "1G"; + }; + }; + disk = { + rootSize = mkOption { + type = size; + default = "100G"; + description = "Root image: the whole OS including /nix. Grows on restart if raised."; + }; + homeSize = mkOption { + type = size; + default = "60G"; + description = "/home image. Grows on restart if raised."; + }; + }; + access = mkOption { + type = tier; + default = { }; + description = "This robot's own tier, added on top of access.shared."; + }; + snapshots = { + enable = mkOption { + type = types.bool; + default = true; + }; + interval = mkOption { + type = types.str; + default = "hourly"; + description = "systemd calendar expression."; + }; + keep = mkOption { + type = types.ints.positive; + default = 24; + description = "Automatic snapshots to keep. Manual ones are never pruned."; + }; + }; + git = { + enable = mkOption { + type = types.bool; + default = true; + description = "Track /etc/nixos in git on first install and auto-commit."; + }; + remoteUrl = mkOption { + type = types.nullOr types.str; + default = null; + example = "git@github.com:ari/genix.git"; + description = '' + Origin for the flake repo, set on first install. null = no remote: + the timer commits locally but never pushes. + ''; + }; + userName = mkOption { + type = types.str; + default = "robotnix"; + }; + userEmail = mkOption { + type = types.str; + default = "robotnix@localhost"; + }; + interval = mkOption { + type = types.str; + default = "daily"; + description = "systemd calendar for the auto-commit (and push, if a remote is set) timer."; + }; + }; + seed = { + configuration = mkOption { + type = types.path; + default = ./seed/configuration.nix; + description = '' + The robot's starting configuration.nix. Copied into /etc/nixos on + first install and on `robotnix reseed`; after that the robot owns it. + ''; + }; + stateVersion = mkOption { + type = types.str; + default = lib.trivial.release; + }; + }; + }; + } + ); +in +{ + imports = [ + ./network.nix + ./robots.nix + ]; + + options.skyg.nixos.server.robotnix = { + enable = mkEnableOption "robotnix self-managed NixOS containers"; + + externalInterface = mkOption { + type = types.nullOr types.str; + default = null; + example = "enp191s0"; + description = "Host LAN uplink. Robots' LAN IPs are proxy-ARPed here."; + }; + + baseDir = mkOption { + type = types.str; + default = "/var/lib/robotnix"; + description = "Must be on btrfs. Each robot gets / as its own subvolume."; + }; + + switch = { + name = mkOption { + type = types.str; + default = "robotnix-br"; + }; + address = mkOption { + type = types.str; + default = "10.78.0.1"; + description = "Host address on the switch: robots' gateway and DNS."; + }; + prefixLength = mkOption { + type = types.int; + default = 24; + }; + }; + + dnsUpstreams = mkOption { + type = types.listOf types.str; + default = [ config.skyg.internalNetworkingMap.appNetwork.gateway ]; + description = "Where the host's per-robot DNS forwards to."; + }; + + localDomains = mkOption { + type = types.listOf types.str; + default = [ + "app.internal" + "lab.internal" + ]; + description = "Always resolvable, even in allowlist mode (reaching them still needs lanAccess)."; + }; + + access.shared = mkOption { + type = tier; + default = { }; + description = "Tier applied to every robot."; + }; + + robots = mkOption { + type = types.attrsOf robot; + default = { }; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = + let + rs = h.robotList; + dupes = f: lib.filter (v: lib.count (x: x == v) (map f rs) > 1) (lib.unique (map f rs)); + apps = config.skyg.internalNetworkingMap.apps; + in + [ + { + assertion = cfg.externalInterface != null; + message = "skyg.nixos.server.robotnix.externalInterface must be set."; + } + { + assertion = dupes (r: r.id) == [ ]; + message = "robotnix: robot ids must be unique (dupes: ${toString (dupes (r: r.id))})."; + } + { + assertion = dupes (r: r.address) == [ ]; + message = "robotnix: robot addresses must be unique (dupes: ${toString (dupes (r: r.address))})."; + } + ] + ++ lib.concatMap + ( + r: + let + clash = lib.attrNames (lib.filterAttrs (an: a: a.ip == r.address && an != r.name) apps); + both = a: b: lib.intersectLists r.access.${a} r.access.${b}; + in + [ + { + assertion = builtins.match "[a-z][a-z0-9-]{0,11}" r.name != null; + message = "robotnix: robot name '${r.name}' must be lowercase, start with a letter, max 12 chars."; + } + { + assertion = clash == [ ]; + message = "robotnix: ${r.name}'s address ${r.address} is already used by app(s): ${toString clash}."; + } + { + assertion = both "allowedDomains" "blockedDomains" == [ ]; + message = "robotnix: ${r.name} both allows and blocks: ${toString (both "allowedDomains" "blockedDomains")}."; + } + { + assertion = both "allowedCidrs" "blockedCidrs" == [ ]; + message = "robotnix: ${r.name} both allows and blocks: ${toString (both "allowedCidrs" "blockedCidrs")}."; + } + { + assertion = r.cpus.min <= r.cpus.max; + message = "robotnix: ${r.name} cpus.min must be <= cpus.max."; + } + { + assertion = h.toBytes r.memory.min <= h.toBytes r.memory.max; + message = "robotnix: ${r.name} memory.min must be <= memory.max."; + } + ] + ) + rs; + }; +} diff --git a/modules/nixos/server/robotnix/helpers.nix b/modules/nixos/server/robotnix/helpers.nix new file mode 100644 index 0000000..567c5b6 --- /dev/null +++ b/modules/nixos/server/robotnix/helpers.nix @@ -0,0 +1,86 @@ +# Shared values for network.nix, robots.nix and the CLI. +{ config, lib }: +let + cfg = config.skyg.nixos.server.robotnix; + + # "16G" -> bytes (IEC, same as systemd and truncate). + toBytes = + s: + let + m = builtins.match "([0-9]+)([KMGT])" s; + mult = { + K = 1024; + M = 1024 * 1024; + G = 1024 * 1024 * 1024; + T = 1024 * 1024 * 1024 * 1024; + }; + in + lib.toInt (builtins.elemAt m 0) * mult.${builtins.elemAt m 1}; + + # nft identifiers can't hold "-". + nftName = name: lib.replaceStrings [ "-" ] [ "_" ] name; + + # Render a string as a Nix string literal (for generated .nix files). + nixStr = s: lib.replaceStrings [ "\${" ] [ "\\\${" ] (builtins.toJSON s); + + # Shared tier + the robot's own tier. Lists are merged, the robot's + # `internet` mode wins when set. + mergeAccess = + r: + let + s = cfg.access.shared; + both = f: lib.unique (f s ++ f r); + in + { + internet = + if r.internet != null then + r.internet + else if s.internet != null then + s.internet + else + "all"; + allowLan = s.allowLan || r.allowLan; + allowedDomains = both (t: t.allowedDomains); + allowedCidrs = both (t: t.allowedCidrs); + blockedDomains = both (t: t.blockedDomains); + blockedCidrs = both (t: t.blockedCidrs); + lanAccess = both (t: t.lanAccess); + hostAccess = both (t: t.hostAccess); + }; + + mkRobot = name: r: { + inherit name; + inherit (r) + id + address + autoStart + privateUsers + cpus + memory + disk + snapshots + seed + user + authorizedKeys + git + ; + nft = nftName name; + port = "vb-${name}"; + dnsPort = 5300 + r.id; + base = "${cfg.baseDir}/${name}"; + run = "/run/robotnix/${name}"; + unit = "robotnix-${name}"; + access = mergeAccess r.access; + }; + + robots = lib.mapAttrs mkRobot cfg.robots; +in +{ + inherit + toBytes + nftName + nixStr + robots + ; + robotList = lib.attrValues robots; +} diff --git a/modules/nixos/server/robotnix/network.nix b/modules/nixos/server/robotnix/network.nix new file mode 100644 index 0000000..6531972 --- /dev/null +++ b/modules/nixos/server/robotnix/network.nix @@ -0,0 +1,294 @@ +# robotnix network: one routed virtual switch, per-robot firewall and DNS. +# +# Robots plug into the switch bridge and hold their LAN IP as a /32. The host +# routes to them and answers ARP for their IPs on the uplink (proxy ARP), so +# every packet passes the host firewall. Robots never see the host's netns. +{ config +, lib +, pkgs +, ... +}: +let + cfg = config.skyg.nixos.server.robotnix; + h = import ./helpers.nix { inherit config lib; }; + rs = h.robotList; + br = cfg.switch.name; + sw = cfg.switch.address; + + set = xs: "{ ${lib.concatStringsSep ", " xs} }"; + + # Never reachable, whatever the tier says. + neverRanges = [ + "0.0.0.0/8" + "127.0.0.0/8" + "224.0.0.0/4" + "240.0.0.0/4" + ]; + # Private ranges: only via lanAccess / allowLan. + lanRanges = [ + "10.0.0.0/8" + "172.16.0.0/12" + "192.168.0.0/16" + "169.254.0.0/16" + "100.64.0.0/10" + ]; + + # Outbound (forwarded) traffic from one robot. Order matters. + outChain = + r: + let + a = r.access; + in + '' + chain out_${r.nft} { + # DNS only via the host (port 53 is redirected in prerouting). + meta l4proto { tcp, udp } th dport 853 drop + meta l4proto { tcp, udp } th dport 53 drop + # The redirected DNS request (now dport = the robot's dnsmasq port) + # must pass before the private-range drop. + ip daddr ${sw} meta l4proto { tcp, udp } th dport ${toString r.dnsPort} accept + ip daddr ${set neverRanges} drop + '' + + lib.optionalString (a.blockedCidrs != [ ]) '' + ip daddr ${set a.blockedCidrs} drop + '' + + lib.concatMapStrings + (e: '' + ip daddr ${e.address} ${e.proto} dport ${toString e.port} accept + '') + a.lanAccess + + ( + if a.allowLan then + '' + ip daddr ${set lanRanges} accept + '' + else + '' + ip daddr ${set lanRanges} drop + '' + ) + + ( + if a.internet == "all" then + '' + accept + '' + else + lib.optionalString (a.allowedCidrs != [ ]) '' + ip daddr ${set a.allowedCidrs} accept + '' + + '' + ip daddr @allow_${r.nft} accept + drop + '' + ) + + '' + } + ''; + + # Traffic from one robot to the host itself. + inChain = r: '' + chain in_${r.nft} { + meta l4proto { tcp, udp } th dport ${toString r.dnsPort} accept + icmp type echo-request accept + '' + + lib.concatMapStrings + (p: '' + ${p.proto} dport ${toString p.port} accept + '') + r.access.hostAccess + + '' + drop + } + ''; + + vmap = prefix: set (map (r: "${r.address} : jump ${prefix}_${r.nft}") rs); + + # Host-side dnsmasq per robot. It is the robot's only resolver. + dnsConf = + r: + let + a = r.access; + servers = d: lib.concatMapStrings (u: "server=/${d}/${u}\n") cfg.dnsUpstreams; + in + pkgs.writeText "robotnix-${r.name}-dnsmasq.conf" ( + '' + listen-address=${sw} + port=${toString r.dnsPort} + bind-dynamic + no-resolv + no-hosts + no-poll + domain-needed + bogus-priv + user=root + cache-size=1000 + # Short TTLs so allowlist IP sets stay fresh after a host reload. + max-ttl=300 + max-cache-ttl=300 + '' + + lib.concatMapStrings servers cfg.localDomains + + ( + if a.internet == "all" then + lib.concatMapStrings (u: "server=${u}\n") cfg.dnsUpstreams + else + '' + # Allowlist: everything else is NXDOMAIN. + address=/#/ + '' + + lib.concatMapStrings + ( + d: servers d + "nftset=/${d}/4#inet#robotnix#allow_${r.nft}\n" + ) + a.allowedDomains + ) + + lib.concatMapStrings (d: "address=/${d}/\n") a.blockedDomains + ); +in +lib.mkIf cfg.enable { + # Same keys and priority networking.nat (and hermenix) use. + boot.kernel.sysctl = { + "net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true; + "net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true; + }; + + # ---------------------------------------------------------------- switch + systemd.network.enable = true; + systemd.network.netdevs."40-${br}".netdevConfig = { + Kind = "bridge"; + Name = br; + }; + systemd.network.networks."40-${br}" = { + matchConfig.Name = br; + address = [ "${sw}/${toString cfg.switch.prefixLength}" ]; + routes = map + (r: { + Destination = "${r.address}/32"; + Scope = "link"; + }) + rs; + networkConfig = { + ConfigureWithoutCarrier = true; + LinkLocalAddressing = "no"; + IPv6AcceptRA = false; + }; + linkConfig.RequiredForOnline = "no"; + }; + # Robot ports (nspawn names them vb-). Isolated: robots can't talk + # to each other at L2, only through the host's rules. + systemd.network.networks."41-${br}-ports" = lib.mkIf (rs != [ ]) { + matchConfig.Name = lib.concatMapStringsSep " " (r: r.port) rs; + networkConfig = { + Bridge = br; + LinkLocalAddressing = "no"; + }; + bridgeConfig.Isolated = true; + linkConfig.RequiredForOnline = "no"; + }; + networking.networkmanager.unmanaged = [ "interface-name:${br}" ] ++ map (r: "interface-name:${r.port}") rs; + + # -------------------------------------------------------------- firewall + networking.nftables.enable = true; + + # L2: each port may only speak IPv4/ARP from its own address. + networking.nftables.tables.robotnix-bridge = lib.mkIf (rs != [ ]) { + family = "bridge"; + content = '' + chain prerouting { + type filter hook prerouting priority filter; policy accept; + '' + + lib.concatMapStrings + (r: '' + iifname "${r.port}" ether type != { ip, arp } drop + iifname "${r.port}" ip saddr != ${r.address} drop + iifname "${r.port}" arp saddr ip != ${r.address} drop + '') + rs + + '' + } + ''; + }; + + # L3. Runs before nixos-fw, so a drop here is final no matter what other + # modules accept. Source addresses are trusted thanks to the bridge table. + networking.nftables.tables.robotnix = lib.mkIf (rs != [ ]) { + family = "inet"; + content = '' + ${lib.concatMapStrings (r: '' + set allow_${r.nft} { type ipv4_addr; } + '') rs} + + chain prerouting { + type nat hook prerouting priority dstnat; policy accept; + ${lib.concatMapStrings (r: '' + iifname "${br}" ip saddr ${r.address} meta l4proto { tcp, udp } th dport 53 dnat ip to ${sw}:${toString r.dnsPort} + '') rs} + } + + chain forward { + type filter hook forward priority filter - 10; policy accept; + iifname "${br}" ct state established,related accept + iifname "${br}" meta nfproto ipv6 drop + oifname "${br}" meta nfproto ipv6 drop + iifname "${br}" ip saddr vmap ${vmap "out"} + iifname "${br}" drop + # Inbound from the LAN to robots: open. + } + + chain input { + type filter hook input priority filter - 10; policy accept; + iifname != "${br}" accept + ct state established,related accept + meta nfproto ipv6 drop + ip saddr vmap ${vmap "in"} + drop + } + + ${lib.concatMapStrings outChain rs} + ${lib.concatMapStrings inChain rs} + ''; + }; + + # The robotnix table already filtered; let nixos-fw pass the rest. + networking.firewall.extraInputRules = '' + iifname "${br}" accept + ''; + networking.firewall.extraForwardRules = '' + iifname "${br}" accept + oifname "${br}" accept + ''; + + # ------------------------------------------------------------------- DNS + systemd.services = lib.listToAttrs ( + map + ( + r: + lib.nameValuePair "${r.unit}-dns" { + description = "robotnix DNS for ${r.name}"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network.target" + "nftables.service" + ]; + serviceConfig = { + ExecStart = "${pkgs.dnsmasq}/bin/dnsmasq --keep-in-foreground --pid-file --conf-file=${dnsConf r}"; + Restart = "always"; + RestartSec = 2; + # Root only for nftset; everything else is locked down. + CapabilityBoundingSet = [ + "CAP_NET_ADMIN" + "CAP_NET_BIND_SERVICE" + "CAP_NET_RAW" + "CAP_SETUID" + "CAP_SETGID" + ]; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + NoNewPrivileges = true; + }; + } + ) + rs + ); +} diff --git a/modules/nixos/server/robotnix/robotnix.nu b/modules/nixos/server/robotnix/robotnix.nu new file mode 100644 index 0000000..c38bceb --- /dev/null +++ b/modules/nixos/server/robotnix/robotnix.nu @@ -0,0 +1,258 @@ + +# robotnix: manage robotnix robots (self-managed NixOS nspawn containers). +# CONFIG and PATH are set by the Nix wrapper (robots.nix) above this line. + +def cfg [] { open $CONFIG } + +def robot [name: string] { + let robots = (cfg).robots + if $name not-in ($robots | columns) { + error make { msg: $"unknown robot '($name)'. Known: ($robots | columns | str join ', ')" } + } + $robots | get $name +} + +def need-root [] { + if not (is-admin) { error make { msg: "this needs root: run it with sudo" } } +} + +def units [r: record] { + { + robot: $"($r.unit).service" + storage: $"($r.unit)-storage.service" + seed: $"($r.unit)-seed.service" + dns: $"($r.unit)-dns.service" + } +} + +def unit-state [unit: string] { + ^systemctl is-active $unit | complete | get stdout | str trim +} + +def is-mounted [dir: string] { + (^mountpoint -q $dir | complete).exit_code == 0 +} + +def disk-usage [dir: string] { + if not (is-mounted $dir) { return "-" } + let f = (^df -B1 --output=used,size $dir | lines | last | str trim | split row -r '\s+') + $"($f.0 | into filesize) / ($f.1 | into filesize)" +} + +def stamp [] { date now | format date "%Y-%m-%dT%H-%M-%S" } + +def confirm-name [name: string, what: string] { + print $what + let answer = (input $"Type '($name)' to continue: ") + if $answer != $name { error make { msg: "aborted" } } +} + +# Stop the robot and unmount its images (seed and robot are bound to storage). +def stop-robot [r: record] { + let u = (units $r) + ^systemctl stop $u.robot $u.seed $u.storage +} + +# Snapshot ids are "-", e.g. 2026-10-23T14-00-00-auto. +def snapshot-list [r: record] { + let dir = $"($r.base)/snapshots" + if not ($dir | path exists) { return [] } + ls $dir + | where type == dir + | each {|e| $e.name | path basename } + | sort + | each {|id| + let p = ($id | parse -r '^(?\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2})-(?.+)$') + if ($p | is-empty) { + { id: $id, kind: "?", created: "?" } + } else { + { id: $id, kind: $p.0.kind, created: ($p.0.stamp | str replace 'T' ' ') } + } + } +} + +# Freeze both filesystems (if mounted), snapshot both subvolumes, thaw. +def take-snapshot [r: record, kind: string] { + let id = $"(stamp)-($kind)" + let dest = $"($r.base)/snapshots/($id)" + mkdir $dest + let mounts = ([$"($r.run)/root" $"($r.run)/home"] | where {|m| is-mounted $m }) + let err = (try { + for m in $mounts { ^fsfreeze -f $m } + for sub in [root home] { + ^btrfs subvolume snapshot -r $"($r.base)/($sub)" $"($dest)/($sub)" + } + null + } catch {|e| $e.msg }) + for m in $mounts { do -i { ^fsfreeze -u $m } } + if $err != null { error make { msg: $"snapshot of ($r.name) failed: ($err)" } } + $id +} + +def delete-snapshot [r: record, id: string] { + let d = $"($r.base)/snapshots/($id)" + if not ($d | path exists) { error make { msg: $"no snapshot '($id)' for ($r.name)" } } + for sub in [root home] { + if ($"($d)/($sub)" | path exists) { ^btrfs subvolume delete $"($d)/($sub)" } + } + ^rmdir $d +} + +# Keep only the newest `keep` automatic snapshots. Others are never pruned. +def prune [r: record] { + let autos = (snapshot-list $r | where kind == "auto" | each {|s| $s.id }) + let extra = ($autos | length) - $r.snapshots.keep + if $extra > 0 { + for id in ($autos | first $extra) { delete-snapshot $r $id } + } +} + +# Manage robotnix robots. Run `robotnix --help` for details. +def main [] { + print "robotnix - self-managed NixOS nspawn robots + + list all robots: state, IP, disk use + status unit status + start|stop|restart + shell [--root] log in without ssh (machinectl) + snapshot [--tag t] take a snapshot now + snapshots list snapshots + forget delete a snapshot + restore [--only root|home] + reseed [--wipe-home] reinstall from the seed config + access effective network rules" +} + +# All robots with state, address and disk use. +def "main list" [] { + (cfg).robots | transpose name r | each {|row| + let r = $row.r + { + name: $row.name + state: (unit-state $"($r.unit).service") + address: $r.address + internet: $r.access.internet + root: (disk-usage $"($r.run)/root") + home: (disk-usage $"($r.run)/home") + snapshots: (snapshot-list $r | length) + } + } +} + +# systemd status of a robot and its helper units. +def "main status" [name: string] { + let u = (units (robot $name)) + do -i { ^systemctl status --no-pager --lines=10 $u.robot $u.dns $u.storage } +} + +def "main start" [name: string] { + need-root + ^systemctl start (units (robot $name)).robot +} + +def "main stop" [name: string] { + need-root + ^systemctl stop (units (robot $name)).robot +} + +def "main restart" [name: string] { + need-root + ^systemctl restart (units (robot $name)).robot +} + +# Log into a robot without ssh. Works even if its network is broken. +def "main shell" [name: string, --root] { + let r = (robot $name) + let who = if $root { "root" } else { $r.user } + ^machinectl shell $"($who)@($r.name)" +} + +# Take a consistent snapshot of the robot's root and home. +def "main snapshot" [ + name: string + --tag: string = "manual" # label for the snapshot id + --auto # timer mode: tag "auto" and prune old ones +] { + need-root + let r = (robot $name) + let kind = if $auto { "auto" } else { $tag } + if not ($kind =~ '^[a-z0-9-]+$') { error make { msg: "--tag may only use a-z, 0-9 and -" } } + let id = (take-snapshot $r $kind) + if $auto { prune $r } + print $"snapshot ($id) taken for ($name)" +} + +# List a robot's snapshots. +def "main snapshots" [name: string] { + snapshot-list (robot $name) +} + +# Delete one snapshot. +def "main forget" [name: string, id: string] { + need-root + delete-snapshot (robot $name) $id + print $"deleted snapshot ($id) of ($name)" +} + +# Roll a robot back to a snapshot. The current state is saved first. +def "main restore" [ + name: string + id: string + --only: string # "root" or "home" (default: both) + --yes # skip the confirmation +] { + need-root + let r = (robot $name) + let subs = if $only == null { + [root home] + } else if $only in [root home] { + [$only] + } else { + error make { msg: "--only must be root or home" } + } + let src = $"($r.base)/snapshots/($id)" + for sub in $subs { + if not ($"($src)/($sub)" | path exists) { error make { msg: $"snapshot ($id) has no ($sub)" } } + } + if not $yes { + confirm-name $name $"This replaces ($name)'s ($subs | str join ' and ') with snapshot ($id).\nThe current state is kept as a pre-restore snapshot." + } + stop-robot $r + let saved = (take-snapshot $r "pre-restore") + for sub in $subs { + ^btrfs subvolume delete $"($r.base)/($sub)" + ^btrfs subvolume snapshot $"($src)/($sub)" $"($r.base)/($sub)" + } + ^systemctl start (units $r).robot + print $"($name) restored to ($id). Undo with: robotnix restore ($name) ($saved)" +} + +# Wipe a robot's OS and reinstall it from its seed config. +def "main reseed" [ + name: string + --wipe-home # also start with an empty /home + --yes # skip the confirmation +] { + need-root + let r = (robot $name) + let what = if $wipe_home { "root AND home" } else { "root (home is kept)" } + if not $yes { + confirm-name $name $"This wipes ($name)'s ($what) and reinstalls it from its seed config.\nA pre-reseed snapshot is taken first." + } + stop-robot $r + let saved = (take-snapshot $r "pre-reseed") + rm -f $"($r.base)/root/root.img" + if $wipe_home { rm -f $"($r.base)/home/home.img" } + ^systemctl start (units $r).robot + print $"($name) reseeded. Undo with: robotnix restore ($name) ($saved)" +} + +# Show a robot's effective (shared + own) network rules. +def "main access" [name: string] { + let r = (robot $name) + print ($r.access | table --expand) + if $r.access.internet == "allowlist" { + print "IPs learned from allowed domains:" + do -i { ^nft list set inet robotnix $"allow_($r.nft)" } + } +} diff --git a/modules/nixos/server/robotnix/robots.nix b/modules/nixos/server/robotnix/robots.nix new file mode 100644 index 0000000..a6474db --- /dev/null +++ b/modules/nixos/server/robotnix/robots.nix @@ -0,0 +1,429 @@ +# robotnix robots: storage, first install, nspawn units, snapshots, CLI. +# +# Layout per robot (all btrfs subvolumes): +# //root/root.img whole OS incl. /nix (hard size limit) +# //home/home.img /home (hard size limit) +# //snapshots//{root,home} +# The host mounts the images under /run/robotnix// so it can freeze +# them for consistent snapshots while the robot runs. +{ config +, lib +, pkgs +, ... +}: +let + cfg = config.skyg.nixos.server.robotnix; + h = import ./helpers.nix { inherit config lib; }; + rs = h.robotList; + sw = cfg.switch.address; + br = cfg.switch.name; + ext = cfg.externalInterface; + hostSystem = pkgs.stdenv.hostPlatform.system; + + # The robot's flake pins the same nixpkgs the host used to build its seed. + nixpkgsRef = + let + rev = config.system.nixos.revision; + in + if rev != null then builtins.unsafeDiscardStringContext rev else "nixos-unstable"; + + keys = r: lib.concatMapStringsSep " " h.nixStr r.authorizedKeys; + tz = lib.optionalString + ( + config.time.timeZone != null + ) "time.timeZone = ${h.nixStr config.time.timeZone};"; + + # The host-owned part of the robot's config -> /etc/nixos/robotnix.nix. + robotnixNix = + r: + builtins.toFile "robotnix-${r.name}.nix" '' + # Generated by robotnix on the host (first install / reseed, or a host + # rebuild if you changed the robot's host-side settings). + # The network and git-sync values must match the host's robotnix + # settings; change them and this robot loses its network or its + # auto-commits stop. Everything else is yours. + { lib, pkgs, ... }: + { + boot.isContainer = true; + nixpkgs.hostPlatform = ${h.nixStr hostSystem}; + networking.hostName = ${h.nixStr r.name}; + + # Address and route handed out by the host. IPv4 only. + networking.useDHCP = false; + networking.useNetworkd = true; + networking.enableIPv6 = false; + networking.useHostResolvConf = lib.mkForce false; + networking.nameservers = [ ${h.nixStr sw} ]; + systemd.network.networks."10-host0" = { + matchConfig.Name = "host0"; + address = [ ${h.nixStr "${r.address}/32"} ]; + routes = [ { Gateway = ${h.nixStr sw}; GatewayOnLink = true; } ]; + networkConfig.LinkLocalAddressing = "no"; + }; + + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + + services.openssh = { + enable = true; + settings.PasswordAuthentication = false; + settings.KbdInteractiveAuthentication = false; + }; + users.users.${h.nixStr r.user} = { + isNormalUser = true; + extraGroups = [ "wheel" ]; + openssh.authorizedKeys.keys = [ ${keys r} ]; + }; + users.users.root.openssh.authorizedKeys.keys = [ ${keys r} ]; + ${tz} + system.stateVersion = ${h.nixStr r.seed.stateVersion}; + ${lib.optionalString r.git.enable '' + # git-tracking your flake: auto-commit, and push when a remote is set. + environment.systemPackages = [ pkgs.git ]; + systemd.services."robotnix-flake-gitsync" = { + path = [ pkgs.git ]; + script = ''' + cd /etc/nixos + git add -A + git diff --cached --quiet || git commit -m "robotnix auto-sync $(date +'%Y-%m-%d %H:%M:%S')" + if git remote get-url origin >/dev/null 2>&1; then + git push -q + fi + '''; + serviceConfig = { + Type = "oneshot"; + User = "root"; + }; + }; + systemd.timers."robotnix-flake-gitsync" = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = ${h.nixStr r.git.interval}; + Persistent = true; + Unit = "robotnix-flake-gitsync.service"; + }; + }; + ''} + } + ''; + + flakeNix = + r: + builtins.toFile "robotnix-${r.name}-flake.nix" '' + # ${r.name}'s own flake. Apply changes with: + # sudo nixos-rebuild switch --flake /etc/nixos + { + inputs.nixpkgs.url = "github:NixOS/nixpkgs/${nixpkgsRef}"; + outputs = { nixpkgs, ... }: { + nixosConfigurations.${h.nixStr r.name} = nixpkgs.lib.nixosSystem { + modules = [ ./robotnix.nix ./configuration.nix ]; + }; + }; + } + ''; + + # The first system, built on the host exactly as the robot's own flake + # builds it later: lib.nixosSystem is eval-config with system = null and + # nixpkgs' lib, so this mirrors flake.nix's `nixpkgs.lib.nixosSystem` call. + seedSystem = + r: + (import "${pkgs.path}/nixos/lib/eval-config.nix" { + system = null; + inherit lib; + modules = [ + (robotnixNix r) + r.seed.configuration + ]; + }).config.system.build.toplevel; + + storageStart = + r: + pkgs.writeShellScript "${r.unit}-storage-start" '' + set -euo pipefail + base=${r.base} + mkdir -p ${cfg.baseDir} + if [ "$(stat -f -c %T ${cfg.baseDir})" != btrfs ]; then + echo "robotnix: ${cfg.baseDir} must be on btrfs (needed for snapshots)" >&2 + exit 1 + fi + for sub in "$base" "$base/root" "$base/home" "$base/snapshots"; do + [ -d "$sub" ] || btrfs subvolume create "$sub" + done + # No copy-on-write for the images (only affects new files). + chattr +C "$base/root" "$base/home" + + image() { # + img=$1 size=$2 dir=$3 + if [ ! -e "$img" ]; then + truncate -s "$size" "$img" + mkfs.ext4 -q -F "$img" + fi + mkdir -p -m 0755 "$dir" + if ! mountpoint -q "$dir"; then + rc=0 + e2fsck -p "$img" || rc=$? + if [ "$rc" -ge 4 ]; then + echo "robotnix: $img needs a manual e2fsck" >&2 + exit 1 + fi + mount -o loop,noatime "$img" "$dir" + fi + # Grow when the configured size goes up. Shrinking is not supported. + want=$(numfmt --from=iec "$size") + have=$(stat -c %s "$img") + if [ "$want" -gt "$have" ]; then + truncate -s "$size" "$img" + dev=$(findmnt -n -o SOURCE "$dir") + losetup -c "$dev" + resize2fs "$dev" + elif [ "$want" -lt "$have" ]; then + echo "robotnix: $img is bigger than $size; shrinking is not supported" >&2 + fi + } + + mkdir -p /run/robotnix ${r.run} + chmod 0755 /run/robotnix ${r.run} + image "$base/root/root.img" ${r.disk.rootSize} ${r.run}/root + image "$base/home/home.img" ${r.disk.homeSize} ${r.run}/home + chmod 0755 ${r.run}/root + ''; + + storageStop = + r: + pkgs.writeShellScript "${r.unit}-storage-stop" '' + for d in ${r.run}/home ${r.run}/root; do + if mountpoint -q "$d"; then umount "$d"; fi + done + ''; + + # Installs the seed only when the root image has no system yet (first + # start, or after `robotnix reseed`). Never touches an installed robot. + seedScript = + r: + pkgs.writeShellScript "${r.unit}-seed" '' + set -euo pipefail + root=${r.run}/root + if [ ! -e "$root/nix/var/nix/profiles/system" ]; then + echo "robotnix: installing the seed system into ${r.name}" + nixos-install --root "$root" --system ${seedSystem r} \ + --no-root-passwd --no-bootloader --no-channel-copy + install -d -m 0755 "$root/etc/nixos" + install -m 0644 ${flakeNix r} "$root/etc/nixos/flake.nix" + install -m 0644 ${robotnixNix r} "$root/etc/nixos/robotnix.nix" + install -m 0644 ${r.seed.configuration} "$root/etc/nixos/configuration.nix" + ${lib.optionalString r.git.enable '' + echo "robotnix: setting up the git repo for ${r.name}'s flake" + git -C "$root/etc/nixos" init -q + git -C "$root/etc/nixos" config user.name ${h.nixStr r.git.userName} + git -C "$root/etc/nixos" config user.email ${h.nixStr r.git.userEmail} + ${lib.optionalString (r.git.remoteUrl != null) '' + git -C "$root/etc/nixos" remote add origin ${h.nixStr r.git.remoteUrl} + ''} + git -C "$root/etc/nixos" add -A + git -C "$root/etc/nixos" commit -q -m "robotnix initial seed" || true + ''} + fi + # What nspawn and NixOS's init expect (same as nixos-containers). + mkdir -p "$root/etc" "$root/var/lib" "$root/root" "$root/home" + chmod 0700 "$root/root" + [ -e "$root/etc/os-release" ] || [ -L "$root/etc/os-release" ] || touch "$root/etc/os-release" + [ -e "$root/etc/machine-id" ] || touch "$root/etc/machine-id" + ''; + + preStart = + r: + pkgs.writeShellScript "${r.unit}-pre" '' + set -euo pipefail + for _ in $(seq 1 30); do + [ -e /sys/class/net/${br} ] && [ -e /sys/class/net/${ext} ] && break + sleep 1 + done + [ -e /sys/class/net/${br} ] || { echo "robotnix: ${br} is missing" >&2; exit 1; } + # Answer ARP for the robot's LAN IP on the uplink (this IP only). + ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true + ip neigh add proxy ${r.address} dev ${ext} + ''; + + postStop = + r: + pkgs.writeShellScript "${r.unit}-post" '' + ip neigh del proxy ${r.address} dev ${ext} 2>/dev/null || true + ''; + + nspawnArgs = + r: + lib.escapeShellArgs ( + [ + "--keep-unit" + "--machine=${r.name}" + "--directory=${r.run}/root" + "--notify-ready=yes" + "--kill-signal=SIGRTMIN+3" + "--network-bridge=${br}" + "--link-journal=try-guest" + "--resolv-conf=off" + "--timezone=off" + "--bind=${r.run}/home:/home${lib.optionalString r.privateUsers ":idmap"}" + ] + ++ lib.optionals r.privateUsers [ + "--private-users=pick" + "--private-users-ownership=auto" + ] + ++ [ "/nix/var/nix/profiles/system/init" ] + ); + + robotUnits = r: { + "${r.unit}-storage" = { + description = "robotnix storage for ${r.name}"; + path = with pkgs; [ + btrfs-progs + coreutils + e2fsprogs + util-linux + ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = storageStart r; + ExecStop = storageStop r; + }; + }; + + "${r.unit}-seed" = { + description = "robotnix first install for ${r.name}"; + requires = [ "${r.unit}-storage.service" ]; + bindsTo = [ "${r.unit}-storage.service" ]; + after = [ "${r.unit}-storage.service" ]; + path = [ + config.system.build.nixos-install + config.nix.package + pkgs.coreutils + pkgs.util-linux + pkgs.git + ]; + environment.HOME = "/root"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = seedScript r; + TimeoutStartSec = "30min"; + }; + }; + + ${r.unit} = + let + mem = h.toBytes r.memory.max; + deps = [ + "${r.unit}-storage.service" + "${r.unit}-seed.service" + "${r.unit}-dns.service" + ]; + in + { + description = "robotnix robot ${r.name}"; + wantedBy = lib.optional r.autoStart "machines.target"; + requires = deps; + bindsTo = [ "${r.unit}-storage.service" ]; + after = deps ++ [ + "network.target" + "systemd-networkd.service" + "nftables.service" + ]; + path = [ + pkgs.iproute2 + pkgs.coreutils + ]; + environment.SYSTEMD_NSPAWN_UNIFIED_HIERARCHY = "1"; + serviceConfig = { + ExecStartPre = preStart r; + ExecStart = "${config.systemd.package}/bin/systemd-nspawn ${nspawnArgs r}"; + # Best effort: refresh LAN ARP caches. + ExecStartPost = "-${pkgs.iputils}/bin/arping -U -c 2 -I ${ext} ${r.address}"; + ExecStopPost = postStop r; + + # Same as nixos-containers. + Type = "notify"; + RestartForceExitStatus = "133"; + SuccessExitStatus = "133"; + Restart = "on-failure"; + TimeoutStartSec = "5min"; + Slice = "machine.slice"; + Delegate = true; + KillMode = "mixed"; + KillSignal = "TERM"; + DevicePolicy = "closed"; + + # Resources. + CPUQuota = "${toString (r.cpus.max * 100)}%"; + CPUWeight = toString (lib.min 10000 (100 * r.cpus.min)); + MemoryLow = r.memory.min; + MemoryHigh = toString (mem - mem / 16); + MemoryMax = r.memory.max; + MemorySwapMax = r.memory.swapMax; + TasksMax = 16384; + }; + }; + }; + + cliConfig = pkgs.writeText "robotnix.json" ( + builtins.toJSON { + inherit (cfg) baseDir; + robots = lib.mapAttrs + (_: r: { + inherit (r) + name + address + port + dnsPort + base + run + unit + user + nft + access + ; + snapshots = { inherit (r.snapshots) enable interval keep; }; + }) + h.robots; + } + ); + + cliPath = lib.makeBinPath [ + pkgs.btrfs-progs + pkgs.util-linux + pkgs.coreutils + pkgs.nftables + config.systemd.package + ]; + + cli = pkgs.writers.writeNuBin "robotnix" ( + '' + const CONFIG = "${cliConfig}" + $env.PATH = ("${cliPath}" | split row ":" | append ($env.PATH | split row (char esep))) + '' + + builtins.readFile ./robotnix.nu + ); +in +lib.mkIf cfg.enable { + systemd.services = lib.mkMerge (map robotUnits rs); + + systemd.targets.multi-user.wants = [ "machines.target" ]; + + # Memory protection only works if the parent slice is protected too. + systemd.slices.machine.sliceConfig.MemoryLow = lib.mkIf (rs != [ ]) ( + lib.mkDefault (toString (lib.foldl' (acc: r: acc + h.toBytes r.memory.min) 0 rs)) + ); + + skyg.server.timers = lib.listToAttrs ( + map + ( + r: + lib.nameValuePair "${r.unit}-snapshot" { + script = "${cli}/bin/robotnix snapshot ${r.name} --auto"; + OnCalendar = r.snapshots.interval; + } + ) + (lib.filter (r: r.snapshots.enable) rs) + ); + + environment.systemPackages = [ cli ]; +} diff --git a/modules/nixos/server/robotnix/seed/configuration.nix b/modules/nixos/server/robotnix/seed/configuration.nix new file mode 100644 index 0000000..6056f1d --- /dev/null +++ b/modules/nixos/server/robotnix/seed/configuration.nix @@ -0,0 +1,24 @@ +# A robot's own NixOS config (default seed). +# +# The host copies this into the robot's /etc/nixos on first install and on +# `robotnix reseed`. After that the robot owns it. Inside the robot: +# sudo nixos-rebuild switch --flake /etc/nixos +# Host-managed basics (network, ssh, user) live in /etc/nixos/robotnix.nix. +{ pkgs, ... }: +{ + # The user has no password; ssh keys only. + security.sudo.wheelNeedsPassword = false; + + environment.systemPackages = with pkgs; [ + git + vim + curl + htop + ]; + + nix.gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 14d"; + }; +}