From bb3c73b3105c88eb339412f71514c955ecc60af8 Mon Sep 17 00:00:00 2001 From: Hermenix Date: Mon, 26 Oct 2026 00:28:45 +0000 Subject: [PATCH 1/5] hl-minipc2: cap the systemd journal at 1G One stuck Scrypted prune loop was writing ~180k lines (62 MB) a day and pushed every other fault out of a four-minute window. --- hosts/hl-minipc2/systemd.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hosts/hl-minipc2/systemd.nix b/hosts/hl-minipc2/systemd.nix index 0057ccf..6a1f2ea 100644 --- a/hosts/hl-minipc2/systemd.nix +++ b/hosts/hl-minipc2/systemd.nix @@ -2,4 +2,8 @@ { boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; + + services.journald.extraConfig = '' + SystemMaxUse=1G + ''; } From 9b537c9f62458f08c3ad6477424189c82d8495bf Mon Sep 17 00:00:00 2001 From: Hermenix Date: Mon, 26 Oct 2026 00:28:45 +0000 Subject: [PATCH 2/5] hl-minipc2/scrypted: pass /dev/dri into the container Without it VAAPI cannot be enumerated and the 4608x1728 H.265 stream is software-decoded, which is what times out Living Room analysis. --- hosts/hl-minipc2/services/scrypted.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/hosts/hl-minipc2/services/scrypted.nix b/hosts/hl-minipc2/services/scrypted.nix index 659640f..8b86d30 100644 --- a/hosts/hl-minipc2/services/scrypted.nix +++ b/hosts/hl-minipc2/services/scrypted.nix @@ -16,6 +16,7 @@ in "/opt/homelab/scrypted/db:/server/volume" "scrypted-nvr:/nvr" ]; + devices = [ "/dev/dri:/dev/dri" ]; environment.SCRYPTED_NVR_VOLUME = "/nvr"; }; volumes.scrypted-nvr = { From 520f804738329f967c34ae43af4c7dab7e3eb073 Mon Sep 17 00:00:00 2001 From: Hermenix Date: Mon, 26 Oct 2026 00:28:45 +0000 Subject: [PATCH 3/5] secrets: declare a hermenix recipient --- secrets.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/secrets.nix b/secrets.nix index 66cbabb..b348bc6 100644 --- a/secrets.nix +++ b/secrets.nix @@ -7,6 +7,7 @@ let fwdesk = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHbBcdMpNPaELUzHRIoRkTUQde+mBQ87Ra2R+974FuUy root@hl-fwdesk"; bigbox1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFl5xiH4aFlmDWdoU+KmERLYOeOQK2aKP7jenY2G3rFb root@nixos"; bigbox2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6l6VqQ23PkGkWVABBoLp1KA6PzkX5oKTy5RBHbQXet root@hl-bigbox2"; + hermenix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPfMrT6dDPw3+J22k3X5aR9ERQ8RMRLeBGgqQjmuL7EI hermenix@lab"; in { "secrets/dns-addresses.conf.age".publicKeys = [ From 86f9ae010e018372f2aae49f8ebaba017b2dc1ab Mon Sep 17 00:00:00 2001 From: Hermenix Date: Mon, 26 Oct 2026 00:45:42 +0000 Subject: [PATCH 4/5] robotnix: give the robot root a /usr tree before nspawn starts robotnix-genix.service on hub was in start-limit-hit with exec_main_status 1. systemd-nspawn refused the root: "Directory /run/robotnix/genix/root doesnt look like it has an OS tree (/usr/ directory is missing). Refusing." The storage and seed units were healthy (image mounted, "installation finished!"), so the tree existed and was populated - it was just missing /usr. The seed script ends with a tail block that runs on every invocation, not only when it installs: it creates etc, var/lib, root and home, and touches os-release and machine-id. That list was copied from nixos-containers but dropped its first line, mkdir -p "$root/usr/bin" (nixos-containers.nix:102). nspawn treats the presence of /usr as the marker of an OS tree, so every start died before the init ran, and because the unit restarts on failure it burned through the start limit instead of recovering. Add that mkdir with an explicit 0755 mode to the same tail block. Because the block is outside the first-install branch, the directory is created on the next storage+seed run for an already seeded root, so this needs no robotnix reseed and loses no data. The module layout is unchanged. ABOUTME.md gains a Notes/decisions bullet recording why the root gets /usr/bin and that no reseed is required. --- modules/nixos/server/robotnix/ABOUTME.md | 4 ++++ modules/nixos/server/robotnix/robots.nix | 1 + 2 files changed, 5 insertions(+) diff --git a/modules/nixos/server/robotnix/ABOUTME.md b/modules/nixos/server/robotnix/ABOUTME.md index d472fd0..8097986 100644 --- a/modules/nixos/server/robotnix/ABOUTME.md +++ b/modules/nixos/server/robotnix/ABOUTME.md @@ -104,6 +104,10 @@ Runtime state comes from `/etc/robotnix.json`, written by the module. the `containers.*` module: that module shares the host store, which would break the robot's self-rebuilds. The unit mirrors `nixos-containers.nix` (`Type=notify`, SIGRTMIN+3, exit status 133, `machine.slice`). +- The robot's root gets `/usr/bin` before its first start: `systemd-nspawn` + refuses a tree without `/usr` ("doesn't look like it has an OS tree"), the + same reason `nixos-containers.nix` creates it. The seed's tail block runs on + every start, so no reseed is needed after this landed. - `systemd-nspawn` runs in `machine.slice` (systemd's nspawn home), not `system.slice`. - Timers use `skyg.server.timers`; eval asserts unique robot ids/IPs, name diff --git a/modules/nixos/server/robotnix/robots.nix b/modules/nixos/server/robotnix/robots.nix index a6474db..9183d1b 100644 --- a/modules/nixos/server/robotnix/robots.nix +++ b/modules/nixos/server/robotnix/robots.nix @@ -224,6 +224,7 @@ let fi # What nspawn and NixOS's init expect (same as nixos-containers). mkdir -p "$root/etc" "$root/var/lib" "$root/root" "$root/home" + mkdir -p -m 0755 "$root/usr/bin" chmod 0700 "$root/root" [ -e "$root/etc/os-release" ] || [ -L "$root/etc/os-release" ] || touch "$root/etc/os-release" [ -e "$root/etc/machine-id" ] || touch "$root/etc/machine-id" From 57fbf70b7c79bd41460b90008278bd8fbde4f647 Mon Sep 17 00:00:00 2001 From: Hermenix Date: Mon, 26 Oct 2026 00:46:08 +0000 Subject: [PATCH 5/5] hl-fwdesk: ignore hermes VM fabric links in networkd wait-online systemd-networkd-wait-online fails on every boot of hl-fwdesk with "Timeout occurred while waiting for network connectivity" and status=1/FAILURE. The hermes VM runs on this host via services.hermenix (ari/hermenix flake input), and its fabric interfaces -- the bridge hermes-br and the tap vm-hermes -- are managed by systemd-networkd (10-hermes-br.network, 11-vm-hermes.network). networkd-managed links default to RequiredForOnline=yes, but those two links only gain carrier when the VM starts, which on the sampled boot was 0.2s after the 120s wait-online timeout fired. The only networkd link that never blocks is robotnix's own bridge, and the physical uplink enp191s0 is owned by NetworkManager, so it is invisible to networkd. The result is a unit that always burns the full timeout and then fails. hermenix now passes --ignore=hermes-br --ignore=vm-hermes to the unit, so it no longer treats the VM's fabric links as a condition for "online": with every candidate ignored the wait returns immediately instead of blocking and then failing. The unit itself stays enabled and anyInterface is untouched. The durable fix belongs in ari/hermenix: set RequiredForOnline = "no" on the hermes-br and vm-hermes .network files, the same way modules/nixos/server/robotnix/network.nix already does for the robotnix fabric. That repository is out of scope for this change; once it lands this ignoredInterfaces entry can be dropped. Found by hermenix from the fleet monitor's persistent failure record for unit:hub:systemd-networkd-wait-online.service. --- hosts/hl-fwdesk/networking.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hosts/hl-fwdesk/networking.nix b/hosts/hl-fwdesk/networking.nix index 2cd96b3..ca6e7cf 100644 --- a/hosts/hl-fwdesk/networking.nix +++ b/hosts/hl-fwdesk/networking.nix @@ -1,4 +1,9 @@ { ... }: { services.tailscale.enable = true; + + systemd.network.wait-online.ignoredInterfaces = [ + "hermes-br" + "vm-hermes" + ]; }