{ config, lib, pkgs, ... }: let inherit (lib) mkOption mkEnableOption mkIf types optionalString ; cfg = config.services.hermenix; net = cfg.network; vmUser = "hermes-vm"; runDir = "/run/hermes-vm"; stateDir = "/var/lib/hermes-vm"; systemImage = "${cfg.disk.baseDir}/system/system.raw"; stateImage = "${cfg.disk.baseDir}/state/state.raw"; snapDir = "${cfg.disk.baseDir}/snapshots"; pt = cfg.passthrough; anyPci = pt.pci != [ ]; anyUsb = pt.usb != [ ]; earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci); usbControllerOn = cfg.devices.usbController == "on" || (cfg.devices.usbController == "auto" && anyUsb); tpmStateDir = "/var/lib/hermes-vm-tpm"; # 18501 == 0x4845 ("HE"); marks guest traffic from the tap (bridged mode). bridgeMark = 18501; # dnsmasq needs an explicit mask for a one-address DHCP range. netmaskFromPrefix = p: lib.concatStringsSep "." ( lib.genList ( b: toString ( builtins.elemAt [ 0 128 192 224 240 248 252 254 255 ] (lib.max 0 (lib.min 8 (p - b * 8))) ) ) 4 ); stopScript = pkgs.writeShellScript "hermes-vm-stop" '' set -eu printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \ | ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0 ''; qga = verb: pkgs.writeShellScript "hermes-vm-fs${verb}" '' set -eu printf '%s' '{"execute":"guest-fs${verb}"}' \ | ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true ''; in { options.services.hermenix = { enable = mkEnableOption "the Hermes Agent VM"; name = mkOption { type = types.str; default = "hermes"; }; vcpu = mkOption { type = types.ints.positive; default = 4; }; mem = mkOption { type = types.ints.positive; default = 8192; description = "RAM in MiB."; }; disk = { baseDir = mkOption { type = types.str; default = "/var/lib/hermes-vm"; description = '' Directory holding the guest's disk images. On btrfs this should be laid out as three subvolumes - system/, state/ and snapshots/ - with nodatacow set. `hermenix setup` does this for you. ''; }; systemSize = mkOption { type = types.str; default = "80G"; description = '' System disk: the NixOS install and the Nix store. Never snapshotted, because it is entirely rebuildable from the flake. Size it for an agent that builds things. ''; }; stateSize = mkOption { type = types.str; default = "20G"; description = '' State disk, mounted at /var/lib/hermes in the guest. Holds Hermes's identity, memories, workspace, and the NixOS config it maintains for itself. This is the only irreplaceable data, so it is the only thing snapshotted and backed up. ''; }; quiesce = mkOption { type = types.bool; default = true; description = "fsfreeze the guest via qemu-guest-agent around each snapshot."; }; }; snapshots = { enable = mkOption { type = types.bool; default = true; description = '' Periodic btrfs snapshots of the state disk. The agent can persist changes, which means it can persist a mistake, and these are the fastest way back. ''; }; interval = mkOption { type = types.str; default = "hourly"; description = "systemd calendar expression."; }; keep = mkOption { type = types.ints.positive; default = 48; description = '' How many to retain. Keep this tighter than you would on ZFS: the state image is nodatacow, and every live snapshot reintroduces copy-on-write for the blocks it still references. ''; }; }; backup = { enable = mkEnableOption "restic backups of the guest's state"; repository = mkOption { type = types.str; default = ""; example = "sftp:backup@nas:/hermes"; description = "Any restic repository string."; }; passwordFile = mkOption { type = types.path; default = "/var/lib/secrets/hermes-restic"; description = "File containing the restic repository password, mode 0400."; }; environmentFile = mkOption { type = types.nullOr types.path; default = null; description = "Optional file with credentials for the backend (S3 keys and so on)."; }; interval = mkOption { type = types.str; default = "daily"; }; keep = mkOption { type = types.listOf types.str; default = [ "--keep-daily" "14" "--keep-weekly" "8" "--keep-monthly" "12" ]; description = "Arguments passed to restic forget."; }; }; hostCache = { enable = mkOption { type = types.bool; default = true; description = '' Serve the host's /nix/store to the guest as a read-only binary cache, so the guest doesn't re-download from the internet what the host already has. This is the ONLY route from guest to host in this design. Turning it off leaves the guest with literally no reachable host service, at the cost of bandwidth on every guest build. Note the guest can read every store path on the host through this. Store paths are world-readable anyway, but check nothing sensitive is baked into your host's store. ''; }; port = mkOption { type = types.port; default = 5000; }; secretKeyFile = mkOption { type = types.path; default = "/var/lib/secrets/hermes-cache.secret"; description = "Generate with nix-store --generate-binary-cache-key. See README."; }; }; authorizedKeys = mkOption { type = types.listOf types.str; default = [ ]; }; allowLan = mkOption { type = types.bool; default = false; description = '' Blanket permission for the guest to reach RFC1918 space. Almost always the wrong tool - use lanAccess to open single endpoints instead. ''; }; lanAccess = mkOption { default = [ ]; example = [ { address = "192.168.1.50"; port = 11434; } ]; description = '' Specific endpoints elsewhere on your network that the guest may reach, punched through the default RFC1918 drop. One entry per service. ''; type = types.listOf ( types.submodule { options = { address = mkOption { type = types.str; }; port = mkOption { type = types.port; }; proto = mkOption { type = types.enum [ "tcp" "udp" ]; default = "tcp"; }; }; } ); }; hostAccess = mkOption { default = [ ]; example = [ { port = 8188; } ]; description = '' Ports on this host itself that the guest may reach. The service must also be bound to an address the guest can see - see the README. ''; type = types.listOf ( types.submodule { options = { port = mkOption { type = types.port; }; proto = mkOption { type = types.enum [ "tcp" "udp" ]; default = "tcp"; }; }; } ); }; publish = mkOption { default = [ ]; example = [ { hostPort = 8080; guestPort = 8080; } ]; description = '' Guest ports to publish on the host's uplink, reachable from the wider network. You do not need this to reach the guest from the host itself. ''; type = types.listOf ( types.submodule { options = { hostPort = mkOption { type = types.port; }; guestPort = mkOption { type = types.port; }; proto = mkOption { type = types.enum [ "tcp" "udp" ]; default = "tcp"; }; }; } ); }; network = { bridge = mkOption { type = types.str; default = "hermes-br"; }; tapId = mkOption { type = types.str; default = "vm-hermes"; }; mac = mkOption { type = types.str; default = "02:00:00:7e:00:01"; }; hostAddress = mkOption { type = types.str; default = "10.77.0.1"; }; guestAddress = mkOption { type = types.str; default = "10.77.0.2"; }; prefixLength = mkOption { type = types.int; default = 24; }; lanAddress = mkOption { type = types.str; default = ""; example = "10.0.12.50"; description = "The guest's fixed address on your LAN. Required when `bridged = true`."; }; hostLanAddress = mkOption { type = types.str; default = ""; example = "10.0.12.10"; description = '' This host's own address on your LAN. When `bridged` is enabled the uplink becomes a bridge port and this address moves onto the bridge (which clones the uplink's MAC, so the host keeps its LAN identity). Required when `bridged = true`. ''; }; lanGateway = mkOption { type = types.str; default = ""; example = "10.0.12.1"; description = '' Your LAN's router. Used as the default gateway for the guest and, while `bridged`, for this host's bridge too (DNS unless `lanDns` is set). Required when `bridged = true`. ''; }; lanPrefixLength = mkOption { type = types.int; default = 24; description = "Prefix length of your LAN subnet (netmask)."; }; lanDns = mkOption { type = types.listOf types.str; default = [ ]; example = [ "1.1.1.1" ]; description = '' DNS servers for this host while `bridged` (the bridge no longer takes a DHCP lease). Defaults to the LAN gateway. ''; }; bridged = mkOption { type = types.bool; default = false; description = '' Put the guest directly on your LAN instead of behind the host's NAT. The bridge absorbs `externalInterface`, so any device on the LAN can reach any port the guest exposes. Addressing is static from the operator's point of view: you pick fixed addresses for the guest (`lanAddress`), this host (`hostLanAddress` — it moves onto the bridge), and the LAN gateway (`lanGateway`). A DHCP server this module runs on the bridge hands `lanAddress` to the guest, keyed to `network.mac`, so the guest stays a plain DHCP client and never needs to know its own address. That lease never leaves the bridge; the LAN's own DHCP cannot race it. Inbound is no longer limited to `publish` (which is ignored in this mode): the LAN may reach every guest port. Outbound is still governed by `lanAccess` / `allowLan`, and `hostAccess` plus the host cache still work over the host-guest service net (`hostAddress` stays configured on the bridge). ''; }; externalInterface = mkOption { type = types.nullOr types.str; default = null; example = "eno1"; description = "Host uplink. Required."; }; }; passthrough = { iommu = mkOption { type = types.nullOr ( types.enum [ "intel" "amd" ] ); default = null; description = '' Enable IOMMU kernel parameters for your CPU vendor. Required for any PCI passthrough. The IOMMU is the only thing stopping a passed-through device from reading host memory, so this is not optional. ''; }; pci = mkOption { default = [ ]; example = [ { address = "0000:01:00.0"; id = "10de:2204"; } { address = "0000:01:00.1"; id = "10de:1aef"; } ]; description = '' PCI devices handed to the guest via VFIO. The host loses them entirely while the VM runs. You must pass every function of the device and, in practice, every device in its IOMMU group. A GPU is usually at least two entries: the graphics function and its HDMI audio function. Set `id` (vendor:device) to have vfio-pci claim the device in the initrd, before a host driver can. Strongly recommended for GPUs; unbinding an active amdgpu or nvidia driver later is unreliable. ''; type = types.listOf ( types.submodule { options = { address = mkOption { type = types.str; example = "0000:01:00.0"; description = "Full PCI address, as shown by lspci -D."; }; id = mkOption { type = types.nullOr types.str; default = null; example = "10de:2204"; description = "vendor:device, from lspci -nn. Enables early binding."; }; }; } ); }; usb = mkOption { default = [ ]; example = [ { vendorId = "8087"; productId = "0026"; } ]; description = '' USB devices forwarded to the guest, matched by vendor and product ID from lsusb. Integrated Bluetooth radios are USB devices and go here too, though the host driver must be kept off them. QEMU resolves these at start, so a device that is unplugged when the VM boots will not appear later without a restart. ''; type = types.listOf ( types.submodule { options = { vendorId = mkOption { type = types.str; example = "046d"; }; productId = mkOption { type = types.str; example = "c52b"; }; }; } ); }; }; devices = { rng.enable = mkOption { type = types.bool; default = true; description = "Attach a virtio entropy source (-device virtio-rng-pci)."; }; vsock.enable = mkOption { type = types.bool; default = true; description = '' Attach the host<->guest vsock (-device vhost-vsock-pci). `hermenix ctl` and the SSH-based backup both reach the guest over it, so turning this off breaks them. ''; }; video = { enable = mkOption { type = types.bool; default = false; description = '' Attach a virtual display adapter, giving the guest a DRM device (/dev/dri). Needed by a Wayland compositor or anything that wants a render node. The console stays serial either way. ''; }; model = mkOption { type = types.enum [ "virtio" "qxl" "cirrus" "vmware" "std" ]; default = "virtio"; description = '' Adapter model passed to -vga. Only "virtio" exposes a render node (/dev/dri/renderD128); the legacy models give card0 without one. ''; }; }; balloon.enable = mkOption { type = types.bool; default = false; description = "Attach a virtio memory balloon (-device virtio-balloon-pci)."; }; watchdog = { enable = mkOption { type = types.bool; default = false; description = '' Attach a hardware watchdog. The guest must also arm it (systemd's RuntimeWatchdogSec) for it to do anything. ''; }; model = mkOption { type = types.enum [ "i6300esb" "ib700" ]; default = "i6300esb"; }; action = mkOption { type = types.enum [ "reset" "shutdown" "poweroff" "pause" "debug" "none" ]; default = "reset"; description = "What QEMU does when the watchdog fires."; }; }; input = { enable = mkOption { type = types.bool; default = false; description = "Attach virtio input devices. Only useful with devices.video.enable."; }; keyboard = mkOption { type = types.bool; default = true; }; tablet = mkOption { type = types.bool; default = false; }; }; audio = { enable = mkOption { type = types.bool; default = false; description = '' Attach an audio device. NOTE: the VM unit is hardened (ProtectHome, PrivateTmp, a dedicated system user) and cannot reach the operator's audio daemon without loosening that; leave this off to keep the hardening intact. ''; }; backend = mkOption { type = types.enum [ "none" "pipewire" "alsa" "pa" "sdl" ]; default = "pipewire"; }; model = mkOption { type = types.enum [ "intel-hda" "virtio-sound" ]; default = "intel-hda"; }; }; tpm = { enable = mkOption { type = types.bool; default = false; description = '' Attach a TPM 2.0 via swtpm. A hermes-vm-swtpm unit starts first and QEMU connects to it over a unix socket. Useful for measured boot or sealing secrets to the VM. ''; }; stateDir = mkOption { type = types.str; default = tpmStateDir; description = '' Where swtpm keeps its state. Must live under /var/lib/ so it can be created and owned by the unit's StateDirectory. ''; }; }; cdroms = mkOption { default = [ ]; description = "Read-only CD-ROM images to attach, on a virtio-scsi controller."; type = types.listOf ( types.submodule { options.path = mkOption { type = types.str; }; } ); }; extraDisks = mkOption { default = [ ]; description = "Extra raw disk images, each on its own virtio-blk device."; type = types.listOf ( types.submodule { options = { name = mkOption { type = types.str; description = "Unique QEMU drive id, e.g. scratch."; }; path = mkOption { type = types.str; }; readonly = mkOption { type = types.bool; default = false; }; aio = mkOption { type = types.enum [ "threads" "native" "io_uring" ]; default = "threads"; }; cache = mkOption { type = types.enum [ "writeback" "none" "directsync" "unsafe" ]; default = "writeback"; }; }; } ); }; usbController = mkOption { type = types.enum [ "auto" "on" "off" ]; default = "auto"; description = '' Whether to attach a qemu-xhci USB controller. "auto" adds one only when passthrough.usb is non-empty; "on"/"off" force the decision. ''; }; }; extraQemuArgs = mkOption { type = types.listOf types.str; default = [ ]; description = '' Raw additional QEMU arguments, appended to the VM command line. Unvalidated escape hatch - prefer the typed devices.* options. ''; }; gatewayUnit = mkOption { type = types.str; default = "hermes-agent"; description = '' The systemd *user* unit that `hermes gateway install` created in the guest. Check with `hermenix ctl units` after installing; upstream has used more than one name for this. ''; }; hermesHome = mkOption { type = types.str; # Guest path (not host disk.baseDir). default = "/var/lib/hermes/.hermes"; description = '' Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/, state.db, and so on) lives inside the guest. Defaults to the seeded layout under the state disk mount (`/var/lib/hermes/.hermes`). Override when carrying over an existing flat HERMES_HOME from elsewhere (e.g. a prior Docker deployment) that you're placing directly at the root of the state disk instead of under .hermes — see `hermenix migrate --help`. This only changes where the host-side `hermenix ctl` commands (soul, memory, conf, secrets, reseed, pull) look. It does not change what the gateway process itself uses — export HERMES_HOME to match inside the guest when you install it. ''; }; vsockCid = mkOption { type = types.int; default = 77; description = "Host reaches the guest over vsock, so no inbound network path is needed."; }; version = mkOption { type = types.str; default = "dev"; description = '' "YYYY.MM.DD.hh.mm-" reported by `hermenix version`. `version` subcommand. Normally supplied for free by this flake's own `nixosModules.default` (from its git metadata) - the "dev" default here only shows up if this module is imported some other way. ''; }; }; config = mkIf cfg.enable { assertions = [ { assertion = net.externalInterface != null; message = "services.hermenix.network.externalInterface must be set."; } { assertion = !net.bridged || (net.lanAddress != "" && net.hostLanAddress != "" && net.lanGateway != ""); message = '' services.hermenix.network.bridged = true requires a fully static LAN setup (DHCP is no longer supported in bridged mode). Set network.lanAddress (the guest's fixed LAN address), network.hostLanAddress (this host's own LAN address, which moves onto the bridge) and network.lanGateway (your router's address). ''; } { assertion = cfg.backup.enable -> (cfg.backup.repository != ""); message = "services.hermenix.backup.repository must be set when backups are enabled."; } { assertion = anyPci -> (pt.iommu != null); message = '' services.hermenix.passthrough.pci needs passthrough.iommu set to "intel" or "amd". Without the IOMMU, a passed-through device can read host memory directly and the VM boundary is meaningless. ''; } { assertion = cfg.devices.vsock.enable || !cfg.backup.enable; message = '' services.hermenix.backup.enable needs the vsock (the backup runs over SSH multiplexed on it). Leave devices.vsock.enable at its default, or turn backups off. ''; } { assertion = cfg.devices.usbController != "off" || !anyUsb; message = '' services.hermenix.passthrough.usb is set but services.hermenix.devices.usbController is "off", so those devices have no controller to attach to. ''; } { assertion = !cfg.devices.tpm.enable || lib.hasPrefix "/var/lib/" cfg.devices.tpm.stateDir; message = '' services.hermenix.devices.tpm.stateDir must live under /var/lib/ so the swtpm unit's StateDirectory can create and own it. ''; } ]; users.users.${vmUser} = { isSystemUser = true; group = vmUser; description = "Runs the Hermes guest VM"; }; users.groups.${vmUser} = { }; systemd.services.hermes-vm = { description = "Hermes Agent VM"; wantedBy = [ "multi-user.target" ]; after = [ "network-online.target" "hermes-vm-tap.service" ] ++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service"; requires = [ "hermes-vm-tap.service" ] ++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service"; wants = [ "network-online.target" ]; preStart = '' for img in ${systemImage} ${stateImage}; do if [ ! -e "$img" ]; then echo "$img is missing. Run 'hermenix setup' first." >&2 exit 1 fi done if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd fi ''; serviceConfig = { User = vmUser; Group = vmUser; SupplementaryGroups = [ "kvm" ]; RuntimeDirectory = "hermes-vm"; ReadWritePaths = [ cfg.disk.baseDir ]; StateDirectory = "hermes-vm"; RuntimeDirectoryMode = "0750"; ExecStart = lib.concatStringsSep " " ( [ "${pkgs.qemu_kvm}/bin/qemu-system-x86_64" "-name ${cfg.name}" "-machine q35,accel=kvm,smm=on" "-cpu host" "-smp ${toString cfg.vcpu}" "-m ${toString cfg.mem}" "-nodefaults" "-no-user-config" "-display none" "-serial stdio" "-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny" "-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}" "-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd" # aio=threads+writeback: images are files, not blockdevs. "-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap" "-device virtio-blk-pci,drive=system" "-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap" "-device virtio-blk-pci,drive=state" # vhost=off: keep packet handling out of the host kernel. "-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off" "-device virtio-net-pci,netdev=net0,mac=${net.mac}" "-qmp unix:${runDir}/qmp.sock,server=on,wait=off" ] ++ lib.optionals cfg.devices.rng.enable [ "-device virtio-rng-pci" ] ++ lib.optionals cfg.devices.vsock.enable [ "-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}" ] ++ lib.optionals cfg.devices.video.enable [ "-vga ${cfg.devices.video.model}" ] ++ lib.optionals cfg.devices.balloon.enable [ "-device virtio-balloon-pci" ] ++ lib.optionals cfg.devices.watchdog.enable [ "-device ${cfg.devices.watchdog.model}" "-watchdog-action ${cfg.devices.watchdog.action}" ] ++ lib.optionals cfg.devices.input.enable ( lib.optional cfg.devices.input.keyboard "-device virtio-keyboard-pci" ++ lib.optional cfg.devices.input.tablet "-device virtio-tablet-pci" ) ++ lib.optionals cfg.devices.audio.enable ( [ "-audiodev ${cfg.devices.audio.backend},id=audio0" ] ++ ( if cfg.devices.audio.model == "virtio-sound" then [ "-device virtio-sound-pci,audiodev=audio0" ] else [ "-device intel-hda" "-device hda-duplex,audiodev=audio0" ] ) ) ++ lib.optionals cfg.devices.tpm.enable [ "-chardev socket,id=chrtpm,path=${runDir}/swtpm.sock" "-tpmdev emulator,id=tpm0,chardev=chrtpm" "-device tpm-crb,tpmdev=tpm0" ] ++ lib.optionals (cfg.devices.cdroms != [ ]) [ "-device virtio-scsi-pci,id=scsi0" ] ++ lib.concatLists ( lib.imap0 (i: c: [ "-drive if=none,id=cd${toString i},media=cdrom,readonly=on,file=${c.path}" "-device scsi-cd,drive=cd${toString i},bus=scsi0.0" ]) cfg.devices.cdroms ) ++ lib.concatMap (d: [ "-drive if=none,id=${d.name},format=raw,file=${d.path},cache=${d.cache},aio=${d.aio}${lib.optionalString d.readonly ",readonly=on"}" "-device virtio-blk-pci,drive=${d.name}" ]) cfg.devices.extraDisks ++ lib.optionals usbControllerOn [ "-device qemu-xhci,id=xhci" ] ++ lib.optionals cfg.disk.quiesce [ "-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0" "-device virtio-serial-pci" "-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0" ] ++ map (d: "-device vfio-pci,host=${d.address}") pt.pci ++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb ++ cfg.extraQemuArgs ); ExecStop = stopScript; TimeoutStopSec = 90; Restart = "always"; RestartSec = 5; DevicePolicy = "closed"; DeviceAllow = [ "/dev/kvm rw" "/dev/net/tun rw" "/dev/vhost-vsock rw" ] ++ lib.optionals anyPci [ "char-vfio rw" ] ++ lib.optionals anyUsb [ "char-usb_device rw" ]; # VFIO pins guest RAM; default memlock is too low. LimitMEMLOCK = lib.mkIf anyPci "infinity"; NoNewPrivileges = true; ProtectSystem = "strict"; ProtectHome = true; ProtectProc = "invisible"; ProcSubset = "pid"; PrivateTmp = true; ProtectKernelTunables = true; ProtectKernelModules = true; ProtectKernelLogs = true; ProtectClock = true; ProtectControlGroups = true; ProtectHostname = true; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; RestrictAddressFamilies = [ "AF_UNIX" "AF_NETLINK" "AF_VSOCK" ]; LockPersonality = true; SystemCallArchitectures = "native"; SystemCallFilter = [ "@system-service" "~@resources" "~@obsolete" ]; # Safe under KVM (no JIT). First flip if QEMU won't start. MemoryDenyWriteExecute = true; CapabilityBoundingSet = [ "" ]; AmbientCapabilities = [ "" ]; }; }; systemd.services.hermes-vm-swtpm = mkIf cfg.devices.tpm.enable { description = "swtpm TPM emulator for the Hermes VM"; wantedBy = [ "multi-user.target" ]; before = [ "hermes-vm.service" ]; requiredBy = [ "hermes-vm.service" ]; serviceConfig = { User = vmUser; Group = vmUser; RuntimeDirectory = "hermes-vm"; RuntimeDirectoryMode = "0750"; StateDirectory = baseNameOf cfg.devices.tpm.stateDir; Restart = "always"; RestartSec = 5; DevicePolicy = "closed"; NoNewPrivileges = true; PrivateTmp = true; ProtectSystem = "strict"; ProtectHome = true; ProtectProc = "invisible"; ProcSubset = "pid"; ProtectKernelTunables = true; ProtectKernelModules = true; ProtectKernelLogs = true; ProtectClock = true; ProtectControlGroups = true; ProtectHostname = true; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; RestrictAddressFamilies = [ "AF_UNIX" ]; LockPersonality = true; SystemCallArchitectures = "native"; SystemCallFilter = [ "@system-service" ]; MemoryDenyWriteExecute = true; CapabilityBoundingSet = [ "" ]; AmbientCapabilities = [ "" ]; }; script = '' exec ${pkgs.swtpm}/bin/swtpm socket \ --tpm2 \ --tpmstate dir=${cfg.devices.tpm.stateDir} \ --ctrl type=unixio,path=${runDir}/swtpm.sock \ --flags not-need-init,startup-clear ''; }; systemd.services.hermes-vm-tap = { description = "TAP interface for the Hermes VM"; after = [ "systemd-networkd.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; path = [ pkgs.iproute2 ]; script = '' ip link show ${net.tapId} >/dev/null 2>&1 \ || ip tuntap add ${net.tapId} mode tap user ${vmUser} ip link set ${net.tapId} up ''; preStop = '' ${pkgs.iproute2}/bin/ip link del ${net.tapId} || true ''; }; boot.kernelParams = lib.optionals (pt.iommu != null) [ (if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on") "iommu=pt" ]; boot.initrd.kernelModules = lib.optionals anyPci [ "vfio_pci" "vfio" "vfio_iommu_type1" ]; boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) '' options vfio-pci ids=${lib.concatStringsSep "," earlyIds} ''; systemd.services.hermes-vm-vfio = mkIf anyPci { description = "Bind passthrough PCI devices to vfio-pci"; before = [ "hermes-vm.service" ]; requiredBy = [ "hermes-vm.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; path = [ pkgs.kmod ]; script = '' modprobe vfio-pci '' + lib.concatMapStrings (d: '' dev=/sys/bus/pci/devices/${d.address} if [ ! -e "$dev" ]; then echo "${d.address} is not present on this machine" >&2 exit 1 fi cur=none [ -e "$dev/driver" ] && cur=$(basename "$(readlink -f "$dev/driver")") if [ "$cur" != "vfio-pci" ]; then [ "$cur" != "none" ] && echo ${d.address} > "$dev/driver/unbind" echo vfio-pci > "$dev/driver_override" echo ${d.address} > /sys/bus/pci/drivers_probe fi '') pt.pci; }; services.udev.extraRules = lib.optionalString anyPci '' SUBSYSTEM=="vfio", OWNER="${vmUser}", GROUP="${vmUser}", MODE="0660" '' + lib.concatMapStrings (u: '' SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660" '') pt.usb; networking.useNetworkd = true; systemd.network.netdevs."10-${net.bridge}".netdevConfig = { Kind = "bridge"; Name = net.bridge; }; systemd.network.networks."10-${net.bridge}" = { matchConfig.Name = net.bridge; # hostAddress = service net; bridged also adds hostLanAddress (static only). address = [ "${net.hostAddress}/${toString net.prefixLength}" ] ++ lib.optional net.bridged "${net.hostLanAddress}/${toString net.lanPrefixLength}"; networkConfig = { ConfigureWithoutCarrier = true; IPv4Forwarding = true; } // lib.optionalAttrs net.bridged { Gateway = net.lanGateway; DNS = if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns; }; }; # mkDefault: operator nameservers win if set. networking.nameservers = lib.mkIf net.bridged ( lib.mkDefault (if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns) ); systemd.network.networks."11-${net.tapId}" = { matchConfig.Name = net.tapId; networkConfig.Bridge = net.bridge; }; systemd.network.networks."12-${net.externalInterface}" = lib.mkIf net.bridged { matchConfig.Name = net.externalInterface; networkConfig.Bridge = net.bridge; }; # Bridge would inherit tap MAC (first port) and collide with guest — clone uplink MAC. systemd.services.hermes-bridge-mac = lib.mkIf net.bridged { description = "Clone the uplink MAC onto the hermenix bridge"; after = [ "systemd-networkd.service" "sys-subsystem-net-devices-${net.bridge}.device" "sys-subsystem-net-devices-${net.externalInterface}.device" ]; wants = [ "systemd-networkd.service" ]; before = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; unitConfig.ConditionPathExists = "/sys/class/net/${net.externalInterface}"; script = '' set -euo pipefail for _ in 1 2 3 4 5 6 7 8 9 10; do [ -e "/sys/class/net/${net.bridge}" ] && break sleep 0.5 done [ -e "/sys/class/net/${net.bridge}" ] || exit 1 mac="$(${pkgs.coreutils}/bin/cat /sys/class/net/${net.externalInterface}/address)" ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} down ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} address "$mac" ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} up ${pkgs.systemd}/bin/networkctl reconfigure ${net.bridge} || true ''; }; # Single-address DHCP on the bridge (MAC-keyed). Broadcasts stay on-bridge. systemd.services.hermes-lan-dhcp = lib.mkIf net.bridged { description = "Static-lease DHCP for the bridged hermenix guest"; after = [ "systemd-networkd.service" "sys-subsystem-net-devices-${net.bridge}.device" ]; wants = [ "systemd-networkd.service" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "simple"; Restart = "on-failure"; RestartSec = 5; }; script = '' for _ in 1 2 3 4 5 6 7 8 9 10; do [ -e "/sys/class/net/${net.bridge}" ] && break sleep 0.5 done exec ${pkgs.dnsmasq}/bin/dnsmasq --keep-in-foreground \ --conf-file=/dev/null \ --interface=${net.bridge} \ --bind-dynamic \ --port=0 \ --dhcp-authoritative \ --dhcp-range=${net.lanAddress},${net.lanAddress},${netmaskFromPrefix net.lanPrefixLength},12h \ --dhcp-host=${net.mac},${net.lanAddress} \ --dhcp-option=option:router,${net.lanGateway} ''; }; # mkOverride 99: same keys as networking.nat, win over weaker defaults. boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true; boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true; # Explicit nft nat (not networking.nat). DNAT iifname != bridge so extra paths still match. networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null) { family = "ip"; content = if net.bridged then # No DNAT when bridged. Scoped masquerade covers service-net fallback if DHCP lapses. '' chain postrouting { type nat hook postrouting priority srcnat; iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade } '' else '' chain prerouting { type nat hook prerouting priority dstnat; '' + lib.concatMapStrings (s: '' iifname != "${net.bridge}" ${s.proto} dport ${toString s.hostPort} dnat to ${net.guestAddress}:${toString s.guestPort} '') cfg.publish + '' } chain postrouting { type nat hook postrouting priority srcnat; iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade } ''; }; networking.nftables.tables."hermenix-bridge" = lib.mkIf net.bridged { family = "bridge"; content = '' chain prerouting { type filter hook prerouting priority filter; policy accept; iifname "${net.tapId}" meta mark set ${toString bridgeMark} # Guest DHCP stays on-bridge only (no race with LAN DHCP). iifname "${net.tapId}" oifname "${net.externalInterface}" ether type ip ip protocol udp udp sport 68 udp dport 67 drop iifname "${net.externalInterface}" oifname "${net.tapId}" ether type ip ip protocol udp udp sport 67 udp dport 68 drop } chain forward { type filter hook forward priority filter; policy accept; oifname "${net.tapId}" accept '' + lib.concatMapStrings (s: '' iifname "${net.tapId}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept '') cfg.lanAccess + ( if cfg.allowLan then '' iifname "${net.tapId}" accept '' else '' iifname "${net.tapId}" ip daddr { 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 169.254.0.0/16, 127.0.0.0/8, 100.64.0.0/10 } drop iifname "${net.tapId}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop iifname "${net.tapId}" accept '' ) + '' } ''; }; networking.nftables.enable = true; networking.firewall.enable = true; networking.firewall.filterForward = true; # Guest→host holes. NAT: guestAddress. Bridged: tap mark (guest may use LAN IP). networking.firewall.extraInputRules = if net.bridged then optionalString cfg.hostCache.enable '' meta mark ${toString bridgeMark} tcp dport ${toString cfg.hostCache.port} accept '' + lib.concatMapStrings (s: '' meta mark ${toString bridgeMark} ${s.proto} dport ${toString s.port} accept '') cfg.hostAccess else optionalString cfg.hostCache.enable '' iifname "${net.bridge}" ip saddr ${net.guestAddress} tcp dport ${toString cfg.hostCache.port} accept '' + lib.concatMapStrings (s: '' iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept '') cfg.hostAccess; # Allowlists before RFC1918 drop. Bridged LAN is L2 (hermenix-bridge table). networking.firewall.extraForwardRules = '' iifname != "${net.bridge}" accept ct state established,related accept '' + lib.concatMapStrings (s: '' iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept '') cfg.lanAccess + lib.optionalString (!net.bridged) ( lib.concatMapStrings (s: '' oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept '') cfg.publish ) + ( if cfg.allowLan then '' iifname "${net.bridge}" accept '' else '' iifname "${net.bridge}" ip daddr { 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 169.254.0.0/16, 127.0.0.0/8, 100.64.0.0/10 } drop iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop iifname "${net.bridge}" accept '' ); services.nix-serve = mkIf cfg.hostCache.enable { enable = true; package = pkgs.nix-serve-ng; bindAddress = net.hostAddress; inherit (cfg.hostCache) port secretKeyFile; openFirewall = false; }; systemd.services.hermes-vm-snapshot = mkIf cfg.snapshots.enable { description = "Snapshot the Hermes VM state disk"; path = with pkgs; [ btrfs-progs coreutils socat ]; serviceConfig.Type = "oneshot"; script = '' set -euo pipefail stamp=$(date +%Y%m%d-%H%M%S) mkdir -p ${snapDir} ${optionalString cfg.disk.quiesce '' ${qga "freeze"} trap '${qga "thaw"}' EXIT ''} btrfs subvolume snapshot -r ${cfg.disk.baseDir}/state ${snapDir}/state-$stamp ${optionalString cfg.disk.quiesce '' ${qga "thaw"} trap - EXIT ''} # Live snapshots reintroduce CoW on nodatacow images — keep retention tight. ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \ | while read -r old; do btrfs subvolume delete "$old"; done ''; }; systemd.timers.hermes-vm-snapshot = mkIf cfg.snapshots.enable { wantedBy = [ "timers.target" ]; timerConfig = { OnCalendar = cfg.snapshots.interval; Persistent = true; RandomizedDelaySec = "5m"; }; }; systemd.services.hermes-vm-backup = mkIf cfg.backup.enable { description = "Back up Hermes state"; path = with pkgs; [ restic openssh socat gnutar ]; serviceConfig = { Type = "oneshot"; EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile; }; script = '' set -euo pipefail export RESTIC_REPOSITORY=${lib.escapeShellArg cfg.backup.repository} export RESTIC_PASSWORD_FILE=${cfg.backup.passwordFile} restic snapshots >/dev/null 2>&1 || restic init ssh -o ProxyCommand="socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22" \ -o StrictHostKeyChecking=accept-new \ hermes@${cfg.name} \ 'tar -C /var/lib/hermes \ --exclude=./.cache \ --exclude=./workspace/**/.direnv \ --exclude=./workspace/**/result \ -cf - .' \ | restic backup --stdin --stdin-filename hermes-state.tar \ --tag hermes --host ${cfg.name} restic forget --tag hermes --host ${cfg.name} --prune ${lib.concatStringsSep " " cfg.backup.keep} ''; }; systemd.timers.hermes-vm-backup = mkIf cfg.backup.enable { wantedBy = [ "timers.target" ]; timerConfig = { OnCalendar = cfg.backup.interval; Persistent = true; RandomizedDelaySec = "30m"; }; }; environment.systemPackages = [ (pkgs.writeTextFile { name = "hermenix-completions"; destination = "/share/nushell/vendor/autoload/hermenix.nu"; text = builtins.readFile ../nu/completions.nu; }) (pkgs.symlinkJoin { name = "hermenix"; paths = [ pkgs.hermenix ]; nativeBuildInputs = [ pkgs.makeWrapper ]; postBuild = '' wrapProgram $out/bin/hermenix \ --set HERMES_VSOCK_CID ${toString cfg.vsockCid} \ --set HERMES_GUEST_NAME ${cfg.name} \ --set HERMES_DISK_BASE_DIR ${cfg.disk.baseDir} \ --set HERMES_HOME_DIR ${cfg.hermesHome} \ --set HERMES_GATEWAY_UNIT ${cfg.gatewayUnit} ''; }) ]; }; }