diff --git a/modules/hermenix/backup.nix b/modules/hermenix/backup.nix new file mode 100644 index 0000000..4d43c5a --- /dev/null +++ b/modules/hermenix/backup.nix @@ -0,0 +1,71 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + inherit (import ./common.nix { inherit config lib pkgs; }) cfg; +in +{ + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = cfg.backup.enable -> (cfg.backup.repository != ""); + message = "services.hermenix.backup.repository must be set when backups are enabled."; + } + { + assertion = cfg.devices.vsock.enable || !cfg.backup.enable; + message = '' + services.hermenix.backup.enable needs the vsock (the backup runs over + SSH multiplexed on it). Leave devices.vsock.enable at its default, or + turn backups off. + ''; + } + ]; + + systemd.services.hermes-vm-backup = lib.mkIf cfg.backup.enable { + description = "Back up Hermes state"; + path = with pkgs; [ + restic + openssh + socat + gnutar + ]; + serviceConfig = { + Type = "oneshot"; + EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile; + }; + script = '' + set -euo pipefail + export RESTIC_REPOSITORY=${lib.escapeShellArg cfg.backup.repository} + export RESTIC_PASSWORD_FILE=${cfg.backup.passwordFile} + + restic snapshots >/dev/null 2>&1 || restic init + + ssh -o ProxyCommand="socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22" \ + -o StrictHostKeyChecking=accept-new \ + hermes@${cfg.name} \ + 'tar -C /var/lib/hermes \ + --exclude=./.cache \ + --exclude=./workspace/**/.direnv \ + --exclude=./workspace/**/result \ + -cf - .' \ + | restic backup --stdin --stdin-filename hermes-state.tar \ + --tag hermes --host ${cfg.name} + + restic forget --tag hermes --host ${cfg.name} --prune ${lib.concatStringsSep " " cfg.backup.keep} + ''; + }; + + systemd.timers.hermes-vm-backup = lib.mkIf cfg.backup.enable { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = cfg.backup.interval; + Persistent = true; + RandomizedDelaySec = "30m"; + }; + }; + }; +} diff --git a/modules/hermenix/backup.options.nix b/modules/hermenix/backup.options.nix new file mode 100644 index 0000000..9fcabab --- /dev/null +++ b/modules/hermenix/backup.options.nix @@ -0,0 +1,51 @@ +{ lib, ... }: + +let + inherit (lib) + mkOption + mkEnableOption + types + ; +in +{ + options.services.hermenix.backup = { + enable = mkEnableOption "restic backups of the guest's state"; + + repository = mkOption { + type = types.str; + default = ""; + example = "sftp:backup@nas:/hermes"; + description = "Any restic repository string."; + }; + + passwordFile = mkOption { + type = types.path; + default = "/var/lib/secrets/hermes-restic"; + description = "File containing the restic repository password, mode 0400."; + }; + + environmentFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Optional file with credentials for the backend (S3 keys and so on)."; + }; + + interval = mkOption { + type = types.str; + default = "daily"; + }; + + keep = mkOption { + type = types.listOf types.str; + default = [ + "--keep-daily" + "14" + "--keep-weekly" + "8" + "--keep-monthly" + "12" + ]; + description = "Arguments passed to restic forget."; + }; + }; +} diff --git a/modules/hermenix/bridge.nix b/modules/hermenix/bridge.nix new file mode 100644 index 0000000..f8857b9 --- /dev/null +++ b/modules/hermenix/bridge.nix @@ -0,0 +1,165 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + inherit (import ./common.nix { inherit config lib pkgs; }) + cfg + net + bridgeMark + netmaskFromPrefix + ; +in +{ + config = lib.mkIf (cfg.enable && net.bridged) { + assertions = [ + { + assertion = net.lanAddress != "" && net.hostLanAddress != "" && net.lanGateway != ""; + message = '' + services.hermenix.network.bridged = true requires a fully static + LAN setup (DHCP is no longer supported in bridged mode). Set + network.lanAddress (the guest's fixed LAN address), + network.hostLanAddress (this host's own LAN address, which moves + onto the bridge) and network.lanGateway (your router's address). + ''; + } + ]; + + # mkDefault: operator nameservers win if set. + networking.nameservers = lib.mkDefault ( + if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns + ); + + systemd.network.networks."12-${net.externalInterface}" = { + matchConfig.Name = net.externalInterface; + networkConfig.Bridge = net.bridge; + }; + + # Bridge would inherit tap MAC (first port) and collide with guest — clone uplink MAC. + systemd.services.hermes-bridge-mac = { + description = "Clone the uplink MAC onto the hermenix bridge"; + after = [ + "systemd-networkd.service" + "sys-subsystem-net-devices-${net.bridge}.device" + "sys-subsystem-net-devices-${net.externalInterface}.device" + ]; + wants = [ "systemd-networkd.service" ]; + before = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + unitConfig.ConditionPathExists = "/sys/class/net/${net.externalInterface}"; + script = '' + set -euo pipefail + for _ in 1 2 3 4 5 6 7 8 9 10; do + [ -e "/sys/class/net/${net.bridge}" ] && break + sleep 0.5 + done + [ -e "/sys/class/net/${net.bridge}" ] || exit 1 + mac="$(${pkgs.coreutils}/bin/cat /sys/class/net/${net.externalInterface}/address)" + ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} down + ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} address "$mac" + ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} up + ${pkgs.systemd}/bin/networkctl reconfigure ${net.bridge} || true + ''; + }; + + # Single-address DHCP on the bridge (MAC-keyed). Broadcasts stay on-bridge. + systemd.services.hermes-lan-dhcp = { + description = "Static-lease DHCP for the bridged hermenix guest"; + after = [ + "systemd-networkd.service" + "sys-subsystem-net-devices-${net.bridge}.device" + ]; + wants = [ "systemd-networkd.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "simple"; + Restart = "on-failure"; + RestartSec = 5; + }; + script = '' + for _ in 1 2 3 4 5 6 7 8 9 10; do + [ -e "/sys/class/net/${net.bridge}" ] && break + sleep 0.5 + done + exec ${pkgs.dnsmasq}/bin/dnsmasq --keep-in-foreground \ + --conf-file=/dev/null \ + --interface=${net.bridge} \ + --bind-dynamic \ + --port=0 \ + --dhcp-authoritative \ + --dhcp-range=${net.lanAddress},${net.lanAddress},${netmaskFromPrefix net.lanPrefixLength},12h \ + --dhcp-host=${net.mac},${net.lanAddress} \ + --dhcp-option=option:router,${net.lanGateway} + ''; + }; + + # No DNAT when bridged. Scoped masquerade covers service-net fallback if DHCP lapses. + networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null) { + family = "ip"; + content = '' + chain postrouting { + type nat hook postrouting priority srcnat; + iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade + } + ''; + }; + + networking.nftables.tables."hermenix-bridge" = { + family = "bridge"; + content = '' + chain prerouting { + type filter hook prerouting priority filter; policy accept; + iifname "${net.tapId}" meta mark set ${toString bridgeMark} + + # Guest DHCP stays on-bridge only (no race with LAN DHCP). + iifname "${net.tapId}" oifname "${net.externalInterface}" ether type ip ip protocol udp udp sport 68 udp dport 67 drop + iifname "${net.externalInterface}" oifname "${net.tapId}" ether type ip ip protocol udp udp sport 67 udp dport 68 drop + } + chain forward { + type filter hook forward priority filter; policy accept; + oifname "${net.tapId}" accept + '' + + lib.concatMapStrings (s: '' + iifname "${net.tapId}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept + '') cfg.lanAccess + + ( + if cfg.allowLan then + '' + iifname "${net.tapId}" accept + '' + else + '' + iifname "${net.tapId}" ip daddr { + 10.0.0.0/8, + 172.16.0.0/12, + 192.168.0.0/16, + 169.254.0.0/16, + 127.0.0.0/8, + 100.64.0.0/10 + } drop + iifname "${net.tapId}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop + iifname "${net.tapId}" accept + '' + ) + + '' + } + ''; + }; + + # Guest→host holes (bridged: match the tap mark; the guest may use its LAN IP). + networking.firewall.extraInputRules = + lib.optionalString cfg.hostCache.enable '' + meta mark ${toString bridgeMark} tcp dport ${toString cfg.hostCache.port} accept + '' + + lib.concatMapStrings (s: '' + meta mark ${toString bridgeMark} ${s.proto} dport ${toString s.port} accept + '') cfg.hostAccess; + }; +} diff --git a/modules/hermenix/bridge.options.nix b/modules/hermenix/bridge.options.nix new file mode 100644 index 0000000..689abff --- /dev/null +++ b/modules/hermenix/bridge.options.nix @@ -0,0 +1,77 @@ +{ lib, ... }: + +let + inherit (lib) + mkOption + types + ; +in +{ + options.services.hermenix.network = { + lanAddress = mkOption { + type = types.str; + default = ""; + example = "10.0.12.50"; + description = "The guest's fixed address on your LAN. Required when `bridged = true`."; + }; + hostLanAddress = mkOption { + type = types.str; + default = ""; + example = "10.0.12.10"; + description = '' + This host's own address on your LAN. When `bridged` is enabled the + uplink becomes a bridge port and this address moves onto the bridge + (which clones the uplink's MAC, so the host keeps its LAN identity). + Required when `bridged = true`. + ''; + }; + lanGateway = mkOption { + type = types.str; + default = ""; + example = "10.0.12.1"; + description = '' + Your LAN's router. Used as the default gateway for the guest and, + while `bridged`, for this host's bridge too (DNS unless `lanDns` is + set). Required when `bridged = true`. + ''; + }; + lanPrefixLength = mkOption { + type = types.int; + default = 24; + description = "Prefix length of your LAN subnet (netmask)."; + }; + lanDns = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "1.1.1.1" ]; + description = '' + DNS servers for this host while `bridged` (the bridge no longer + takes a DHCP lease). Defaults to the LAN gateway. + ''; + }; + bridged = mkOption { + type = types.bool; + default = false; + description = '' + Put the guest directly on your LAN instead of behind the host's + NAT. The bridge absorbs `externalInterface`, so any device on the + LAN can reach any port the guest exposes. + + Addressing is static from the operator's point of view: you pick + fixed addresses for the guest (`lanAddress`), this host + (`hostLanAddress` — it moves onto the bridge), and the LAN + gateway (`lanGateway`). A DHCP server this module runs on the + bridge hands `lanAddress` to the guest, keyed to `network.mac`, + so the guest stays a plain DHCP client and never needs to know + its own address. That lease never leaves the bridge; the LAN's + own DHCP cannot race it. + + Inbound is no longer limited to `publish` (which is ignored in + this mode): the LAN may reach every guest port. Outbound is still + governed by `lanAccess` / `allowLan`, and `hostAccess` plus the + host cache still work over the host-guest service net + (`hostAddress` stays configured on the bridge). + ''; + }; + }; +} diff --git a/modules/hermenix/common.nix b/modules/hermenix/common.nix new file mode 100644 index 0000000..e666fa4 --- /dev/null +++ b/modules/hermenix/common.nix @@ -0,0 +1,85 @@ +{ + config, + lib, + pkgs, +}: + +let + cfg = config.services.hermenix; + net = cfg.network; + pt = cfg.passthrough; + + vmUser = "hermes-vm"; + runDir = "/run/hermes-vm"; + stateDir = "/var/lib/hermes-vm"; + systemImage = "${cfg.disk.baseDir}/system/system.raw"; + stateImage = "${cfg.disk.baseDir}/state/state.raw"; + snapDir = "${cfg.disk.baseDir}/snapshots"; + + anyPci = pt.pci != [ ]; + anyUsb = pt.usb != [ ]; + earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci); + + usbControllerOn = + cfg.devices.usbController == "on" || (cfg.devices.usbController == "auto" && anyUsb); + + # 18501 == 0x4845 ("HE"); marks guest traffic from the tap (bridged mode). + bridgeMark = 18501; + + # dnsmasq needs an explicit mask for a one-address DHCP range. + netmaskFromPrefix = + p: + lib.concatStringsSep "." ( + lib.genList ( + b: + toString ( + builtins.elemAt [ + 0 + 128 + 192 + 224 + 240 + 248 + 252 + 254 + 255 + ] (lib.max 0 (lib.min 8 (p - b * 8))) + ) + ) 4 + ); + + stopScript = pkgs.writeShellScript "hermes-vm-stop" '' + set -eu + printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \ + | ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0 + ''; + + qga = + verb: + pkgs.writeShellScript "hermes-vm-fs${verb}" '' + set -eu + printf '%s' '{"execute":"guest-fs${verb}"}' \ + | ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true + ''; +in +{ + inherit + cfg + net + pt + vmUser + runDir + stateDir + systemImage + stateImage + snapDir + anyPci + anyUsb + earlyIds + usbControllerOn + bridgeMark + netmaskFromPrefix + stopScript + qga + ; +} diff --git a/modules/hermenix/network.nix b/modules/hermenix/network.nix new file mode 100644 index 0000000..fb1f0bc --- /dev/null +++ b/modules/hermenix/network.nix @@ -0,0 +1,127 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + inherit (import ./common.nix { inherit config lib pkgs; }) cfg net; +in +{ + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = net.externalInterface != null; + message = "services.hermenix.network.externalInterface must be set."; + } + ]; + + networking.useNetworkd = true; + + systemd.network.netdevs."10-${net.bridge}".netdevConfig = { + Kind = "bridge"; + Name = net.bridge; + }; + + systemd.network.networks."10-${net.bridge}" = { + matchConfig.Name = net.bridge; + # hostAddress = service net; bridged also adds hostLanAddress (static only). + address = [ + "${net.hostAddress}/${toString net.prefixLength}" + ] + ++ lib.optional net.bridged "${net.hostLanAddress}/${toString net.lanPrefixLength}"; + networkConfig = { + ConfigureWithoutCarrier = true; + IPv4Forwarding = true; + } + // lib.optionalAttrs net.bridged { + Gateway = net.lanGateway; + DNS = if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns; + }; + }; + + systemd.network.networks."11-${net.tapId}" = { + matchConfig.Name = net.tapId; + networkConfig.Bridge = net.bridge; + }; + + # mkOverride 99: same keys as networking.nat, win over weaker defaults. + boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true; + boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true; + + # Explicit nft nat (not networking.nat). DNAT iifname != bridge so extra paths still match. + networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null && !net.bridged) { + family = "ip"; + content = '' + chain prerouting { + type nat hook prerouting priority dstnat; + '' + + lib.concatMapStrings (s: '' + iifname != "${net.bridge}" ${s.proto} dport ${toString s.hostPort} dnat to ${net.guestAddress}:${toString s.guestPort} + '') cfg.publish + + '' + } + chain postrouting { + type nat hook postrouting priority srcnat; + iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade + } + ''; + }; + + networking.nftables.enable = true; + networking.firewall.enable = true; + networking.firewall.filterForward = true; + + # Guest→host holes (NAT mode: match the service-net address). + networking.firewall.extraInputRules = lib.mkIf (!net.bridged) ( + lib.optionalString cfg.hostCache.enable '' + iifname "${net.bridge}" ip saddr ${net.guestAddress} tcp dport ${toString cfg.hostCache.port} accept + '' + + lib.concatMapStrings (s: '' + iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept + '') cfg.hostAccess + ); + + # Shared by both modes; the bridged LAN is L2 (see bridge.nix). + networking.firewall.extraForwardRules = '' + iifname != "${net.bridge}" accept + ct state established,related accept + '' + + lib.concatMapStrings (s: '' + iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept + '') cfg.lanAccess + + lib.optionalString (!net.bridged) ( + lib.concatMapStrings (s: '' + oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept + '') cfg.publish + ) + + ( + if cfg.allowLan then + '' + iifname "${net.bridge}" accept + '' + else + '' + iifname "${net.bridge}" ip daddr { + 10.0.0.0/8, + 172.16.0.0/12, + 192.168.0.0/16, + 169.254.0.0/16, + 127.0.0.0/8, + 100.64.0.0/10 + } drop + iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop + iifname "${net.bridge}" accept + '' + ); + + services.nix-serve = lib.mkIf cfg.hostCache.enable { + enable = true; + package = pkgs.nix-serve-ng; + bindAddress = net.hostAddress; + inherit (cfg.hostCache) port secretKeyFile; + openFirewall = false; + }; + }; +} diff --git a/modules/hermenix/network.options.nix b/modules/hermenix/network.options.nix new file mode 100644 index 0000000..864da52 --- /dev/null +++ b/modules/hermenix/network.options.nix @@ -0,0 +1,163 @@ +{ lib, ... }: + +let + inherit (lib) + mkOption + types + ; +in +{ + options.services.hermenix = { + hostCache = { + enable = mkOption { + type = types.bool; + default = true; + description = '' + Serve the host's /nix/store to the guest as a read-only binary + cache, so the guest doesn't re-download from the internet what the + host already has. + + This is the ONLY route from guest to host in this design. Turning it + off leaves the guest with literally no reachable host service, at + the cost of bandwidth on every guest build. + + Note the guest can read every store path on the host through this. + Store paths are world-readable anyway, but check nothing sensitive + is baked into your host's store. + ''; + }; + port = mkOption { + type = types.port; + default = 5000; + }; + secretKeyFile = mkOption { + type = types.path; + default = "/var/lib/secrets/hermes-cache.secret"; + description = "Generate with nix-store --generate-binary-cache-key. See README."; + }; + }; + + allowLan = mkOption { + type = types.bool; + default = false; + description = '' + Blanket permission for the guest to reach RFC1918 space. Almost always + the wrong tool - use lanAccess to open single endpoints instead. + ''; + }; + + lanAccess = mkOption { + default = [ ]; + example = [ + { + address = "192.168.1.50"; + port = 11434; + } + ]; + description = '' + Specific endpoints elsewhere on your network that the guest may reach, + punched through the default RFC1918 drop. One entry per service. + ''; + type = types.listOf ( + types.submodule { + options = { + address = mkOption { type = types.str; }; + port = mkOption { type = types.port; }; + proto = mkOption { + type = types.enum [ + "tcp" + "udp" + ]; + default = "tcp"; + }; + }; + } + ); + }; + + hostAccess = mkOption { + default = [ ]; + example = [ { port = 8188; } ]; + description = '' + Ports on this host itself that the guest may reach. The service must + also be bound to an address the guest can see - see the README. + ''; + type = types.listOf ( + types.submodule { + options = { + port = mkOption { type = types.port; }; + proto = mkOption { + type = types.enum [ + "tcp" + "udp" + ]; + default = "tcp"; + }; + }; + } + ); + }; + + publish = mkOption { + default = [ ]; + example = [ + { + hostPort = 8080; + guestPort = 8080; + } + ]; + description = '' + Guest ports to publish on the host's uplink, reachable from the wider + network. You do not need this to reach the guest from the host itself. + ''; + type = types.listOf ( + types.submodule { + options = { + hostPort = mkOption { type = types.port; }; + guestPort = mkOption { type = types.port; }; + proto = mkOption { + type = types.enum [ + "tcp" + "udp" + ]; + default = "tcp"; + }; + }; + } + ); + }; + + network = { + bridge = mkOption { + type = types.str; + default = "hermes-br"; + }; + tapId = mkOption { + type = types.str; + default = "vm-hermes"; + }; + mac = mkOption { + type = types.str; + default = "02:00:00:7e:00:01"; + }; + hostAddress = mkOption { + type = types.str; + default = "10.77.0.1"; + }; + guestAddress = mkOption { + type = types.str; + default = "10.77.0.2"; + }; + prefixLength = mkOption { + type = types.int; + default = 24; + }; + externalInterface = mkOption { + type = types.nullOr types.str; + default = null; + example = "eno1"; + description = "Host uplink. Required."; + }; + }; + }; +} diff --git a/modules/hermenix/packages.nix b/modules/hermenix/packages.nix new file mode 100644 index 0000000..bd45f13 --- /dev/null +++ b/modules/hermenix/packages.nix @@ -0,0 +1,35 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + inherit (import ./common.nix { inherit config lib pkgs; }) cfg; +in +{ + config = lib.mkIf cfg.enable { + environment.systemPackages = [ + (pkgs.writeTextFile { + name = "hermenix-completions"; + destination = "/share/nushell/vendor/autoload/hermenix.nu"; + text = builtins.readFile ../nu/completions.nu; + }) + + (pkgs.symlinkJoin { + name = "hermenix"; + paths = [ pkgs.hermenix ]; + nativeBuildInputs = [ pkgs.makeWrapper ]; + postBuild = '' + wrapProgram $out/bin/hermenix \ + --set HERMES_VSOCK_CID ${toString cfg.vsockCid} \ + --set HERMES_GUEST_NAME ${cfg.name} \ + --set HERMES_DISK_BASE_DIR ${cfg.disk.baseDir} \ + --set HERMES_HOME_DIR ${cfg.hermesHome} \ + --set HERMES_GATEWAY_UNIT ${cfg.gatewayUnit} + ''; + }) + ]; + }; +} diff --git a/modules/hermenix/passthrough.nix b/modules/hermenix/passthrough.nix new file mode 100644 index 0000000..1fb4fa2 --- /dev/null +++ b/modules/hermenix/passthrough.nix @@ -0,0 +1,90 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + inherit (import ./common.nix { inherit config lib pkgs; }) + cfg + pt + anyPci + anyUsb + earlyIds + vmUser + ; +in +{ + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = anyPci -> (pt.iommu != null); + message = '' + services.hermenix.passthrough.pci needs passthrough.iommu set to + "intel" or "amd". Without the IOMMU, a passed-through device can + read host memory directly and the VM boundary is meaningless. + ''; + } + { + assertion = cfg.devices.usbController != "off" || !anyUsb; + message = '' + services.hermenix.passthrough.usb is set but + services.hermenix.devices.usbController is "off", so those devices + have no controller to attach to. + ''; + } + ]; + + boot.kernelParams = lib.optionals (pt.iommu != null) [ + (if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on") + "iommu=pt" + ]; + + boot.initrd.kernelModules = lib.optionals anyPci [ + "vfio_pci" + "vfio" + "vfio_iommu_type1" + ]; + + boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) '' + options vfio-pci ids=${lib.concatStringsSep "," earlyIds} + ''; + + systemd.services.hermes-vm-vfio = lib.mkIf anyPci { + description = "Bind passthrough PCI devices to vfio-pci"; + before = [ "hermes-vm.service" ]; + requiredBy = [ "hermes-vm.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + path = [ pkgs.kmod ]; + script = '' + modprobe vfio-pci + '' + + lib.concatMapStrings (d: '' + dev=/sys/bus/pci/devices/${d.address} + if [ ! -e "$dev" ]; then + echo "${d.address} is not present on this machine" >&2 + exit 1 + fi + cur=none + [ -e "$dev/driver" ] && cur=$(basename "$(readlink -f "$dev/driver")") + if [ "$cur" != "vfio-pci" ]; then + [ "$cur" != "none" ] && echo ${d.address} > "$dev/driver/unbind" + echo vfio-pci > "$dev/driver_override" + echo ${d.address} > /sys/bus/pci/drivers_probe + fi + '') pt.pci; + }; + + services.udev.extraRules = + lib.optionalString anyPci '' + SUBSYSTEM=="vfio", OWNER="${vmUser}", GROUP="${vmUser}", MODE="0660" + '' + + lib.concatMapStrings (u: '' + SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660" + '') pt.usb; + }; +} diff --git a/modules/hermenix/passthrough.options.nix b/modules/hermenix/passthrough.options.nix new file mode 100644 index 0000000..dafe838 --- /dev/null +++ b/modules/hermenix/passthrough.options.nix @@ -0,0 +1,101 @@ +{ lib, ... }: + +let + inherit (lib) + mkOption + types + ; +in +{ + options.services.hermenix.passthrough = { + iommu = mkOption { + type = types.nullOr ( + types.enum [ + "intel" + "amd" + ] + ); + default = null; + description = '' + Enable IOMMU kernel parameters for your CPU vendor. Required for any + PCI passthrough. The IOMMU is the only thing stopping a passed-through + device from reading host memory, so this is not optional. + ''; + }; + + pci = mkOption { + default = [ ]; + example = [ + { + address = "0000:01:00.0"; + id = "10de:2204"; + } + { + address = "0000:01:00.1"; + id = "10de:1aef"; + } + ]; + description = '' + PCI devices handed to the guest via VFIO. The host loses them + entirely while the VM runs. + + You must pass every function of the device and, in practice, every + device in its IOMMU group. A GPU is usually at least two entries: + the graphics function and its HDMI audio function. + + Set `id` (vendor:device) to have vfio-pci claim the device in the + initrd, before a host driver can. Strongly recommended for GPUs; + unbinding an active amdgpu or nvidia driver later is unreliable. + ''; + type = types.listOf ( + types.submodule { + options = { + address = mkOption { + type = types.str; + example = "0000:01:00.0"; + description = "Full PCI address, as shown by lspci -D."; + }; + id = mkOption { + type = types.nullOr types.str; + default = null; + example = "10de:2204"; + description = "vendor:device, from lspci -nn. Enables early binding."; + }; + }; + } + ); + }; + + usb = mkOption { + default = [ ]; + example = [ + { + vendorId = "8087"; + productId = "0026"; + } + ]; + description = '' + USB devices forwarded to the guest, matched by vendor and product + ID from lsusb. Integrated Bluetooth radios are USB devices and go + here too, though the host driver must be kept off them. + + QEMU resolves these at start, so a device that is unplugged when the + VM boots will not appear later without a restart. + ''; + type = types.listOf ( + types.submodule { + options = { + vendorId = mkOption { + type = types.str; + example = "046d"; + }; + productId = mkOption { + type = types.str; + example = "c52b"; + }; + }; + } + ); + }; + }; +} diff --git a/modules/hermenix/shared-options.nix b/modules/hermenix/shared-options.nix new file mode 100644 index 0000000..4449fe6 --- /dev/null +++ b/modules/hermenix/shared-options.nix @@ -0,0 +1,71 @@ +{ lib, ... }: + +let + inherit (lib) + mkOption + mkEnableOption + types + ; +in +{ + options.services.hermenix = { + enable = mkEnableOption "the Hermes Agent VM"; + + name = mkOption { + type = types.str; + default = "hermes"; + }; + + authorizedKeys = mkOption { + type = types.listOf types.str; + default = [ ]; + }; + + gatewayUnit = mkOption { + type = types.str; + default = "hermes-agent"; + description = '' + The systemd *user* unit that `hermes gateway install` created in the + guest. Check with `hermenix ctl units` after installing; upstream has + used more than one name for this. + ''; + }; + + hermesHome = mkOption { + type = types.str; + # Guest path (not host disk.baseDir). + default = "/var/lib/hermes/.hermes"; + description = '' + Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/, + state.db, and so on) lives inside the guest. Defaults to the seeded + layout under the state disk mount (`/var/lib/hermes/.hermes`). + Override when carrying over an existing flat HERMES_HOME from + elsewhere (e.g. a prior Docker deployment) that you're placing + directly at the root of the state disk instead of under .hermes — + see `hermenix migrate --help`. + + This only changes where the host-side `hermenix ctl` commands (soul, + memory, conf, secrets, reseed, pull) look. It does not change what + the gateway process itself uses — export HERMES_HOME to match inside + the guest when you install it. + ''; + }; + + vsockCid = mkOption { + type = types.int; + default = 77; + description = "Host reaches the guest over vsock, so no inbound network path is needed."; + }; + + version = mkOption { + type = types.str; + default = "dev"; + description = '' + "YYYY.MM.DD.hh.mm-" reported by `hermenix version`. + `version` subcommand. Normally supplied for free by this flake's own + `nixosModules.default` (from its git metadata) - the "dev" default + here only shows up if this module is imported some other way. + ''; + }; + }; +} diff --git a/modules/hermenix/snapshots.nix b/modules/hermenix/snapshots.nix new file mode 100644 index 0000000..fa66160 --- /dev/null +++ b/modules/hermenix/snapshots.nix @@ -0,0 +1,57 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + inherit (import ./common.nix { inherit config lib pkgs; }) + cfg + snapDir + qga + ; +in +{ + config = lib.mkIf cfg.enable { + systemd.services.hermes-vm-snapshot = lib.mkIf cfg.snapshots.enable { + description = "Snapshot the Hermes VM state disk"; + path = with pkgs; [ + btrfs-progs + coreutils + socat + ]; + serviceConfig.Type = "oneshot"; + script = '' + set -euo pipefail + stamp=$(date +%Y%m%d-%H%M%S) + mkdir -p ${snapDir} + + ${lib.optionalString cfg.disk.quiesce '' + ${qga "freeze"} + trap '${qga "thaw"}' EXIT + ''} + + btrfs subvolume snapshot -r ${cfg.disk.baseDir}/state ${snapDir}/state-$stamp + + ${lib.optionalString cfg.disk.quiesce '' + ${qga "thaw"} + trap - EXIT + ''} + + # Live snapshots reintroduce CoW on nodatacow images — keep retention tight. + ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \ + | while read -r old; do btrfs subvolume delete "$old"; done + ''; + }; + + systemd.timers.hermes-vm-snapshot = lib.mkIf cfg.snapshots.enable { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = cfg.snapshots.interval; + Persistent = true; + RandomizedDelaySec = "5m"; + }; + }; + }; +} diff --git a/modules/hermenix/snapshots.options.nix b/modules/hermenix/snapshots.options.nix new file mode 100644 index 0000000..63a93fa --- /dev/null +++ b/modules/hermenix/snapshots.options.nix @@ -0,0 +1,37 @@ +{ lib, ... }: + +let + inherit (lib) + mkOption + types + ; +in +{ + options.services.hermenix.snapshots = { + enable = mkOption { + type = types.bool; + default = true; + description = '' + Periodic btrfs snapshots of the state disk. The agent can persist + changes, which means it can persist a mistake, and these are the + fastest way back. + ''; + }; + + interval = mkOption { + type = types.str; + default = "hourly"; + description = "systemd calendar expression."; + }; + + keep = mkOption { + type = types.ints.positive; + default = 48; + description = '' + How many to retain. Keep this tighter than you would on ZFS: the + state image is nodatacow, and every live snapshot reintroduces + copy-on-write for the blocks it still references. + ''; + }; + }; +} diff --git a/modules/hermenix/vm.nix b/modules/hermenix/vm.nix new file mode 100644 index 0000000..922c289 --- /dev/null +++ b/modules/hermenix/vm.nix @@ -0,0 +1,281 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + inherit (import ./common.nix { inherit config lib pkgs; }) + cfg + net + pt + vmUser + runDir + stateDir + systemImage + stateImage + anyPci + anyUsb + usbControllerOn + stopScript + ; +in +{ + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = !cfg.devices.tpm.enable || lib.hasPrefix "/var/lib/" cfg.devices.tpm.stateDir; + message = '' + services.hermenix.devices.tpm.stateDir must live under /var/lib/ so + the swtpm unit's StateDirectory can create and own it. + ''; + } + ]; + + users.users.${vmUser} = { + isSystemUser = true; + group = vmUser; + description = "Runs the Hermes guest VM"; + }; + users.groups.${vmUser} = { }; + + systemd.services.hermes-vm = { + description = "Hermes Agent VM"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "hermes-vm-tap.service" + ] + ++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service"; + requires = [ + "hermes-vm-tap.service" + ] + ++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service"; + wants = [ "network-online.target" ]; + + preStart = '' + for img in ${systemImage} ${stateImage}; do + if [ ! -e "$img" ]; then + echo "$img is missing. Run 'hermenix setup' first." >&2 + exit 1 + fi + done + if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then + install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd + fi + ''; + + serviceConfig = { + User = vmUser; + Group = vmUser; + SupplementaryGroups = [ "kvm" ]; + RuntimeDirectory = "hermes-vm"; + ReadWritePaths = [ cfg.disk.baseDir ]; + StateDirectory = "hermes-vm"; + RuntimeDirectoryMode = "0750"; + + ExecStart = lib.concatStringsSep " " ( + [ + "${pkgs.qemu_kvm}/bin/qemu-system-x86_64" + "-name ${cfg.name}" + "-machine q35,accel=kvm,smm=on" + "-cpu host" + "-smp ${toString cfg.vcpu}" + "-m ${toString cfg.mem}" + "-nodefaults" + "-no-user-config" + "-display none" + "-serial stdio" + "-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny" + "-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}" + "-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd" + # aio=threads+writeback: images are files, not blockdevs. + "-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap" + "-device virtio-blk-pci,drive=system" + "-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap" + "-device virtio-blk-pci,drive=state" + # vhost=off: keep packet handling out of the host kernel. + "-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off" + "-device virtio-net-pci,netdev=net0,mac=${net.mac}" + "-qmp unix:${runDir}/qmp.sock,server=on,wait=off" + ] + ++ lib.optionals cfg.devices.rng.enable [ + "-device virtio-rng-pci" + ] + ++ lib.optionals cfg.devices.vsock.enable [ + "-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}" + ] + ++ lib.optionals cfg.devices.video.enable [ + "-vga ${cfg.devices.video.model}" + ] + ++ lib.optionals cfg.devices.balloon.enable [ + "-device virtio-balloon-pci" + ] + ++ lib.optionals cfg.devices.watchdog.enable [ + "-device ${cfg.devices.watchdog.model}" + "-watchdog-action ${cfg.devices.watchdog.action}" + ] + ++ lib.optionals cfg.devices.input.enable ( + lib.optional cfg.devices.input.keyboard "-device virtio-keyboard-pci" + ++ lib.optional cfg.devices.input.tablet "-device virtio-tablet-pci" + ) + ++ lib.optionals cfg.devices.audio.enable ( + [ "-audiodev ${cfg.devices.audio.backend},id=audio0" ] + ++ ( + if cfg.devices.audio.model == "virtio-sound" then + [ "-device virtio-sound-pci,audiodev=audio0" ] + else + [ + "-device intel-hda" + "-device hda-duplex,audiodev=audio0" + ] + ) + ) + ++ lib.optionals cfg.devices.tpm.enable [ + "-chardev socket,id=chrtpm,path=${runDir}/swtpm.sock" + "-tpmdev emulator,id=tpm0,chardev=chrtpm" + "-device tpm-crb,tpmdev=tpm0" + ] + ++ lib.optionals (cfg.devices.cdroms != [ ]) [ + "-device virtio-scsi-pci,id=scsi0" + ] + ++ lib.concatLists ( + lib.imap0 (i: c: [ + "-drive if=none,id=cd${toString i},media=cdrom,readonly=on,file=${c.path}" + "-device scsi-cd,drive=cd${toString i},bus=scsi0.0" + ]) cfg.devices.cdroms + ) + ++ lib.concatMap (d: [ + "-drive if=none,id=${d.name},format=raw,file=${d.path},cache=${d.cache},aio=${d.aio}${lib.optionalString d.readonly ",readonly=on"}" + "-device virtio-blk-pci,drive=${d.name}" + ]) cfg.devices.extraDisks + ++ lib.optionals usbControllerOn [ + "-device qemu-xhci,id=xhci" + ] + ++ lib.optionals cfg.disk.quiesce [ + "-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0" + "-device virtio-serial-pci" + "-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0" + ] + ++ map (d: "-device vfio-pci,host=${d.address}") pt.pci + ++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb + ++ cfg.extraQemuArgs + ); + + ExecStop = stopScript; + TimeoutStopSec = 90; + Restart = "always"; + RestartSec = 5; + + DevicePolicy = "closed"; + DeviceAllow = [ + "/dev/kvm rw" + "/dev/net/tun rw" + "/dev/vhost-vsock rw" + ] + ++ lib.optionals anyPci [ "char-vfio rw" ] + ++ lib.optionals anyUsb [ "char-usb_device rw" ]; + + # VFIO pins guest RAM; default memlock is too low. + LimitMEMLOCK = lib.mkIf anyPci "infinity"; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + ProtectProc = "invisible"; + ProcSubset = "pid"; + PrivateTmp = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHostname = true; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + RestrictAddressFamilies = [ + "AF_UNIX" + "AF_NETLINK" + "AF_VSOCK" + ]; + LockPersonality = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@resources" + "~@obsolete" + ]; + # Safe under KVM (no JIT). First flip if QEMU won't start. + MemoryDenyWriteExecute = true; + CapabilityBoundingSet = [ "" ]; + AmbientCapabilities = [ "" ]; + }; + }; + + systemd.services.hermes-vm-swtpm = lib.mkIf cfg.devices.tpm.enable { + description = "swtpm TPM emulator for the Hermes VM"; + wantedBy = [ "multi-user.target" ]; + before = [ "hermes-vm.service" ]; + requiredBy = [ "hermes-vm.service" ]; + serviceConfig = { + User = vmUser; + Group = vmUser; + RuntimeDirectory = "hermes-vm"; + RuntimeDirectoryMode = "0750"; + StateDirectory = baseNameOf cfg.devices.tpm.stateDir; + Restart = "always"; + RestartSec = 5; + + DevicePolicy = "closed"; + NoNewPrivileges = true; + PrivateTmp = true; + ProtectSystem = "strict"; + ProtectHome = true; + ProtectProc = "invisible"; + ProcSubset = "pid"; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHostname = true; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + RestrictAddressFamilies = [ "AF_UNIX" ]; + LockPersonality = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ "@system-service" ]; + MemoryDenyWriteExecute = true; + CapabilityBoundingSet = [ "" ]; + AmbientCapabilities = [ "" ]; + }; + script = '' + exec ${pkgs.swtpm}/bin/swtpm socket \ + --tpm2 \ + --tpmstate dir=${cfg.devices.tpm.stateDir} \ + --ctrl type=unixio,path=${runDir}/swtpm.sock \ + --flags not-need-init,startup-clear + ''; + }; + + systemd.services.hermes-vm-tap = { + description = "TAP interface for the Hermes VM"; + after = [ "systemd-networkd.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + path = [ pkgs.iproute2 ]; + script = '' + ip link show ${net.tapId} >/dev/null 2>&1 \ + || ip tuntap add ${net.tapId} mode tap user ${vmUser} + ip link set ${net.tapId} up + ''; + preStop = '' + ${pkgs.iproute2}/bin/ip link del ${net.tapId} || true + ''; + }; + }; +} diff --git a/modules/hermenix/vm.options.nix b/modules/hermenix/vm.options.nix new file mode 100644 index 0000000..08e82bc --- /dev/null +++ b/modules/hermenix/vm.options.nix @@ -0,0 +1,278 @@ +{ lib, ... }: + +let + inherit (lib) + mkOption + types + ; + + tpmStateDir = "/var/lib/hermes-vm-tpm"; +in +{ + options.services.hermenix = { + vcpu = mkOption { + type = types.ints.positive; + default = 4; + }; + + mem = mkOption { + type = types.ints.positive; + default = 8192; + description = "RAM in MiB."; + }; + + disk = { + baseDir = mkOption { + type = types.str; + default = "/var/lib/hermes-vm"; + description = '' + Directory holding the guest's disk images. On btrfs this should be + laid out as three subvolumes - system/, state/ and snapshots/ - with + nodatacow set. `hermenix setup` does this for you. + ''; + }; + + systemSize = mkOption { + type = types.str; + default = "80G"; + description = '' + System disk: the NixOS install and the Nix store. Never snapshotted, + because it is entirely rebuildable from the flake. Size it for an + agent that builds things. + ''; + }; + + stateSize = mkOption { + type = types.str; + default = "20G"; + description = '' + State disk, mounted at /var/lib/hermes in the guest. Holds Hermes's + identity, memories, workspace, and the NixOS config it maintains for + itself. This is the only irreplaceable data, so it is the only thing + snapshotted and backed up. + ''; + }; + + quiesce = mkOption { + type = types.bool; + default = true; + description = "fsfreeze the guest via qemu-guest-agent around each snapshot."; + }; + }; + + devices = { + rng.enable = mkOption { + type = types.bool; + default = true; + description = "Attach a virtio entropy source (-device virtio-rng-pci)."; + }; + + vsock.enable = mkOption { + type = types.bool; + default = true; + description = '' + Attach the host<->guest vsock (-device vhost-vsock-pci). `hermenix + ctl` and the SSH-based backup both reach the guest over it, so + turning this off breaks them. + ''; + }; + + video = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Attach a virtual display adapter, giving the guest a DRM device + (/dev/dri). Needed by a Wayland compositor or anything that wants + a render node. The console stays serial either way. + ''; + }; + model = mkOption { + type = types.enum [ + "virtio" + "qxl" + "cirrus" + "vmware" + "std" + ]; + default = "virtio"; + description = '' + Adapter model passed to -vga. Only "virtio" exposes a render node + (/dev/dri/renderD128); the legacy models give card0 without one. + ''; + }; + }; + + balloon.enable = mkOption { + type = types.bool; + default = false; + description = "Attach a virtio memory balloon (-device virtio-balloon-pci)."; + }; + + watchdog = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Attach a hardware watchdog. The guest must also arm it (systemd's + RuntimeWatchdogSec) for it to do anything. + ''; + }; + model = mkOption { + type = types.enum [ + "i6300esb" + "ib700" + ]; + default = "i6300esb"; + }; + action = mkOption { + type = types.enum [ + "reset" + "shutdown" + "poweroff" + "pause" + "debug" + "none" + ]; + default = "reset"; + description = "What QEMU does when the watchdog fires."; + }; + }; + + input = { + enable = mkOption { + type = types.bool; + default = false; + description = "Attach virtio input devices. Only useful with devices.video.enable."; + }; + keyboard = mkOption { + type = types.bool; + default = true; + }; + tablet = mkOption { + type = types.bool; + default = false; + }; + }; + + audio = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Attach an audio device. NOTE: the VM unit is hardened (ProtectHome, + PrivateTmp, a dedicated system user) and cannot reach the operator's + audio daemon without loosening that; leave this off to keep the + hardening intact. + ''; + }; + backend = mkOption { + type = types.enum [ + "none" + "pipewire" + "alsa" + "pa" + "sdl" + ]; + default = "pipewire"; + }; + model = mkOption { + type = types.enum [ + "intel-hda" + "virtio-sound" + ]; + default = "intel-hda"; + }; + }; + + tpm = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Attach a TPM 2.0 via swtpm. A hermes-vm-swtpm unit starts first and + QEMU connects to it over a unix socket. Useful for measured boot or + sealing secrets to the VM. + ''; + }; + stateDir = mkOption { + type = types.str; + default = tpmStateDir; + description = '' + Where swtpm keeps its state. Must live under /var/lib/ so it can be + created and owned by the unit's StateDirectory. + ''; + }; + }; + + cdroms = mkOption { + default = [ ]; + description = "Read-only CD-ROM images to attach, on a virtio-scsi controller."; + type = types.listOf ( + types.submodule { + options.path = mkOption { type = types.str; }; + } + ); + }; + + extraDisks = mkOption { + default = [ ]; + description = "Extra raw disk images, each on its own virtio-blk device."; + type = types.listOf ( + types.submodule { + options = { + name = mkOption { + type = types.str; + description = "Unique QEMU drive id, e.g. scratch."; + }; + path = mkOption { type = types.str; }; + readonly = mkOption { + type = types.bool; + default = false; + }; + aio = mkOption { + type = types.enum [ + "threads" + "native" + "io_uring" + ]; + default = "threads"; + }; + cache = mkOption { + type = types.enum [ + "writeback" + "none" + "directsync" + "unsafe" + ]; + default = "writeback"; + }; + }; + } + ); + }; + + usbController = mkOption { + type = types.enum [ + "auto" + "on" + "off" + ]; + default = "auto"; + description = '' + Whether to attach a qemu-xhci USB controller. "auto" adds one only + when passthrough.usb is non-empty; "on"/"off" force the decision. + ''; + }; + }; + + extraQemuArgs = mkOption { + type = types.listOf types.str; + default = [ ]; + description = '' + Raw additional QEMU arguments, appended to the VM command line. + Unvalidated escape hatch - prefer the typed devices.* options. + ''; + }; + }; +} diff --git a/modules/host.nix b/modules/host.nix index 44aae76..0d28829 100644 --- a/modules/host.nix +++ b/modules/host.nix @@ -1,1473 +1,19 @@ +{ ... }: { - config, - lib, - pkgs, - ... -}: - -let - inherit (lib) - mkOption - mkEnableOption - mkIf - types - optionalString - ; - cfg = config.services.hermenix; - net = cfg.network; - - vmUser = "hermes-vm"; - runDir = "/run/hermes-vm"; - stateDir = "/var/lib/hermes-vm"; - systemImage = "${cfg.disk.baseDir}/system/system.raw"; - stateImage = "${cfg.disk.baseDir}/state/state.raw"; - snapDir = "${cfg.disk.baseDir}/snapshots"; - - pt = cfg.passthrough; - anyPci = pt.pci != [ ]; - anyUsb = pt.usb != [ ]; - earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci); - - usbControllerOn = - cfg.devices.usbController == "on" || (cfg.devices.usbController == "auto" && anyUsb); - - tpmStateDir = "/var/lib/hermes-vm-tpm"; - - # 18501 == 0x4845 ("HE"); marks guest traffic from the tap (bridged mode). - bridgeMark = 18501; - - # dnsmasq needs an explicit mask for a one-address DHCP range. - netmaskFromPrefix = - p: - lib.concatStringsSep "." ( - lib.genList ( - b: - toString ( - builtins.elemAt [ - 0 - 128 - 192 - 224 - 240 - 248 - 252 - 254 - 255 - ] (lib.max 0 (lib.min 8 (p - b * 8))) - ) - ) 4 - ); - - stopScript = pkgs.writeShellScript "hermes-vm-stop" '' - set -eu - printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \ - | ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0 - ''; - - qga = - verb: - pkgs.writeShellScript "hermes-vm-fs${verb}" '' - set -eu - printf '%s' '{"execute":"guest-fs${verb}"}' \ - | ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true - ''; -in -{ - options.services.hermenix = { - enable = mkEnableOption "the Hermes Agent VM"; - - name = mkOption { - type = types.str; - default = "hermes"; - }; - - vcpu = mkOption { - type = types.ints.positive; - default = 4; - }; - - mem = mkOption { - type = types.ints.positive; - default = 8192; - description = "RAM in MiB."; - }; - - disk = { - baseDir = mkOption { - type = types.str; - default = "/var/lib/hermes-vm"; - description = '' - Directory holding the guest's disk images. On btrfs this should be - laid out as three subvolumes - system/, state/ and snapshots/ - with - nodatacow set. `hermenix setup` does this for you. - ''; - }; - - systemSize = mkOption { - type = types.str; - default = "80G"; - description = '' - System disk: the NixOS install and the Nix store. Never snapshotted, - because it is entirely rebuildable from the flake. Size it for an - agent that builds things. - ''; - }; - - stateSize = mkOption { - type = types.str; - default = "20G"; - description = '' - State disk, mounted at /var/lib/hermes in the guest. Holds Hermes's - identity, memories, workspace, and the NixOS config it maintains for - itself. This is the only irreplaceable data, so it is the only thing - snapshotted and backed up. - ''; - }; - - quiesce = mkOption { - type = types.bool; - default = true; - description = "fsfreeze the guest via qemu-guest-agent around each snapshot."; - }; - }; - - snapshots = { - enable = mkOption { - type = types.bool; - default = true; - description = '' - Periodic btrfs snapshots of the state disk. The agent can persist - changes, which means it can persist a mistake, and these are the - fastest way back. - ''; - }; - - interval = mkOption { - type = types.str; - default = "hourly"; - description = "systemd calendar expression."; - }; - - keep = mkOption { - type = types.ints.positive; - default = 48; - description = '' - How many to retain. Keep this tighter than you would on ZFS: the - state image is nodatacow, and every live snapshot reintroduces - copy-on-write for the blocks it still references. - ''; - }; - }; - - backup = { - enable = mkEnableOption "restic backups of the guest's state"; - - repository = mkOption { - type = types.str; - default = ""; - example = "sftp:backup@nas:/hermes"; - description = "Any restic repository string."; - }; - - passwordFile = mkOption { - type = types.path; - default = "/var/lib/secrets/hermes-restic"; - description = "File containing the restic repository password, mode 0400."; - }; - - environmentFile = mkOption { - type = types.nullOr types.path; - default = null; - description = "Optional file with credentials for the backend (S3 keys and so on)."; - }; - - interval = mkOption { - type = types.str; - default = "daily"; - }; - - keep = mkOption { - type = types.listOf types.str; - default = [ - "--keep-daily" - "14" - "--keep-weekly" - "8" - "--keep-monthly" - "12" - ]; - description = "Arguments passed to restic forget."; - }; - }; - - hostCache = { - enable = mkOption { - type = types.bool; - default = true; - description = '' - Serve the host's /nix/store to the guest as a read-only binary - cache, so the guest doesn't re-download from the internet what the - host already has. - - This is the ONLY route from guest to host in this design. Turning it - off leaves the guest with literally no reachable host service, at - the cost of bandwidth on every guest build. - - Note the guest can read every store path on the host through this. - Store paths are world-readable anyway, but check nothing sensitive - is baked into your host's store. - ''; - }; - port = mkOption { - type = types.port; - default = 5000; - }; - secretKeyFile = mkOption { - type = types.path; - default = "/var/lib/secrets/hermes-cache.secret"; - description = "Generate with nix-store --generate-binary-cache-key. See README."; - }; - }; - - authorizedKeys = mkOption { - type = types.listOf types.str; - default = [ ]; - }; - - allowLan = mkOption { - type = types.bool; - default = false; - description = '' - Blanket permission for the guest to reach RFC1918 space. Almost always - the wrong tool - use lanAccess to open single endpoints instead. - ''; - }; - - lanAccess = mkOption { - default = [ ]; - example = [ - { - address = "192.168.1.50"; - port = 11434; - } - ]; - description = '' - Specific endpoints elsewhere on your network that the guest may reach, - punched through the default RFC1918 drop. One entry per service. - ''; - type = types.listOf ( - types.submodule { - options = { - address = mkOption { type = types.str; }; - port = mkOption { type = types.port; }; - proto = mkOption { - type = types.enum [ - "tcp" - "udp" - ]; - default = "tcp"; - }; - }; - } - ); - }; - - hostAccess = mkOption { - default = [ ]; - example = [ { port = 8188; } ]; - description = '' - Ports on this host itself that the guest may reach. The service must - also be bound to an address the guest can see - see the README. - ''; - type = types.listOf ( - types.submodule { - options = { - port = mkOption { type = types.port; }; - proto = mkOption { - type = types.enum [ - "tcp" - "udp" - ]; - default = "tcp"; - }; - }; - } - ); - }; - - publish = mkOption { - default = [ ]; - example = [ - { - hostPort = 8080; - guestPort = 8080; - } - ]; - description = '' - Guest ports to publish on the host's uplink, reachable from the wider - network. You do not need this to reach the guest from the host itself. - ''; - type = types.listOf ( - types.submodule { - options = { - hostPort = mkOption { type = types.port; }; - guestPort = mkOption { type = types.port; }; - proto = mkOption { - type = types.enum [ - "tcp" - "udp" - ]; - default = "tcp"; - }; - }; - } - ); - }; - - network = { - bridge = mkOption { - type = types.str; - default = "hermes-br"; - }; - tapId = mkOption { - type = types.str; - default = "vm-hermes"; - }; - mac = mkOption { - type = types.str; - default = "02:00:00:7e:00:01"; - }; - hostAddress = mkOption { - type = types.str; - default = "10.77.0.1"; - }; - guestAddress = mkOption { - type = types.str; - default = "10.77.0.2"; - }; - prefixLength = mkOption { - type = types.int; - default = 24; - }; - lanAddress = mkOption { - type = types.str; - default = ""; - example = "10.0.12.50"; - description = "The guest's fixed address on your LAN. Required when `bridged = true`."; - }; - hostLanAddress = mkOption { - type = types.str; - default = ""; - example = "10.0.12.10"; - description = '' - This host's own address on your LAN. When `bridged` is enabled the - uplink becomes a bridge port and this address moves onto the bridge - (which clones the uplink's MAC, so the host keeps its LAN identity). - Required when `bridged = true`. - ''; - }; - lanGateway = mkOption { - type = types.str; - default = ""; - example = "10.0.12.1"; - description = '' - Your LAN's router. Used as the default gateway for the guest and, - while `bridged`, for this host's bridge too (DNS unless `lanDns` is - set). Required when `bridged = true`. - ''; - }; - lanPrefixLength = mkOption { - type = types.int; - default = 24; - description = "Prefix length of your LAN subnet (netmask)."; - }; - lanDns = mkOption { - type = types.listOf types.str; - default = [ ]; - example = [ "1.1.1.1" ]; - description = '' - DNS servers for this host while `bridged` (the bridge no longer - takes a DHCP lease). Defaults to the LAN gateway. - ''; - }; - bridged = mkOption { - type = types.bool; - default = false; - description = '' - Put the guest directly on your LAN instead of behind the host's - NAT. The bridge absorbs `externalInterface`, so any device on the - LAN can reach any port the guest exposes. - - Addressing is static from the operator's point of view: you pick - fixed addresses for the guest (`lanAddress`), this host - (`hostLanAddress` — it moves onto the bridge), and the LAN - gateway (`lanGateway`). A DHCP server this module runs on the - bridge hands `lanAddress` to the guest, keyed to `network.mac`, - so the guest stays a plain DHCP client and never needs to know - its own address. That lease never leaves the bridge; the LAN's - own DHCP cannot race it. - - Inbound is no longer limited to `publish` (which is ignored in - this mode): the LAN may reach every guest port. Outbound is still - governed by `lanAccess` / `allowLan`, and `hostAccess` plus the - host cache still work over the host-guest service net - (`hostAddress` stays configured on the bridge). - ''; - }; - externalInterface = mkOption { - type = types.nullOr types.str; - default = null; - example = "eno1"; - description = "Host uplink. Required."; - }; - }; - - passthrough = { - iommu = mkOption { - type = types.nullOr ( - types.enum [ - "intel" - "amd" - ] - ); - default = null; - description = '' - Enable IOMMU kernel parameters for your CPU vendor. Required for any - PCI passthrough. The IOMMU is the only thing stopping a passed-through - device from reading host memory, so this is not optional. - ''; - }; - - pci = mkOption { - default = [ ]; - example = [ - { - address = "0000:01:00.0"; - id = "10de:2204"; - } - { - address = "0000:01:00.1"; - id = "10de:1aef"; - } - ]; - description = '' - PCI devices handed to the guest via VFIO. The host loses them - entirely while the VM runs. - - You must pass every function of the device and, in practice, every - device in its IOMMU group. A GPU is usually at least two entries: - the graphics function and its HDMI audio function. - - Set `id` (vendor:device) to have vfio-pci claim the device in the - initrd, before a host driver can. Strongly recommended for GPUs; - unbinding an active amdgpu or nvidia driver later is unreliable. - ''; - type = types.listOf ( - types.submodule { - options = { - address = mkOption { - type = types.str; - example = "0000:01:00.0"; - description = "Full PCI address, as shown by lspci -D."; - }; - id = mkOption { - type = types.nullOr types.str; - default = null; - example = "10de:2204"; - description = "vendor:device, from lspci -nn. Enables early binding."; - }; - }; - } - ); - }; - - usb = mkOption { - default = [ ]; - example = [ - { - vendorId = "8087"; - productId = "0026"; - } - ]; - description = '' - USB devices forwarded to the guest, matched by vendor and product - ID from lsusb. Integrated Bluetooth radios are USB devices and go - here too, though the host driver must be kept off them. - - QEMU resolves these at start, so a device that is unplugged when the - VM boots will not appear later without a restart. - ''; - type = types.listOf ( - types.submodule { - options = { - vendorId = mkOption { - type = types.str; - example = "046d"; - }; - productId = mkOption { - type = types.str; - example = "c52b"; - }; - }; - } - ); - }; - }; - - devices = { - rng.enable = mkOption { - type = types.bool; - default = true; - description = "Attach a virtio entropy source (-device virtio-rng-pci)."; - }; - - vsock.enable = mkOption { - type = types.bool; - default = true; - description = '' - Attach the host<->guest vsock (-device vhost-vsock-pci). `hermenix - ctl` and the SSH-based backup both reach the guest over it, so - turning this off breaks them. - ''; - }; - - video = { - enable = mkOption { - type = types.bool; - default = false; - description = '' - Attach a virtual display adapter, giving the guest a DRM device - (/dev/dri). Needed by a Wayland compositor or anything that wants - a render node. The console stays serial either way. - ''; - }; - model = mkOption { - type = types.enum [ - "virtio" - "qxl" - "cirrus" - "vmware" - "std" - ]; - default = "virtio"; - description = '' - Adapter model passed to -vga. Only "virtio" exposes a render node - (/dev/dri/renderD128); the legacy models give card0 without one. - ''; - }; - }; - - balloon.enable = mkOption { - type = types.bool; - default = false; - description = "Attach a virtio memory balloon (-device virtio-balloon-pci)."; - }; - - watchdog = { - enable = mkOption { - type = types.bool; - default = false; - description = '' - Attach a hardware watchdog. The guest must also arm it (systemd's - RuntimeWatchdogSec) for it to do anything. - ''; - }; - model = mkOption { - type = types.enum [ - "i6300esb" - "ib700" - ]; - default = "i6300esb"; - }; - action = mkOption { - type = types.enum [ - "reset" - "shutdown" - "poweroff" - "pause" - "debug" - "none" - ]; - default = "reset"; - description = "What QEMU does when the watchdog fires."; - }; - }; - - input = { - enable = mkOption { - type = types.bool; - default = false; - description = "Attach virtio input devices. Only useful with devices.video.enable."; - }; - keyboard = mkOption { - type = types.bool; - default = true; - }; - tablet = mkOption { - type = types.bool; - default = false; - }; - }; - - audio = { - enable = mkOption { - type = types.bool; - default = false; - description = '' - Attach an audio device. NOTE: the VM unit is hardened (ProtectHome, - PrivateTmp, a dedicated system user) and cannot reach the operator's - audio daemon without loosening that; leave this off to keep the - hardening intact. - ''; - }; - backend = mkOption { - type = types.enum [ - "none" - "pipewire" - "alsa" - "pa" - "sdl" - ]; - default = "pipewire"; - }; - model = mkOption { - type = types.enum [ - "intel-hda" - "virtio-sound" - ]; - default = "intel-hda"; - }; - }; - - tpm = { - enable = mkOption { - type = types.bool; - default = false; - description = '' - Attach a TPM 2.0 via swtpm. A hermes-vm-swtpm unit starts first and - QEMU connects to it over a unix socket. Useful for measured boot or - sealing secrets to the VM. - ''; - }; - stateDir = mkOption { - type = types.str; - default = tpmStateDir; - description = '' - Where swtpm keeps its state. Must live under /var/lib/ so it can be - created and owned by the unit's StateDirectory. - ''; - }; - }; - - cdroms = mkOption { - default = [ ]; - description = "Read-only CD-ROM images to attach, on a virtio-scsi controller."; - type = types.listOf ( - types.submodule { - options.path = mkOption { type = types.str; }; - } - ); - }; - - extraDisks = mkOption { - default = [ ]; - description = "Extra raw disk images, each on its own virtio-blk device."; - type = types.listOf ( - types.submodule { - options = { - name = mkOption { - type = types.str; - description = "Unique QEMU drive id, e.g. scratch."; - }; - path = mkOption { type = types.str; }; - readonly = mkOption { - type = types.bool; - default = false; - }; - aio = mkOption { - type = types.enum [ - "threads" - "native" - "io_uring" - ]; - default = "threads"; - }; - cache = mkOption { - type = types.enum [ - "writeback" - "none" - "directsync" - "unsafe" - ]; - default = "writeback"; - }; - }; - } - ); - }; - - usbController = mkOption { - type = types.enum [ - "auto" - "on" - "off" - ]; - default = "auto"; - description = '' - Whether to attach a qemu-xhci USB controller. "auto" adds one only - when passthrough.usb is non-empty; "on"/"off" force the decision. - ''; - }; - }; - - extraQemuArgs = mkOption { - type = types.listOf types.str; - default = [ ]; - description = '' - Raw additional QEMU arguments, appended to the VM command line. - Unvalidated escape hatch - prefer the typed devices.* options. - ''; - }; - - gatewayUnit = mkOption { - type = types.str; - default = "hermes-agent"; - description = '' - The systemd *user* unit that `hermes gateway install` created in the - guest. Check with `hermenix ctl units` after installing; upstream has - used more than one name for this. - ''; - }; - - hermesHome = mkOption { - type = types.str; - # Guest path (not host disk.baseDir). - default = "/var/lib/hermes/.hermes"; - description = '' - Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/, - state.db, and so on) lives inside the guest. Defaults to the seeded - layout under the state disk mount (`/var/lib/hermes/.hermes`). - Override when carrying over an existing flat HERMES_HOME from - elsewhere (e.g. a prior Docker deployment) that you're placing - directly at the root of the state disk instead of under .hermes — - see `hermenix migrate --help`. - - This only changes where the host-side `hermenix ctl` commands (soul, - memory, conf, secrets, reseed, pull) look. It does not change what - the gateway process itself uses — export HERMES_HOME to match inside - the guest when you install it. - ''; - }; - - vsockCid = mkOption { - type = types.int; - default = 77; - description = "Host reaches the guest over vsock, so no inbound network path is needed."; - }; - - version = mkOption { - type = types.str; - default = "dev"; - description = '' - "YYYY.MM.DD.hh.mm-" reported by `hermenix version`. - `version` subcommand. Normally supplied for free by this flake's own - `nixosModules.default` (from its git metadata) - the "dev" default - here only shows up if this module is imported some other way. - ''; - }; - }; - - config = mkIf cfg.enable { - assertions = [ - { - assertion = net.externalInterface != null; - message = "services.hermenix.network.externalInterface must be set."; - } - { - assertion = - !net.bridged || (net.lanAddress != "" && net.hostLanAddress != "" && net.lanGateway != ""); - message = '' - services.hermenix.network.bridged = true requires a fully static - LAN setup (DHCP is no longer supported in bridged mode). Set - network.lanAddress (the guest's fixed LAN address), - network.hostLanAddress (this host's own LAN address, which moves - onto the bridge) and network.lanGateway (your router's address). - ''; - } - { - assertion = cfg.backup.enable -> (cfg.backup.repository != ""); - message = "services.hermenix.backup.repository must be set when backups are enabled."; - } - { - assertion = anyPci -> (pt.iommu != null); - message = '' - services.hermenix.passthrough.pci needs passthrough.iommu set to - "intel" or "amd". Without the IOMMU, a passed-through device can - read host memory directly and the VM boundary is meaningless. - ''; - } - { - assertion = cfg.devices.vsock.enable || !cfg.backup.enable; - message = '' - services.hermenix.backup.enable needs the vsock (the backup runs over - SSH multiplexed on it). Leave devices.vsock.enable at its default, or - turn backups off. - ''; - } - { - assertion = cfg.devices.usbController != "off" || !anyUsb; - message = '' - services.hermenix.passthrough.usb is set but - services.hermenix.devices.usbController is "off", so those devices - have no controller to attach to. - ''; - } - { - assertion = !cfg.devices.tpm.enable || lib.hasPrefix "/var/lib/" cfg.devices.tpm.stateDir; - message = '' - services.hermenix.devices.tpm.stateDir must live under /var/lib/ so - the swtpm unit's StateDirectory can create and own it. - ''; - } - ]; - - users.users.${vmUser} = { - isSystemUser = true; - group = vmUser; - description = "Runs the Hermes guest VM"; - }; - users.groups.${vmUser} = { }; - - systemd.services.hermes-vm = { - description = "Hermes Agent VM"; - wantedBy = [ "multi-user.target" ]; - after = [ - "network-online.target" - "hermes-vm-tap.service" - ] - ++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service"; - requires = [ - "hermes-vm-tap.service" - ] - ++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service"; - wants = [ "network-online.target" ]; - - preStart = '' - for img in ${systemImage} ${stateImage}; do - if [ ! -e "$img" ]; then - echo "$img is missing. Run 'hermenix setup' first." >&2 - exit 1 - fi - done - if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then - install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd - fi - ''; - - serviceConfig = { - User = vmUser; - Group = vmUser; - SupplementaryGroups = [ "kvm" ]; - RuntimeDirectory = "hermes-vm"; - ReadWritePaths = [ cfg.disk.baseDir ]; - StateDirectory = "hermes-vm"; - RuntimeDirectoryMode = "0750"; - - ExecStart = lib.concatStringsSep " " ( - [ - "${pkgs.qemu_kvm}/bin/qemu-system-x86_64" - "-name ${cfg.name}" - "-machine q35,accel=kvm,smm=on" - "-cpu host" - "-smp ${toString cfg.vcpu}" - "-m ${toString cfg.mem}" - "-nodefaults" - "-no-user-config" - "-display none" - "-serial stdio" - "-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny" - "-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}" - "-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd" - # aio=threads+writeback: images are files, not blockdevs. - "-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap" - "-device virtio-blk-pci,drive=system" - "-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap" - "-device virtio-blk-pci,drive=state" - # vhost=off: keep packet handling out of the host kernel. - "-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off" - "-device virtio-net-pci,netdev=net0,mac=${net.mac}" - "-qmp unix:${runDir}/qmp.sock,server=on,wait=off" - ] - ++ lib.optionals cfg.devices.rng.enable [ - "-device virtio-rng-pci" - ] - ++ lib.optionals cfg.devices.vsock.enable [ - "-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}" - ] - ++ lib.optionals cfg.devices.video.enable [ - "-vga ${cfg.devices.video.model}" - ] - ++ lib.optionals cfg.devices.balloon.enable [ - "-device virtio-balloon-pci" - ] - ++ lib.optionals cfg.devices.watchdog.enable [ - "-device ${cfg.devices.watchdog.model}" - "-watchdog-action ${cfg.devices.watchdog.action}" - ] - ++ lib.optionals cfg.devices.input.enable ( - lib.optional cfg.devices.input.keyboard "-device virtio-keyboard-pci" - ++ lib.optional cfg.devices.input.tablet "-device virtio-tablet-pci" - ) - ++ lib.optionals cfg.devices.audio.enable ( - [ "-audiodev ${cfg.devices.audio.backend},id=audio0" ] - ++ ( - if cfg.devices.audio.model == "virtio-sound" then - [ "-device virtio-sound-pci,audiodev=audio0" ] - else - [ - "-device intel-hda" - "-device hda-duplex,audiodev=audio0" - ] - ) - ) - ++ lib.optionals cfg.devices.tpm.enable [ - "-chardev socket,id=chrtpm,path=${runDir}/swtpm.sock" - "-tpmdev emulator,id=tpm0,chardev=chrtpm" - "-device tpm-crb,tpmdev=tpm0" - ] - ++ lib.optionals (cfg.devices.cdroms != [ ]) [ - "-device virtio-scsi-pci,id=scsi0" - ] - ++ lib.concatLists ( - lib.imap0 (i: c: [ - "-drive if=none,id=cd${toString i},media=cdrom,readonly=on,file=${c.path}" - "-device scsi-cd,drive=cd${toString i},bus=scsi0.0" - ]) cfg.devices.cdroms - ) - ++ lib.concatMap (d: [ - "-drive if=none,id=${d.name},format=raw,file=${d.path},cache=${d.cache},aio=${d.aio}${lib.optionalString d.readonly ",readonly=on"}" - "-device virtio-blk-pci,drive=${d.name}" - ]) cfg.devices.extraDisks - ++ lib.optionals usbControllerOn [ - "-device qemu-xhci,id=xhci" - ] - ++ lib.optionals cfg.disk.quiesce [ - "-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0" - "-device virtio-serial-pci" - "-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0" - ] - ++ map (d: "-device vfio-pci,host=${d.address}") pt.pci - ++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb - ++ cfg.extraQemuArgs - ); - - ExecStop = stopScript; - TimeoutStopSec = 90; - Restart = "always"; - RestartSec = 5; - - DevicePolicy = "closed"; - DeviceAllow = [ - "/dev/kvm rw" - "/dev/net/tun rw" - "/dev/vhost-vsock rw" - ] - ++ lib.optionals anyPci [ "char-vfio rw" ] - ++ lib.optionals anyUsb [ "char-usb_device rw" ]; - - # VFIO pins guest RAM; default memlock is too low. - LimitMEMLOCK = lib.mkIf anyPci "infinity"; - NoNewPrivileges = true; - ProtectSystem = "strict"; - ProtectHome = true; - ProtectProc = "invisible"; - ProcSubset = "pid"; - PrivateTmp = true; - ProtectKernelTunables = true; - ProtectKernelModules = true; - ProtectKernelLogs = true; - ProtectClock = true; - ProtectControlGroups = true; - ProtectHostname = true; - RestrictNamespaces = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - RestrictAddressFamilies = [ - "AF_UNIX" - "AF_NETLINK" - "AF_VSOCK" - ]; - LockPersonality = true; - SystemCallArchitectures = "native"; - SystemCallFilter = [ - "@system-service" - "~@resources" - "~@obsolete" - ]; - # Safe under KVM (no JIT). First flip if QEMU won't start. - MemoryDenyWriteExecute = true; - CapabilityBoundingSet = [ "" ]; - AmbientCapabilities = [ "" ]; - }; - }; - - systemd.services.hermes-vm-swtpm = mkIf cfg.devices.tpm.enable { - description = "swtpm TPM emulator for the Hermes VM"; - wantedBy = [ "multi-user.target" ]; - before = [ "hermes-vm.service" ]; - requiredBy = [ "hermes-vm.service" ]; - serviceConfig = { - User = vmUser; - Group = vmUser; - RuntimeDirectory = "hermes-vm"; - RuntimeDirectoryMode = "0750"; - StateDirectory = baseNameOf cfg.devices.tpm.stateDir; - Restart = "always"; - RestartSec = 5; - - DevicePolicy = "closed"; - NoNewPrivileges = true; - PrivateTmp = true; - ProtectSystem = "strict"; - ProtectHome = true; - ProtectProc = "invisible"; - ProcSubset = "pid"; - ProtectKernelTunables = true; - ProtectKernelModules = true; - ProtectKernelLogs = true; - ProtectClock = true; - ProtectControlGroups = true; - ProtectHostname = true; - RestrictNamespaces = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - RestrictAddressFamilies = [ "AF_UNIX" ]; - LockPersonality = true; - SystemCallArchitectures = "native"; - SystemCallFilter = [ "@system-service" ]; - MemoryDenyWriteExecute = true; - CapabilityBoundingSet = [ "" ]; - AmbientCapabilities = [ "" ]; - }; - script = '' - exec ${pkgs.swtpm}/bin/swtpm socket \ - --tpm2 \ - --tpmstate dir=${cfg.devices.tpm.stateDir} \ - --ctrl type=unixio,path=${runDir}/swtpm.sock \ - --flags not-need-init,startup-clear - ''; - }; - - systemd.services.hermes-vm-tap = { - description = "TAP interface for the Hermes VM"; - after = [ "systemd-networkd.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - path = [ pkgs.iproute2 ]; - script = '' - ip link show ${net.tapId} >/dev/null 2>&1 \ - || ip tuntap add ${net.tapId} mode tap user ${vmUser} - ip link set ${net.tapId} up - ''; - preStop = '' - ${pkgs.iproute2}/bin/ip link del ${net.tapId} || true - ''; - }; - - boot.kernelParams = lib.optionals (pt.iommu != null) [ - (if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on") - "iommu=pt" - ]; - - boot.initrd.kernelModules = lib.optionals anyPci [ - "vfio_pci" - "vfio" - "vfio_iommu_type1" - ]; - - boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) '' - options vfio-pci ids=${lib.concatStringsSep "," earlyIds} - ''; - - systemd.services.hermes-vm-vfio = mkIf anyPci { - description = "Bind passthrough PCI devices to vfio-pci"; - before = [ "hermes-vm.service" ]; - requiredBy = [ "hermes-vm.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - path = [ pkgs.kmod ]; - script = '' - modprobe vfio-pci - '' - + lib.concatMapStrings (d: '' - dev=/sys/bus/pci/devices/${d.address} - if [ ! -e "$dev" ]; then - echo "${d.address} is not present on this machine" >&2 - exit 1 - fi - cur=none - [ -e "$dev/driver" ] && cur=$(basename "$(readlink -f "$dev/driver")") - if [ "$cur" != "vfio-pci" ]; then - [ "$cur" != "none" ] && echo ${d.address} > "$dev/driver/unbind" - echo vfio-pci > "$dev/driver_override" - echo ${d.address} > /sys/bus/pci/drivers_probe - fi - '') pt.pci; - }; - - services.udev.extraRules = - lib.optionalString anyPci '' - SUBSYSTEM=="vfio", OWNER="${vmUser}", GROUP="${vmUser}", MODE="0660" - '' - + lib.concatMapStrings (u: '' - SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660" - '') pt.usb; - - networking.useNetworkd = true; - - systemd.network.netdevs."10-${net.bridge}".netdevConfig = { - Kind = "bridge"; - Name = net.bridge; - }; - - systemd.network.networks."10-${net.bridge}" = { - matchConfig.Name = net.bridge; - # hostAddress = service net; bridged also adds hostLanAddress (static only). - address = [ - "${net.hostAddress}/${toString net.prefixLength}" - ] - ++ lib.optional net.bridged "${net.hostLanAddress}/${toString net.lanPrefixLength}"; - networkConfig = { - ConfigureWithoutCarrier = true; - IPv4Forwarding = true; - } - // lib.optionalAttrs net.bridged { - Gateway = net.lanGateway; - DNS = if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns; - }; - }; - - # mkDefault: operator nameservers win if set. - networking.nameservers = lib.mkIf net.bridged ( - lib.mkDefault (if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns) - ); - - systemd.network.networks."11-${net.tapId}" = { - matchConfig.Name = net.tapId; - networkConfig.Bridge = net.bridge; - }; - - systemd.network.networks."12-${net.externalInterface}" = lib.mkIf net.bridged { - matchConfig.Name = net.externalInterface; - networkConfig.Bridge = net.bridge; - }; - - # Bridge would inherit tap MAC (first port) and collide with guest — clone uplink MAC. - systemd.services.hermes-bridge-mac = lib.mkIf net.bridged { - description = "Clone the uplink MAC onto the hermenix bridge"; - after = [ - "systemd-networkd.service" - "sys-subsystem-net-devices-${net.bridge}.device" - "sys-subsystem-net-devices-${net.externalInterface}.device" - ]; - wants = [ "systemd-networkd.service" ]; - before = [ "network-online.target" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - unitConfig.ConditionPathExists = "/sys/class/net/${net.externalInterface}"; - script = '' - set -euo pipefail - for _ in 1 2 3 4 5 6 7 8 9 10; do - [ -e "/sys/class/net/${net.bridge}" ] && break - sleep 0.5 - done - [ -e "/sys/class/net/${net.bridge}" ] || exit 1 - mac="$(${pkgs.coreutils}/bin/cat /sys/class/net/${net.externalInterface}/address)" - ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} down - ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} address "$mac" - ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} up - ${pkgs.systemd}/bin/networkctl reconfigure ${net.bridge} || true - ''; - }; - - # Single-address DHCP on the bridge (MAC-keyed). Broadcasts stay on-bridge. - systemd.services.hermes-lan-dhcp = lib.mkIf net.bridged { - description = "Static-lease DHCP for the bridged hermenix guest"; - after = [ - "systemd-networkd.service" - "sys-subsystem-net-devices-${net.bridge}.device" - ]; - wants = [ "systemd-networkd.service" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - Type = "simple"; - Restart = "on-failure"; - RestartSec = 5; - }; - script = '' - for _ in 1 2 3 4 5 6 7 8 9 10; do - [ -e "/sys/class/net/${net.bridge}" ] && break - sleep 0.5 - done - exec ${pkgs.dnsmasq}/bin/dnsmasq --keep-in-foreground \ - --conf-file=/dev/null \ - --interface=${net.bridge} \ - --bind-dynamic \ - --port=0 \ - --dhcp-authoritative \ - --dhcp-range=${net.lanAddress},${net.lanAddress},${netmaskFromPrefix net.lanPrefixLength},12h \ - --dhcp-host=${net.mac},${net.lanAddress} \ - --dhcp-option=option:router,${net.lanGateway} - ''; - }; - - # mkOverride 99: same keys as networking.nat, win over weaker defaults. - boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true; - boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true; - - # Explicit nft nat (not networking.nat). DNAT iifname != bridge so extra paths still match. - networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null) { - family = "ip"; - content = - if net.bridged then - # No DNAT when bridged. Scoped masquerade covers service-net fallback if DHCP lapses. - '' - chain postrouting { - type nat hook postrouting priority srcnat; - iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade - } - '' - else - '' - chain prerouting { - type nat hook prerouting priority dstnat; - '' - + lib.concatMapStrings (s: '' - iifname != "${net.bridge}" ${s.proto} dport ${toString s.hostPort} dnat to ${net.guestAddress}:${toString s.guestPort} - '') cfg.publish - + '' - } - chain postrouting { - type nat hook postrouting priority srcnat; - iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade - } - ''; - }; - - networking.nftables.tables."hermenix-bridge" = lib.mkIf net.bridged { - family = "bridge"; - content = '' - chain prerouting { - type filter hook prerouting priority filter; policy accept; - iifname "${net.tapId}" meta mark set ${toString bridgeMark} - - # Guest DHCP stays on-bridge only (no race with LAN DHCP). - iifname "${net.tapId}" oifname "${net.externalInterface}" ether type ip ip protocol udp udp sport 68 udp dport 67 drop - iifname "${net.externalInterface}" oifname "${net.tapId}" ether type ip ip protocol udp udp sport 67 udp dport 68 drop - } - chain forward { - type filter hook forward priority filter; policy accept; - oifname "${net.tapId}" accept - '' - + lib.concatMapStrings (s: '' - iifname "${net.tapId}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept - '') cfg.lanAccess - + ( - if cfg.allowLan then - '' - iifname "${net.tapId}" accept - '' - else - '' - iifname "${net.tapId}" ip daddr { - 10.0.0.0/8, - 172.16.0.0/12, - 192.168.0.0/16, - 169.254.0.0/16, - 127.0.0.0/8, - 100.64.0.0/10 - } drop - iifname "${net.tapId}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop - iifname "${net.tapId}" accept - '' - ) - + '' - } - ''; - }; - - networking.nftables.enable = true; - networking.firewall.enable = true; - networking.firewall.filterForward = true; - - # Guest→host holes. NAT: guestAddress. Bridged: tap mark (guest may use LAN IP). - networking.firewall.extraInputRules = - if net.bridged then - optionalString cfg.hostCache.enable '' - meta mark ${toString bridgeMark} tcp dport ${toString cfg.hostCache.port} accept - '' - + lib.concatMapStrings (s: '' - meta mark ${toString bridgeMark} ${s.proto} dport ${toString s.port} accept - '') cfg.hostAccess - else - optionalString cfg.hostCache.enable '' - iifname "${net.bridge}" ip saddr ${net.guestAddress} tcp dport ${toString cfg.hostCache.port} accept - '' - + lib.concatMapStrings (s: '' - iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept - '') cfg.hostAccess; - - # Allowlists before RFC1918 drop. Bridged LAN is L2 (hermenix-bridge table). - networking.firewall.extraForwardRules = '' - iifname != "${net.bridge}" accept - ct state established,related accept - '' - + lib.concatMapStrings (s: '' - iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept - '') cfg.lanAccess - + lib.optionalString (!net.bridged) ( - lib.concatMapStrings (s: '' - oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept - '') cfg.publish - ) - + ( - if cfg.allowLan then - '' - iifname "${net.bridge}" accept - '' - else - '' - iifname "${net.bridge}" ip daddr { - 10.0.0.0/8, - 172.16.0.0/12, - 192.168.0.0/16, - 169.254.0.0/16, - 127.0.0.0/8, - 100.64.0.0/10 - } drop - iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop - iifname "${net.bridge}" accept - '' - ); - - services.nix-serve = mkIf cfg.hostCache.enable { - enable = true; - package = pkgs.nix-serve-ng; - bindAddress = net.hostAddress; - inherit (cfg.hostCache) port secretKeyFile; - openFirewall = false; - }; - - systemd.services.hermes-vm-snapshot = mkIf cfg.snapshots.enable { - description = "Snapshot the Hermes VM state disk"; - path = with pkgs; [ - btrfs-progs - coreutils - socat - ]; - serviceConfig.Type = "oneshot"; - script = '' - set -euo pipefail - stamp=$(date +%Y%m%d-%H%M%S) - mkdir -p ${snapDir} - - ${optionalString cfg.disk.quiesce '' - ${qga "freeze"} - trap '${qga "thaw"}' EXIT - ''} - - btrfs subvolume snapshot -r ${cfg.disk.baseDir}/state ${snapDir}/state-$stamp - - ${optionalString cfg.disk.quiesce '' - ${qga "thaw"} - trap - EXIT - ''} - - # Live snapshots reintroduce CoW on nodatacow images — keep retention tight. - ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \ - | while read -r old; do btrfs subvolume delete "$old"; done - ''; - }; - - systemd.timers.hermes-vm-snapshot = mkIf cfg.snapshots.enable { - wantedBy = [ "timers.target" ]; - timerConfig = { - OnCalendar = cfg.snapshots.interval; - Persistent = true; - RandomizedDelaySec = "5m"; - }; - }; - - systemd.services.hermes-vm-backup = mkIf cfg.backup.enable { - description = "Back up Hermes state"; - path = with pkgs; [ - restic - openssh - socat - gnutar - ]; - serviceConfig = { - Type = "oneshot"; - EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile; - }; - script = '' - set -euo pipefail - export RESTIC_REPOSITORY=${lib.escapeShellArg cfg.backup.repository} - export RESTIC_PASSWORD_FILE=${cfg.backup.passwordFile} - - restic snapshots >/dev/null 2>&1 || restic init - - ssh -o ProxyCommand="socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22" \ - -o StrictHostKeyChecking=accept-new \ - hermes@${cfg.name} \ - 'tar -C /var/lib/hermes \ - --exclude=./.cache \ - --exclude=./workspace/**/.direnv \ - --exclude=./workspace/**/result \ - -cf - .' \ - | restic backup --stdin --stdin-filename hermes-state.tar \ - --tag hermes --host ${cfg.name} - - restic forget --tag hermes --host ${cfg.name} --prune ${lib.concatStringsSep " " cfg.backup.keep} - ''; - }; - - systemd.timers.hermes-vm-backup = mkIf cfg.backup.enable { - wantedBy = [ "timers.target" ]; - timerConfig = { - OnCalendar = cfg.backup.interval; - Persistent = true; - RandomizedDelaySec = "30m"; - }; - }; - - environment.systemPackages = [ - (pkgs.writeTextFile { - name = "hermenix-completions"; - destination = "/share/nushell/vendor/autoload/hermenix.nu"; - text = builtins.readFile ../nu/completions.nu; - }) - - (pkgs.symlinkJoin { - name = "hermenix"; - paths = [ pkgs.hermenix ]; - nativeBuildInputs = [ pkgs.makeWrapper ]; - postBuild = '' - wrapProgram $out/bin/hermenix \ - --set HERMES_VSOCK_CID ${toString cfg.vsockCid} \ - --set HERMES_GUEST_NAME ${cfg.name} \ - --set HERMES_DISK_BASE_DIR ${cfg.disk.baseDir} \ - --set HERMES_HOME_DIR ${cfg.hermesHome} \ - --set HERMES_GATEWAY_UNIT ${cfg.gatewayUnit} - ''; - }) - ]; - }; + imports = [ + ./hermenix/shared-options.nix + ./hermenix/vm.options.nix + ./hermenix/vm.nix + ./hermenix/passthrough.options.nix + ./hermenix/passthrough.nix + ./hermenix/network.options.nix + ./hermenix/network.nix + ./hermenix/bridge.options.nix + ./hermenix/bridge.nix + ./hermenix/snapshots.options.nix + ./hermenix/snapshots.nix + ./hermenix/backup.options.nix + ./hermenix/backup.nix + ./hermenix/packages.nix + ]; }