diff --git a/README.md b/README.md index 93def99..bfa0739 100644 --- a/README.md +++ b/README.md @@ -685,6 +685,54 @@ lifts `LimitMEMLOCK` for you when PCI passthrough is configured, since VFIO pins the whole guest RAM allocation and the default limit otherwise produces a memorably unhelpful error. +## Virtual devices + +The VM boots a fixed set of virtual hardware, and `services.hermenix.devices` +controls which optional pieces are attached. Everything here is a QEMU device: +none of it touches host hardware, and none of it needs anything the guest +doesn't already ship (the mainline `virtio_*` drivers). + +| Option | Default | Attaches | +| --- | --- | --- | +| `devices.rng.enable` | `true` | virtio entropy source | +| `devices.vsock.enable` | `true` | the host<->guest vsock (`hermenix ctl`, backups) | +| `devices.video.enable` | `false` | a virtual display adapter | +| `devices.video.model` | `"virtio"` | `virtio` (has a render node) or a legacy VGA model | +| `devices.balloon.enable` | `false` | virtio memory balloon | +| `devices.watchdog.enable` | `false` | hardware watchdog (`i6300esb`/`ib700`, action `reset`/…) | +| `devices.input.enable` | `false` | virtio keyboard/tablet | +| `devices.audio.enable` | `false` | audio device (`intel-hda`/`virtio-sound`) | +| `devices.tpm.enable` | `false` | TPM 2.0 via swtpm | +| `devices.cdroms` | `[]` | read-only CD images | +| `devices.extraDisks` | `[]` | extra raw disks | +| `devices.usbController` | `"auto"` | `qemu-xhci` (`auto` = on iff `passthrough.usb` is set) | + +`extraQemuArgs` appends raw arguments to the QEMU command line for anything +not covered above. It is unvalidated - prefer a typed option where one exists. + +### A render node (Wayland, GPU compute, anything wanting /dev/dri) + +```nix +services.hermenix.devices.video.enable = true; # model defaults to "virtio" +``` + +Gives the guest `/dev/dri/card0` and `/dev/dri/renderD128`. The console stays +serial and nothing is shown on the host. Only the `virtio` model exposes the +render node; the legacy models give `card0` alone. + +### TPM + +`devices.tpm.enable = true` starts an `swtpm` unit first and points QEMU at +it. State lives in the unit's `StateDirectory` (`/var/lib/hermes-vm-tpm` by +default), so the guest gets a TPM 2.0 it can seal secrets to. + +### Audio, with a caveat + +`devices.audio.enable` is wired up, but `hermes-vm.service` is deliberately +hardened (`ProtectHome`, `PrivateTmp`, a dedicated system user), so it cannot +reach your PipeWire socket without loosening that hardening. Leave it off +unless you mean to relax the unit for it. + ## Recovery Four layers, cheapest first. The first two live inside the guest and cover diff --git a/modules/host.nix b/modules/host.nix index ed81b05..7c1f19e 100644 --- a/modules/host.nix +++ b/modules/host.nix @@ -36,6 +36,14 @@ let anyUsb = pt.usb != [ ]; earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci); + # "auto" adds a USB controller only when something needs plugging into it; + # "on"/"off" force the decision. + usbControllerOn = + cfg.devices.usbController == "on" || (cfg.devices.usbController == "auto" && anyUsb); + + # swtpm state. Must live under /var/lib/ so the unit's StateDirectory owns it. + tpmStateDir = "/var/lib/hermes-vm-tpm"; + # Netfilter mark tagging guest-originated traffic in bridged mode. The # guest's only way in is the tap interface, so marking at the bridge # prerouting hook identifies the guest regardless of which address it @@ -526,6 +534,221 @@ in }; }; + devices = { + rng.enable = mkOption { + type = types.bool; + default = true; + description = "Attach a virtio entropy source (-device virtio-rng-pci)."; + }; + + vsock.enable = mkOption { + type = types.bool; + default = true; + description = '' + Attach the host<->guest vsock (-device vhost-vsock-pci). `hermenix + ctl` and the SSH-based backup both reach the guest over it, so + turning this off breaks them. + ''; + }; + + video = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Attach a virtual display adapter, giving the guest a DRM device + (/dev/dri). Needed by a Wayland compositor or anything that wants + a render node. The console stays serial either way. + ''; + }; + model = mkOption { + type = types.enum [ + "virtio" + "qxl" + "cirrus" + "vmware" + "std" + ]; + default = "virtio"; + description = '' + Adapter model passed to -vga. Only "virtio" exposes a render node + (/dev/dri/renderD128); the legacy models give card0 without one. + ''; + }; + }; + + balloon.enable = mkOption { + type = types.bool; + default = false; + description = "Attach a virtio memory balloon (-device virtio-balloon-pci)."; + }; + + watchdog = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Attach a hardware watchdog. The guest must also arm it (systemd's + RuntimeWatchdogSec) for it to do anything. + ''; + }; + model = mkOption { + type = types.enum [ + "i6300esb" + "ib700" + ]; + default = "i6300esb"; + }; + action = mkOption { + type = types.enum [ + "reset" + "shutdown" + "poweroff" + "pause" + "debug" + "none" + ]; + default = "reset"; + description = "What QEMU does when the watchdog fires."; + }; + }; + + input = { + enable = mkOption { + type = types.bool; + default = false; + description = "Attach virtio input devices. Only useful with devices.video.enable."; + }; + keyboard = mkOption { + type = types.bool; + default = true; + }; + tablet = mkOption { + type = types.bool; + default = false; + }; + }; + + audio = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Attach an audio device. NOTE: the VM unit is hardened (ProtectHome, + PrivateTmp, a dedicated system user) and cannot reach the operator's + audio daemon without loosening that; leave this off to keep the + hardening intact. + ''; + }; + backend = mkOption { + type = types.enum [ + "none" + "pipewire" + "alsa" + "pa" + "sdl" + ]; + default = "pipewire"; + }; + model = mkOption { + type = types.enum [ + "intel-hda" + "virtio-sound" + ]; + default = "intel-hda"; + }; + }; + + tpm = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Attach a TPM 2.0 via swtpm. A hermes-vm-swtpm unit starts first and + QEMU connects to it over a unix socket. Useful for measured boot or + sealing secrets to the VM. + ''; + }; + stateDir = mkOption { + type = types.str; + default = tpmStateDir; + description = '' + Where swtpm keeps its state. Must live under /var/lib/ so it can be + created and owned by the unit's StateDirectory. + ''; + }; + }; + + cdroms = mkOption { + default = [ ]; + description = "Read-only CD-ROM images to attach, on a virtio-scsi controller."; + type = types.listOf ( + types.submodule { + options.path = mkOption { type = types.str; }; + } + ); + }; + + extraDisks = mkOption { + default = [ ]; + description = "Extra raw disk images, each on its own virtio-blk device."; + type = types.listOf ( + types.submodule { + options = { + name = mkOption { + type = types.str; + description = "Unique QEMU drive id, e.g. scratch."; + }; + path = mkOption { type = types.str; }; + readonly = mkOption { + type = types.bool; + default = false; + }; + aio = mkOption { + type = types.enum [ + "threads" + "native" + "io_uring" + ]; + default = "threads"; + }; + cache = mkOption { + type = types.enum [ + "writeback" + "none" + "directsync" + "unsafe" + ]; + default = "writeback"; + }; + }; + } + ); + }; + + usbController = mkOption { + type = types.enum [ + "auto" + "on" + "off" + ]; + default = "auto"; + description = '' + Whether to attach a qemu-xhci USB controller. "auto" adds one only + when passthrough.usb is non-empty; "on"/"off" force the decision. + ''; + }; + }; + + extraQemuArgs = mkOption { + type = types.listOf types.str; + default = [ ]; + description = '' + Raw additional QEMU arguments, appended to the VM command line. + Unvalidated escape hatch - prefer the typed devices.* options. + ''; + }; + gatewayUnit = mkOption { type = types.str; default = "hermes-agent"; @@ -601,6 +824,29 @@ in read host memory directly and the VM boundary is meaningless. ''; } + { + assertion = cfg.devices.vsock.enable || !cfg.backup.enable; + message = '' + services.hermenix.backup.enable needs the vsock (the backup runs over + SSH multiplexed on it). Leave devices.vsock.enable at its default, or + turn backups off. + ''; + } + { + assertion = cfg.devices.usbController != "off" || !anyUsb; + message = '' + services.hermenix.passthrough.usb is set but + services.hermenix.devices.usbController is "off", so those devices + have no controller to attach to. + ''; + } + { + assertion = !cfg.devices.tpm.enable || lib.hasPrefix "/var/lib/" cfg.devices.tpm.stateDir; + message = '' + services.hermenix.devices.tpm.stateDir must live under /var/lib/ so + the swtpm unit's StateDirectory can create and own it. + ''; + } ]; users.users.${vmUser} = { @@ -622,8 +868,12 @@ in after = [ "network-online.target" "hermes-vm-tap.service" - ]; - requires = [ "hermes-vm-tap.service" ]; + ] + ++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service"; + requires = [ + "hermes-vm-tap.service" + ] + ++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service"; wants = [ "network-online.target" ]; preStart = '' @@ -660,6 +910,7 @@ in "-m ${toString cfg.mem}" "-nodefaults" "-no-user-config" + # Display Settings "-display none" "-serial stdio" # QEMU's own seccomp sandbox, everything denied that we don't need. @@ -681,10 +932,63 @@ in "-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off" "-device virtio-net-pci,netdev=net0,mac=${net.mac}" # Host -> guest management, no network involved. - "-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}" - "-device virtio-rng-pci" "-qmp unix:${runDir}/qmp.sock,server=on,wait=off" ] + # Optional devices. rng and vsock are on by default; everything else + # is off unless enabled via services.hermenix.devices. + ++ lib.optionals cfg.devices.rng.enable [ + "-device virtio-rng-pci" + ] + ++ lib.optionals cfg.devices.vsock.enable [ + "-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}" + ] + ++ lib.optionals cfg.devices.video.enable [ + "-vga ${cfg.devices.video.model}" + ] + ++ lib.optionals cfg.devices.balloon.enable [ + "-device virtio-balloon-pci" + ] + ++ lib.optionals cfg.devices.watchdog.enable [ + "-device ${cfg.devices.watchdog.model}" + "-watchdog-action ${cfg.devices.watchdog.action}" + ] + ++ lib.optionals cfg.devices.input.enable ( + lib.optional cfg.devices.input.keyboard "-device virtio-keyboard-pci" + ++ lib.optional cfg.devices.input.tablet "-device virtio-tablet-pci" + ) + ++ lib.optionals cfg.devices.audio.enable ( + [ "-audiodev ${cfg.devices.audio.backend},id=audio0" ] + ++ ( + if cfg.devices.audio.model == "virtio-sound" then + [ "-device virtio-sound-pci,audiodev=audio0" ] + else + [ + "-device intel-hda" + "-device hda-duplex,audiodev=audio0" + ] + ) + ) + ++ lib.optionals cfg.devices.tpm.enable [ + "-chardev socket,id=chrtpm,path=${runDir}/swtpm.sock" + "-tpmdev emulator,id=tpm0,chardev=chrtpm" + "-device tpm-crb,tpmdev=tpm0" + ] + ++ lib.optionals (cfg.devices.cdroms != [ ]) [ + "-device virtio-scsi-pci,id=scsi0" + ] + ++ lib.concatLists ( + lib.imap0 (i: c: [ + "-drive if=none,id=cd${toString i},media=cdrom,readonly=on,file=${c.path}" + "-device scsi-cd,drive=cd${toString i},bus=scsi0.0" + ]) cfg.devices.cdroms + ) + ++ lib.concatMap (d: [ + "-drive if=none,id=${d.name},format=raw,file=${d.path},cache=${d.cache},aio=${d.aio}${lib.optionalString d.readonly ",readonly=on"}" + "-device virtio-blk-pci,drive=${d.name}" + ]) cfg.devices.extraDisks + ++ lib.optionals usbControllerOn [ + "-device qemu-xhci,id=xhci" + ] ++ lib.optionals cfg.disk.quiesce [ "-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0" "-device virtio-serial-pci" @@ -693,9 +997,9 @@ in # Passed-through PCI devices. The host cannot use these while the VM # is running. ++ map (d: "-device vfio-pci,host=${d.address}") pt.pci - # A USB controller, only if something needs plugging into it. - ++ lib.optional anyUsb "-device qemu-xhci,id=xhci" ++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb + # Raw escape hatch. Unvalidated - prefer the typed devices.* options. + ++ cfg.extraQemuArgs ); ExecStop = stopScript; @@ -751,6 +1055,55 @@ in }; }; + # TPM 2.0 emulator (swtpm), started before the VM. QEMU connects to its + # control socket in the runtime dir; state persists under its StateDirectory. + systemd.services.hermes-vm-swtpm = mkIf cfg.devices.tpm.enable { + description = "swtpm TPM emulator for the Hermes VM"; + wantedBy = [ "multi-user.target" ]; + before = [ "hermes-vm.service" ]; + requiredBy = [ "hermes-vm.service" ]; + serviceConfig = { + User = vmUser; + Group = vmUser; + RuntimeDirectory = "hermes-vm"; + RuntimeDirectoryMode = "0750"; + StateDirectory = baseNameOf cfg.devices.tpm.stateDir; + Restart = "always"; + RestartSec = 5; + + DevicePolicy = "closed"; + NoNewPrivileges = true; + PrivateTmp = true; + ProtectSystem = "strict"; + ProtectHome = true; + ProtectProc = "invisible"; + ProcSubset = "pid"; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHostname = true; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + RestrictAddressFamilies = [ "AF_UNIX" ]; + LockPersonality = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ "@system-service" ]; + MemoryDenyWriteExecute = true; + CapabilityBoundingSet = [ "" ]; + AmbientCapabilities = [ "" ]; + }; + script = '' + exec ${pkgs.swtpm}/bin/swtpm socket \ + --tpm2 \ + --tpmstate dir=${cfg.devices.tpm.stateDir} \ + --ctrl type=unixio,path=${runDir}/swtpm.sock \ + --flags not-need-init,startup-clear + ''; + }; + # Tap device, created as root, owned by the VM user, bridged by networkd. systemd.services.hermes-vm-tap = { description = "TAP interface for the Hermes VM";