added flags for virt devices

This commit is contained in:
ari committed 2026-09-29 11:33:33 -04:00
1 parent d5afd6510f
commit 32edfbd111
2 files changed
+407 -6

No files matched your search

+48
View File
@@ -685,6 +685,54 @@ lifts `LimitMEMLOCK` for you when PCI passthrough is configured, since VFIO
pins the whole guest RAM allocation and the default limit otherwise produces
a memorably unhelpful error.
## Virtual devices
The VM boots a fixed set of virtual hardware, and `services.hermenix.devices`
controls which optional pieces are attached. Everything here is a QEMU device:
none of it touches host hardware, and none of it needs anything the guest
doesn't already ship (the mainline `virtio_*` drivers).
| Option | Default | Attaches |
| --- | --- | --- |
| `devices.rng.enable` | `true` | virtio entropy source |
| `devices.vsock.enable` | `true` | the host<->guest vsock (`hermenix ctl`, backups) |
| `devices.video.enable` | `false` | a virtual display adapter |
| `devices.video.model` | `"virtio"` | `virtio` (has a render node) or a legacy VGA model |
| `devices.balloon.enable` | `false` | virtio memory balloon |
| `devices.watchdog.enable` | `false` | hardware watchdog (`i6300esb`/`ib700`, action `reset`/…) |
| `devices.input.enable` | `false` | virtio keyboard/tablet |
| `devices.audio.enable` | `false` | audio device (`intel-hda`/`virtio-sound`) |
| `devices.tpm.enable` | `false` | TPM 2.0 via swtpm |
| `devices.cdroms` | `[]` | read-only CD images |
| `devices.extraDisks` | `[]` | extra raw disks |
| `devices.usbController` | `"auto"` | `qemu-xhci` (`auto` = on iff `passthrough.usb` is set) |
`extraQemuArgs` appends raw arguments to the QEMU command line for anything
not covered above. It is unvalidated - prefer a typed option where one exists.
### A render node (Wayland, GPU compute, anything wanting /dev/dri)
```nix
services.hermenix.devices.video.enable = true; # model defaults to "virtio"
```
Gives the guest `/dev/dri/card0` and `/dev/dri/renderD128`. The console stays
serial and nothing is shown on the host. Only the `virtio` model exposes the
render node; the legacy models give `card0` alone.
### TPM
`devices.tpm.enable = true` starts an `swtpm` unit first and points QEMU at
it. State lives in the unit's `StateDirectory` (`/var/lib/hermes-vm-tpm` by
default), so the guest gets a TPM 2.0 it can seal secrets to.
### Audio, with a caveat
`devices.audio.enable` is wired up, but `hermes-vm.service` is deliberately
hardened (`ProtectHome`, `PrivateTmp`, a dedicated system user), so it cannot
reach your PipeWire socket without loosening that hardening. Leave it off
unless you mean to relax the unit for it.
## Recovery
Four layers, cheapest first. The first two live inside the guest and cover
+359 -6
View File
@@ -36,6 +36,14 @@ let
anyUsb = pt.usb != [ ];
earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci);
# "auto" adds a USB controller only when something needs plugging into it;
# "on"/"off" force the decision.
usbControllerOn =
cfg.devices.usbController == "on" || (cfg.devices.usbController == "auto" && anyUsb);
# swtpm state. Must live under /var/lib/ so the unit's StateDirectory owns it.
tpmStateDir = "/var/lib/hermes-vm-tpm";
# Netfilter mark tagging guest-originated traffic in bridged mode. The
# guest's only way in is the tap interface, so marking at the bridge
# prerouting hook identifies the guest regardless of which address it
@@ -526,6 +534,221 @@ in
};
};
devices = {
rng.enable = mkOption {
type = types.bool;
default = true;
description = "Attach a virtio entropy source (-device virtio-rng-pci).";
};
vsock.enable = mkOption {
type = types.bool;
default = true;
description = ''
Attach the host<->guest vsock (-device vhost-vsock-pci). `hermenix
ctl` and the SSH-based backup both reach the guest over it, so
turning this off breaks them.
'';
};
video = {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Attach a virtual display adapter, giving the guest a DRM device
(/dev/dri). Needed by a Wayland compositor or anything that wants
a render node. The console stays serial either way.
'';
};
model = mkOption {
type = types.enum [
"virtio"
"qxl"
"cirrus"
"vmware"
"std"
];
default = "virtio";
description = ''
Adapter model passed to -vga. Only "virtio" exposes a render node
(/dev/dri/renderD128); the legacy models give card0 without one.
'';
};
};
balloon.enable = mkOption {
type = types.bool;
default = false;
description = "Attach a virtio memory balloon (-device virtio-balloon-pci).";
};
watchdog = {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Attach a hardware watchdog. The guest must also arm it (systemd's
RuntimeWatchdogSec) for it to do anything.
'';
};
model = mkOption {
type = types.enum [
"i6300esb"
"ib700"
];
default = "i6300esb";
};
action = mkOption {
type = types.enum [
"reset"
"shutdown"
"poweroff"
"pause"
"debug"
"none"
];
default = "reset";
description = "What QEMU does when the watchdog fires.";
};
};
input = {
enable = mkOption {
type = types.bool;
default = false;
description = "Attach virtio input devices. Only useful with devices.video.enable.";
};
keyboard = mkOption {
type = types.bool;
default = true;
};
tablet = mkOption {
type = types.bool;
default = false;
};
};
audio = {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Attach an audio device. NOTE: the VM unit is hardened (ProtectHome,
PrivateTmp, a dedicated system user) and cannot reach the operator's
audio daemon without loosening that; leave this off to keep the
hardening intact.
'';
};
backend = mkOption {
type = types.enum [
"none"
"pipewire"
"alsa"
"pa"
"sdl"
];
default = "pipewire";
};
model = mkOption {
type = types.enum [
"intel-hda"
"virtio-sound"
];
default = "intel-hda";
};
};
tpm = {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Attach a TPM 2.0 via swtpm. A hermes-vm-swtpm unit starts first and
QEMU connects to it over a unix socket. Useful for measured boot or
sealing secrets to the VM.
'';
};
stateDir = mkOption {
type = types.str;
default = tpmStateDir;
description = ''
Where swtpm keeps its state. Must live under /var/lib/ so it can be
created and owned by the unit's StateDirectory.
'';
};
};
cdroms = mkOption {
default = [ ];
description = "Read-only CD-ROM images to attach, on a virtio-scsi controller.";
type = types.listOf (
types.submodule {
options.path = mkOption { type = types.str; };
}
);
};
extraDisks = mkOption {
default = [ ];
description = "Extra raw disk images, each on its own virtio-blk device.";
type = types.listOf (
types.submodule {
options = {
name = mkOption {
type = types.str;
description = "Unique QEMU drive id, e.g. scratch.";
};
path = mkOption { type = types.str; };
readonly = mkOption {
type = types.bool;
default = false;
};
aio = mkOption {
type = types.enum [
"threads"
"native"
"io_uring"
];
default = "threads";
};
cache = mkOption {
type = types.enum [
"writeback"
"none"
"directsync"
"unsafe"
];
default = "writeback";
};
};
}
);
};
usbController = mkOption {
type = types.enum [
"auto"
"on"
"off"
];
default = "auto";
description = ''
Whether to attach a qemu-xhci USB controller. "auto" adds one only
when passthrough.usb is non-empty; "on"/"off" force the decision.
'';
};
};
extraQemuArgs = mkOption {
type = types.listOf types.str;
default = [ ];
description = ''
Raw additional QEMU arguments, appended to the VM command line.
Unvalidated escape hatch - prefer the typed devices.* options.
'';
};
gatewayUnit = mkOption {
type = types.str;
default = "hermes-agent";
@@ -601,6 +824,29 @@ in
read host memory directly and the VM boundary is meaningless.
'';
}
{
assertion = cfg.devices.vsock.enable || !cfg.backup.enable;
message = ''
services.hermenix.backup.enable needs the vsock (the backup runs over
SSH multiplexed on it). Leave devices.vsock.enable at its default, or
turn backups off.
'';
}
{
assertion = cfg.devices.usbController != "off" || !anyUsb;
message = ''
services.hermenix.passthrough.usb is set but
services.hermenix.devices.usbController is "off", so those devices
have no controller to attach to.
'';
}
{
assertion = !cfg.devices.tpm.enable || lib.hasPrefix "/var/lib/" cfg.devices.tpm.stateDir;
message = ''
services.hermenix.devices.tpm.stateDir must live under /var/lib/ so
the swtpm unit's StateDirectory can create and own it.
'';
}
];
users.users.${vmUser} = {
@@ -622,8 +868,12 @@ in
after = [
"network-online.target"
"hermes-vm-tap.service"
];
requires = [ "hermes-vm-tap.service" ];
]
++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service";
requires = [
"hermes-vm-tap.service"
]
++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service";
wants = [ "network-online.target" ];
preStart = ''
@@ -660,6 +910,7 @@ in
"-m ${toString cfg.mem}"
"-nodefaults"
"-no-user-config"
# Display Settings
"-display none"
"-serial stdio"
# QEMU's own seccomp sandbox, everything denied that we don't need.
@@ -681,10 +932,63 @@ in
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
# Host -> guest management, no network involved.
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
"-device virtio-rng-pci"
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
]
# Optional devices. rng and vsock are on by default; everything else
# is off unless enabled via services.hermenix.devices.
++ lib.optionals cfg.devices.rng.enable [
"-device virtio-rng-pci"
]
++ lib.optionals cfg.devices.vsock.enable [
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
]
++ lib.optionals cfg.devices.video.enable [
"-vga ${cfg.devices.video.model}"
]
++ lib.optionals cfg.devices.balloon.enable [
"-device virtio-balloon-pci"
]
++ lib.optionals cfg.devices.watchdog.enable [
"-device ${cfg.devices.watchdog.model}"
"-watchdog-action ${cfg.devices.watchdog.action}"
]
++ lib.optionals cfg.devices.input.enable (
lib.optional cfg.devices.input.keyboard "-device virtio-keyboard-pci"
++ lib.optional cfg.devices.input.tablet "-device virtio-tablet-pci"
)
++ lib.optionals cfg.devices.audio.enable (
[ "-audiodev ${cfg.devices.audio.backend},id=audio0" ]
++ (
if cfg.devices.audio.model == "virtio-sound" then
[ "-device virtio-sound-pci,audiodev=audio0" ]
else
[
"-device intel-hda"
"-device hda-duplex,audiodev=audio0"
]
)
)
++ lib.optionals cfg.devices.tpm.enable [
"-chardev socket,id=chrtpm,path=${runDir}/swtpm.sock"
"-tpmdev emulator,id=tpm0,chardev=chrtpm"
"-device tpm-crb,tpmdev=tpm0"
]
++ lib.optionals (cfg.devices.cdroms != [ ]) [
"-device virtio-scsi-pci,id=scsi0"
]
++ lib.concatLists (
lib.imap0 (i: c: [
"-drive if=none,id=cd${toString i},media=cdrom,readonly=on,file=${c.path}"
"-device scsi-cd,drive=cd${toString i},bus=scsi0.0"
]) cfg.devices.cdroms
)
++ lib.concatMap (d: [
"-drive if=none,id=${d.name},format=raw,file=${d.path},cache=${d.cache},aio=${d.aio}${lib.optionalString d.readonly ",readonly=on"}"
"-device virtio-blk-pci,drive=${d.name}"
]) cfg.devices.extraDisks
++ lib.optionals usbControllerOn [
"-device qemu-xhci,id=xhci"
]
++ lib.optionals cfg.disk.quiesce [
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
"-device virtio-serial-pci"
@@ -693,9 +997,9 @@ in
# Passed-through PCI devices. The host cannot use these while the VM
# is running.
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
# A USB controller, only if something needs plugging into it.
++ lib.optional anyUsb "-device qemu-xhci,id=xhci"
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
# Raw escape hatch. Unvalidated - prefer the typed devices.* options.
++ cfg.extraQemuArgs
);
ExecStop = stopScript;
@@ -751,6 +1055,55 @@ in
};
};
# TPM 2.0 emulator (swtpm), started before the VM. QEMU connects to its
# control socket in the runtime dir; state persists under its StateDirectory.
systemd.services.hermes-vm-swtpm = mkIf cfg.devices.tpm.enable {
description = "swtpm TPM emulator for the Hermes VM";
wantedBy = [ "multi-user.target" ];
before = [ "hermes-vm.service" ];
requiredBy = [ "hermes-vm.service" ];
serviceConfig = {
User = vmUser;
Group = vmUser;
RuntimeDirectory = "hermes-vm";
RuntimeDirectoryMode = "0750";
StateDirectory = baseNameOf cfg.devices.tpm.stateDir;
Restart = "always";
RestartSec = 5;
DevicePolicy = "closed";
NoNewPrivileges = true;
PrivateTmp = true;
ProtectSystem = "strict";
ProtectHome = true;
ProtectProc = "invisible";
ProcSubset = "pid";
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHostname = true;
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
RestrictAddressFamilies = [ "AF_UNIX" ];
LockPersonality = true;
SystemCallArchitectures = "native";
SystemCallFilter = [ "@system-service" ];
MemoryDenyWriteExecute = true;
CapabilityBoundingSet = [ "" ];
AmbientCapabilities = [ "" ];
};
script = ''
exec ${pkgs.swtpm}/bin/swtpm socket \
--tpm2 \
--tpmstate dir=${cfg.devices.tpm.stateDir} \
--ctrl type=unixio,path=${runDir}/swtpm.sock \
--flags not-need-init,startup-clear
'';
};
# Tap device, created as root, owned by the VM user, bridged by networkd.
systemd.services.hermes-vm-tap = {
description = "TAP interface for the Hermes VM";