added flags for virt devices
This commit is contained in:
1 parent
d5afd6510f
commit
32edfbd111
2 files changed
+407
-6
No files matched your search
@@ -685,6 +685,54 @@ lifts `LimitMEMLOCK` for you when PCI passthrough is configured, since VFIO
|
||||
pins the whole guest RAM allocation and the default limit otherwise produces
|
||||
a memorably unhelpful error.
|
||||
|
||||
## Virtual devices
|
||||
|
||||
The VM boots a fixed set of virtual hardware, and `services.hermenix.devices`
|
||||
controls which optional pieces are attached. Everything here is a QEMU device:
|
||||
none of it touches host hardware, and none of it needs anything the guest
|
||||
doesn't already ship (the mainline `virtio_*` drivers).
|
||||
|
||||
| Option | Default | Attaches |
|
||||
| --- | --- | --- |
|
||||
| `devices.rng.enable` | `true` | virtio entropy source |
|
||||
| `devices.vsock.enable` | `true` | the host<->guest vsock (`hermenix ctl`, backups) |
|
||||
| `devices.video.enable` | `false` | a virtual display adapter |
|
||||
| `devices.video.model` | `"virtio"` | `virtio` (has a render node) or a legacy VGA model |
|
||||
| `devices.balloon.enable` | `false` | virtio memory balloon |
|
||||
| `devices.watchdog.enable` | `false` | hardware watchdog (`i6300esb`/`ib700`, action `reset`/…) |
|
||||
| `devices.input.enable` | `false` | virtio keyboard/tablet |
|
||||
| `devices.audio.enable` | `false` | audio device (`intel-hda`/`virtio-sound`) |
|
||||
| `devices.tpm.enable` | `false` | TPM 2.0 via swtpm |
|
||||
| `devices.cdroms` | `[]` | read-only CD images |
|
||||
| `devices.extraDisks` | `[]` | extra raw disks |
|
||||
| `devices.usbController` | `"auto"` | `qemu-xhci` (`auto` = on iff `passthrough.usb` is set) |
|
||||
|
||||
`extraQemuArgs` appends raw arguments to the QEMU command line for anything
|
||||
not covered above. It is unvalidated - prefer a typed option where one exists.
|
||||
|
||||
### A render node (Wayland, GPU compute, anything wanting /dev/dri)
|
||||
|
||||
```nix
|
||||
services.hermenix.devices.video.enable = true; # model defaults to "virtio"
|
||||
```
|
||||
|
||||
Gives the guest `/dev/dri/card0` and `/dev/dri/renderD128`. The console stays
|
||||
serial and nothing is shown on the host. Only the `virtio` model exposes the
|
||||
render node; the legacy models give `card0` alone.
|
||||
|
||||
### TPM
|
||||
|
||||
`devices.tpm.enable = true` starts an `swtpm` unit first and points QEMU at
|
||||
it. State lives in the unit's `StateDirectory` (`/var/lib/hermes-vm-tpm` by
|
||||
default), so the guest gets a TPM 2.0 it can seal secrets to.
|
||||
|
||||
### Audio, with a caveat
|
||||
|
||||
`devices.audio.enable` is wired up, but `hermes-vm.service` is deliberately
|
||||
hardened (`ProtectHome`, `PrivateTmp`, a dedicated system user), so it cannot
|
||||
reach your PipeWire socket without loosening that hardening. Leave it off
|
||||
unless you mean to relax the unit for it.
|
||||
|
||||
## Recovery
|
||||
|
||||
Four layers, cheapest first. The first two live inside the guest and cover
|
||||
|
||||
+359
-6
@@ -36,6 +36,14 @@ let
|
||||
anyUsb = pt.usb != [ ];
|
||||
earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci);
|
||||
|
||||
# "auto" adds a USB controller only when something needs plugging into it;
|
||||
# "on"/"off" force the decision.
|
||||
usbControllerOn =
|
||||
cfg.devices.usbController == "on" || (cfg.devices.usbController == "auto" && anyUsb);
|
||||
|
||||
# swtpm state. Must live under /var/lib/ so the unit's StateDirectory owns it.
|
||||
tpmStateDir = "/var/lib/hermes-vm-tpm";
|
||||
|
||||
# Netfilter mark tagging guest-originated traffic in bridged mode. The
|
||||
# guest's only way in is the tap interface, so marking at the bridge
|
||||
# prerouting hook identifies the guest regardless of which address it
|
||||
@@ -526,6 +534,221 @@ in
|
||||
};
|
||||
};
|
||||
|
||||
devices = {
|
||||
rng.enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Attach a virtio entropy source (-device virtio-rng-pci).";
|
||||
};
|
||||
|
||||
vsock.enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Attach the host<->guest vsock (-device vhost-vsock-pci). `hermenix
|
||||
ctl` and the SSH-based backup both reach the guest over it, so
|
||||
turning this off breaks them.
|
||||
'';
|
||||
};
|
||||
|
||||
video = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Attach a virtual display adapter, giving the guest a DRM device
|
||||
(/dev/dri). Needed by a Wayland compositor or anything that wants
|
||||
a render node. The console stays serial either way.
|
||||
'';
|
||||
};
|
||||
model = mkOption {
|
||||
type = types.enum [
|
||||
"virtio"
|
||||
"qxl"
|
||||
"cirrus"
|
||||
"vmware"
|
||||
"std"
|
||||
];
|
||||
default = "virtio";
|
||||
description = ''
|
||||
Adapter model passed to -vga. Only "virtio" exposes a render node
|
||||
(/dev/dri/renderD128); the legacy models give card0 without one.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
balloon.enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Attach a virtio memory balloon (-device virtio-balloon-pci).";
|
||||
};
|
||||
|
||||
watchdog = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Attach a hardware watchdog. The guest must also arm it (systemd's
|
||||
RuntimeWatchdogSec) for it to do anything.
|
||||
'';
|
||||
};
|
||||
model = mkOption {
|
||||
type = types.enum [
|
||||
"i6300esb"
|
||||
"ib700"
|
||||
];
|
||||
default = "i6300esb";
|
||||
};
|
||||
action = mkOption {
|
||||
type = types.enum [
|
||||
"reset"
|
||||
"shutdown"
|
||||
"poweroff"
|
||||
"pause"
|
||||
"debug"
|
||||
"none"
|
||||
];
|
||||
default = "reset";
|
||||
description = "What QEMU does when the watchdog fires.";
|
||||
};
|
||||
};
|
||||
|
||||
input = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Attach virtio input devices. Only useful with devices.video.enable.";
|
||||
};
|
||||
keyboard = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
};
|
||||
tablet = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
};
|
||||
|
||||
audio = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Attach an audio device. NOTE: the VM unit is hardened (ProtectHome,
|
||||
PrivateTmp, a dedicated system user) and cannot reach the operator's
|
||||
audio daemon without loosening that; leave this off to keep the
|
||||
hardening intact.
|
||||
'';
|
||||
};
|
||||
backend = mkOption {
|
||||
type = types.enum [
|
||||
"none"
|
||||
"pipewire"
|
||||
"alsa"
|
||||
"pa"
|
||||
"sdl"
|
||||
];
|
||||
default = "pipewire";
|
||||
};
|
||||
model = mkOption {
|
||||
type = types.enum [
|
||||
"intel-hda"
|
||||
"virtio-sound"
|
||||
];
|
||||
default = "intel-hda";
|
||||
};
|
||||
};
|
||||
|
||||
tpm = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Attach a TPM 2.0 via swtpm. A hermes-vm-swtpm unit starts first and
|
||||
QEMU connects to it over a unix socket. Useful for measured boot or
|
||||
sealing secrets to the VM.
|
||||
'';
|
||||
};
|
||||
stateDir = mkOption {
|
||||
type = types.str;
|
||||
default = tpmStateDir;
|
||||
description = ''
|
||||
Where swtpm keeps its state. Must live under /var/lib/ so it can be
|
||||
created and owned by the unit's StateDirectory.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
cdroms = mkOption {
|
||||
default = [ ];
|
||||
description = "Read-only CD-ROM images to attach, on a virtio-scsi controller.";
|
||||
type = types.listOf (
|
||||
types.submodule {
|
||||
options.path = mkOption { type = types.str; };
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
extraDisks = mkOption {
|
||||
default = [ ];
|
||||
description = "Extra raw disk images, each on its own virtio-blk device.";
|
||||
type = types.listOf (
|
||||
types.submodule {
|
||||
options = {
|
||||
name = mkOption {
|
||||
type = types.str;
|
||||
description = "Unique QEMU drive id, e.g. scratch.";
|
||||
};
|
||||
path = mkOption { type = types.str; };
|
||||
readonly = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
aio = mkOption {
|
||||
type = types.enum [
|
||||
"threads"
|
||||
"native"
|
||||
"io_uring"
|
||||
];
|
||||
default = "threads";
|
||||
};
|
||||
cache = mkOption {
|
||||
type = types.enum [
|
||||
"writeback"
|
||||
"none"
|
||||
"directsync"
|
||||
"unsafe"
|
||||
];
|
||||
default = "writeback";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
usbController = mkOption {
|
||||
type = types.enum [
|
||||
"auto"
|
||||
"on"
|
||||
"off"
|
||||
];
|
||||
default = "auto";
|
||||
description = ''
|
||||
Whether to attach a qemu-xhci USB controller. "auto" adds one only
|
||||
when passthrough.usb is non-empty; "on"/"off" force the decision.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
extraQemuArgs = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ ];
|
||||
description = ''
|
||||
Raw additional QEMU arguments, appended to the VM command line.
|
||||
Unvalidated escape hatch - prefer the typed devices.* options.
|
||||
'';
|
||||
};
|
||||
|
||||
gatewayUnit = mkOption {
|
||||
type = types.str;
|
||||
default = "hermes-agent";
|
||||
@@ -601,6 +824,29 @@ in
|
||||
read host memory directly and the VM boundary is meaningless.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = cfg.devices.vsock.enable || !cfg.backup.enable;
|
||||
message = ''
|
||||
services.hermenix.backup.enable needs the vsock (the backup runs over
|
||||
SSH multiplexed on it). Leave devices.vsock.enable at its default, or
|
||||
turn backups off.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = cfg.devices.usbController != "off" || !anyUsb;
|
||||
message = ''
|
||||
services.hermenix.passthrough.usb is set but
|
||||
services.hermenix.devices.usbController is "off", so those devices
|
||||
have no controller to attach to.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = !cfg.devices.tpm.enable || lib.hasPrefix "/var/lib/" cfg.devices.tpm.stateDir;
|
||||
message = ''
|
||||
services.hermenix.devices.tpm.stateDir must live under /var/lib/ so
|
||||
the swtpm unit's StateDirectory can create and own it.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
users.users.${vmUser} = {
|
||||
@@ -622,8 +868,12 @@ in
|
||||
after = [
|
||||
"network-online.target"
|
||||
"hermes-vm-tap.service"
|
||||
];
|
||||
requires = [ "hermes-vm-tap.service" ];
|
||||
]
|
||||
++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service";
|
||||
requires = [
|
||||
"hermes-vm-tap.service"
|
||||
]
|
||||
++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service";
|
||||
wants = [ "network-online.target" ];
|
||||
|
||||
preStart = ''
|
||||
@@ -660,6 +910,7 @@ in
|
||||
"-m ${toString cfg.mem}"
|
||||
"-nodefaults"
|
||||
"-no-user-config"
|
||||
# Display Settings
|
||||
"-display none"
|
||||
"-serial stdio"
|
||||
# QEMU's own seccomp sandbox, everything denied that we don't need.
|
||||
@@ -681,10 +932,63 @@ in
|
||||
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
|
||||
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
|
||||
# Host -> guest management, no network involved.
|
||||
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
|
||||
"-device virtio-rng-pci"
|
||||
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
|
||||
]
|
||||
# Optional devices. rng and vsock are on by default; everything else
|
||||
# is off unless enabled via services.hermenix.devices.
|
||||
++ lib.optionals cfg.devices.rng.enable [
|
||||
"-device virtio-rng-pci"
|
||||
]
|
||||
++ lib.optionals cfg.devices.vsock.enable [
|
||||
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
|
||||
]
|
||||
++ lib.optionals cfg.devices.video.enable [
|
||||
"-vga ${cfg.devices.video.model}"
|
||||
]
|
||||
++ lib.optionals cfg.devices.balloon.enable [
|
||||
"-device virtio-balloon-pci"
|
||||
]
|
||||
++ lib.optionals cfg.devices.watchdog.enable [
|
||||
"-device ${cfg.devices.watchdog.model}"
|
||||
"-watchdog-action ${cfg.devices.watchdog.action}"
|
||||
]
|
||||
++ lib.optionals cfg.devices.input.enable (
|
||||
lib.optional cfg.devices.input.keyboard "-device virtio-keyboard-pci"
|
||||
++ lib.optional cfg.devices.input.tablet "-device virtio-tablet-pci"
|
||||
)
|
||||
++ lib.optionals cfg.devices.audio.enable (
|
||||
[ "-audiodev ${cfg.devices.audio.backend},id=audio0" ]
|
||||
++ (
|
||||
if cfg.devices.audio.model == "virtio-sound" then
|
||||
[ "-device virtio-sound-pci,audiodev=audio0" ]
|
||||
else
|
||||
[
|
||||
"-device intel-hda"
|
||||
"-device hda-duplex,audiodev=audio0"
|
||||
]
|
||||
)
|
||||
)
|
||||
++ lib.optionals cfg.devices.tpm.enable [
|
||||
"-chardev socket,id=chrtpm,path=${runDir}/swtpm.sock"
|
||||
"-tpmdev emulator,id=tpm0,chardev=chrtpm"
|
||||
"-device tpm-crb,tpmdev=tpm0"
|
||||
]
|
||||
++ lib.optionals (cfg.devices.cdroms != [ ]) [
|
||||
"-device virtio-scsi-pci,id=scsi0"
|
||||
]
|
||||
++ lib.concatLists (
|
||||
lib.imap0 (i: c: [
|
||||
"-drive if=none,id=cd${toString i},media=cdrom,readonly=on,file=${c.path}"
|
||||
"-device scsi-cd,drive=cd${toString i},bus=scsi0.0"
|
||||
]) cfg.devices.cdroms
|
||||
)
|
||||
++ lib.concatMap (d: [
|
||||
"-drive if=none,id=${d.name},format=raw,file=${d.path},cache=${d.cache},aio=${d.aio}${lib.optionalString d.readonly ",readonly=on"}"
|
||||
"-device virtio-blk-pci,drive=${d.name}"
|
||||
]) cfg.devices.extraDisks
|
||||
++ lib.optionals usbControllerOn [
|
||||
"-device qemu-xhci,id=xhci"
|
||||
]
|
||||
++ lib.optionals cfg.disk.quiesce [
|
||||
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
|
||||
"-device virtio-serial-pci"
|
||||
@@ -693,9 +997,9 @@ in
|
||||
# Passed-through PCI devices. The host cannot use these while the VM
|
||||
# is running.
|
||||
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
|
||||
# A USB controller, only if something needs plugging into it.
|
||||
++ lib.optional anyUsb "-device qemu-xhci,id=xhci"
|
||||
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
|
||||
# Raw escape hatch. Unvalidated - prefer the typed devices.* options.
|
||||
++ cfg.extraQemuArgs
|
||||
);
|
||||
|
||||
ExecStop = stopScript;
|
||||
@@ -751,6 +1055,55 @@ in
|
||||
};
|
||||
};
|
||||
|
||||
# TPM 2.0 emulator (swtpm), started before the VM. QEMU connects to its
|
||||
# control socket in the runtime dir; state persists under its StateDirectory.
|
||||
systemd.services.hermes-vm-swtpm = mkIf cfg.devices.tpm.enable {
|
||||
description = "swtpm TPM emulator for the Hermes VM";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [ "hermes-vm.service" ];
|
||||
requiredBy = [ "hermes-vm.service" ];
|
||||
serviceConfig = {
|
||||
User = vmUser;
|
||||
Group = vmUser;
|
||||
RuntimeDirectory = "hermes-vm";
|
||||
RuntimeDirectoryMode = "0750";
|
||||
StateDirectory = baseNameOf cfg.devices.tpm.stateDir;
|
||||
Restart = "always";
|
||||
RestartSec = 5;
|
||||
|
||||
DevicePolicy = "closed";
|
||||
NoNewPrivileges = true;
|
||||
PrivateTmp = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
ProtectProc = "invisible";
|
||||
ProcSubset = "pid";
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHostname = true;
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
RestrictAddressFamilies = [ "AF_UNIX" ];
|
||||
LockPersonality = true;
|
||||
SystemCallArchitectures = "native";
|
||||
SystemCallFilter = [ "@system-service" ];
|
||||
MemoryDenyWriteExecute = true;
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
AmbientCapabilities = [ "" ];
|
||||
};
|
||||
script = ''
|
||||
exec ${pkgs.swtpm}/bin/swtpm socket \
|
||||
--tpm2 \
|
||||
--tpmstate dir=${cfg.devices.tpm.stateDir} \
|
||||
--ctrl type=unixio,path=${runDir}/swtpm.sock \
|
||||
--flags not-need-init,startup-clear
|
||||
'';
|
||||
};
|
||||
|
||||
# Tap device, created as root, owned by the VM user, bridged by networkd.
|
||||
systemd.services.hermes-vm-tap = {
|
||||
description = "TAP interface for the Hermes VM";
|
||||
|
||||
Reference in new issue
Block a user