splitting host.nix

This commit is contained in:
ari committed 2026-10-10 22:56:24 -04:00
1 parent 053b9c4c95
commit f1622fae84
16 files changed
+1706 -1471

No files matched your search

+71
View File
@@ -0,0 +1,71 @@
{
config,
lib,
pkgs,
...
}:
let
inherit (import ./common.nix { inherit config lib pkgs; }) cfg;
in
{
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = cfg.backup.enable -> (cfg.backup.repository != "");
message = "services.hermenix.backup.repository must be set when backups are enabled.";
}
{
assertion = cfg.devices.vsock.enable || !cfg.backup.enable;
message = ''
services.hermenix.backup.enable needs the vsock (the backup runs over
SSH multiplexed on it). Leave devices.vsock.enable at its default, or
turn backups off.
'';
}
];
systemd.services.hermes-vm-backup = lib.mkIf cfg.backup.enable {
description = "Back up Hermes state";
path = with pkgs; [
restic
openssh
socat
gnutar
];
serviceConfig = {
Type = "oneshot";
EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile;
};
script = ''
set -euo pipefail
export RESTIC_REPOSITORY=${lib.escapeShellArg cfg.backup.repository}
export RESTIC_PASSWORD_FILE=${cfg.backup.passwordFile}
restic snapshots >/dev/null 2>&1 || restic init
ssh -o ProxyCommand="socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22" \
-o StrictHostKeyChecking=accept-new \
hermes@${cfg.name} \
'tar -C /var/lib/hermes \
--exclude=./.cache \
--exclude=./workspace/**/.direnv \
--exclude=./workspace/**/result \
-cf - .' \
| restic backup --stdin --stdin-filename hermes-state.tar \
--tag hermes --host ${cfg.name}
restic forget --tag hermes --host ${cfg.name} --prune ${lib.concatStringsSep " " cfg.backup.keep}
'';
};
systemd.timers.hermes-vm-backup = lib.mkIf cfg.backup.enable {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.backup.interval;
Persistent = true;
RandomizedDelaySec = "30m";
};
};
};
}
+51
View File
@@ -0,0 +1,51 @@
{ lib, ... }:
let
inherit (lib)
mkOption
mkEnableOption
types
;
in
{
options.services.hermenix.backup = {
enable = mkEnableOption "restic backups of the guest's state";
repository = mkOption {
type = types.str;
default = "";
example = "sftp:backup@nas:/hermes";
description = "Any restic repository string.";
};
passwordFile = mkOption {
type = types.path;
default = "/var/lib/secrets/hermes-restic";
description = "File containing the restic repository password, mode 0400.";
};
environmentFile = mkOption {
type = types.nullOr types.path;
default = null;
description = "Optional file with credentials for the backend (S3 keys and so on).";
};
interval = mkOption {
type = types.str;
default = "daily";
};
keep = mkOption {
type = types.listOf types.str;
default = [
"--keep-daily"
"14"
"--keep-weekly"
"8"
"--keep-monthly"
"12"
];
description = "Arguments passed to restic forget.";
};
};
}
+165
View File
@@ -0,0 +1,165 @@
{
config,
lib,
pkgs,
...
}:
let
inherit (import ./common.nix { inherit config lib pkgs; })
cfg
net
bridgeMark
netmaskFromPrefix
;
in
{
config = lib.mkIf (cfg.enable && net.bridged) {
assertions = [
{
assertion = net.lanAddress != "" && net.hostLanAddress != "" && net.lanGateway != "";
message = ''
services.hermenix.network.bridged = true requires a fully static
LAN setup (DHCP is no longer supported in bridged mode). Set
network.lanAddress (the guest's fixed LAN address),
network.hostLanAddress (this host's own LAN address, which moves
onto the bridge) and network.lanGateway (your router's address).
'';
}
];
# mkDefault: operator nameservers win if set.
networking.nameservers = lib.mkDefault (
if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns
);
systemd.network.networks."12-${net.externalInterface}" = {
matchConfig.Name = net.externalInterface;
networkConfig.Bridge = net.bridge;
};
# Bridge would inherit tap MAC (first port) and collide with guest — clone uplink MAC.
systemd.services.hermes-bridge-mac = {
description = "Clone the uplink MAC onto the hermenix bridge";
after = [
"systemd-networkd.service"
"sys-subsystem-net-devices-${net.bridge}.device"
"sys-subsystem-net-devices-${net.externalInterface}.device"
];
wants = [ "systemd-networkd.service" ];
before = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
unitConfig.ConditionPathExists = "/sys/class/net/${net.externalInterface}";
script = ''
set -euo pipefail
for _ in 1 2 3 4 5 6 7 8 9 10; do
[ -e "/sys/class/net/${net.bridge}" ] && break
sleep 0.5
done
[ -e "/sys/class/net/${net.bridge}" ] || exit 1
mac="$(${pkgs.coreutils}/bin/cat /sys/class/net/${net.externalInterface}/address)"
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} down
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} address "$mac"
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} up
${pkgs.systemd}/bin/networkctl reconfigure ${net.bridge} || true
'';
};
# Single-address DHCP on the bridge (MAC-keyed). Broadcasts stay on-bridge.
systemd.services.hermes-lan-dhcp = {
description = "Static-lease DHCP for the bridged hermenix guest";
after = [
"systemd-networkd.service"
"sys-subsystem-net-devices-${net.bridge}.device"
];
wants = [ "systemd-networkd.service" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "simple";
Restart = "on-failure";
RestartSec = 5;
};
script = ''
for _ in 1 2 3 4 5 6 7 8 9 10; do
[ -e "/sys/class/net/${net.bridge}" ] && break
sleep 0.5
done
exec ${pkgs.dnsmasq}/bin/dnsmasq --keep-in-foreground \
--conf-file=/dev/null \
--interface=${net.bridge} \
--bind-dynamic \
--port=0 \
--dhcp-authoritative \
--dhcp-range=${net.lanAddress},${net.lanAddress},${netmaskFromPrefix net.lanPrefixLength},12h \
--dhcp-host=${net.mac},${net.lanAddress} \
--dhcp-option=option:router,${net.lanGateway}
'';
};
# No DNAT when bridged. Scoped masquerade covers service-net fallback if DHCP lapses.
networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null) {
family = "ip";
content = ''
chain postrouting {
type nat hook postrouting priority srcnat;
iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade
}
'';
};
networking.nftables.tables."hermenix-bridge" = {
family = "bridge";
content = ''
chain prerouting {
type filter hook prerouting priority filter; policy accept;
iifname "${net.tapId}" meta mark set ${toString bridgeMark}
# Guest DHCP stays on-bridge only (no race with LAN DHCP).
iifname "${net.tapId}" oifname "${net.externalInterface}" ether type ip ip protocol udp udp sport 68 udp dport 67 drop
iifname "${net.externalInterface}" oifname "${net.tapId}" ether type ip ip protocol udp udp sport 67 udp dport 68 drop
}
chain forward {
type filter hook forward priority filter; policy accept;
oifname "${net.tapId}" accept
''
+ lib.concatMapStrings (s: ''
iifname "${net.tapId}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
'') cfg.lanAccess
+ (
if cfg.allowLan then
''
iifname "${net.tapId}" accept
''
else
''
iifname "${net.tapId}" ip daddr {
10.0.0.0/8,
172.16.0.0/12,
192.168.0.0/16,
169.254.0.0/16,
127.0.0.0/8,
100.64.0.0/10
} drop
iifname "${net.tapId}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
iifname "${net.tapId}" accept
''
)
+ ''
}
'';
};
# Guest→host holes (bridged: match the tap mark; the guest may use its LAN IP).
networking.firewall.extraInputRules =
lib.optionalString cfg.hostCache.enable ''
meta mark ${toString bridgeMark} tcp dport ${toString cfg.hostCache.port} accept
''
+ lib.concatMapStrings (s: ''
meta mark ${toString bridgeMark} ${s.proto} dport ${toString s.port} accept
'') cfg.hostAccess;
};
}
+77
View File
@@ -0,0 +1,77 @@
{ lib, ... }:
let
inherit (lib)
mkOption
types
;
in
{
options.services.hermenix.network = {
lanAddress = mkOption {
type = types.str;
default = "";
example = "10.0.12.50";
description = "The guest's fixed address on your LAN. Required when `bridged = true`.";
};
hostLanAddress = mkOption {
type = types.str;
default = "";
example = "10.0.12.10";
description = ''
This host's own address on your LAN. When `bridged` is enabled the
uplink becomes a bridge port and this address moves onto the bridge
(which clones the uplink's MAC, so the host keeps its LAN identity).
Required when `bridged = true`.
'';
};
lanGateway = mkOption {
type = types.str;
default = "";
example = "10.0.12.1";
description = ''
Your LAN's router. Used as the default gateway for the guest and,
while `bridged`, for this host's bridge too (DNS unless `lanDns` is
set). Required when `bridged = true`.
'';
};
lanPrefixLength = mkOption {
type = types.int;
default = 24;
description = "Prefix length of your LAN subnet (netmask).";
};
lanDns = mkOption {
type = types.listOf types.str;
default = [ ];
example = [ "1.1.1.1" ];
description = ''
DNS servers for this host while `bridged` (the bridge no longer
takes a DHCP lease). Defaults to the LAN gateway.
'';
};
bridged = mkOption {
type = types.bool;
default = false;
description = ''
Put the guest directly on your LAN instead of behind the host's
NAT. The bridge absorbs `externalInterface`, so any device on the
LAN can reach any port the guest exposes.
Addressing is static from the operator's point of view: you pick
fixed addresses for the guest (`lanAddress`), this host
(`hostLanAddress` — it moves onto the bridge), and the LAN
gateway (`lanGateway`). A DHCP server this module runs on the
bridge hands `lanAddress` to the guest, keyed to `network.mac`,
so the guest stays a plain DHCP client and never needs to know
its own address. That lease never leaves the bridge; the LAN's
own DHCP cannot race it.
Inbound is no longer limited to `publish` (which is ignored in
this mode): the LAN may reach every guest port. Outbound is still
governed by `lanAccess` / `allowLan`, and `hostAccess` plus the
host cache still work over the host-guest service net
(`hostAddress` stays configured on the bridge).
'';
};
};
}
+85
View File
@@ -0,0 +1,85 @@
{
config,
lib,
pkgs,
}:
let
cfg = config.services.hermenix;
net = cfg.network;
pt = cfg.passthrough;
vmUser = "hermes-vm";
runDir = "/run/hermes-vm";
stateDir = "/var/lib/hermes-vm";
systemImage = "${cfg.disk.baseDir}/system/system.raw";
stateImage = "${cfg.disk.baseDir}/state/state.raw";
snapDir = "${cfg.disk.baseDir}/snapshots";
anyPci = pt.pci != [ ];
anyUsb = pt.usb != [ ];
earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci);
usbControllerOn =
cfg.devices.usbController == "on" || (cfg.devices.usbController == "auto" && anyUsb);
# 18501 == 0x4845 ("HE"); marks guest traffic from the tap (bridged mode).
bridgeMark = 18501;
# dnsmasq needs an explicit mask for a one-address DHCP range.
netmaskFromPrefix =
p:
lib.concatStringsSep "." (
lib.genList (
b:
toString (
builtins.elemAt [
0
128
192
224
240
248
252
254
255
] (lib.max 0 (lib.min 8 (p - b * 8)))
)
) 4
);
stopScript = pkgs.writeShellScript "hermes-vm-stop" ''
set -eu
printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \
| ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0
'';
qga =
verb:
pkgs.writeShellScript "hermes-vm-fs${verb}" ''
set -eu
printf '%s' '{"execute":"guest-fs${verb}"}' \
| ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true
'';
in
{
inherit
cfg
net
pt
vmUser
runDir
stateDir
systemImage
stateImage
snapDir
anyPci
anyUsb
earlyIds
usbControllerOn
bridgeMark
netmaskFromPrefix
stopScript
qga
;
}
+127
View File
@@ -0,0 +1,127 @@
{
config,
lib,
pkgs,
...
}:
let
inherit (import ./common.nix { inherit config lib pkgs; }) cfg net;
in
{
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = net.externalInterface != null;
message = "services.hermenix.network.externalInterface must be set.";
}
];
networking.useNetworkd = true;
systemd.network.netdevs."10-${net.bridge}".netdevConfig = {
Kind = "bridge";
Name = net.bridge;
};
systemd.network.networks."10-${net.bridge}" = {
matchConfig.Name = net.bridge;
# hostAddress = service net; bridged also adds hostLanAddress (static only).
address = [
"${net.hostAddress}/${toString net.prefixLength}"
]
++ lib.optional net.bridged "${net.hostLanAddress}/${toString net.lanPrefixLength}";
networkConfig = {
ConfigureWithoutCarrier = true;
IPv4Forwarding = true;
}
// lib.optionalAttrs net.bridged {
Gateway = net.lanGateway;
DNS = if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns;
};
};
systemd.network.networks."11-${net.tapId}" = {
matchConfig.Name = net.tapId;
networkConfig.Bridge = net.bridge;
};
# mkOverride 99: same keys as networking.nat, win over weaker defaults.
boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true;
boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true;
# Explicit nft nat (not networking.nat). DNAT iifname != bridge so extra paths still match.
networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null && !net.bridged) {
family = "ip";
content = ''
chain prerouting {
type nat hook prerouting priority dstnat;
''
+ lib.concatMapStrings (s: ''
iifname != "${net.bridge}" ${s.proto} dport ${toString s.hostPort} dnat to ${net.guestAddress}:${toString s.guestPort}
'') cfg.publish
+ ''
}
chain postrouting {
type nat hook postrouting priority srcnat;
iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade
}
'';
};
networking.nftables.enable = true;
networking.firewall.enable = true;
networking.firewall.filterForward = true;
# Guest→host holes (NAT mode: match the service-net address).
networking.firewall.extraInputRules = lib.mkIf (!net.bridged) (
lib.optionalString cfg.hostCache.enable ''
iifname "${net.bridge}" ip saddr ${net.guestAddress} tcp dport ${toString cfg.hostCache.port} accept
''
+ lib.concatMapStrings (s: ''
iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept
'') cfg.hostAccess
);
# Shared by both modes; the bridged LAN is L2 (see bridge.nix).
networking.firewall.extraForwardRules = ''
iifname != "${net.bridge}" accept
ct state established,related accept
''
+ lib.concatMapStrings (s: ''
iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
'') cfg.lanAccess
+ lib.optionalString (!net.bridged) (
lib.concatMapStrings (s: ''
oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept
'') cfg.publish
)
+ (
if cfg.allowLan then
''
iifname "${net.bridge}" accept
''
else
''
iifname "${net.bridge}" ip daddr {
10.0.0.0/8,
172.16.0.0/12,
192.168.0.0/16,
169.254.0.0/16,
127.0.0.0/8,
100.64.0.0/10
} drop
iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
iifname "${net.bridge}" accept
''
);
services.nix-serve = lib.mkIf cfg.hostCache.enable {
enable = true;
package = pkgs.nix-serve-ng;
bindAddress = net.hostAddress;
inherit (cfg.hostCache) port secretKeyFile;
openFirewall = false;
};
};
}
+163
View File
@@ -0,0 +1,163 @@
{ lib, ... }:
let
inherit (lib)
mkOption
types
;
in
{
options.services.hermenix = {
hostCache = {
enable = mkOption {
type = types.bool;
default = true;
description = ''
Serve the host's /nix/store to the guest as a read-only binary
cache, so the guest doesn't re-download from the internet what the
host already has.
This is the ONLY route from guest to host in this design. Turning it
off leaves the guest with literally no reachable host service, at
the cost of bandwidth on every guest build.
Note the guest can read every store path on the host through this.
Store paths are world-readable anyway, but check nothing sensitive
is baked into your host's store.
'';
};
port = mkOption {
type = types.port;
default = 5000;
};
secretKeyFile = mkOption {
type = types.path;
default = "/var/lib/secrets/hermes-cache.secret";
description = "Generate with nix-store --generate-binary-cache-key. See README.";
};
};
allowLan = mkOption {
type = types.bool;
default = false;
description = ''
Blanket permission for the guest to reach RFC1918 space. Almost always
the wrong tool - use lanAccess to open single endpoints instead.
'';
};
lanAccess = mkOption {
default = [ ];
example = [
{
address = "192.168.1.50";
port = 11434;
}
];
description = ''
Specific endpoints elsewhere on your network that the guest may reach,
punched through the default RFC1918 drop. One entry per service.
'';
type = types.listOf (
types.submodule {
options = {
address = mkOption { type = types.str; };
port = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
}
);
};
hostAccess = mkOption {
default = [ ];
example = [ { port = 8188; } ];
description = ''
Ports on this host itself that the guest may reach. The service must
also be bound to an address the guest can see - see the README.
'';
type = types.listOf (
types.submodule {
options = {
port = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
}
);
};
publish = mkOption {
default = [ ];
example = [
{
hostPort = 8080;
guestPort = 8080;
}
];
description = ''
Guest ports to publish on the host's uplink, reachable from the wider
network. You do not need this to reach the guest from the host itself.
'';
type = types.listOf (
types.submodule {
options = {
hostPort = mkOption { type = types.port; };
guestPort = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
}
);
};
network = {
bridge = mkOption {
type = types.str;
default = "hermes-br";
};
tapId = mkOption {
type = types.str;
default = "vm-hermes";
};
mac = mkOption {
type = types.str;
default = "02:00:00:7e:00:01";
};
hostAddress = mkOption {
type = types.str;
default = "10.77.0.1";
};
guestAddress = mkOption {
type = types.str;
default = "10.77.0.2";
};
prefixLength = mkOption {
type = types.int;
default = 24;
};
externalInterface = mkOption {
type = types.nullOr types.str;
default = null;
example = "eno1";
description = "Host uplink. Required.";
};
};
};
}
+35
View File
@@ -0,0 +1,35 @@
{
config,
lib,
pkgs,
...
}:
let
inherit (import ./common.nix { inherit config lib pkgs; }) cfg;
in
{
config = lib.mkIf cfg.enable {
environment.systemPackages = [
(pkgs.writeTextFile {
name = "hermenix-completions";
destination = "/share/nushell/vendor/autoload/hermenix.nu";
text = builtins.readFile ../nu/completions.nu;
})
(pkgs.symlinkJoin {
name = "hermenix";
paths = [ pkgs.hermenix ];
nativeBuildInputs = [ pkgs.makeWrapper ];
postBuild = ''
wrapProgram $out/bin/hermenix \
--set HERMES_VSOCK_CID ${toString cfg.vsockCid} \
--set HERMES_GUEST_NAME ${cfg.name} \
--set HERMES_DISK_BASE_DIR ${cfg.disk.baseDir} \
--set HERMES_HOME_DIR ${cfg.hermesHome} \
--set HERMES_GATEWAY_UNIT ${cfg.gatewayUnit}
'';
})
];
};
}
+90
View File
@@ -0,0 +1,90 @@
{
config,
lib,
pkgs,
...
}:
let
inherit (import ./common.nix { inherit config lib pkgs; })
cfg
pt
anyPci
anyUsb
earlyIds
vmUser
;
in
{
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = anyPci -> (pt.iommu != null);
message = ''
services.hermenix.passthrough.pci needs passthrough.iommu set to
"intel" or "amd". Without the IOMMU, a passed-through device can
read host memory directly and the VM boundary is meaningless.
'';
}
{
assertion = cfg.devices.usbController != "off" || !anyUsb;
message = ''
services.hermenix.passthrough.usb is set but
services.hermenix.devices.usbController is "off", so those devices
have no controller to attach to.
'';
}
];
boot.kernelParams = lib.optionals (pt.iommu != null) [
(if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on")
"iommu=pt"
];
boot.initrd.kernelModules = lib.optionals anyPci [
"vfio_pci"
"vfio"
"vfio_iommu_type1"
];
boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) ''
options vfio-pci ids=${lib.concatStringsSep "," earlyIds}
'';
systemd.services.hermes-vm-vfio = lib.mkIf anyPci {
description = "Bind passthrough PCI devices to vfio-pci";
before = [ "hermes-vm.service" ];
requiredBy = [ "hermes-vm.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
path = [ pkgs.kmod ];
script = ''
modprobe vfio-pci
''
+ lib.concatMapStrings (d: ''
dev=/sys/bus/pci/devices/${d.address}
if [ ! -e "$dev" ]; then
echo "${d.address} is not present on this machine" >&2
exit 1
fi
cur=none
[ -e "$dev/driver" ] && cur=$(basename "$(readlink -f "$dev/driver")")
if [ "$cur" != "vfio-pci" ]; then
[ "$cur" != "none" ] && echo ${d.address} > "$dev/driver/unbind"
echo vfio-pci > "$dev/driver_override"
echo ${d.address} > /sys/bus/pci/drivers_probe
fi
'') pt.pci;
};
services.udev.extraRules =
lib.optionalString anyPci ''
SUBSYSTEM=="vfio", OWNER="${vmUser}", GROUP="${vmUser}", MODE="0660"
''
+ lib.concatMapStrings (u: ''
SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660"
'') pt.usb;
};
}
+101
View File
@@ -0,0 +1,101 @@
{ lib, ... }:
let
inherit (lib)
mkOption
types
;
in
{
options.services.hermenix.passthrough = {
iommu = mkOption {
type = types.nullOr (
types.enum [
"intel"
"amd"
]
);
default = null;
description = ''
Enable IOMMU kernel parameters for your CPU vendor. Required for any
PCI passthrough. The IOMMU is the only thing stopping a passed-through
device from reading host memory, so this is not optional.
'';
};
pci = mkOption {
default = [ ];
example = [
{
address = "0000:01:00.0";
id = "10de:2204";
}
{
address = "0000:01:00.1";
id = "10de:1aef";
}
];
description = ''
PCI devices handed to the guest via VFIO. The host loses them
entirely while the VM runs.
You must pass every function of the device and, in practice, every
device in its IOMMU group. A GPU is usually at least two entries:
the graphics function and its HDMI audio function.
Set `id` (vendor:device) to have vfio-pci claim the device in the
initrd, before a host driver can. Strongly recommended for GPUs;
unbinding an active amdgpu or nvidia driver later is unreliable.
'';
type = types.listOf (
types.submodule {
options = {
address = mkOption {
type = types.str;
example = "0000:01:00.0";
description = "Full PCI address, as shown by lspci -D.";
};
id = mkOption {
type = types.nullOr types.str;
default = null;
example = "10de:2204";
description = "vendor:device, from lspci -nn. Enables early binding.";
};
};
}
);
};
usb = mkOption {
default = [ ];
example = [
{
vendorId = "8087";
productId = "0026";
}
];
description = ''
USB devices forwarded to the guest, matched by vendor and product
ID from lsusb. Integrated Bluetooth radios are USB devices and go
here too, though the host driver must be kept off them.
QEMU resolves these at start, so a device that is unplugged when the
VM boots will not appear later without a restart.
'';
type = types.listOf (
types.submodule {
options = {
vendorId = mkOption {
type = types.str;
example = "046d";
};
productId = mkOption {
type = types.str;
example = "c52b";
};
};
}
);
};
};
}
+71
View File
@@ -0,0 +1,71 @@
{ lib, ... }:
let
inherit (lib)
mkOption
mkEnableOption
types
;
in
{
options.services.hermenix = {
enable = mkEnableOption "the Hermes Agent VM";
name = mkOption {
type = types.str;
default = "hermes";
};
authorizedKeys = mkOption {
type = types.listOf types.str;
default = [ ];
};
gatewayUnit = mkOption {
type = types.str;
default = "hermes-agent";
description = ''
The systemd *user* unit that `hermes gateway install` created in the
guest. Check with `hermenix ctl units` after installing; upstream has
used more than one name for this.
'';
};
hermesHome = mkOption {
type = types.str;
# Guest path (not host disk.baseDir).
default = "/var/lib/hermes/.hermes";
description = ''
Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/,
state.db, and so on) lives inside the guest. Defaults to the seeded
layout under the state disk mount (`/var/lib/hermes/.hermes`).
Override when carrying over an existing flat HERMES_HOME from
elsewhere (e.g. a prior Docker deployment) that you're placing
directly at the root of the state disk instead of under .hermes —
see `hermenix migrate --help`.
This only changes where the host-side `hermenix ctl` commands (soul,
memory, conf, secrets, reseed, pull) look. It does not change what
the gateway process itself uses — export HERMES_HOME to match inside
the guest when you install it.
'';
};
vsockCid = mkOption {
type = types.int;
default = 77;
description = "Host reaches the guest over vsock, so no inbound network path is needed.";
};
version = mkOption {
type = types.str;
default = "dev";
description = ''
"YYYY.MM.DD.hh.mm-<commit-sha>" reported by `hermenix version`.
`version` subcommand. Normally supplied for free by this flake's own
`nixosModules.default` (from its git metadata) - the "dev" default
here only shows up if this module is imported some other way.
'';
};
};
}
+57
View File
@@ -0,0 +1,57 @@
{
config,
lib,
pkgs,
...
}:
let
inherit (import ./common.nix { inherit config lib pkgs; })
cfg
snapDir
qga
;
in
{
config = lib.mkIf cfg.enable {
systemd.services.hermes-vm-snapshot = lib.mkIf cfg.snapshots.enable {
description = "Snapshot the Hermes VM state disk";
path = with pkgs; [
btrfs-progs
coreutils
socat
];
serviceConfig.Type = "oneshot";
script = ''
set -euo pipefail
stamp=$(date +%Y%m%d-%H%M%S)
mkdir -p ${snapDir}
${lib.optionalString cfg.disk.quiesce ''
${qga "freeze"}
trap '${qga "thaw"}' EXIT
''}
btrfs subvolume snapshot -r ${cfg.disk.baseDir}/state ${snapDir}/state-$stamp
${lib.optionalString cfg.disk.quiesce ''
${qga "thaw"}
trap - EXIT
''}
# Live snapshots reintroduce CoW on nodatacow images — keep retention tight.
ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \
| while read -r old; do btrfs subvolume delete "$old"; done
'';
};
systemd.timers.hermes-vm-snapshot = lib.mkIf cfg.snapshots.enable {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.snapshots.interval;
Persistent = true;
RandomizedDelaySec = "5m";
};
};
};
}
+37
View File
@@ -0,0 +1,37 @@
{ lib, ... }:
let
inherit (lib)
mkOption
types
;
in
{
options.services.hermenix.snapshots = {
enable = mkOption {
type = types.bool;
default = true;
description = ''
Periodic btrfs snapshots of the state disk. The agent can persist
changes, which means it can persist a mistake, and these are the
fastest way back.
'';
};
interval = mkOption {
type = types.str;
default = "hourly";
description = "systemd calendar expression.";
};
keep = mkOption {
type = types.ints.positive;
default = 48;
description = ''
How many to retain. Keep this tighter than you would on ZFS: the
state image is nodatacow, and every live snapshot reintroduces
copy-on-write for the blocks it still references.
'';
};
};
}
+281
View File
@@ -0,0 +1,281 @@
{
config,
lib,
pkgs,
...
}:
let
inherit (import ./common.nix { inherit config lib pkgs; })
cfg
net
pt
vmUser
runDir
stateDir
systemImage
stateImage
anyPci
anyUsb
usbControllerOn
stopScript
;
in
{
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = !cfg.devices.tpm.enable || lib.hasPrefix "/var/lib/" cfg.devices.tpm.stateDir;
message = ''
services.hermenix.devices.tpm.stateDir must live under /var/lib/ so
the swtpm unit's StateDirectory can create and own it.
'';
}
];
users.users.${vmUser} = {
isSystemUser = true;
group = vmUser;
description = "Runs the Hermes guest VM";
};
users.groups.${vmUser} = { };
systemd.services.hermes-vm = {
description = "Hermes Agent VM";
wantedBy = [ "multi-user.target" ];
after = [
"network-online.target"
"hermes-vm-tap.service"
]
++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service";
requires = [
"hermes-vm-tap.service"
]
++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service";
wants = [ "network-online.target" ];
preStart = ''
for img in ${systemImage} ${stateImage}; do
if [ ! -e "$img" ]; then
echo "$img is missing. Run 'hermenix setup' first." >&2
exit 1
fi
done
if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then
install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd
fi
'';
serviceConfig = {
User = vmUser;
Group = vmUser;
SupplementaryGroups = [ "kvm" ];
RuntimeDirectory = "hermes-vm";
ReadWritePaths = [ cfg.disk.baseDir ];
StateDirectory = "hermes-vm";
RuntimeDirectoryMode = "0750";
ExecStart = lib.concatStringsSep " " (
[
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
"-name ${cfg.name}"
"-machine q35,accel=kvm,smm=on"
"-cpu host"
"-smp ${toString cfg.vcpu}"
"-m ${toString cfg.mem}"
"-nodefaults"
"-no-user-config"
"-display none"
"-serial stdio"
"-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
"-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd"
# aio=threads+writeback: images are files, not blockdevs.
"-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=system"
"-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=state"
# vhost=off: keep packet handling out of the host kernel.
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
]
++ lib.optionals cfg.devices.rng.enable [
"-device virtio-rng-pci"
]
++ lib.optionals cfg.devices.vsock.enable [
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
]
++ lib.optionals cfg.devices.video.enable [
"-vga ${cfg.devices.video.model}"
]
++ lib.optionals cfg.devices.balloon.enable [
"-device virtio-balloon-pci"
]
++ lib.optionals cfg.devices.watchdog.enable [
"-device ${cfg.devices.watchdog.model}"
"-watchdog-action ${cfg.devices.watchdog.action}"
]
++ lib.optionals cfg.devices.input.enable (
lib.optional cfg.devices.input.keyboard "-device virtio-keyboard-pci"
++ lib.optional cfg.devices.input.tablet "-device virtio-tablet-pci"
)
++ lib.optionals cfg.devices.audio.enable (
[ "-audiodev ${cfg.devices.audio.backend},id=audio0" ]
++ (
if cfg.devices.audio.model == "virtio-sound" then
[ "-device virtio-sound-pci,audiodev=audio0" ]
else
[
"-device intel-hda"
"-device hda-duplex,audiodev=audio0"
]
)
)
++ lib.optionals cfg.devices.tpm.enable [
"-chardev socket,id=chrtpm,path=${runDir}/swtpm.sock"
"-tpmdev emulator,id=tpm0,chardev=chrtpm"
"-device tpm-crb,tpmdev=tpm0"
]
++ lib.optionals (cfg.devices.cdroms != [ ]) [
"-device virtio-scsi-pci,id=scsi0"
]
++ lib.concatLists (
lib.imap0 (i: c: [
"-drive if=none,id=cd${toString i},media=cdrom,readonly=on,file=${c.path}"
"-device scsi-cd,drive=cd${toString i},bus=scsi0.0"
]) cfg.devices.cdroms
)
++ lib.concatMap (d: [
"-drive if=none,id=${d.name},format=raw,file=${d.path},cache=${d.cache},aio=${d.aio}${lib.optionalString d.readonly ",readonly=on"}"
"-device virtio-blk-pci,drive=${d.name}"
]) cfg.devices.extraDisks
++ lib.optionals usbControllerOn [
"-device qemu-xhci,id=xhci"
]
++ lib.optionals cfg.disk.quiesce [
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
"-device virtio-serial-pci"
"-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
]
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
++ cfg.extraQemuArgs
);
ExecStop = stopScript;
TimeoutStopSec = 90;
Restart = "always";
RestartSec = 5;
DevicePolicy = "closed";
DeviceAllow = [
"/dev/kvm rw"
"/dev/net/tun rw"
"/dev/vhost-vsock rw"
]
++ lib.optionals anyPci [ "char-vfio rw" ]
++ lib.optionals anyUsb [ "char-usb_device rw" ];
# VFIO pins guest RAM; default memlock is too low.
LimitMEMLOCK = lib.mkIf anyPci "infinity";
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
ProtectProc = "invisible";
ProcSubset = "pid";
PrivateTmp = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHostname = true;
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
RestrictAddressFamilies = [
"AF_UNIX"
"AF_NETLINK"
"AF_VSOCK"
];
LockPersonality = true;
SystemCallArchitectures = "native";
SystemCallFilter = [
"@system-service"
"~@resources"
"~@obsolete"
];
# Safe under KVM (no JIT). First flip if QEMU won't start.
MemoryDenyWriteExecute = true;
CapabilityBoundingSet = [ "" ];
AmbientCapabilities = [ "" ];
};
};
systemd.services.hermes-vm-swtpm = lib.mkIf cfg.devices.tpm.enable {
description = "swtpm TPM emulator for the Hermes VM";
wantedBy = [ "multi-user.target" ];
before = [ "hermes-vm.service" ];
requiredBy = [ "hermes-vm.service" ];
serviceConfig = {
User = vmUser;
Group = vmUser;
RuntimeDirectory = "hermes-vm";
RuntimeDirectoryMode = "0750";
StateDirectory = baseNameOf cfg.devices.tpm.stateDir;
Restart = "always";
RestartSec = 5;
DevicePolicy = "closed";
NoNewPrivileges = true;
PrivateTmp = true;
ProtectSystem = "strict";
ProtectHome = true;
ProtectProc = "invisible";
ProcSubset = "pid";
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHostname = true;
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
RestrictAddressFamilies = [ "AF_UNIX" ];
LockPersonality = true;
SystemCallArchitectures = "native";
SystemCallFilter = [ "@system-service" ];
MemoryDenyWriteExecute = true;
CapabilityBoundingSet = [ "" ];
AmbientCapabilities = [ "" ];
};
script = ''
exec ${pkgs.swtpm}/bin/swtpm socket \
--tpm2 \
--tpmstate dir=${cfg.devices.tpm.stateDir} \
--ctrl type=unixio,path=${runDir}/swtpm.sock \
--flags not-need-init,startup-clear
'';
};
systemd.services.hermes-vm-tap = {
description = "TAP interface for the Hermes VM";
after = [ "systemd-networkd.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
path = [ pkgs.iproute2 ];
script = ''
ip link show ${net.tapId} >/dev/null 2>&1 \
|| ip tuntap add ${net.tapId} mode tap user ${vmUser}
ip link set ${net.tapId} up
'';
preStop = ''
${pkgs.iproute2}/bin/ip link del ${net.tapId} || true
'';
};
};
}
+278
View File
@@ -0,0 +1,278 @@
{ lib, ... }:
let
inherit (lib)
mkOption
types
;
tpmStateDir = "/var/lib/hermes-vm-tpm";
in
{
options.services.hermenix = {
vcpu = mkOption {
type = types.ints.positive;
default = 4;
};
mem = mkOption {
type = types.ints.positive;
default = 8192;
description = "RAM in MiB.";
};
disk = {
baseDir = mkOption {
type = types.str;
default = "/var/lib/hermes-vm";
description = ''
Directory holding the guest's disk images. On btrfs this should be
laid out as three subvolumes - system/, state/ and snapshots/ - with
nodatacow set. `hermenix setup` does this for you.
'';
};
systemSize = mkOption {
type = types.str;
default = "80G";
description = ''
System disk: the NixOS install and the Nix store. Never snapshotted,
because it is entirely rebuildable from the flake. Size it for an
agent that builds things.
'';
};
stateSize = mkOption {
type = types.str;
default = "20G";
description = ''
State disk, mounted at /var/lib/hermes in the guest. Holds Hermes's
identity, memories, workspace, and the NixOS config it maintains for
itself. This is the only irreplaceable data, so it is the only thing
snapshotted and backed up.
'';
};
quiesce = mkOption {
type = types.bool;
default = true;
description = "fsfreeze the guest via qemu-guest-agent around each snapshot.";
};
};
devices = {
rng.enable = mkOption {
type = types.bool;
default = true;
description = "Attach a virtio entropy source (-device virtio-rng-pci).";
};
vsock.enable = mkOption {
type = types.bool;
default = true;
description = ''
Attach the host<->guest vsock (-device vhost-vsock-pci). `hermenix
ctl` and the SSH-based backup both reach the guest over it, so
turning this off breaks them.
'';
};
video = {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Attach a virtual display adapter, giving the guest a DRM device
(/dev/dri). Needed by a Wayland compositor or anything that wants
a render node. The console stays serial either way.
'';
};
model = mkOption {
type = types.enum [
"virtio"
"qxl"
"cirrus"
"vmware"
"std"
];
default = "virtio";
description = ''
Adapter model passed to -vga. Only "virtio" exposes a render node
(/dev/dri/renderD128); the legacy models give card0 without one.
'';
};
};
balloon.enable = mkOption {
type = types.bool;
default = false;
description = "Attach a virtio memory balloon (-device virtio-balloon-pci).";
};
watchdog = {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Attach a hardware watchdog. The guest must also arm it (systemd's
RuntimeWatchdogSec) for it to do anything.
'';
};
model = mkOption {
type = types.enum [
"i6300esb"
"ib700"
];
default = "i6300esb";
};
action = mkOption {
type = types.enum [
"reset"
"shutdown"
"poweroff"
"pause"
"debug"
"none"
];
default = "reset";
description = "What QEMU does when the watchdog fires.";
};
};
input = {
enable = mkOption {
type = types.bool;
default = false;
description = "Attach virtio input devices. Only useful with devices.video.enable.";
};
keyboard = mkOption {
type = types.bool;
default = true;
};
tablet = mkOption {
type = types.bool;
default = false;
};
};
audio = {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Attach an audio device. NOTE: the VM unit is hardened (ProtectHome,
PrivateTmp, a dedicated system user) and cannot reach the operator's
audio daemon without loosening that; leave this off to keep the
hardening intact.
'';
};
backend = mkOption {
type = types.enum [
"none"
"pipewire"
"alsa"
"pa"
"sdl"
];
default = "pipewire";
};
model = mkOption {
type = types.enum [
"intel-hda"
"virtio-sound"
];
default = "intel-hda";
};
};
tpm = {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Attach a TPM 2.0 via swtpm. A hermes-vm-swtpm unit starts first and
QEMU connects to it over a unix socket. Useful for measured boot or
sealing secrets to the VM.
'';
};
stateDir = mkOption {
type = types.str;
default = tpmStateDir;
description = ''
Where swtpm keeps its state. Must live under /var/lib/ so it can be
created and owned by the unit's StateDirectory.
'';
};
};
cdroms = mkOption {
default = [ ];
description = "Read-only CD-ROM images to attach, on a virtio-scsi controller.";
type = types.listOf (
types.submodule {
options.path = mkOption { type = types.str; };
}
);
};
extraDisks = mkOption {
default = [ ];
description = "Extra raw disk images, each on its own virtio-blk device.";
type = types.listOf (
types.submodule {
options = {
name = mkOption {
type = types.str;
description = "Unique QEMU drive id, e.g. scratch.";
};
path = mkOption { type = types.str; };
readonly = mkOption {
type = types.bool;
default = false;
};
aio = mkOption {
type = types.enum [
"threads"
"native"
"io_uring"
];
default = "threads";
};
cache = mkOption {
type = types.enum [
"writeback"
"none"
"directsync"
"unsafe"
];
default = "writeback";
};
};
}
);
};
usbController = mkOption {
type = types.enum [
"auto"
"on"
"off"
];
default = "auto";
description = ''
Whether to attach a qemu-xhci USB controller. "auto" adds one only
when passthrough.usb is non-empty; "on"/"off" force the decision.
'';
};
};
extraQemuArgs = mkOption {
type = types.listOf types.str;
default = [ ];
description = ''
Raw additional QEMU arguments, appended to the VM command line.
Unvalidated escape hatch - prefer the typed devices.* options.
'';
};
};
}
+17 -1471
View File
File diff suppressed because it is too large. Load diff