Files
hermenix/modules/host.nix
T
2026-09-23 19:29:28 -04:00

1285 lines
46 KiB
Nix

{
config,
lib,
pkgs,
...
}:
let
inherit (lib)
mkOption
mkEnableOption
mkIf
types
optionalString
;
cfg = config.services.hermenix;
net = cfg.network;
vmUser = "hermes-vm";
runDir = "/run/hermes-vm";
stateDir = "/var/lib/hermes-vm";
# Two disks, deliberately.
#
# The system disk is large, churny and entirely rebuildable - it is the
# NixOS install and the Nix store. We never snapshot it.
#
# The state disk holds the only things that cannot be reproduced: Hermes's
# identity and memories, its workspace, and the NixOS config it has written
# for itself. Small, so snapshots are cheap and backups are quick.
systemImage = "${cfg.disk.baseDir}/system/system.raw";
stateImage = "${cfg.disk.baseDir}/state/state.raw";
snapDir = "${cfg.disk.baseDir}/snapshots";
pt = cfg.passthrough;
anyPci = pt.pci != [ ];
anyUsb = pt.usb != [ ];
earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci);
# Netfilter mark tagging guest-originated traffic in bridged mode. The
# guest's only way in is the tap interface, so marking at the bridge
# prerouting hook identifies the guest regardless of which address it
# currently presents - its static service-net address, or the fixed LAN
# address served to it by the host-side DHCP on the bridge. 18501 ==
# 0x4845 ("HE"), for hermenix.
bridgeMark = 18501;
# Dotted-quad netmask for a prefix length, used by the bridged-mode DHCP
# server's single-address lease (dnsmasq needs an explicit mask for a
# one-address range).
netmaskFromPrefix =
p:
lib.concatStringsSep "." (
lib.genList (
b:
toString (
builtins.elemAt [
0
128
192
224
240
248
252
254
255
] (lib.max 0 (lib.min 8 (p - b * 8)))
)
) 4
);
# Graceful ACPI power button via QMP, then wait for QEMU to exit.
stopScript = pkgs.writeShellScript "hermes-vm-stop" ''
set -eu
printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \
| ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0
'';
# Freeze the guest filesystems around a snapshot so it captures a consistent
# state rather than the equivalent of pulling the power.
qga =
verb:
pkgs.writeShellScript "hermes-vm-fs${verb}" ''
set -eu
printf '%s' '{"execute":"guest-fs${verb}"}' \
| ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true
'';
in
{
options.services.hermenix = {
enable = mkEnableOption "the Hermes Agent VM";
name = mkOption {
type = types.str;
default = "hermes";
};
vcpu = mkOption {
type = types.ints.positive;
default = 4;
};
mem = mkOption {
type = types.ints.positive;
default = 8192;
description = "RAM in MiB.";
};
disk = {
baseDir = mkOption {
type = types.str;
default = "/var/lib/hermes-vm";
description = ''
Directory holding the guest's disk images. On btrfs this should be
laid out as three subvolumes - system/, state/ and snapshots/ - with
nodatacow set. `hermenix setup` does this for you.
'';
};
systemSize = mkOption {
type = types.str;
default = "80G";
description = ''
System disk: the NixOS install and the Nix store. Never snapshotted,
because it is entirely rebuildable from the flake. Size it for an
agent that builds things.
'';
};
stateSize = mkOption {
type = types.str;
default = "20G";
description = ''
State disk, mounted at /var/lib/hermes in the guest. Holds Hermes's
identity, memories, workspace, and the NixOS config it maintains for
itself. This is the only irreplaceable data, so it is the only thing
snapshotted and backed up.
'';
};
quiesce = mkOption {
type = types.bool;
default = true;
description = "fsfreeze the guest via qemu-guest-agent around each snapshot.";
};
};
snapshots = {
enable = mkOption {
type = types.bool;
default = true;
description = ''
Periodic btrfs snapshots of the state disk. The agent can persist
changes, which means it can persist a mistake, and these are the
fastest way back.
'';
};
interval = mkOption {
type = types.str;
default = "hourly";
description = "systemd calendar expression.";
};
keep = mkOption {
type = types.ints.positive;
default = 48;
description = ''
How many to retain. Keep this tighter than you would on ZFS: the
state image is nodatacow, and every live snapshot reintroduces
copy-on-write for the blocks it still references.
'';
};
};
backup = {
enable = mkEnableOption "restic backups of the guest's state";
repository = mkOption {
type = types.str;
default = "";
example = "sftp:backup@nas:/hermes";
description = "Any restic repository string.";
};
passwordFile = mkOption {
type = types.path;
default = "/var/lib/secrets/hermes-restic";
description = "File containing the restic repository password, mode 0400.";
};
environmentFile = mkOption {
type = types.nullOr types.path;
default = null;
description = "Optional file with credentials for the backend (S3 keys and so on).";
};
interval = mkOption {
type = types.str;
default = "daily";
};
keep = mkOption {
type = types.listOf types.str;
default = [
"--keep-daily"
"14"
"--keep-weekly"
"8"
"--keep-monthly"
"12"
];
description = "Arguments passed to restic forget.";
};
};
hostCache = {
enable = mkOption {
type = types.bool;
default = true;
description = ''
Serve the host's /nix/store to the guest as a read-only binary
cache, so the guest doesn't re-download from the internet what the
host already has.
This is the ONLY route from guest to host in this design. Turning it
off leaves the guest with literally no reachable host service, at
the cost of bandwidth on every guest build.
Note the guest can read every store path on the host through this.
Store paths are world-readable anyway, but check nothing sensitive
is baked into your host's store.
'';
};
port = mkOption {
type = types.port;
default = 5000;
};
secretKeyFile = mkOption {
type = types.path;
default = "/var/lib/secrets/hermes-cache.secret";
description = "Generate with nix-store --generate-binary-cache-key. See README.";
};
};
authorizedKeys = mkOption {
type = types.listOf types.str;
default = [ ];
};
allowLan = mkOption {
type = types.bool;
default = false;
description = ''
Blanket permission for the guest to reach RFC1918 space. Almost always
the wrong tool - use lanAccess to open single endpoints instead.
'';
};
lanAccess = mkOption {
default = [ ];
example = [
{
address = "192.168.1.50";
port = 11434;
}
];
description = ''
Specific endpoints elsewhere on your network that the guest may reach,
punched through the default RFC1918 drop. One entry per service.
'';
type = types.listOf (
types.submodule {
options = {
address = mkOption { type = types.str; };
port = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
}
);
};
hostAccess = mkOption {
default = [ ];
example = [ { port = 8188; } ];
description = ''
Ports on this host itself that the guest may reach. The service must
also be bound to an address the guest can see - see the README.
'';
type = types.listOf (
types.submodule {
options = {
port = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
}
);
};
publish = mkOption {
default = [ ];
example = [
{
hostPort = 8080;
guestPort = 8080;
}
];
description = ''
Guest ports to publish on the host's uplink, reachable from the wider
network. You do not need this to reach the guest from the host itself.
'';
type = types.listOf (
types.submodule {
options = {
hostPort = mkOption { type = types.port; };
guestPort = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
}
);
};
network = {
bridge = mkOption {
type = types.str;
default = "hermes-br";
};
tapId = mkOption {
type = types.str;
default = "vm-hermes";
};
mac = mkOption {
type = types.str;
default = "02:00:00:7e:00:01";
};
hostAddress = mkOption {
type = types.str;
default = "10.77.0.1";
};
guestAddress = mkOption {
type = types.str;
default = "10.77.0.2";
};
prefixLength = mkOption {
type = types.int;
default = 24;
};
lanAddress = mkOption {
type = types.str;
default = "";
example = "10.0.12.50";
description = "The guest's fixed address on your LAN. Required when `bridged = true`.";
};
hostLanAddress = mkOption {
type = types.str;
default = "";
example = "10.0.12.10";
description = ''
This host's own address on your LAN. When `bridged` is enabled the
uplink becomes a bridge port and this address moves onto the bridge
(which clones the uplink's MAC, so the host keeps its LAN identity).
Required when `bridged = true`.
'';
};
lanGateway = mkOption {
type = types.str;
default = "";
example = "10.0.12.1";
description = ''
Your LAN's router. Used as the default gateway for the guest and,
while `bridged`, for this host's bridge too (DNS unless `lanDns` is
set). Required when `bridged = true`.
'';
};
lanPrefixLength = mkOption {
type = types.int;
default = 24;
description = "Prefix length of your LAN subnet (netmask).";
};
lanDns = mkOption {
type = types.listOf types.str;
default = [ ];
example = [ "1.1.1.1" ];
description = ''
DNS servers for this host while `bridged` (the bridge no longer
takes a DHCP lease). Defaults to the LAN gateway.
'';
};
bridged = mkOption {
type = types.bool;
default = false;
description = ''
Put the guest directly on your LAN instead of behind the host's
NAT. The bridge absorbs `externalInterface`, so any device on the
LAN can reach any port the guest exposes.
Addressing is fully static - no DHCP is involved. You must give
the guest a fixed LAN address (`lanAddress`), the host's own LAN
address (`hostLanAddress` - it moves onto the bridge), and the
LAN gateway (`lanGateway`). The guest is handed `lanAddress` by a
DHCP server this module runs on the bridge, keyed to the guest's
MAC (`network.mac`), so the guest itself stays a plain DHCP
client and never needs to know its own address.
Inbound is no longer limited to `publish` (which is ignored in
this mode): the LAN may reach every guest port. Outbound is still
governed by `lanAccess` / `allowLan`, and `hostAccess` plus the
host cache still work over the host-guest service net
(`hostAddress` stays configured on the bridge).
'';
};
externalInterface = mkOption {
type = types.nullOr types.str;
default = null;
example = "eno1";
description = "Host uplink. Required.";
};
};
passthrough = {
iommu = mkOption {
type = types.nullOr (
types.enum [
"intel"
"amd"
]
);
default = null;
description = ''
Enable IOMMU kernel parameters for your CPU vendor. Required for any
PCI passthrough. The IOMMU is the only thing stopping a passed-through
device from reading host memory, so this is not optional.
'';
};
pci = mkOption {
default = [ ];
example = [
{
address = "0000:01:00.0";
id = "10de:2204";
}
{
address = "0000:01:00.1";
id = "10de:1aef";
}
];
description = ''
PCI devices handed to the guest via VFIO. The host loses them
entirely while the VM runs.
You must pass every function of the device and, in practice, every
device in its IOMMU group. A GPU is usually at least two entries:
the graphics function and its HDMI audio function.
Set `id` (vendor:device) to have vfio-pci claim the device in the
initrd, before a host driver can. Strongly recommended for GPUs;
unbinding an active amdgpu or nvidia driver later is unreliable.
'';
type = types.listOf (
types.submodule {
options = {
address = mkOption {
type = types.str;
example = "0000:01:00.0";
description = "Full PCI address, as shown by lspci -D.";
};
id = mkOption {
type = types.nullOr types.str;
default = null;
example = "10de:2204";
description = "vendor:device, from lspci -nn. Enables early binding.";
};
};
}
);
};
usb = mkOption {
default = [ ];
example = [
{
vendorId = "8087";
productId = "0026";
}
];
description = ''
USB devices forwarded to the guest, matched by vendor and product
ID from lsusb. Integrated Bluetooth radios are USB devices and go
here too, though the host driver must be kept off them.
QEMU resolves these at start, so a device that is unplugged when the
VM boots will not appear later without a restart.
'';
type = types.listOf (
types.submodule {
options = {
vendorId = mkOption {
type = types.str;
example = "046d";
};
productId = mkOption {
type = types.str;
example = "c52b";
};
};
}
);
};
};
gatewayUnit = mkOption {
type = types.str;
default = "hermes-agent";
description = ''
The systemd *user* unit that `hermes gateway install` created in the
guest. Check with `hermenix ctl units` after installing; upstream has
used more than one name for this.
'';
};
hermesHome = mkOption {
type = types.str;
default = "${stateDir}/.hermes";
description = ''
Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/,
state.db, and so on) lives inside the guest. Defaults to the seeded
layout under the state disk mount. Override this when carrying over
an existing flat HERMES_HOME from elsewhere (e.g. a prior Docker
deployment) that you're placing directly at the root of the state
disk instead of under .hermes - see `hermenix migrate --help`.
This only changes where the host-side `hermenix ctl` commands (soul,
memory, conf, secrets, reseed, pull) looks. It does not change what
the gateway process itself uses - export HERMES_HOME to match inside
the guest when you install it.
'';
};
vsockCid = mkOption {
type = types.int;
default = 77;
description = "Host reaches the guest over vsock, so no inbound network path is needed.";
};
version = mkOption {
type = types.str;
default = "dev";
description = ''
"YYYY.MM.DD.hh.mm-<commit-sha>" reported by `hermenix version`.
`version` subcommand. Normally supplied for free by this flake's own
`nixosModules.default` (from its git metadata) - the "dev" default
here only shows up if this module is imported some other way.
'';
};
};
config = mkIf cfg.enable {
assertions = [
{
assertion = net.externalInterface != null;
message = "services.hermenix.network.externalInterface must be set.";
}
{
assertion =
!net.bridged || (net.lanAddress != "" && net.hostLanAddress != "" && net.lanGateway != "");
message = ''
services.hermenix.network.bridged = true requires a fully static
LAN setup (DHCP is no longer supported in bridged mode). Set
network.lanAddress (the guest's fixed LAN address),
network.hostLanAddress (this host's own LAN address, which moves
onto the bridge) and network.lanGateway (your router's address).
'';
}
{
assertion = cfg.backup.enable -> (cfg.backup.repository != "");
message = "services.hermenix.backup.repository must be set when backups are enabled.";
}
{
assertion = anyPci -> (pt.iommu != null);
message = ''
services.hermenix.passthrough.pci needs passthrough.iommu set to
"intel" or "amd". Without the IOMMU, a passed-through device can
read host memory directly and the VM boundary is meaningless.
'';
}
];
users.users.${vmUser} = {
isSystemUser = true;
group = vmUser;
description = "Runs the Hermes guest VM";
};
users.groups.${vmUser} = { };
# =====================================================================
# The VM itself
#
# No virtiofs, no shared filesystem of any kind, no vhost-net. The guest
# gets a block device, a tap, a vsock and a serial port. That is all.
# =====================================================================
systemd.services.hermes-vm = {
description = "Hermes Agent VM";
wantedBy = [ "multi-user.target" ];
after = [
"network-online.target"
"hermes-vm-tap.service"
];
requires = [ "hermes-vm-tap.service" ];
wants = [ "network-online.target" ];
preStart = ''
for img in ${systemImage} ${stateImage}; do
if [ ! -e "$img" ]; then
echo "$img is missing. Run 'hermenix setup' first." >&2
exit 1
fi
done
# Per-VM writable UEFI variable store, seeded once.
if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then
install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd
fi
'';
serviceConfig = {
User = vmUser;
Group = vmUser;
SupplementaryGroups = [ "kvm" ];
RuntimeDirectory = "hermes-vm";
# ProtectSystem=strict makes everything read-only; punch through for
# the images and the UEFI variable store only.
ReadWritePaths = [ cfg.disk.baseDir ];
StateDirectory = "hermes-vm";
RuntimeDirectoryMode = "0750";
ExecStart = lib.concatStringsSep " " (
[
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
"-name ${cfg.name}"
"-machine q35,accel=kvm,smm=on"
"-cpu host"
"-smp ${toString cfg.vcpu}"
"-m ${toString cfg.mem}"
"-nodefaults"
"-no-user-config"
"-display none"
"-serial stdio"
# QEMU's own seccomp sandbox, everything denied that we don't need.
"-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"
# UEFI: firmware read-only, variables per-VM and writable.
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
"-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd"
# System disk: NixOS and the Nix store. Rebuildable, never snapshotted.
# aio=threads and a writeback cache because these are files on a
# filesystem, not block devices; aio=native wants the latter.
"-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=system"
# State disk: everything irreplaceable. Mounted at /var/lib/hermes.
"-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=state"
# Pre-created tap. vhost=off on purpose: vhost-net would move packet
# handling into the host kernel, which is exactly the surface we are
# trying not to expose. Costs throughput we do not need.
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
# Host -> guest management, no network involved.
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
"-device virtio-rng-pci"
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
]
++ lib.optionals cfg.disk.quiesce [
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
"-device virtio-serial-pci"
"-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
]
# Passed-through PCI devices. The host cannot use these while the VM
# is running.
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
# A USB controller, only if something needs plugging into it.
++ lib.optional anyUsb "-device qemu-xhci,id=xhci"
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
);
ExecStop = stopScript;
TimeoutStopSec = 90;
Restart = "always";
RestartSec = 5;
# ---- hardening -------------------------------------------------
DevicePolicy = "closed";
DeviceAllow = [
"/dev/kvm rw"
"/dev/net/tun rw"
"/dev/vhost-vsock rw"
]
++ lib.optionals anyPci [ "char-vfio rw" ]
++ lib.optionals anyUsb [ "char-usb_device rw" ];
# VFIO pins the entire guest RAM allocation, so the default memlock
# limit has to go or QEMU fails to start with an obscure error.
LimitMEMLOCK = lib.mkIf anyPci "infinity";
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
ProtectProc = "invisible";
ProcSubset = "pid";
PrivateTmp = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHostname = true;
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
RestrictAddressFamilies = [
"AF_UNIX"
"AF_NETLINK"
"AF_VSOCK"
];
LockPersonality = true;
SystemCallArchitectures = "native";
SystemCallFilter = [
"@system-service"
"~@resources"
"~@obsolete"
];
# KVM means no JIT, so W^X is safe. If QEMU refuses to start, this is
# the first thing to flip.
MemoryDenyWriteExecute = true;
CapabilityBoundingSet = [ "" ];
AmbientCapabilities = [ "" ];
};
};
# Tap device, created as root, owned by the VM user, bridged by networkd.
systemd.services.hermes-vm-tap = {
description = "TAP interface for the Hermes VM";
after = [ "systemd-networkd.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
path = [ pkgs.iproute2 ];
script = ''
ip link show ${net.tapId} >/dev/null 2>&1 \
|| ip tuntap add ${net.tapId} mode tap user ${vmUser}
ip link set ${net.tapId} up
'';
preStop = ''
${pkgs.iproute2}/bin/ip link del ${net.tapId} || true
'';
};
# =====================================================================
# Device passthrough
# =====================================================================
# The IOMMU is what keeps a passed-through device from reading host
# memory. Without it, PCI passthrough is not a boundary at all.
boot.kernelParams = lib.optionals (pt.iommu != null) [
(if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on")
"iommu=pt"
];
boot.initrd.kernelModules = lib.optionals anyPci [
"vfio_pci"
"vfio"
"vfio_iommu_type1"
];
# Claim the device before any host driver gets to it.
boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) ''
options vfio-pci ids=${lib.concatStringsSep "," earlyIds}
'';
# Catch anything that was not early-bound, by address. Idempotent.
systemd.services.hermes-vm-vfio = mkIf anyPci {
description = "Bind passthrough PCI devices to vfio-pci";
before = [ "hermes-vm.service" ];
requiredBy = [ "hermes-vm.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
path = [ pkgs.kmod ];
script = ''
modprobe vfio-pci
''
+ lib.concatMapStrings (d: ''
dev=/sys/bus/pci/devices/${d.address}
if [ ! -e "$dev" ]; then
echo "${d.address} is not present on this machine" >&2
exit 1
fi
cur=none
[ -e "$dev/driver" ] && cur=$(basename "$(readlink -f "$dev/driver")")
if [ "$cur" != "vfio-pci" ]; then
[ "$cur" != "none" ] && echo ${d.address} > "$dev/driver/unbind"
echo vfio-pci > "$dev/driver_override"
echo ${d.address} > /sys/bus/pci/drivers_probe
fi
'') pt.pci;
};
services.udev.extraRules =
lib.optionalString anyPci ''
SUBSYSTEM=="vfio", OWNER="${vmUser}", GROUP="${vmUser}", MODE="0660"
''
+ lib.concatMapStrings (u: ''
SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660"
'') pt.usb;
# =====================================================================
# Networking
# =====================================================================
networking.useNetworkd = true;
systemd.network.netdevs."10-${net.bridge}".netdevConfig = {
Kind = "bridge";
Name = net.bridge;
};
systemd.network.networks."10-${net.bridge}" = {
matchConfig.Name = net.bridge;
# hostAddress stays on the bridge in both modes: it is the guest's
# service net (host cache, hostAccess). In bridged mode the host's
# static LAN address is added on top of it - the bridge only ever has
# static addresses, never a DHCP lease.
address = [
"${net.hostAddress}/${toString net.prefixLength}"
]
++ lib.optional net.bridged "${net.hostLanAddress}/${toString net.lanPrefixLength}";
networkConfig = {
ConfigureWithoutCarrier = true;
IPv4Forwarding = true;
}
// lib.optionalAttrs net.bridged {
# Static LAN config for the host: the bridge no longer takes a DHCP
# lease (bridged mode is static-only), so its gateway and DNS come
# from the module options instead.
Gateway = net.lanGateway;
DNS = if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns;
};
};
# The host's DNS, now that the bridge cannot learn it from a DHCP lease.
# mkDefault so the operator's own nameservers win if they set any.
networking.nameservers = lib.mkIf net.bridged (
lib.mkDefault (if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns)
);
systemd.network.networks."11-${net.tapId}" = {
matchConfig.Name = net.tapId;
networkConfig.Bridge = net.bridge;
};
# Bridged mode: the physical uplink becomes a bridge port so the LAN
# shares an L2 domain with the guest. The port must not carry its own
# address config - the bridge owns the LAN address now (static, via
# hostLanAddress; there is no DHCP on the bridge).
systemd.network.networks."12-${net.externalInterface}" = lib.mkIf net.bridged {
matchConfig.Name = net.externalInterface;
networkConfig.Bridge = net.bridge;
};
# The bridge is created with no MAC of its own, so Linux makes it adopt
# the lowest-numbered port's MAC - and the tap is always the first port.
# That is how the host once ended up presenting the VM's MAC on the LAN
# (a duplicate address, since the guest uses the same MAC). Clone the
# uplink's MAC onto the bridge so the host keeps its LAN identity; the
# guest keeps its own tap MAC (net.mac).
systemd.services.hermes-bridge-mac = lib.mkIf net.bridged {
description = "Clone the uplink MAC onto the hermenix bridge";
after = [
"systemd-networkd.service"
"sys-subsystem-net-devices-${net.bridge}.device"
"sys-subsystem-net-devices-${net.externalInterface}.device"
];
wants = [ "systemd-networkd.service" ];
before = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
unitConfig.ConditionPathExists = "/sys/class/net/${net.externalInterface}";
script = ''
set -euo pipefail
# networkd creates the bridge netdev asynchronously; give it a moment.
for _ in 1 2 3 4 5 6 7 8 9 10; do
[ -e "/sys/class/net/${net.bridge}" ] && break
sleep 0.5
done
[ -e "/sys/class/net/${net.bridge}" ] || exit 1
mac="$(${pkgs.coreutils}/bin/cat /sys/class/net/${net.externalInterface}/address)"
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} down
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} address "$mac"
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} up
# Re-apply the bridge's static address config under the new MAC.
${pkgs.systemd}/bin/networkctl reconfigure ${net.bridge} || true
'';
};
# The guest is handed its fixed LAN address (lanAddress) by a dedicated
# DHCP server bound to the bridge, keyed to the guest's MAC. This is the
# guest's ONLY DHCP: its broadcast is blocked from leaving the bridge
# (see the hermenix-bridge table), so the LAN's own DHCP server can
# never race the reservation. DNS is disabled (port=0) - the guest runs
# static QUAD9 resolvers, and this host gets its DNS from lanDns.
systemd.services.hermes-lan-dhcp = lib.mkIf net.bridged {
description = "Static-lease DHCP for the bridged hermenix guest";
after = [
"systemd-networkd.service"
"sys-subsystem-net-devices-${net.bridge}.device"
];
wants = [ "systemd-networkd.service" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "simple";
Restart = "on-failure";
RestartSec = 5;
};
script = ''
for _ in 1 2 3 4 5 6 7 8 9 10; do
[ -e "/sys/class/net/${net.bridge}" ] && break
sleep 0.5
done
exec ${pkgs.dnsmasq}/bin/dnsmasq --keep-in-foreground \
--conf-file=/dev/null \
--interface=${net.bridge} \
--bind-dynamic \
--port=0 \
--dhcp-authoritative \
--dhcp-range=${net.lanAddress},${net.lanAddress},${netmaskFromPrefix net.lanPrefixLength},12h \
--dhcp-host=${net.mac},${net.lanAddress} \
--dhcp-option=option:router,${net.lanGateway}
'';
};
# ip_forward: use the same sysctl keys networking.nat used (mkOverride 99
# so they win over any lower-priority defaults elsewhere in the config).
boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true;
boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true;
# Masquerade outbound guest traffic and DNAT published ports inbound.
# We use an explicit nftables table instead of networking.nat so that the
# rules are certain to appear in the prerouting/postrouting hooks.
#
# NOTE: DNAT uses 'iifname != bridge' rather than 'iifname externalInterface'.
# Scoping to a single interface means traffic arriving on any other path
# (Tailscale, a second NIC, WiFi, loopback) silently skips DNAT: the
# packet stays addressed to the host's own IP, falls into the INPUT chain,
# and gets dropped because those ports are not in allowedTCPPorts.
# Excluding only the bridge itself prevents loops from inside the guest
# while covering every legitimate external path.
networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null) {
family = "ip";
content =
if net.bridged then
# Bridged mode: no DNAT (the guest is directly reachable on the
# LAN) and no primary masquerade. Keep one scoped masquerade as a
# fallback: if the guest's DHCP lease lapses it falls back to its
# static default route via the host, and this keeps that path
# working instead of blackholing at the uplink. With a healthy
# lease the guest's traffic is L2-bridged and never reaches this
# chain. Only traffic handed up from the bridge (i.e. the guest's
# service-net traffic) is matched, never the LAN itself.
''
chain postrouting {
type nat hook postrouting priority srcnat;
iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade
}
''
else
''
chain prerouting {
type nat hook prerouting priority dstnat;
''
+ lib.concatMapStrings (s: ''
iifname != "${net.bridge}" ${s.proto} dport ${toString s.hostPort} dnat to ${net.guestAddress}:${toString s.guestPort}
'') cfg.publish
+ ''
}
chain postrouting {
type nat hook postrouting priority srcnat;
iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade
}
'';
};
# =====================================================================
# Bridged mode: bridge-family enforcement.
#
# With the uplink bridged, guest <-> LAN traffic is switched at L2 and
# never traverses the ip-family hooks the NAT-mode rules rely on, so
# bridged mode gets its own table where the tap port is unambiguous.
# lanAccess / allowLan still gate what the guest may reach; inbound
# from the LAN to the guest is open on every port.
# =====================================================================
networking.nftables.tables."hermenix-bridge" = lib.mkIf net.bridged {
family = "bridge";
content = ''
chain prerouting {
type filter hook prerouting priority filter; policy accept;
# Tag anything entering from the guest. ip-family input rules
# (host cache, hostAccess) match this mark.
iifname "${net.tapId}" meta mark set ${toString bridgeMark}
# The guest's address is static - served by hermes-lan-dhcp on
# this bridge - so keep its DHCP traffic off the LAN and keep the
# LAN's DHCP answers out of the guest. Only the host-side server
# may talk DHCP to it.
iifname "${net.tapId}" oifname "${net.externalInterface}" ether type ip ip protocol udp udp sport 68 udp dport 67 drop
iifname "${net.externalInterface}" oifname "${net.tapId}" ether type ip ip protocol udp udp sport 67 udp dport 68 drop
}
chain forward {
type filter hook forward priority filter; policy accept;
# Inbound from the LAN to the guest: any device, any port.
oifname "${net.tapId}" accept
''
+ lib.concatMapStrings (s: ''
iifname "${net.tapId}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
'') cfg.lanAccess
+ (
if cfg.allowLan then
''
iifname "${net.tapId}" accept
''
else
''
iifname "${net.tapId}" ip daddr {
10.0.0.0/8,
172.16.0.0/12,
192.168.0.0/16,
169.254.0.0/16,
127.0.0.0/8,
100.64.0.0/10
} drop
iifname "${net.tapId}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
iifname "${net.tapId}" accept
''
)
+ ''
}
'';
};
# =====================================================================
# Firewall
#
# The guest owns its own kernel and its own config now, so nothing inside
# it constrains it. These rules are the boundary.
# =====================================================================
networking.nftables.enable = true;
networking.firewall.enable = true;
networking.firewall.filterForward = true;
# The deliberate guest -> host holes: the Nix cache, plus anything listed
# in hostAccess.
#
# NAT mode: the guest always arrives from its fixed service-net address,
# so match address + bridge. Bridged mode: the guest presents either the
# service-net address or its LAN address (served by the bridge's DHCP),
# so the bridge table above tags guest traffic with a netfilter mark as
# it enters the tap - the rules match that mark instead. The tap is
# exclusive to the guest, so the mark is as specific as an address
# match: LAN devices can never inherit these holes.
networking.firewall.extraInputRules =
if net.bridged then
optionalString cfg.hostCache.enable ''
meta mark ${toString bridgeMark} tcp dport ${toString cfg.hostCache.port} accept
''
+ lib.concatMapStrings (s: ''
meta mark ${toString bridgeMark} ${s.proto} dport ${toString s.port} accept
'') cfg.hostAccess
else
optionalString cfg.hostCache.enable ''
iifname "${net.bridge}" ip saddr ${net.guestAddress} tcp dport ${toString cfg.hostCache.port} accept
''
+ lib.concatMapStrings (s: ''
iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept
'') cfg.hostAccess;
# Order matters here. Allowlists must precede the RFC1918 drop.
#
# These ip-family rules only ever see routed traffic. They are the whole
# story in NAT mode; in bridged mode the guest's LAN traffic is
# L2-bridged instead and is governed by the hermenix-bridge table.
networking.firewall.extraForwardRules = ''
iifname != "${net.bridge}" accept
ct state established,related accept
''
# Guest reaching named services elsewhere on your network.
+ lib.concatMapStrings (s: ''
iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
'') cfg.lanAccess
# Inbound to guest services published on the uplink. NAT mode only; in
# bridged mode the bridge table opens every inbound port instead.
+ lib.optionalString (!net.bridged) (
lib.concatMapStrings (s: ''
oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept
'') cfg.publish
)
+ (
if cfg.allowLan then
''
iifname "${net.bridge}" accept
''
else
''
iifname "${net.bridge}" ip daddr {
10.0.0.0/8,
172.16.0.0/12,
192.168.0.0/16,
169.254.0.0/16,
127.0.0.0/8,
100.64.0.0/10
} drop
iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
iifname "${net.bridge}" accept
''
);
# =====================================================================
# Host store as a read-only cache for the guest
# =====================================================================
services.nix-serve = mkIf cfg.hostCache.enable {
enable = true;
package = pkgs.nix-serve-ng;
bindAddress = net.hostAddress;
inherit (cfg.hostCache) port secretKeyFile;
openFirewall = false; # handled above, scoped to the guest address
};
# =====================================================================
# Snapshots of the state disk only.
#
# The system disk is not snapshotted on purpose. It is large, it is where
# all the write churn happens, and `hermenix setup` can rebuild it from the flake
# in minutes. Snapshotting it would cost a lot and buy nothing.
# =====================================================================
systemd.services.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
description = "Snapshot the Hermes VM state disk";
path = with pkgs; [
btrfs-progs
coreutils
socat
];
serviceConfig.Type = "oneshot";
script = ''
set -euo pipefail
stamp=$(date +%Y%m%d-%H%M%S)
mkdir -p ${snapDir}
${optionalString cfg.disk.quiesce ''
# Flush the guest's filesystems so the snapshot is consistent rather
# than the equivalent of pulling the power. Takes milliseconds; the
# VM keeps running throughout.
${qga "freeze"}
trap '${qga "thaw"}' EXIT
''}
btrfs subvolume snapshot -r ${cfg.disk.baseDir}/state ${snapDir}/state-$stamp
${optionalString cfg.disk.quiesce ''
${qga "thaw"}
trap - EXIT
''}
# Prune oldest first. Keep this list short: every live snapshot
# reintroduces copy-on-write on a nodatacow image.
ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \
| while read -r old; do btrfs subvolume delete "$old"; done
'';
};
systemd.timers.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.snapshots.interval;
Persistent = true;
RandomizedDelaySec = "5m";
};
};
# =====================================================================
# Backups: the guest's state, not its disk.
#
# Pulled file-by-file over the SSH channel that already exists, rather
# than backing up the image. That keeps the backup portable - it restores
# onto any filesystem, or a different machine entirely - and avoids
# loop-mounting a guest-written filesystem on the host, which would mean
# the host kernel parsing data the agent controls.
# =====================================================================
systemd.services.hermes-vm-backup = mkIf cfg.backup.enable {
description = "Back up Hermes state";
path = with pkgs; [
restic
openssh
socat
gnutar
];
serviceConfig = {
Type = "oneshot";
EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile;
};
script = ''
set -euo pipefail
export RESTIC_REPOSITORY=${lib.escapeShellArg cfg.backup.repository}
export RESTIC_PASSWORD_FILE=${cfg.backup.passwordFile}
restic snapshots >/dev/null 2>&1 || restic init
ssh -o ProxyCommand="socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22" \
-o StrictHostKeyChecking=accept-new \
hermes@${cfg.name} \
'tar -C /var/lib/hermes \
--exclude=./.cache \
--exclude=./workspace/**/.direnv \
--exclude=./workspace/**/result \
-cf - .' \
| restic backup --stdin --stdin-filename hermes-state.tar \
--tag hermes --host ${cfg.name}
restic forget --tag hermes --host ${cfg.name} --prune ${lib.concatStringsSep " " cfg.backup.keep}
'';
};
systemd.timers.hermes-vm-backup = mkIf cfg.backup.enable {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.backup.interval;
Persistent = true;
RandomizedDelaySec = "30m";
};
};
environment.systemPackages = [
# Nushell tab-completion definitions for hermenix. Installed to
# vendor/autoload so nushell picks them up automatically.
(pkgs.writeTextFile {
name = "hermenix-completions";
destination = "/share/nushell/vendor/autoload/hermenix.nu";
text = builtins.readFile ../nu/completions.nu;
})
# Single consolidated CLI: `hermenix vm *` controls the VM,
# `hermenix ctl *` drives Hermes inside it. Per-host values are
# added on top of the base package via makeWrapper.
(pkgs.symlinkJoin {
name = "hermenix";
paths = [ pkgs.hermenix ];
nativeBuildInputs = [ pkgs.makeWrapper ];
postBuild = ''
wrapProgram $out/bin/hermenix \
--set HERMES_VSOCK_CID ${toString cfg.vsockCid} \
--set HERMES_GUEST_NAME ${cfg.name} \
--set HERMES_DISK_BASE_DIR ${cfg.disk.baseDir} \
--set HERMES_HOME_DIR ${cfg.hermesHome} \
--set HERMES_GATEWAY_UNIT ${cfg.gatewayUnit}
'';
})
];
};
}