doc clean

This commit is contained in:
ari committed 2026-10-09 14:35:12 -04:00
1 parent c2a7f94821
commit 30427e3975
34 files changed
+1119 -1758

No files matched your search

+35 -199
View File
@@ -19,14 +19,6 @@ let
vmUser = "hermes-vm";
runDir = "/run/hermes-vm";
stateDir = "/var/lib/hermes-vm";
# Two disks, deliberately.
#
# The system disk is large, churny and entirely rebuildable - it is the
# NixOS install and the Nix store. We never snapshot it.
#
# The state disk holds the only things that cannot be reproduced: Hermes's
# identity and memories, its workspace, and the NixOS config it has written
# for itself. Small, so snapshots are cheap and backups are quick.
systemImage = "${cfg.disk.baseDir}/system/system.raw";
stateImage = "${cfg.disk.baseDir}/state/state.raw";
snapDir = "${cfg.disk.baseDir}/snapshots";
@@ -36,25 +28,15 @@ let
anyUsb = pt.usb != [ ];
earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci);
# "auto" adds a USB controller only when something needs plugging into it;
# "on"/"off" force the decision.
usbControllerOn =
cfg.devices.usbController == "on" || (cfg.devices.usbController == "auto" && anyUsb);
# swtpm state. Must live under /var/lib/ so the unit's StateDirectory owns it.
tpmStateDir = "/var/lib/hermes-vm-tpm";
# Netfilter mark tagging guest-originated traffic in bridged mode. The
# guest's only way in is the tap interface, so marking at the bridge
# prerouting hook identifies the guest regardless of which address it
# currently presents - its static service-net address, or the fixed LAN
# address served to it by the host-side DHCP on the bridge. 18501 ==
# 0x4845 ("HE"), for hermenix.
# 18501 == 0x4845 ("HE"); marks guest traffic from the tap (bridged mode).
bridgeMark = 18501;
# Dotted-quad netmask for a prefix length, used by the bridged-mode DHCP
# server's single-address lease (dnsmasq needs an explicit mask for a
# one-address range).
# dnsmasq needs an explicit mask for a one-address DHCP range.
netmaskFromPrefix =
p:
lib.concatStringsSep "." (
@@ -76,15 +58,12 @@ let
) 4
);
# Graceful ACPI power button via QMP, then wait for QEMU to exit.
stopScript = pkgs.writeShellScript "hermes-vm-stop" ''
set -eu
printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \
| ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0
'';
# Freeze the guest filesystems around a snapshot so it captures a consistent
# state rather than the equivalent of pulling the power.
qga =
verb:
pkgs.writeShellScript "hermes-vm-fs${verb}" ''
@@ -419,13 +398,14 @@ in
NAT. The bridge absorbs `externalInterface`, so any device on the
LAN can reach any port the guest exposes.
Addressing is fully static - no DHCP is involved. You must give
the guest a fixed LAN address (`lanAddress`), the host's own LAN
address (`hostLanAddress` - it moves onto the bridge), and the
LAN gateway (`lanGateway`). The guest is handed `lanAddress` by a
DHCP server this module runs on the bridge, keyed to the guest's
MAC (`network.mac`), so the guest itself stays a plain DHCP
client and never needs to know its own address.
Addressing is static from the operator's point of view: you pick
fixed addresses for the guest (`lanAddress`), this host
(`hostLanAddress` — it moves onto the bridge), and the LAN
gateway (`lanGateway`). A DHCP server this module runs on the
bridge hands `lanAddress` to the guest, keyed to `network.mac`,
so the guest stays a plain DHCP client and never needs to know
its own address. That lease never leaves the bridge; the LAN's
own DHCP cannot race it.
Inbound is no longer limited to `publish` (which is ignored in
this mode): the LAN may reach every guest port. Outbound is still
@@ -761,18 +741,20 @@ in
hermesHome = mkOption {
type = types.str;
default = "${stateDir}/.hermes";
# Guest path (not host disk.baseDir).
default = "/var/lib/hermes/.hermes";
description = ''
Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/,
state.db, and so on) lives inside the guest. Defaults to the seeded
layout under the state disk mount. Override this when carrying over
an existing flat HERMES_HOME from elsewhere (e.g. a prior Docker
deployment) that you're placing directly at the root of the state
disk instead of under .hermes - see `hermenix migrate --help`.
layout under the state disk mount (`/var/lib/hermes/.hermes`).
Override when carrying over an existing flat HERMES_HOME from
elsewhere (e.g. a prior Docker deployment) that you're placing
directly at the root of the state disk instead of under .hermes —
see `hermenix migrate --help`.
This only changes where the host-side `hermenix ctl` commands (soul,
memory, conf, secrets, reseed, pull) looks. It does not change what
the gateway process itself uses - export HERMES_HOME to match inside
memory, conf, secrets, reseed, pull) look. It does not change what
the gateway process itself uses — export HERMES_HOME to match inside
the guest when you install it.
'';
};
@@ -856,12 +838,6 @@ in
};
users.groups.${vmUser} = { };
# =====================================================================
# The VM itself
#
# No virtiofs, no shared filesystem of any kind, no vhost-net. The guest
# gets a block device, a tap, a vsock and a serial port. That is all.
# =====================================================================
systemd.services.hermes-vm = {
description = "Hermes Agent VM";
wantedBy = [ "multi-user.target" ];
@@ -883,7 +859,6 @@ in
exit 1
fi
done
# Per-VM writable UEFI variable store, seeded once.
if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then
install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd
fi
@@ -894,8 +869,6 @@ in
Group = vmUser;
SupplementaryGroups = [ "kvm" ];
RuntimeDirectory = "hermes-vm";
# ProtectSystem=strict makes everything read-only; punch through for
# the images and the UEFI variable store only.
ReadWritePaths = [ cfg.disk.baseDir ];
StateDirectory = "hermes-vm";
RuntimeDirectoryMode = "0750";
@@ -910,32 +883,21 @@ in
"-m ${toString cfg.mem}"
"-nodefaults"
"-no-user-config"
# Display Settings
"-display none"
"-serial stdio"
# QEMU's own seccomp sandbox, everything denied that we don't need.
"-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"
# UEFI: firmware read-only, variables per-VM and writable.
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
"-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd"
# System disk: NixOS and the Nix store. Rebuildable, never snapshotted.
# aio=threads and a writeback cache because these are files on a
# filesystem, not block devices; aio=native wants the latter.
# aio=threads+writeback: images are files, not blockdevs.
"-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=system"
# State disk: everything irreplaceable. Mounted at /var/lib/hermes.
"-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=state"
# Pre-created tap. vhost=off on purpose: vhost-net would move packet
# handling into the host kernel, which is exactly the surface we are
# trying not to expose. Costs throughput we do not need.
# vhost=off: keep packet handling out of the host kernel.
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
# Host -> guest management, no network involved.
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
]
# Optional devices. rng and vsock are on by default; everything else
# is off unless enabled via services.hermenix.devices.
++ lib.optionals cfg.devices.rng.enable [
"-device virtio-rng-pci"
]
@@ -994,11 +956,8 @@ in
"-device virtio-serial-pci"
"-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
]
# Passed-through PCI devices. The host cannot use these while the VM
# is running.
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
# Raw escape hatch. Unvalidated - prefer the typed devices.* options.
++ cfg.extraQemuArgs
);
@@ -1007,7 +966,6 @@ in
Restart = "always";
RestartSec = 5;
# ---- hardening -------------------------------------------------
DevicePolicy = "closed";
DeviceAllow = [
"/dev/kvm rw"
@@ -1017,8 +975,7 @@ in
++ lib.optionals anyPci [ "char-vfio rw" ]
++ lib.optionals anyUsb [ "char-usb_device rw" ];
# VFIO pins the entire guest RAM allocation, so the default memlock
# limit has to go or QEMU fails to start with an obscure error.
# VFIO pins guest RAM; default memlock is too low.
LimitMEMLOCK = lib.mkIf anyPci "infinity";
NoNewPrivileges = true;
ProtectSystem = "strict";
@@ -1047,16 +1004,13 @@ in
"~@resources"
"~@obsolete"
];
# KVM means no JIT, so W^X is safe. If QEMU refuses to start, this is
# the first thing to flip.
# Safe under KVM (no JIT). First flip if QEMU won't start.
MemoryDenyWriteExecute = true;
CapabilityBoundingSet = [ "" ];
AmbientCapabilities = [ "" ];
};
};
# TPM 2.0 emulator (swtpm), started before the VM. QEMU connects to its
# control socket in the runtime dir; state persists under its StateDirectory.
systemd.services.hermes-vm-swtpm = mkIf cfg.devices.tpm.enable {
description = "swtpm TPM emulator for the Hermes VM";
wantedBy = [ "multi-user.target" ];
@@ -1104,7 +1058,6 @@ in
'';
};
# Tap device, created as root, owned by the VM user, bridged by networkd.
systemd.services.hermes-vm-tap = {
description = "TAP interface for the Hermes VM";
after = [ "systemd-networkd.service" ];
@@ -1123,12 +1076,6 @@ in
'';
};
# =====================================================================
# Device passthrough
# =====================================================================
# The IOMMU is what keeps a passed-through device from reading host
# memory. Without it, PCI passthrough is not a boundary at all.
boot.kernelParams = lib.optionals (pt.iommu != null) [
(if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on")
"iommu=pt"
@@ -1140,12 +1087,10 @@ in
"vfio_iommu_type1"
];
# Claim the device before any host driver gets to it.
boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) ''
options vfio-pci ids=${lib.concatStringsSep "," earlyIds}
'';
# Catch anything that was not early-bound, by address. Idempotent.
systemd.services.hermes-vm-vfio = mkIf anyPci {
description = "Bind passthrough PCI devices to vfio-pci";
before = [ "hermes-vm.service" ];
@@ -1182,9 +1127,6 @@ in
SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660"
'') pt.usb;
# =====================================================================
# Networking
# =====================================================================
networking.useNetworkd = true;
systemd.network.netdevs."10-${net.bridge}".netdevConfig = {
@@ -1194,10 +1136,7 @@ in
systemd.network.networks."10-${net.bridge}" = {
matchConfig.Name = net.bridge;
# hostAddress stays on the bridge in both modes: it is the guest's
# service net (host cache, hostAccess). In bridged mode the host's
# static LAN address is added on top of it - the bridge only ever has
# static addresses, never a DHCP lease.
# hostAddress = service net; bridged also adds hostLanAddress (static only).
address = [
"${net.hostAddress}/${toString net.prefixLength}"
]
@@ -1207,16 +1146,12 @@ in
IPv4Forwarding = true;
}
// lib.optionalAttrs net.bridged {
# Static LAN config for the host: the bridge no longer takes a DHCP
# lease (bridged mode is static-only), so its gateway and DNS come
# from the module options instead.
Gateway = net.lanGateway;
DNS = if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns;
};
};
# The host's DNS, now that the bridge cannot learn it from a DHCP lease.
# mkDefault so the operator's own nameservers win if they set any.
# mkDefault: operator nameservers win if set.
networking.nameservers = lib.mkIf net.bridged (
lib.mkDefault (if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns)
);
@@ -1226,21 +1161,12 @@ in
networkConfig.Bridge = net.bridge;
};
# Bridged mode: the physical uplink becomes a bridge port so the LAN
# shares an L2 domain with the guest. The port must not carry its own
# address config - the bridge owns the LAN address now (static, via
# hostLanAddress; there is no DHCP on the bridge).
systemd.network.networks."12-${net.externalInterface}" = lib.mkIf net.bridged {
matchConfig.Name = net.externalInterface;
networkConfig.Bridge = net.bridge;
};
# The bridge is created with no MAC of its own, so Linux makes it adopt
# the lowest-numbered port's MAC - and the tap is always the first port.
# That is how the host once ended up presenting the VM's MAC on the LAN
# (a duplicate address, since the guest uses the same MAC). Clone the
# uplink's MAC onto the bridge so the host keeps its LAN identity; the
# guest keeps its own tap MAC (net.mac).
# Bridge would inherit tap MAC (first port) and collide with guest — clone uplink MAC.
systemd.services.hermes-bridge-mac = lib.mkIf net.bridged {
description = "Clone the uplink MAC onto the hermenix bridge";
after = [
@@ -1258,7 +1184,6 @@ in
unitConfig.ConditionPathExists = "/sys/class/net/${net.externalInterface}";
script = ''
set -euo pipefail
# networkd creates the bridge netdev asynchronously; give it a moment.
for _ in 1 2 3 4 5 6 7 8 9 10; do
[ -e "/sys/class/net/${net.bridge}" ] && break
sleep 0.5
@@ -1268,17 +1193,11 @@ in
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} down
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} address "$mac"
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} up
# Re-apply the bridge's static address config under the new MAC.
${pkgs.systemd}/bin/networkctl reconfigure ${net.bridge} || true
'';
};
# The guest is handed its fixed LAN address (lanAddress) by a dedicated
# DHCP server bound to the bridge, keyed to the guest's MAC. This is the
# guest's ONLY DHCP: its broadcast is blocked from leaving the bridge
# (see the hermenix-bridge table), so the LAN's own DHCP server can
# never race the reservation. DNS is disabled (port=0) - the guest runs
# static QUAD9 resolvers, and this host gets its DNS from lanDns.
# Single-address DHCP on the bridge (MAC-keyed). Broadcasts stay on-bridge.
systemd.services.hermes-lan-dhcp = lib.mkIf net.bridged {
description = "Static-lease DHCP for the bridged hermenix guest";
after = [
@@ -1309,34 +1228,16 @@ in
'';
};
# ip_forward: use the same sysctl keys networking.nat used (mkOverride 99
# so they win over any lower-priority defaults elsewhere in the config).
# mkOverride 99: same keys as networking.nat, win over weaker defaults.
boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true;
boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true;
# Masquerade outbound guest traffic and DNAT published ports inbound.
# We use an explicit nftables table instead of networking.nat so that the
# rules are certain to appear in the prerouting/postrouting hooks.
#
# NOTE: DNAT uses 'iifname != bridge' rather than 'iifname externalInterface'.
# Scoping to a single interface means traffic arriving on any other path
# (Tailscale, a second NIC, WiFi, loopback) silently skips DNAT: the
# packet stays addressed to the host's own IP, falls into the INPUT chain,
# and gets dropped because those ports are not in allowedTCPPorts.
# Excluding only the bridge itself prevents loops from inside the guest
# while covering every legitimate external path.
# Explicit nft nat (not networking.nat). DNAT iifname != bridge so extra paths still match.
networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null) {
family = "ip";
content =
if net.bridged then
# Bridged mode: no DNAT (the guest is directly reachable on the
# LAN) and no primary masquerade. Keep one scoped masquerade as a
# fallback: if the guest's DHCP lease lapses it falls back to its
# static default route via the host, and this keeps that path
# working instead of blackholing at the uplink. With a healthy
# lease the guest's traffic is L2-bridged and never reaches this
# chain. Only traffic handed up from the bridge (i.e. the guest's
# service-net traffic) is matched, never the LAN itself.
# No DNAT when bridged. Scoped masquerade covers service-net fallback if DHCP lapses.
''
chain postrouting {
type nat hook postrouting priority srcnat;
@@ -1360,34 +1261,19 @@ in
'';
};
# =====================================================================
# Bridged mode: bridge-family enforcement.
#
# With the uplink bridged, guest <-> LAN traffic is switched at L2 and
# never traverses the ip-family hooks the NAT-mode rules rely on, so
# bridged mode gets its own table where the tap port is unambiguous.
# lanAccess / allowLan still gate what the guest may reach; inbound
# from the LAN to the guest is open on every port.
# =====================================================================
networking.nftables.tables."hermenix-bridge" = lib.mkIf net.bridged {
family = "bridge";
content = ''
chain prerouting {
type filter hook prerouting priority filter; policy accept;
# Tag anything entering from the guest. ip-family input rules
# (host cache, hostAccess) match this mark.
iifname "${net.tapId}" meta mark set ${toString bridgeMark}
# The guest's address is static - served by hermes-lan-dhcp on
# this bridge - so keep its DHCP traffic off the LAN and keep the
# LAN's DHCP answers out of the guest. Only the host-side server
# may talk DHCP to it.
# Guest DHCP stays on-bridge only (no race with LAN DHCP).
iifname "${net.tapId}" oifname "${net.externalInterface}" ether type ip ip protocol udp udp sport 68 udp dport 67 drop
iifname "${net.externalInterface}" oifname "${net.tapId}" ether type ip ip protocol udp udp sport 67 udp dport 68 drop
}
chain forward {
type filter hook forward priority filter; policy accept;
# Inbound from the LAN to the guest: any device, any port.
oifname "${net.tapId}" accept
''
+ lib.concatMapStrings (s: ''
@@ -1417,26 +1303,11 @@ in
'';
};
# =====================================================================
# Firewall
#
# The guest owns its own kernel and its own config now, so nothing inside
# it constrains it. These rules are the boundary.
# =====================================================================
networking.nftables.enable = true;
networking.firewall.enable = true;
networking.firewall.filterForward = true;
# The deliberate guest -> host holes: the Nix cache, plus anything listed
# in hostAccess.
#
# NAT mode: the guest always arrives from its fixed service-net address,
# so match address + bridge. Bridged mode: the guest presents either the
# service-net address or its LAN address (served by the bridge's DHCP),
# so the bridge table above tags guest traffic with a netfilter mark as
# it enters the tap - the rules match that mark instead. The tap is
# exclusive to the guest, so the mark is as specific as an address
# match: LAN devices can never inherit these holes.
# Guest→host holes. NAT: guestAddress. Bridged: tap mark (guest may use LAN IP).
networking.firewall.extraInputRules =
if net.bridged then
optionalString cfg.hostCache.enable ''
@@ -1453,21 +1324,14 @@ in
iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept
'') cfg.hostAccess;
# Order matters here. Allowlists must precede the RFC1918 drop.
#
# These ip-family rules only ever see routed traffic. They are the whole
# story in NAT mode; in bridged mode the guest's LAN traffic is
# L2-bridged instead and is governed by the hermenix-bridge table.
# Allowlists before RFC1918 drop. Bridged LAN is L2 (hermenix-bridge table).
networking.firewall.extraForwardRules = ''
iifname != "${net.bridge}" accept
ct state established,related accept
''
# Guest reaching named services elsewhere on your network.
+ lib.concatMapStrings (s: ''
iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
'') cfg.lanAccess
# Inbound to guest services published on the uplink. NAT mode only; in
# bridged mode the bridge table opens every inbound port instead.
+ lib.optionalString (!net.bridged) (
lib.concatMapStrings (s: ''
oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept
@@ -1493,24 +1357,14 @@ in
''
);
# =====================================================================
# Host store as a read-only cache for the guest
# =====================================================================
services.nix-serve = mkIf cfg.hostCache.enable {
enable = true;
package = pkgs.nix-serve-ng;
bindAddress = net.hostAddress;
inherit (cfg.hostCache) port secretKeyFile;
openFirewall = false; # handled above, scoped to the guest address
openFirewall = false;
};
# =====================================================================
# Snapshots of the state disk only.
#
# The system disk is not snapshotted on purpose. It is large, it is where
# all the write churn happens, and `hermenix setup` can rebuild it from the flake
# in minutes. Snapshotting it would cost a lot and buy nothing.
# =====================================================================
systemd.services.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
description = "Snapshot the Hermes VM state disk";
path = with pkgs; [
@@ -1525,9 +1379,6 @@ in
mkdir -p ${snapDir}
${optionalString cfg.disk.quiesce ''
# Flush the guest's filesystems so the snapshot is consistent rather
# than the equivalent of pulling the power. Takes milliseconds; the
# VM keeps running throughout.
${qga "freeze"}
trap '${qga "thaw"}' EXIT
''}
@@ -1539,8 +1390,7 @@ in
trap - EXIT
''}
# Prune oldest first. Keep this list short: every live snapshot
# reintroduces copy-on-write on a nodatacow image.
# Live snapshots reintroduce CoW on nodatacow images — keep retention tight.
ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \
| while read -r old; do btrfs subvolume delete "$old"; done
'';
@@ -1555,15 +1405,6 @@ in
};
};
# =====================================================================
# Backups: the guest's state, not its disk.
#
# Pulled file-by-file over the SSH channel that already exists, rather
# than backing up the image. That keeps the backup portable - it restores
# onto any filesystem, or a different machine entirely - and avoids
# loop-mounting a guest-written filesystem on the host, which would mean
# the host kernel parsing data the agent controls.
# =====================================================================
systemd.services.hermes-vm-backup = mkIf cfg.backup.enable {
description = "Back up Hermes state";
path = with pkgs; [
@@ -1608,17 +1449,12 @@ in
};
environment.systemPackages = [
# Nushell tab-completion definitions for hermenix. Installed to
# vendor/autoload so nushell picks them up automatically.
(pkgs.writeTextFile {
name = "hermenix-completions";
destination = "/share/nushell/vendor/autoload/hermenix.nu";
text = builtins.readFile ../nu/completions.nu;
})
# Single consolidated CLI: `hermenix vm *` controls the VM,
# `hermenix ctl *` drives Hermes inside it. Per-host values are
# added on top of the base package via makeWrapper.
(pkgs.symlinkJoin {
name = "hermenix";
paths = [ pkgs.hermenix ];