Files
hermenix/modules/host.nix
T
2026-09-18 12:33:33 -04:00

978 lines
33 KiB
Nix

{
config,
lib,
pkgs,
...
}:
let
inherit (lib)
mkOption
mkEnableOption
mkIf
types
optionalString
;
cfg = config.services.hermenix;
net = cfg.network;
vmUser = "hermes-vm";
runDir = "/run/hermes-vm";
stateDir = "/var/lib/hermes-vm";
# Two disks, deliberately.
#
# The system disk is large, churny and entirely rebuildable - it is the
# NixOS install and the Nix store. We never snapshot it.
#
# The state disk holds the only things that cannot be reproduced: Hermes's
# identity and memories, its workspace, and the NixOS config it has written
# for itself. Small, so snapshots are cheap and backups are quick.
systemImage = "${cfg.disk.baseDir}/system/system.raw";
stateImage = "${cfg.disk.baseDir}/state/state.raw";
snapDir = "${cfg.disk.baseDir}/snapshots";
pt = cfg.passthrough;
anyPci = pt.pci != [ ];
anyUsb = pt.usb != [ ];
earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci);
# Graceful ACPI power button via QMP, then wait for QEMU to exit.
stopScript = pkgs.writeShellScript "hermes-vm-stop" ''
set -eu
printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \
| ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0
'';
# Freeze the guest filesystems around a snapshot so it captures a consistent
# state rather than the equivalent of pulling the power.
qga =
verb:
pkgs.writeShellScript "hermes-vm-fs${verb}" ''
set -eu
printf '%s' '{"execute":"guest-fs${verb}"}' \
| ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true
'';
in
{
options.services.hermenix = {
enable = mkEnableOption "the Hermes Agent VM";
name = mkOption {
type = types.str;
default = "hermes";
};
vcpu = mkOption {
type = types.ints.positive;
default = 4;
};
mem = mkOption {
type = types.ints.positive;
default = 8192;
description = "RAM in MiB.";
};
disk = {
baseDir = mkOption {
type = types.str;
default = "/var/lib/hermes-vm";
description = ''
Directory holding the guest's disk images. On btrfs this should be
laid out as three subvolumes - system/, state/ and snapshots/ - with
nodatacow set. `hermenix setup` does this for you.
'';
};
systemSize = mkOption {
type = types.str;
default = "80G";
description = ''
System disk: the NixOS install and the Nix store. Never snapshotted,
because it is entirely rebuildable from the flake. Size it for an
agent that builds things.
'';
};
stateSize = mkOption {
type = types.str;
default = "20G";
description = ''
State disk, mounted at /var/lib/hermes in the guest. Holds Hermes's
identity, memories, workspace, and the NixOS config it maintains for
itself. This is the only irreplaceable data, so it is the only thing
snapshotted and backed up.
'';
};
quiesce = mkOption {
type = types.bool;
default = true;
description = "fsfreeze the guest via qemu-guest-agent around each snapshot.";
};
};
snapshots = {
enable = mkOption {
type = types.bool;
default = true;
description = ''
Periodic btrfs snapshots of the state disk. The agent can persist
changes, which means it can persist a mistake, and these are the
fastest way back.
'';
};
interval = mkOption {
type = types.str;
default = "hourly";
description = "systemd calendar expression.";
};
keep = mkOption {
type = types.ints.positive;
default = 48;
description = ''
How many to retain. Keep this tighter than you would on ZFS: the
state image is nodatacow, and every live snapshot reintroduces
copy-on-write for the blocks it still references.
'';
};
};
backup = {
enable = mkEnableOption "restic backups of the guest's state";
repository = mkOption {
type = types.str;
default = "";
example = "sftp:backup@nas:/hermes";
description = "Any restic repository string.";
};
passwordFile = mkOption {
type = types.path;
default = "/var/lib/secrets/hermes-restic";
description = "File containing the restic repository password, mode 0400.";
};
environmentFile = mkOption {
type = types.nullOr types.path;
default = null;
description = "Optional file with credentials for the backend (S3 keys and so on).";
};
interval = mkOption {
type = types.str;
default = "daily";
};
keep = mkOption {
type = types.listOf types.str;
default = [
"--keep-daily"
"14"
"--keep-weekly"
"8"
"--keep-monthly"
"12"
];
description = "Arguments passed to restic forget.";
};
};
hostCache = {
enable = mkOption {
type = types.bool;
default = true;
description = ''
Serve the host's /nix/store to the guest as a read-only binary
cache, so the guest doesn't re-download from the internet what the
host already has.
This is the ONLY route from guest to host in this design. Turning it
off leaves the guest with literally no reachable host service, at
the cost of bandwidth on every guest build.
Note the guest can read every store path on the host through this.
Store paths are world-readable anyway, but check nothing sensitive
is baked into your host's store.
'';
};
port = mkOption {
type = types.port;
default = 5000;
};
secretKeyFile = mkOption {
type = types.path;
default = "/var/lib/secrets/hermes-cache.secret";
description = "Generate with nix-store --generate-binary-cache-key. See README.";
};
};
authorizedKeys = mkOption {
type = types.listOf types.str;
default = [ ];
};
allowLan = mkOption {
type = types.bool;
default = false;
description = ''
Blanket permission for the guest to reach RFC1918 space. Almost always
the wrong tool - use lanAccess to open single endpoints instead.
'';
};
lanAccess = mkOption {
default = [ ];
example = [
{
address = "192.168.1.50";
port = 11434;
}
];
description = ''
Specific endpoints elsewhere on your network that the guest may reach,
punched through the default RFC1918 drop. One entry per service.
'';
type = types.listOf (
types.submodule {
options = {
address = mkOption { type = types.str; };
port = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
}
);
};
hostAccess = mkOption {
default = [ ];
example = [ { port = 8188; } ];
description = ''
Ports on this host itself that the guest may reach. The service must
also be bound to an address the guest can see - see the README.
'';
type = types.listOf (
types.submodule {
options = {
port = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
}
);
};
publish = mkOption {
default = [ ];
example = [
{
hostPort = 8080;
guestPort = 8080;
}
];
description = ''
Guest ports to publish on the host's uplink, reachable from the wider
network. You do not need this to reach the guest from the host itself.
'';
type = types.listOf (
types.submodule {
options = {
hostPort = mkOption { type = types.port; };
guestPort = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
}
);
};
network = {
bridge = mkOption {
type = types.str;
default = "hermes-br";
};
tapId = mkOption {
type = types.str;
default = "vm-hermes";
};
mac = mkOption {
type = types.str;
default = "02:00:00:7e:00:01";
};
hostAddress = mkOption {
type = types.str;
default = "10.77.0.1";
};
guestAddress = mkOption {
type = types.str;
default = "10.77.0.2";
};
prefixLength = mkOption {
type = types.int;
default = 24;
};
externalInterface = mkOption {
type = types.nullOr types.str;
default = null;
example = "eno1";
description = "Host uplink. Required.";
};
};
passthrough = {
iommu = mkOption {
type = types.nullOr (
types.enum [
"intel"
"amd"
]
);
default = null;
description = ''
Enable IOMMU kernel parameters for your CPU vendor. Required for any
PCI passthrough. The IOMMU is the only thing stopping a passed-through
device from reading host memory, so this is not optional.
'';
};
pci = mkOption {
default = [ ];
example = [
{
address = "0000:01:00.0";
id = "10de:2204";
}
{
address = "0000:01:00.1";
id = "10de:1aef";
}
];
description = ''
PCI devices handed to the guest via VFIO. The host loses them
entirely while the VM runs.
You must pass every function of the device and, in practice, every
device in its IOMMU group. A GPU is usually at least two entries:
the graphics function and its HDMI audio function.
Set `id` (vendor:device) to have vfio-pci claim the device in the
initrd, before a host driver can. Strongly recommended for GPUs;
unbinding an active amdgpu or nvidia driver later is unreliable.
'';
type = types.listOf (
types.submodule {
options = {
address = mkOption {
type = types.str;
example = "0000:01:00.0";
description = "Full PCI address, as shown by lspci -D.";
};
id = mkOption {
type = types.nullOr types.str;
default = null;
example = "10de:2204";
description = "vendor:device, from lspci -nn. Enables early binding.";
};
};
}
);
};
usb = mkOption {
default = [ ];
example = [
{
vendorId = "8087";
productId = "0026";
}
];
description = ''
USB devices forwarded to the guest, matched by vendor and product
ID from lsusb. Integrated Bluetooth radios are USB devices and go
here too, though the host driver must be kept off them.
QEMU resolves these at start, so a device that is unplugged when the
VM boots will not appear later without a restart.
'';
type = types.listOf (
types.submodule {
options = {
vendorId = mkOption {
type = types.str;
example = "046d";
};
productId = mkOption {
type = types.str;
example = "c52b";
};
};
}
);
};
};
gatewayUnit = mkOption {
type = types.str;
default = "hermes-agent";
description = ''
The systemd *user* unit that `hermes gateway install` created in the
guest. Check with `hermenix ctl units` after installing; upstream has
used more than one name for this.
'';
};
hermesHome = mkOption {
type = types.str;
default = "${stateDir}/.hermes";
description = ''
Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/,
state.db, and so on) lives inside the guest. Defaults to the seeded
layout under the state disk mount. Override this when carrying over
an existing flat HERMES_HOME from elsewhere (e.g. a prior Docker
deployment) that you're placing directly at the root of the state
disk instead of under .hermes - see `hermenix migrate --help`.
This only changes where the host-side `hermenix ctl` commands (soul,
memory, conf, secrets, reseed, pull) looks. It does not change what
the gateway process itself uses - export HERMES_HOME to match inside
the guest when you install it.
'';
};
vsockCid = mkOption {
type = types.int;
default = 77;
description = "Host reaches the guest over vsock, so no inbound network path is needed.";
};
version = mkOption {
type = types.str;
default = "dev";
description = ''
"YYYY.MM.DD.hh.mm-<commit-sha>" reported by `hermenix version`.
`version` subcommand. Normally supplied for free by this flake's own
`nixosModules.default` (from its git metadata) - the "dev" default
here only shows up if this module is imported some other way.
'';
};
};
config = mkIf cfg.enable {
assertions = [
{
assertion = net.externalInterface != null;
message = "services.hermenix.network.externalInterface must be set.";
}
{
assertion = cfg.backup.enable -> (cfg.backup.repository != "");
message = "services.hermenix.backup.repository must be set when backups are enabled.";
}
{
assertion = anyPci -> (pt.iommu != null);
message = ''
services.hermenix.passthrough.pci needs passthrough.iommu set to
"intel" or "amd". Without the IOMMU, a passed-through device can
read host memory directly and the VM boundary is meaningless.
'';
}
];
users.users.${vmUser} = {
isSystemUser = true;
group = vmUser;
description = "Runs the Hermes guest VM";
};
users.groups.${vmUser} = { };
# =====================================================================
# The VM itself
#
# No virtiofs, no shared filesystem of any kind, no vhost-net. The guest
# gets a block device, a tap, a vsock and a serial port. That is all.
# =====================================================================
systemd.services.hermes-vm = {
description = "Hermes Agent VM";
wantedBy = [ "multi-user.target" ];
after = [
"network-online.target"
"hermes-vm-tap.service"
];
requires = [ "hermes-vm-tap.service" ];
wants = [ "network-online.target" ];
preStart = ''
for img in ${systemImage} ${stateImage}; do
if [ ! -e "$img" ]; then
echo "$img is missing. Run 'hermenix setup' first." >&2
exit 1
fi
done
# Per-VM writable UEFI variable store, seeded once.
if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then
install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd
fi
'';
serviceConfig = {
User = vmUser;
Group = vmUser;
SupplementaryGroups = [ "kvm" ];
RuntimeDirectory = "hermes-vm";
# ProtectSystem=strict makes everything read-only; punch through for
# the images and the UEFI variable store only.
ReadWritePaths = [ cfg.disk.baseDir ];
StateDirectory = "hermes-vm";
RuntimeDirectoryMode = "0750";
ExecStart = lib.concatStringsSep " " (
[
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
"-name ${cfg.name}"
"-machine q35,accel=kvm,smm=on"
"-cpu host"
"-smp ${toString cfg.vcpu}"
"-m ${toString cfg.mem}"
"-nodefaults"
"-no-user-config"
"-display none"
"-serial stdio"
# QEMU's own seccomp sandbox, everything denied that we don't need.
"-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"
# UEFI: firmware read-only, variables per-VM and writable.
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
"-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd"
# System disk: NixOS and the Nix store. Rebuildable, never snapshotted.
# aio=threads and a writeback cache because these are files on a
# filesystem, not block devices; aio=native wants the latter.
"-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=system"
# State disk: everything irreplaceable. Mounted at /var/lib/hermes.
"-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=state"
# Pre-created tap. vhost=off on purpose: vhost-net would move packet
# handling into the host kernel, which is exactly the surface we are
# trying not to expose. Costs throughput we do not need.
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
# Host -> guest management, no network involved.
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
"-device virtio-rng-pci"
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
]
++ lib.optionals cfg.disk.quiesce [
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
"-device virtio-serial-pci"
"-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
]
# Passed-through PCI devices. The host cannot use these while the VM
# is running.
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
# A USB controller, only if something needs plugging into it.
++ lib.optional anyUsb "-device qemu-xhci,id=xhci"
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
);
ExecStop = stopScript;
TimeoutStopSec = 90;
Restart = "always";
RestartSec = 5;
# ---- hardening -------------------------------------------------
DevicePolicy = "closed";
DeviceAllow = [
"/dev/kvm rw"
"/dev/net/tun rw"
"/dev/vhost-vsock rw"
]
++ lib.optionals anyPci [ "char-vfio rw" ]
++ lib.optionals anyUsb [ "char-usb_device rw" ];
# VFIO pins the entire guest RAM allocation, so the default memlock
# limit has to go or QEMU fails to start with an obscure error.
LimitMEMLOCK = lib.mkIf anyPci "infinity";
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
ProtectProc = "invisible";
ProcSubset = "pid";
PrivateTmp = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHostname = true;
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
RestrictAddressFamilies = [
"AF_UNIX"
"AF_NETLINK"
"AF_VSOCK"
];
LockPersonality = true;
SystemCallArchitectures = "native";
SystemCallFilter = [
"@system-service"
"~@resources"
"~@obsolete"
];
# KVM means no JIT, so W^X is safe. If QEMU refuses to start, this is
# the first thing to flip.
MemoryDenyWriteExecute = true;
CapabilityBoundingSet = [ "" ];
AmbientCapabilities = [ "" ];
};
};
# Tap device, created as root, owned by the VM user, bridged by networkd.
systemd.services.hermes-vm-tap = {
description = "TAP interface for the Hermes VM";
after = [ "systemd-networkd.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
path = [ pkgs.iproute2 ];
script = ''
ip link show ${net.tapId} >/dev/null 2>&1 \
|| ip tuntap add ${net.tapId} mode tap user ${vmUser}
ip link set ${net.tapId} up
'';
preStop = ''
${pkgs.iproute2}/bin/ip link del ${net.tapId} || true
'';
};
# =====================================================================
# Device passthrough
# =====================================================================
# The IOMMU is what keeps a passed-through device from reading host
# memory. Without it, PCI passthrough is not a boundary at all.
boot.kernelParams = lib.optionals (pt.iommu != null) [
(if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on")
"iommu=pt"
];
boot.initrd.kernelModules = lib.optionals anyPci [
"vfio_pci"
"vfio"
"vfio_iommu_type1"
];
# Claim the device before any host driver gets to it.
boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) ''
options vfio-pci ids=${lib.concatStringsSep "," earlyIds}
'';
# Catch anything that was not early-bound, by address. Idempotent.
systemd.services.hermes-vm-vfio = mkIf anyPci {
description = "Bind passthrough PCI devices to vfio-pci";
before = [ "hermes-vm.service" ];
requiredBy = [ "hermes-vm.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
path = [ pkgs.kmod ];
script = ''
modprobe vfio-pci
''
+ lib.concatMapStrings (d: ''
dev=/sys/bus/pci/devices/${d.address}
if [ ! -e "$dev" ]; then
echo "${d.address} is not present on this machine" >&2
exit 1
fi
cur=none
[ -e "$dev/driver" ] && cur=$(basename "$(readlink -f "$dev/driver")")
if [ "$cur" != "vfio-pci" ]; then
[ "$cur" != "none" ] && echo ${d.address} > "$dev/driver/unbind"
echo vfio-pci > "$dev/driver_override"
echo ${d.address} > /sys/bus/pci/drivers_probe
fi
'') pt.pci;
};
services.udev.extraRules =
lib.optionalString anyPci ''
SUBSYSTEM=="vfio", OWNER="${vmUser}", GROUP="${vmUser}", MODE="0660"
''
+ lib.concatMapStrings (u: ''
SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660"
'') pt.usb;
# =====================================================================
# Networking
# =====================================================================
networking.useNetworkd = true;
systemd.network.netdevs."10-${net.bridge}".netdevConfig = {
Kind = "bridge";
Name = net.bridge;
};
systemd.network.networks."10-${net.bridge}" = {
matchConfig.Name = net.bridge;
address = [ "${net.hostAddress}/${toString net.prefixLength}" ];
networkConfig = {
ConfigureWithoutCarrier = true;
IPv4Forwarding = true;
};
};
systemd.network.networks."11-${net.tapId}" = {
matchConfig.Name = net.tapId;
networkConfig.Bridge = net.bridge;
};
# ip_forward: use the same sysctl keys networking.nat used (mkOverride 99
# so they win over any lower-priority defaults elsewhere in the config).
boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true;
boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true;
# Masquerade outbound guest traffic and DNAT published ports inbound.
# We use an explicit nftables table instead of networking.nat so that the
# rules are certain to appear in the prerouting/postrouting hooks.
#
# NOTE: DNAT uses 'iifname != bridge' rather than 'iifname externalInterface'.
# Scoping to a single interface means traffic arriving on any other path
# (Tailscale, a second NIC, WiFi, loopback) silently skips DNAT: the
# packet stays addressed to the host's own IP, falls into the INPUT chain,
# and gets dropped because those ports are not in allowedTCPPorts.
# Excluding only the bridge itself prevents loops from inside the guest
# while covering every legitimate external path.
networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null) {
family = "ip";
content = ''
chain prerouting {
type nat hook prerouting priority dstnat;
''
+ lib.concatMapStrings (s: ''
iifname != "${net.bridge}" ${s.proto} dport ${toString s.hostPort} dnat to ${net.guestAddress}:${toString s.guestPort}
'') cfg.publish
+ ''
}
chain postrouting {
type nat hook postrouting priority srcnat;
iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade
}
'';
};
# =====================================================================
# Firewall
#
# The guest owns its own kernel and its own config now, so nothing inside
# it constrains it. These rules are the boundary.
# =====================================================================
networking.nftables.enable = true;
networking.firewall.enable = true;
networking.firewall.filterForward = true;
# The deliberate guest -> host holes: the Nix cache, plus anything listed
# in hostAccess.
networking.firewall.extraInputRules =
optionalString cfg.hostCache.enable ''
iifname "${net.bridge}" ip saddr ${net.guestAddress} tcp dport ${toString cfg.hostCache.port} accept
''
+ lib.concatMapStrings (s: ''
iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept
'') cfg.hostAccess;
# Order matters here. Allowlists must precede the RFC1918 drop.
networking.firewall.extraForwardRules = ''
iifname != "${net.bridge}" accept
ct state established,related accept
''
# Guest reaching named services elsewhere on your network.
+ lib.concatMapStrings (s: ''
iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
'') cfg.lanAccess
# Inbound to guest services published on the uplink.
+ lib.concatMapStrings (s: ''
oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept
'') cfg.publish
+ (
if cfg.allowLan then
''
iifname "${net.bridge}" accept
''
else
''
iifname "${net.bridge}" ip daddr {
10.0.0.0/8,
172.16.0.0/12,
192.168.0.0/16,
169.254.0.0/16,
127.0.0.0/8,
100.64.0.0/10
} drop
iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
iifname "${net.bridge}" accept
''
);
# =====================================================================
# Host store as a read-only cache for the guest
# =====================================================================
services.nix-serve = mkIf cfg.hostCache.enable {
enable = true;
package = pkgs.nix-serve-ng;
bindAddress = net.hostAddress;
inherit (cfg.hostCache) port secretKeyFile;
openFirewall = false; # handled above, scoped to the guest address
};
# =====================================================================
# Snapshots of the state disk only.
#
# The system disk is not snapshotted on purpose. It is large, it is where
# all the write churn happens, and `hermenix setup` can rebuild it from the flake
# in minutes. Snapshotting it would cost a lot and buy nothing.
# =====================================================================
systemd.services.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
description = "Snapshot the Hermes VM state disk";
path = with pkgs; [
btrfs-progs
coreutils
socat
];
serviceConfig.Type = "oneshot";
script = ''
set -euo pipefail
stamp=$(date +%Y%m%d-%H%M%S)
mkdir -p ${snapDir}
${optionalString cfg.disk.quiesce ''
# Flush the guest's filesystems so the snapshot is consistent rather
# than the equivalent of pulling the power. Takes milliseconds; the
# VM keeps running throughout.
${qga "freeze"}
trap '${qga "thaw"}' EXIT
''}
btrfs subvolume snapshot -r ${cfg.disk.baseDir}/state ${snapDir}/state-$stamp
${optionalString cfg.disk.quiesce ''
${qga "thaw"}
trap - EXIT
''}
# Prune oldest first. Keep this list short: every live snapshot
# reintroduces copy-on-write on a nodatacow image.
ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \
| while read -r old; do btrfs subvolume delete "$old"; done
'';
};
systemd.timers.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.snapshots.interval;
Persistent = true;
RandomizedDelaySec = "5m";
};
};
# =====================================================================
# Backups: the guest's state, not its disk.
#
# Pulled file-by-file over the SSH channel that already exists, rather
# than backing up the image. That keeps the backup portable - it restores
# onto any filesystem, or a different machine entirely - and avoids
# loop-mounting a guest-written filesystem on the host, which would mean
# the host kernel parsing data the agent controls.
# =====================================================================
systemd.services.hermes-vm-backup = mkIf cfg.backup.enable {
description = "Back up Hermes state";
path = with pkgs; [
restic
openssh
socat
gnutar
];
serviceConfig = {
Type = "oneshot";
EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile;
};
script = ''
set -euo pipefail
export RESTIC_REPOSITORY=${lib.escapeShellArg cfg.backup.repository}
export RESTIC_PASSWORD_FILE=${cfg.backup.passwordFile}
restic snapshots >/dev/null 2>&1 || restic init
ssh -o ProxyCommand="socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22" \
-o StrictHostKeyChecking=accept-new \
hermes@${cfg.name} \
'tar -C /var/lib/hermes \
--exclude=./.cache \
--exclude=./workspace/**/.direnv \
--exclude=./workspace/**/result \
-cf - .' \
| restic backup --stdin --stdin-filename hermes-state.tar \
--tag hermes --host ${cfg.name}
restic forget --tag hermes --host ${cfg.name} --prune ${lib.concatStringsSep " " cfg.backup.keep}
'';
};
systemd.timers.hermes-vm-backup = mkIf cfg.backup.enable {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.backup.interval;
Persistent = true;
RandomizedDelaySec = "30m";
};
};
environment.systemPackages = [
# Nushell tab-completion definitions for hermenix. Installed to
# vendor/autoload so nushell picks them up automatically.
(pkgs.writeTextFile {
name = "hermenix-completions";
destination = "/share/nushell/vendor/autoload/hermenix.nu";
text = builtins.readFile ../nu/completions.nu;
})
# Single consolidated CLI: `hermenix vm *` controls the VM,
# `hermenix ctl *` drives Hermes inside it. Per-host values are
# added on top of the base package via makeWrapper.
(pkgs.symlinkJoin {
name = "hermenix";
paths = [ pkgs.hermenix ];
nativeBuildInputs = [ pkgs.makeWrapper ];
postBuild = ''
wrapProgram $out/bin/hermenix \
--set HERMES_VSOCK_CID ${toString cfg.vsockCid} \
--set HERMES_GUEST_NAME ${cfg.name} \
--set HERMES_DISK_BASE_DIR ${cfg.disk.baseDir} \
--set HERMES_HOME_DIR ${cfg.hermesHome} \
--set HERMES_GATEWAY_UNIT ${cfg.gatewayUnit}
'';
})
];
};
}