services.hermesVm -> services.hermenix throughout. setup.sh and
migrate-docker-hermes.sh are replaced by nu/hermenix.nu (setup + migrate
subcommands), packaged as a flake output (packages.${system}.hermenix,
pkgs.writers.writeNuBin) so it's installable without cloning conventions
beyond the repo itself. Both subcommands verified end-to-end against a
sandboxed dry run (stubbed disk/cache tools, a local ssh stand-in for the
migrate path) before landing.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
995 lines
37 KiB
Nix
995 lines
37 KiB
Nix
{ config, lib, pkgs, ... }:
|
|
|
|
let
|
|
inherit (lib) mkOption mkEnableOption mkIf types optionalString;
|
|
cfg = config.services.hermenix;
|
|
net = cfg.network;
|
|
|
|
vmUser = "hermes-vm";
|
|
runDir = "/run/hermes-vm";
|
|
stateDir = "/var/lib/hermes-vm";
|
|
# Two disks, deliberately.
|
|
#
|
|
# The system disk is large, churny and entirely rebuildable - it is the
|
|
# NixOS install and the Nix store. We never snapshot it.
|
|
#
|
|
# The state disk holds the only things that cannot be reproduced: Hermes's
|
|
# identity and memories, its workspace, and the NixOS config it has written
|
|
# for itself. Small, so snapshots are cheap and backups are quick.
|
|
systemImage = "${cfg.disk.baseDir}/system/system.raw";
|
|
stateImage = "${cfg.disk.baseDir}/state/state.raw";
|
|
snapDir = "${cfg.disk.baseDir}/snapshots";
|
|
|
|
pt = cfg.passthrough;
|
|
anyPci = pt.pci != [ ];
|
|
anyUsb = pt.usb != [ ];
|
|
earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci);
|
|
|
|
# Graceful ACPI power button via QMP, then wait for QEMU to exit.
|
|
stopScript = pkgs.writeShellScript "hermes-vm-stop" ''
|
|
set -eu
|
|
printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \
|
|
| ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0
|
|
'';
|
|
|
|
# Freeze the guest filesystems around a snapshot so it captures a consistent
|
|
# state rather than the equivalent of pulling the power.
|
|
qga = verb: pkgs.writeShellScript "hermes-vm-fs${verb}" ''
|
|
set -eu
|
|
printf '%s' '{"execute":"guest-fs${verb}"}' \
|
|
| ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true
|
|
'';
|
|
in
|
|
{
|
|
options.services.hermenix = {
|
|
enable = mkEnableOption "the Hermes Agent VM";
|
|
|
|
name = mkOption {
|
|
type = types.str;
|
|
default = "hermes";
|
|
};
|
|
|
|
vcpu = mkOption {
|
|
type = types.ints.positive;
|
|
default = 4;
|
|
};
|
|
|
|
mem = mkOption {
|
|
type = types.ints.positive;
|
|
default = 8192;
|
|
description = "RAM in MiB.";
|
|
};
|
|
|
|
disk = {
|
|
baseDir = mkOption {
|
|
type = types.str;
|
|
default = "/var/lib/hermes-vm";
|
|
description = ''
|
|
Directory holding the guest's disk images. On btrfs this should be
|
|
laid out as three subvolumes - system/, state/ and snapshots/ - with
|
|
nodatacow set. `hermenix setup` does this for you.
|
|
'';
|
|
};
|
|
|
|
systemSize = mkOption {
|
|
type = types.str;
|
|
default = "80G";
|
|
description = ''
|
|
System disk: the NixOS install and the Nix store. Never snapshotted,
|
|
because it is entirely rebuildable from the flake. Size it for an
|
|
agent that builds things.
|
|
'';
|
|
};
|
|
|
|
stateSize = mkOption {
|
|
type = types.str;
|
|
default = "20G";
|
|
description = ''
|
|
State disk, mounted at /var/lib/hermes in the guest. Holds Hermes's
|
|
identity, memories, workspace, and the NixOS config it maintains for
|
|
itself. This is the only irreplaceable data, so it is the only thing
|
|
snapshotted and backed up.
|
|
'';
|
|
};
|
|
|
|
quiesce = mkOption {
|
|
type = types.bool;
|
|
default = true;
|
|
description = "fsfreeze the guest via qemu-guest-agent around each snapshot.";
|
|
};
|
|
};
|
|
|
|
snapshots = {
|
|
enable = mkOption {
|
|
type = types.bool;
|
|
default = true;
|
|
description = ''
|
|
Periodic btrfs snapshots of the state disk. The agent can persist
|
|
changes, which means it can persist a mistake, and these are the
|
|
fastest way back.
|
|
'';
|
|
};
|
|
|
|
interval = mkOption {
|
|
type = types.str;
|
|
default = "hourly";
|
|
description = "systemd calendar expression.";
|
|
};
|
|
|
|
keep = mkOption {
|
|
type = types.ints.positive;
|
|
default = 48;
|
|
description = ''
|
|
How many to retain. Keep this tighter than you would on ZFS: the
|
|
state image is nodatacow, and every live snapshot reintroduces
|
|
copy-on-write for the blocks it still references.
|
|
'';
|
|
};
|
|
};
|
|
|
|
backup = {
|
|
enable = mkEnableOption "restic backups of the guest's state";
|
|
|
|
repository = mkOption {
|
|
type = types.str;
|
|
default = "";
|
|
example = "sftp:backup@nas:/hermes";
|
|
description = "Any restic repository string.";
|
|
};
|
|
|
|
passwordFile = mkOption {
|
|
type = types.path;
|
|
default = "/var/lib/secrets/hermes-restic";
|
|
description = "File containing the restic repository password, mode 0400.";
|
|
};
|
|
|
|
environmentFile = mkOption {
|
|
type = types.nullOr types.path;
|
|
default = null;
|
|
description = "Optional file with credentials for the backend (S3 keys and so on).";
|
|
};
|
|
|
|
interval = mkOption {
|
|
type = types.str;
|
|
default = "daily";
|
|
};
|
|
|
|
keep = mkOption {
|
|
type = types.listOf types.str;
|
|
default = [ "--keep-daily" "14" "--keep-weekly" "8" "--keep-monthly" "12" ];
|
|
description = "Arguments passed to restic forget.";
|
|
};
|
|
};
|
|
|
|
hostCache = {
|
|
enable = mkOption {
|
|
type = types.bool;
|
|
default = true;
|
|
description = ''
|
|
Serve the host's /nix/store to the guest as a read-only binary
|
|
cache, so the guest doesn't re-download from the internet what the
|
|
host already has.
|
|
|
|
This is the ONLY route from guest to host in this design. Turning it
|
|
off leaves the guest with literally no reachable host service, at
|
|
the cost of bandwidth on every guest build.
|
|
|
|
Note the guest can read every store path on the host through this.
|
|
Store paths are world-readable anyway, but check nothing sensitive
|
|
is baked into your host's store.
|
|
'';
|
|
};
|
|
port = mkOption {
|
|
type = types.port;
|
|
default = 5000;
|
|
};
|
|
secretKeyFile = mkOption {
|
|
type = types.path;
|
|
default = "/var/lib/secrets/hermes-cache.secret";
|
|
description = "Generate with nix-store --generate-binary-cache-key. See README.";
|
|
};
|
|
};
|
|
|
|
authorizedKeys = mkOption {
|
|
type = types.listOf types.str;
|
|
default = [ ];
|
|
};
|
|
|
|
allowLan = mkOption {
|
|
type = types.bool;
|
|
default = false;
|
|
description = ''
|
|
Blanket permission for the guest to reach RFC1918 space. Almost always
|
|
the wrong tool - use lanAccess to open single endpoints instead.
|
|
'';
|
|
};
|
|
|
|
lanAccess = mkOption {
|
|
default = [ ];
|
|
example = [ { address = "192.168.1.50"; port = 11434; } ];
|
|
description = ''
|
|
Specific endpoints elsewhere on your network that the guest may reach,
|
|
punched through the default RFC1918 drop. One entry per service.
|
|
'';
|
|
type = types.listOf (types.submodule {
|
|
options = {
|
|
address = mkOption { type = types.str; };
|
|
port = mkOption { type = types.port; };
|
|
proto = mkOption {
|
|
type = types.enum [ "tcp" "udp" ];
|
|
default = "tcp";
|
|
};
|
|
};
|
|
});
|
|
};
|
|
|
|
hostAccess = mkOption {
|
|
default = [ ];
|
|
example = [ { port = 8188; } ];
|
|
description = ''
|
|
Ports on this host itself that the guest may reach. The service must
|
|
also be bound to an address the guest can see - see the README.
|
|
'';
|
|
type = types.listOf (types.submodule {
|
|
options = {
|
|
port = mkOption { type = types.port; };
|
|
proto = mkOption {
|
|
type = types.enum [ "tcp" "udp" ];
|
|
default = "tcp";
|
|
};
|
|
};
|
|
});
|
|
};
|
|
|
|
publish = mkOption {
|
|
default = [ ];
|
|
example = [ { hostPort = 8080; guestPort = 8080; } ];
|
|
description = ''
|
|
Guest ports to publish on the host's uplink, reachable from the wider
|
|
network. You do not need this to reach the guest from the host itself.
|
|
'';
|
|
type = types.listOf (types.submodule {
|
|
options = {
|
|
hostPort = mkOption { type = types.port; };
|
|
guestPort = mkOption { type = types.port; };
|
|
proto = mkOption {
|
|
type = types.enum [ "tcp" "udp" ];
|
|
default = "tcp";
|
|
};
|
|
};
|
|
});
|
|
};
|
|
|
|
network = {
|
|
bridge = mkOption { type = types.str; default = "hermes-br"; };
|
|
tapId = mkOption { type = types.str; default = "vm-hermes"; };
|
|
mac = mkOption { type = types.str; default = "02:00:00:7e:00:01"; };
|
|
hostAddress = mkOption { type = types.str; default = "10.77.0.1"; };
|
|
guestAddress = mkOption { type = types.str; default = "10.77.0.2"; };
|
|
prefixLength = mkOption { type = types.int; default = 24; };
|
|
externalInterface = mkOption {
|
|
type = types.nullOr types.str;
|
|
default = null;
|
|
example = "eno1";
|
|
description = "Host uplink. Required.";
|
|
};
|
|
};
|
|
|
|
passthrough = {
|
|
iommu = mkOption {
|
|
type = types.nullOr (types.enum [ "intel" "amd" ]);
|
|
default = null;
|
|
description = ''
|
|
Enable IOMMU kernel parameters for your CPU vendor. Required for any
|
|
PCI passthrough. The IOMMU is the only thing stopping a passed-through
|
|
device from reading host memory, so this is not optional.
|
|
'';
|
|
};
|
|
|
|
pci = mkOption {
|
|
default = [ ];
|
|
example = [
|
|
{ address = "0000:01:00.0"; id = "10de:2204"; }
|
|
{ address = "0000:01:00.1"; id = "10de:1aef"; }
|
|
];
|
|
description = ''
|
|
PCI devices handed to the guest via VFIO. The host loses them
|
|
entirely while the VM runs.
|
|
|
|
You must pass every function of the device and, in practice, every
|
|
device in its IOMMU group. A GPU is usually at least two entries:
|
|
the graphics function and its HDMI audio function.
|
|
|
|
Set `id` (vendor:device) to have vfio-pci claim the device in the
|
|
initrd, before a host driver can. Strongly recommended for GPUs;
|
|
unbinding an active amdgpu or nvidia driver later is unreliable.
|
|
'';
|
|
type = types.listOf (types.submodule {
|
|
options = {
|
|
address = mkOption {
|
|
type = types.str;
|
|
example = "0000:01:00.0";
|
|
description = "Full PCI address, as shown by lspci -D.";
|
|
};
|
|
id = mkOption {
|
|
type = types.nullOr types.str;
|
|
default = null;
|
|
example = "10de:2204";
|
|
description = "vendor:device, from lspci -nn. Enables early binding.";
|
|
};
|
|
};
|
|
});
|
|
};
|
|
|
|
usb = mkOption {
|
|
default = [ ];
|
|
example = [ { vendorId = "8087"; productId = "0026"; } ];
|
|
description = ''
|
|
USB devices forwarded to the guest, matched by vendor and product
|
|
ID from lsusb. Integrated Bluetooth radios are USB devices and go
|
|
here too, though the host driver must be kept off them.
|
|
|
|
QEMU resolves these at start, so a device that is unplugged when the
|
|
VM boots will not appear later without a restart.
|
|
'';
|
|
type = types.listOf (types.submodule {
|
|
options = {
|
|
vendorId = mkOption { type = types.str; example = "046d"; };
|
|
productId = mkOption { type = types.str; example = "c52b"; };
|
|
};
|
|
});
|
|
};
|
|
};
|
|
|
|
cliName = mkOption {
|
|
type = types.str;
|
|
default = "hermes-ctl";
|
|
description = ''
|
|
Name of the host-side wrapper that drives Hermes inside the guest.
|
|
Set to "hermes" if you want it to feel local and are sure nothing
|
|
else on the host claims that name.
|
|
'';
|
|
};
|
|
|
|
gatewayUnit = mkOption {
|
|
type = types.str;
|
|
default = "hermes-agent";
|
|
description = ''
|
|
The systemd *user* unit that `hermes gateway install` created in the
|
|
guest. Check with `hermes-ctl units` after installing; upstream has
|
|
used more than one name for this.
|
|
'';
|
|
};
|
|
|
|
hermesHome = mkOption {
|
|
type = types.str;
|
|
default = "${stateDir}/.hermes";
|
|
description = ''
|
|
Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/,
|
|
state.db, and so on) lives inside the guest. Defaults to the seeded
|
|
layout under the state disk mount. Override this when carrying over
|
|
an existing flat HERMES_HOME from elsewhere (e.g. a prior Docker
|
|
deployment) that you're placing directly at the root of the state
|
|
disk instead of under .hermes - see `hermenix migrate --help`.
|
|
|
|
This only changes where the host-side hermes-ctl wrapper (soul,
|
|
memory, conf, secrets, reseed, pull) looks. It does not change what
|
|
the gateway process itself uses - export HERMES_HOME to match inside
|
|
the guest when you install it.
|
|
'';
|
|
};
|
|
|
|
vsockCid = mkOption {
|
|
type = types.int;
|
|
default = 77;
|
|
description = "Host reaches the guest over vsock, so no inbound network path is needed.";
|
|
};
|
|
};
|
|
|
|
config = mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = net.externalInterface != null;
|
|
message = "services.hermenix.network.externalInterface must be set.";
|
|
}
|
|
{
|
|
assertion = cfg.backup.enable -> (cfg.backup.repository != "");
|
|
message = "services.hermenix.backup.repository must be set when backups are enabled.";
|
|
}
|
|
{
|
|
assertion = anyPci -> (pt.iommu != null);
|
|
message = ''
|
|
services.hermenix.passthrough.pci needs passthrough.iommu set to
|
|
"intel" or "amd". Without the IOMMU, a passed-through device can
|
|
read host memory directly and the VM boundary is meaningless.
|
|
'';
|
|
}
|
|
];
|
|
|
|
users.users.${vmUser} = {
|
|
isSystemUser = true;
|
|
group = vmUser;
|
|
description = "Runs the Hermes guest VM";
|
|
};
|
|
users.groups.${vmUser} = { };
|
|
|
|
# =====================================================================
|
|
# The VM itself
|
|
#
|
|
# No virtiofs, no shared filesystem of any kind, no vhost-net. The guest
|
|
# gets a block device, a tap, a vsock and a serial port. That is all.
|
|
# =====================================================================
|
|
systemd.services.hermes-vm = {
|
|
description = "Hermes Agent VM";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network-online.target" "hermes-vm-tap.service" ];
|
|
requires = [ "hermes-vm-tap.service" ];
|
|
wants = [ "network-online.target" ];
|
|
|
|
preStart = ''
|
|
for img in ${systemImage} ${stateImage}; do
|
|
if [ ! -e "$img" ]; then
|
|
echo "$img is missing. Run 'hermenix setup' first." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
# Per-VM writable UEFI variable store, seeded once.
|
|
if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then
|
|
install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd
|
|
fi
|
|
'';
|
|
|
|
serviceConfig = {
|
|
User = vmUser;
|
|
Group = vmUser;
|
|
SupplementaryGroups = [ "kvm" ];
|
|
RuntimeDirectory = "hermes-vm";
|
|
# ProtectSystem=strict makes everything read-only; punch through for
|
|
# the images and the UEFI variable store only.
|
|
ReadWritePaths = [ cfg.disk.baseDir ];
|
|
StateDirectory = "hermes-vm";
|
|
RuntimeDirectoryMode = "0750";
|
|
|
|
ExecStart = lib.concatStringsSep " " ([
|
|
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
|
|
"-name ${cfg.name}"
|
|
"-machine q35,accel=kvm,smm=on"
|
|
"-cpu host"
|
|
"-smp ${toString cfg.vcpu}"
|
|
"-m ${toString cfg.mem}"
|
|
"-nodefaults"
|
|
"-no-user-config"
|
|
"-display none"
|
|
"-serial stdio"
|
|
# QEMU's own seccomp sandbox, everything denied that we don't need.
|
|
"-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"
|
|
# UEFI: firmware read-only, variables per-VM and writable.
|
|
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
|
|
"-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd"
|
|
# System disk: NixOS and the Nix store. Rebuildable, never snapshotted.
|
|
# aio=threads and a writeback cache because these are files on a
|
|
# filesystem, not block devices; aio=native wants the latter.
|
|
"-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap"
|
|
"-device virtio-blk-pci,drive=system"
|
|
# State disk: everything irreplaceable. Mounted at /var/lib/hermes.
|
|
"-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap"
|
|
"-device virtio-blk-pci,drive=state"
|
|
# Pre-created tap. vhost=off on purpose: vhost-net would move packet
|
|
# handling into the host kernel, which is exactly the surface we are
|
|
# trying not to expose. Costs throughput we do not need.
|
|
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
|
|
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
|
|
# Host -> guest management, no network involved.
|
|
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
|
|
"-device virtio-rng-pci"
|
|
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
|
|
] ++ lib.optionals cfg.disk.quiesce [
|
|
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
|
|
"-device virtio-serial-pci"
|
|
"-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
|
|
]
|
|
# Passed-through PCI devices. The host cannot use these while the VM
|
|
# is running.
|
|
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
|
|
# A USB controller, only if something needs plugging into it.
|
|
++ lib.optional anyUsb "-device qemu-xhci,id=xhci"
|
|
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
|
|
);
|
|
|
|
ExecStop = stopScript;
|
|
TimeoutStopSec = 90;
|
|
Restart = "always";
|
|
RestartSec = 5;
|
|
|
|
# ---- hardening -------------------------------------------------
|
|
DevicePolicy = "closed";
|
|
DeviceAllow = [
|
|
"/dev/kvm rw"
|
|
"/dev/net/tun rw"
|
|
"/dev/vhost-vsock rw"
|
|
]
|
|
++ lib.optionals anyPci [ "char-vfio rw" ]
|
|
++ lib.optionals anyUsb [ "char-usb_device rw" ];
|
|
|
|
# VFIO pins the entire guest RAM allocation, so the default memlock
|
|
# limit has to go or QEMU fails to start with an obscure error.
|
|
LimitMEMLOCK = lib.mkIf anyPci "infinity";
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
ProtectProc = "invisible";
|
|
ProcSubset = "pid";
|
|
PrivateTmp = true;
|
|
ProtectKernelTunables = true;
|
|
ProtectKernelModules = true;
|
|
ProtectKernelLogs = true;
|
|
ProtectClock = true;
|
|
ProtectControlGroups = true;
|
|
ProtectHostname = true;
|
|
RestrictNamespaces = true;
|
|
RestrictRealtime = true;
|
|
RestrictSUIDSGID = true;
|
|
RestrictAddressFamilies = [ "AF_UNIX" "AF_NETLINK" "AF_VSOCK" ];
|
|
LockPersonality = true;
|
|
SystemCallArchitectures = "native";
|
|
SystemCallFilter = [ "@system-service" "~@resources" "~@obsolete" ];
|
|
# KVM means no JIT, so W^X is safe. If QEMU refuses to start, this is
|
|
# the first thing to flip.
|
|
MemoryDenyWriteExecute = true;
|
|
CapabilityBoundingSet = [ "" ];
|
|
AmbientCapabilities = [ "" ];
|
|
};
|
|
};
|
|
|
|
# Tap device, created as root, owned by the VM user, bridged by networkd.
|
|
systemd.services.hermes-vm-tap = {
|
|
description = "TAP interface for the Hermes VM";
|
|
after = [ "systemd-networkd.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
};
|
|
path = [ pkgs.iproute2 ];
|
|
script = ''
|
|
ip link show ${net.tapId} >/dev/null 2>&1 \
|
|
|| ip tuntap add ${net.tapId} mode tap user ${vmUser} multi_queue
|
|
ip link set ${net.tapId} up
|
|
'';
|
|
preStop = ''
|
|
${pkgs.iproute2}/bin/ip link del ${net.tapId} || true
|
|
'';
|
|
};
|
|
|
|
# =====================================================================
|
|
# Device passthrough
|
|
# =====================================================================
|
|
|
|
# The IOMMU is what keeps a passed-through device from reading host
|
|
# memory. Without it, PCI passthrough is not a boundary at all.
|
|
boot.kernelParams = lib.optionals (pt.iommu != null) [
|
|
(if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on")
|
|
"iommu=pt"
|
|
];
|
|
|
|
boot.initrd.kernelModules = lib.optionals anyPci [
|
|
"vfio_pci"
|
|
"vfio"
|
|
"vfio_iommu_type1"
|
|
];
|
|
|
|
# Claim the device before any host driver gets to it.
|
|
boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) ''
|
|
options vfio-pci ids=${lib.concatStringsSep "," earlyIds}
|
|
'';
|
|
|
|
# Catch anything that was not early-bound, by address. Idempotent.
|
|
systemd.services.hermes-vm-vfio = mkIf anyPci {
|
|
description = "Bind passthrough PCI devices to vfio-pci";
|
|
before = [ "hermes-vm.service" ];
|
|
requiredBy = [ "hermes-vm.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
};
|
|
path = [ pkgs.kmod ];
|
|
script = ''
|
|
modprobe vfio-pci
|
|
'' + lib.concatMapStrings (d: ''
|
|
dev=/sys/bus/pci/devices/${d.address}
|
|
if [ ! -e "$dev" ]; then
|
|
echo "${d.address} is not present on this machine" >&2
|
|
exit 1
|
|
fi
|
|
cur=none
|
|
[ -e "$dev/driver" ] && cur=$(basename "$(readlink -f "$dev/driver")")
|
|
if [ "$cur" != "vfio-pci" ]; then
|
|
[ "$cur" != "none" ] && echo ${d.address} > "$dev/driver/unbind"
|
|
echo vfio-pci > "$dev/driver_override"
|
|
echo ${d.address} > /sys/bus/pci/drivers_probe
|
|
fi
|
|
'') pt.pci;
|
|
};
|
|
|
|
services.udev.extraRules =
|
|
lib.optionalString anyPci ''
|
|
SUBSYSTEM=="vfio", OWNER="${vmUser}", GROUP="${vmUser}", MODE="0660"
|
|
''
|
|
+ lib.concatMapStrings (u: ''
|
|
SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660"
|
|
'') pt.usb;
|
|
|
|
# =====================================================================
|
|
# Networking
|
|
# =====================================================================
|
|
networking.useNetworkd = true;
|
|
|
|
systemd.network.netdevs."10-${net.bridge}".netdevConfig = {
|
|
Kind = "bridge";
|
|
Name = net.bridge;
|
|
};
|
|
|
|
systemd.network.networks."10-${net.bridge}" = {
|
|
matchConfig.Name = net.bridge;
|
|
address = [ "${net.hostAddress}/${toString net.prefixLength}" ];
|
|
networkConfig = {
|
|
ConfigureWithoutCarrier = true;
|
|
IPv4Forwarding = true;
|
|
};
|
|
};
|
|
|
|
systemd.network.networks."11-${net.tapId}" = {
|
|
matchConfig.Name = net.tapId;
|
|
networkConfig.Bridge = net.bridge;
|
|
};
|
|
|
|
networking.nat = {
|
|
enable = true;
|
|
internalInterfaces = [ net.bridge ];
|
|
externalInterface = net.externalInterface;
|
|
forwardPorts = map (s: {
|
|
sourcePort = s.hostPort;
|
|
destination = "${net.guestAddress}:${toString s.guestPort}";
|
|
inherit (s) proto;
|
|
}) cfg.publish;
|
|
};
|
|
|
|
# =====================================================================
|
|
# Firewall
|
|
#
|
|
# The guest owns its own kernel and its own config now, so nothing inside
|
|
# it constrains it. These rules are the boundary.
|
|
# =====================================================================
|
|
networking.nftables.enable = true;
|
|
networking.firewall.enable = true;
|
|
networking.firewall.filterForward = true;
|
|
|
|
# The deliberate guest -> host holes: the Nix cache, plus anything listed
|
|
# in hostAccess.
|
|
networking.firewall.extraInputRules =
|
|
optionalString cfg.hostCache.enable ''
|
|
iifname "${net.bridge}" ip saddr ${net.guestAddress} tcp dport ${toString cfg.hostCache.port} accept
|
|
''
|
|
+ lib.concatMapStrings (s: ''
|
|
iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept
|
|
'') cfg.hostAccess;
|
|
|
|
# Order matters here. Allowlists must precede the RFC1918 drop.
|
|
networking.firewall.extraForwardRules =
|
|
''
|
|
ct state established,related accept
|
|
''
|
|
# Guest reaching named services elsewhere on your network.
|
|
+ lib.concatMapStrings (s: ''
|
|
iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
|
|
'') cfg.lanAccess
|
|
# Inbound to guest services published on the uplink.
|
|
+ lib.concatMapStrings (s: ''
|
|
oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept
|
|
'') cfg.publish
|
|
+ (if cfg.allowLan then ''
|
|
iifname "${net.bridge}" accept
|
|
'' else ''
|
|
iifname "${net.bridge}" ip daddr {
|
|
10.0.0.0/8,
|
|
172.16.0.0/12,
|
|
192.168.0.0/16,
|
|
169.254.0.0/16,
|
|
127.0.0.0/8,
|
|
100.64.0.0/10
|
|
} drop
|
|
iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
|
|
iifname "${net.bridge}" accept
|
|
'');
|
|
|
|
# =====================================================================
|
|
# Host store as a read-only cache for the guest
|
|
# =====================================================================
|
|
services.nix-serve = mkIf cfg.hostCache.enable {
|
|
enable = true;
|
|
package = pkgs.nix-serve-ng;
|
|
bindAddress = net.hostAddress;
|
|
inherit (cfg.hostCache) port secretKeyFile;
|
|
openFirewall = false; # handled above, scoped to the guest address
|
|
};
|
|
|
|
# =====================================================================
|
|
# Snapshots of the state disk only.
|
|
#
|
|
# The system disk is not snapshotted on purpose. It is large, it is where
|
|
# all the write churn happens, and `hermenix setup` can rebuild it from the flake
|
|
# in minutes. Snapshotting it would cost a lot and buy nothing.
|
|
# =====================================================================
|
|
systemd.services.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
|
|
description = "Snapshot the Hermes VM state disk";
|
|
path = with pkgs; [ btrfs-progs coreutils socat ];
|
|
serviceConfig.Type = "oneshot";
|
|
script = ''
|
|
set -euo pipefail
|
|
stamp=$(date +%Y%m%d-%H%M%S)
|
|
mkdir -p ${snapDir}
|
|
|
|
${optionalString cfg.disk.quiesce ''
|
|
# Flush the guest's filesystems so the snapshot is consistent rather
|
|
# than the equivalent of pulling the power. Takes milliseconds; the
|
|
# VM keeps running throughout.
|
|
${qga "freeze"}
|
|
trap '${qga "thaw"}' EXIT
|
|
''}
|
|
|
|
btrfs subvolume snapshot -r ${cfg.disk.baseDir}/state ${snapDir}/state-$stamp
|
|
|
|
${optionalString cfg.disk.quiesce ''
|
|
${qga "thaw"}
|
|
trap - EXIT
|
|
''}
|
|
|
|
# Prune oldest first. Keep this list short: every live snapshot
|
|
# reintroduces copy-on-write on a nodatacow image.
|
|
ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \
|
|
| while read -r old; do btrfs subvolume delete "$old"; done
|
|
'';
|
|
};
|
|
|
|
systemd.timers.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnCalendar = cfg.snapshots.interval;
|
|
Persistent = true;
|
|
RandomizedDelaySec = "5m";
|
|
};
|
|
};
|
|
|
|
# =====================================================================
|
|
# Backups: the guest's state, not its disk.
|
|
#
|
|
# Pulled file-by-file over the SSH channel that already exists, rather
|
|
# than backing up the image. That keeps the backup portable - it restores
|
|
# onto any filesystem, or a different machine entirely - and avoids
|
|
# loop-mounting a guest-written filesystem on the host, which would mean
|
|
# the host kernel parsing data the agent controls.
|
|
# =====================================================================
|
|
systemd.services.hermes-vm-backup = mkIf cfg.backup.enable {
|
|
description = "Back up Hermes state";
|
|
path = with pkgs; [ restic openssh socat gnutar ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile;
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
export RESTIC_REPOSITORY=${lib.escapeShellArg cfg.backup.repository}
|
|
export RESTIC_PASSWORD_FILE=${cfg.backup.passwordFile}
|
|
|
|
restic snapshots >/dev/null 2>&1 || restic init
|
|
|
|
ssh -o ProxyCommand="socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22" \
|
|
-o StrictHostKeyChecking=accept-new \
|
|
hermes@${cfg.name} \
|
|
'tar -C /var/lib/hermes \
|
|
--exclude=./.cache \
|
|
--exclude=./workspace/**/.direnv \
|
|
--exclude=./workspace/**/result \
|
|
-cf - .' \
|
|
| restic backup --stdin --stdin-filename hermes-state.tar \
|
|
--tag hermes --host ${cfg.name}
|
|
|
|
restic forget --tag hermes --host ${cfg.name} --prune ${lib.concatStringsSep " " cfg.backup.keep}
|
|
'';
|
|
};
|
|
|
|
systemd.timers.hermes-vm-backup = mkIf cfg.backup.enable {
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnCalendar = cfg.backup.interval;
|
|
Persistent = true;
|
|
RandomizedDelaySec = "30m";
|
|
};
|
|
};
|
|
|
|
# =====================================================================
|
|
# Host GC is now safe: the guest has its own store and shares nothing.
|
|
# =====================================================================
|
|
nix.gc.automatic = lib.mkDefault true;
|
|
nix.gc.options = lib.mkDefault "--delete-older-than 30d";
|
|
|
|
environment.systemPackages = [
|
|
(pkgs.writeShellScriptBin "hermes-vm" ''
|
|
set -euo pipefail
|
|
SSH=(${pkgs.openssh}/bin/ssh
|
|
-o ProxyCommand="${pkgs.socat}/bin/socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22"
|
|
-o StrictHostKeyChecking=accept-new
|
|
hermes@${cfg.name})
|
|
cmd="''${1-}"; shift || true
|
|
case "$cmd" in
|
|
ssh) exec "''${SSH[@]}" "$@" ;;
|
|
console) exec journalctl -u hermes-vm -f -o cat ;;
|
|
start) exec systemctl start hermes-vm ;;
|
|
stop) exec systemctl stop hermes-vm ;;
|
|
restart) exec systemctl restart hermes-vm ;;
|
|
|
|
# What is the guest actually running right now?
|
|
config)
|
|
"''${SSH[@]}" 'echo "== current generation =="; \
|
|
readlink -f /run/current-system; \
|
|
echo; echo "== generations =="; \
|
|
sudo nix-env --list-generations -p /nix/var/nix/profiles/system | tail -n 10; \
|
|
echo; echo "== /etc/nixos history =="; \
|
|
git -C /etc/nixos log --oneline -n 15; \
|
|
echo; echo "== uncommitted =="; \
|
|
git -C /etc/nixos status --short'
|
|
;;
|
|
# Copy the guest's live config here for reading or diffing.
|
|
config-pull)
|
|
dest="''${1:-./hermes-etc-nixos}"
|
|
mkdir -p "$dest"
|
|
"''${SSH[@]}" 'tar -C /etc/nixos -cf - .' | tar -C "$dest" -xf -
|
|
echo "pulled to $dest"
|
|
;;
|
|
|
|
snapshot) exec systemctl start hermes-vm-snapshot ;;
|
|
snapshots)
|
|
${pkgs.util-linux}/bin/column -t <<<"$(
|
|
echo "SNAPSHOT SIZE CREATED"
|
|
for d in ${snapDir}/state-*; do
|
|
[ -e "$d" ] || continue
|
|
echo "$(basename "$d") $(du -sh "$d" 2>/dev/null | cut -f1) $(stat -c %y "$d" | cut -d. -f1)"
|
|
done
|
|
)" ;;
|
|
|
|
# Restore the state disk from a snapshot. The system disk is left
|
|
# alone - it holds nothing worth restoring.
|
|
rollback)
|
|
[ $# -ge 1 ] || { echo "usage: hermes-vm rollback <snapshot>" >&2; exit 1; }
|
|
snap=${snapDir}/"$1"
|
|
[ -d "$snap" ] || { echo "no such snapshot: $1" >&2; exit 1; }
|
|
echo "Restores Hermes's identity, memories, workspace and NixOS" >&2
|
|
echo "config to $1. Everything since is lost." >&2
|
|
read -r -p "type 'rollback' to confirm: " c
|
|
[ "$c" = rollback ] || { echo "aborted" >&2; exit 1; }
|
|
systemctl stop hermes-vm
|
|
mv ${cfg.disk.baseDir}/state ${cfg.disk.baseDir}/state.old-$(date +%s)
|
|
${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot "$snap" ${cfg.disk.baseDir}/state
|
|
systemctl start hermes-vm
|
|
echo "restored; previous state kept as state.old-*" ;;
|
|
|
|
backup) exec systemctl start hermes-vm-backup ;;
|
|
backups)
|
|
exec ${pkgs.restic}/bin/restic \
|
|
-r "$(systemctl show hermes-vm-backup -p Environment --value | tr ' ' '\n' | sed -n 's/^RESTIC_REPOSITORY=//p')" \
|
|
snapshots --tag hermes ;;
|
|
*)
|
|
echo "usage: hermes-vm {ssh [cmd]|console|config|config-pull [dir]|start|stop|restart|snapshot|snapshots|rollback <snap>|backup|backups}" >&2
|
|
exit 1 ;;
|
|
esac
|
|
'')
|
|
|
|
# ---------------------------------------------------------------
|
|
# Driving Hermes itself, as opposed to the VM around it.
|
|
# ---------------------------------------------------------------
|
|
(pkgs.writeShellScriptBin cfg.cliName ''
|
|
set -euo pipefail
|
|
SSH=(${pkgs.openssh}/bin/ssh
|
|
-o ProxyCommand="${pkgs.socat}/bin/socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22"
|
|
-o StrictHostKeyChecking=accept-new
|
|
hermes@${cfg.name})
|
|
HOME_DIR=${cfg.hermesHome}
|
|
UNIT=${cfg.gatewayUnit}
|
|
|
|
# Pull a file out of the guest, open it in your editor, push it back.
|
|
edit_remote() {
|
|
remote="$1"; mode="''${2:-0644}"
|
|
tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT
|
|
"''${SSH[@]}" "cat $remote" > "$tmp" 2>/dev/null || true
|
|
before=$(${pkgs.coreutils}/bin/sha256sum < "$tmp")
|
|
"''${EDITOR:-${pkgs.vim}/bin/vim}" "$tmp"
|
|
if [ "$(${pkgs.coreutils}/bin/sha256sum < "$tmp")" = "$before" ]; then
|
|
echo "unchanged"; return
|
|
fi
|
|
"''${SSH[@]}" "install -m $mode /dev/stdin $remote" < "$tmp"
|
|
echo "written to $remote"
|
|
read -r -p "restart the gateway now? [y/N] " yn
|
|
case "$yn" in [yY]*) "''${SSH[@]}" "systemctl --user restart $UNIT" ;; esac
|
|
}
|
|
|
|
cmd="''${1-}"; shift || true
|
|
case "$cmd" in
|
|
soul)
|
|
case "''${1-}" in
|
|
edit) edit_remote "$HOME_DIR/SOUL.md" ;;
|
|
*) "''${SSH[@]}" "cat $HOME_DIR/SOUL.md" ;;
|
|
esac ;;
|
|
|
|
memory)
|
|
case "''${1-}" in
|
|
edit) edit_remote "$HOME_DIR/memories/USER.md" ;;
|
|
*) "''${SSH[@]}" "cat $HOME_DIR/memories/USER.md" ;;
|
|
esac ;;
|
|
|
|
conf)
|
|
case "''${1-}" in
|
|
edit) edit_remote "$HOME_DIR/config.yaml" ;;
|
|
*) "''${SSH[@]}" "cat $HOME_DIR/config.yaml" ;;
|
|
esac ;;
|
|
|
|
# API keys. Mode 0600, never enters the Nix store, and not printed
|
|
# unless you ask for it by name.
|
|
secrets)
|
|
case "''${1-}" in
|
|
edit) edit_remote "$HOME_DIR/.env" 0600 ;;
|
|
show) "''${SSH[@]}" "cat $HOME_DIR/.env" ;;
|
|
*) "''${SSH[@]}" "sed 's/=.*/=<set>/' $HOME_DIR/.env" ;;
|
|
esac ;;
|
|
|
|
status) "''${SSH[@]}" "systemctl --user status $UNIT --no-pager" ;;
|
|
logs) "''${SSH[@]}" -t "journalctl --user -u $UNIT -f" ;;
|
|
restart) "''${SSH[@]}" "systemctl --user restart $UNIT" ;;
|
|
units) "''${SSH[@]}" "systemctl --user list-units 'hermes*' --all --no-pager" ;;
|
|
|
|
# Overwrite the agent's identity from the seed. Destructive: this
|
|
# is how you undo whatever it has decided about itself.
|
|
reseed)
|
|
echo "This overwrites SOUL.md, memories and config.yaml in the guest" >&2
|
|
echo "with the seed from /etc/nixos/hermes. Anything Hermes has" >&2
|
|
echo "learned and written there is lost." >&2
|
|
read -r -p "type 'reseed' to confirm: " confirm
|
|
[ "$confirm" = "reseed" ] || { echo "aborted" >&2; exit 1; }
|
|
"''${SSH[@]}" "cp -rT /etc/nixos/hermes/home $HOME_DIR && \
|
|
cp -rT /etc/nixos/hermes/workspace /var/lib/hermes/workspace && \
|
|
systemctl --user restart $UNIT"
|
|
echo "reseeded" ;;
|
|
|
|
# Copy the agent's live identity back here, so you can fold what it
|
|
# has learned into the seed in guest/hermes/.
|
|
pull)
|
|
dest="''${1:-./hermes-home}"
|
|
mkdir -p "$dest"
|
|
"''${SSH[@]}" "tar -C $HOME_DIR --exclude=.env --exclude=sessions --exclude=state.db -cf - ." \
|
|
| tar -C "$dest" -xf -
|
|
echo "pulled to $dest (secrets and sessions excluded)" ;;
|
|
|
|
""|help|-h|--help)
|
|
cat >&2 <<'USAGE'
|
|
${cfg.cliName} - drive Hermes inside the guest
|
|
|
|
soul [edit] show or edit SOUL.md
|
|
memory [edit] show or edit memories/USER.md
|
|
conf [edit] show or edit config.yaml
|
|
secrets [edit|show] manage .env, keys masked by default
|
|
status | logs | restart | units
|
|
reseed overwrite identity from /etc/nixos/hermes
|
|
pull [dir] copy live identity back to the host
|
|
|
|
Anything else goes straight to the hermes CLI in the guest:
|
|
${cfg.cliName} chat
|
|
${cfg.cliName} update
|
|
${cfg.cliName} sessions list
|
|
USAGE
|
|
exit 1 ;;
|
|
|
|
*) exec "''${SSH[@]}" -t hermes "$cmd" "$@" ;;
|
|
esac
|
|
'')
|
|
];
|
|
};
|
|
}
|