Initial hermex: Hermes Agent in a hardened, self-managing NixOS VM
Two-disk (system/state) btrfs-backed QEMU VM, nftables-isolated by default, adapted from a design provided as loose files. Adds a hermesHome option to the host module and migrate-docker-hermes.sh so an existing flat HERMES_HOME (e.g. from a prior Docker deployment) can be carried over intact. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
commit
12227460fd
15 files changed
+2765
No files matched your search
@@ -0,0 +1,994 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
inherit (lib) mkOption mkEnableOption mkIf types optionalString;
|
||||
cfg = config.services.hermesVm;
|
||||
net = cfg.network;
|
||||
|
||||
vmUser = "hermes-vm";
|
||||
runDir = "/run/hermes-vm";
|
||||
stateDir = "/var/lib/hermes-vm";
|
||||
# Two disks, deliberately.
|
||||
#
|
||||
# The system disk is large, churny and entirely rebuildable - it is the
|
||||
# NixOS install and the Nix store. We never snapshot it.
|
||||
#
|
||||
# The state disk holds the only things that cannot be reproduced: Hermes's
|
||||
# identity and memories, its workspace, and the NixOS config it has written
|
||||
# for itself. Small, so snapshots are cheap and backups are quick.
|
||||
systemImage = "${cfg.disk.baseDir}/system/system.raw";
|
||||
stateImage = "${cfg.disk.baseDir}/state/state.raw";
|
||||
snapDir = "${cfg.disk.baseDir}/snapshots";
|
||||
|
||||
pt = cfg.passthrough;
|
||||
anyPci = pt.pci != [ ];
|
||||
anyUsb = pt.usb != [ ];
|
||||
earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci);
|
||||
|
||||
# Graceful ACPI power button via QMP, then wait for QEMU to exit.
|
||||
stopScript = pkgs.writeShellScript "hermes-vm-stop" ''
|
||||
set -eu
|
||||
printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \
|
||||
| ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0
|
||||
'';
|
||||
|
||||
# Freeze the guest filesystems around a snapshot so it captures a consistent
|
||||
# state rather than the equivalent of pulling the power.
|
||||
qga = verb: pkgs.writeShellScript "hermes-vm-fs${verb}" ''
|
||||
set -eu
|
||||
printf '%s' '{"execute":"guest-fs${verb}"}' \
|
||||
| ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.services.hermesVm = {
|
||||
enable = mkEnableOption "the Hermes Agent VM";
|
||||
|
||||
name = mkOption {
|
||||
type = types.str;
|
||||
default = "hermes";
|
||||
};
|
||||
|
||||
vcpu = mkOption {
|
||||
type = types.ints.positive;
|
||||
default = 4;
|
||||
};
|
||||
|
||||
mem = mkOption {
|
||||
type = types.ints.positive;
|
||||
default = 8192;
|
||||
description = "RAM in MiB.";
|
||||
};
|
||||
|
||||
disk = {
|
||||
baseDir = mkOption {
|
||||
type = types.str;
|
||||
default = "/var/lib/hermes-vm";
|
||||
description = ''
|
||||
Directory holding the guest's disk images. On btrfs this should be
|
||||
laid out as three subvolumes - system/, state/ and snapshots/ - with
|
||||
nodatacow set. setup.sh does this for you.
|
||||
'';
|
||||
};
|
||||
|
||||
systemSize = mkOption {
|
||||
type = types.str;
|
||||
default = "80G";
|
||||
description = ''
|
||||
System disk: the NixOS install and the Nix store. Never snapshotted,
|
||||
because it is entirely rebuildable from the flake. Size it for an
|
||||
agent that builds things.
|
||||
'';
|
||||
};
|
||||
|
||||
stateSize = mkOption {
|
||||
type = types.str;
|
||||
default = "20G";
|
||||
description = ''
|
||||
State disk, mounted at /var/lib/hermes in the guest. Holds Hermes's
|
||||
identity, memories, workspace, and the NixOS config it maintains for
|
||||
itself. This is the only irreplaceable data, so it is the only thing
|
||||
snapshotted and backed up.
|
||||
'';
|
||||
};
|
||||
|
||||
quiesce = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "fsfreeze the guest via qemu-guest-agent around each snapshot.";
|
||||
};
|
||||
};
|
||||
|
||||
snapshots = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Periodic btrfs snapshots of the state disk. The agent can persist
|
||||
changes, which means it can persist a mistake, and these are the
|
||||
fastest way back.
|
||||
'';
|
||||
};
|
||||
|
||||
interval = mkOption {
|
||||
type = types.str;
|
||||
default = "hourly";
|
||||
description = "systemd calendar expression.";
|
||||
};
|
||||
|
||||
keep = mkOption {
|
||||
type = types.ints.positive;
|
||||
default = 48;
|
||||
description = ''
|
||||
How many to retain. Keep this tighter than you would on ZFS: the
|
||||
state image is nodatacow, and every live snapshot reintroduces
|
||||
copy-on-write for the blocks it still references.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
backup = {
|
||||
enable = mkEnableOption "restic backups of the guest's state";
|
||||
|
||||
repository = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
example = "sftp:backup@nas:/hermes";
|
||||
description = "Any restic repository string.";
|
||||
};
|
||||
|
||||
passwordFile = mkOption {
|
||||
type = types.path;
|
||||
default = "/var/lib/secrets/hermes-restic";
|
||||
description = "File containing the restic repository password, mode 0400.";
|
||||
};
|
||||
|
||||
environmentFile = mkOption {
|
||||
type = types.nullOr types.path;
|
||||
default = null;
|
||||
description = "Optional file with credentials for the backend (S3 keys and so on).";
|
||||
};
|
||||
|
||||
interval = mkOption {
|
||||
type = types.str;
|
||||
default = "daily";
|
||||
};
|
||||
|
||||
keep = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ "--keep-daily" "14" "--keep-weekly" "8" "--keep-monthly" "12" ];
|
||||
description = "Arguments passed to restic forget.";
|
||||
};
|
||||
};
|
||||
|
||||
hostCache = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Serve the host's /nix/store to the guest as a read-only binary
|
||||
cache, so the guest doesn't re-download from the internet what the
|
||||
host already has.
|
||||
|
||||
This is the ONLY route from guest to host in this design. Turning it
|
||||
off leaves the guest with literally no reachable host service, at
|
||||
the cost of bandwidth on every guest build.
|
||||
|
||||
Note the guest can read every store path on the host through this.
|
||||
Store paths are world-readable anyway, but check nothing sensitive
|
||||
is baked into your host's store.
|
||||
'';
|
||||
};
|
||||
port = mkOption {
|
||||
type = types.port;
|
||||
default = 5000;
|
||||
};
|
||||
secretKeyFile = mkOption {
|
||||
type = types.path;
|
||||
default = "/var/lib/secrets/hermes-cache.secret";
|
||||
description = "Generate with nix-store --generate-binary-cache-key. See README.";
|
||||
};
|
||||
};
|
||||
|
||||
authorizedKeys = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ ];
|
||||
};
|
||||
|
||||
allowLan = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Blanket permission for the guest to reach RFC1918 space. Almost always
|
||||
the wrong tool - use lanAccess to open single endpoints instead.
|
||||
'';
|
||||
};
|
||||
|
||||
lanAccess = mkOption {
|
||||
default = [ ];
|
||||
example = [ { address = "192.168.1.50"; port = 11434; } ];
|
||||
description = ''
|
||||
Specific endpoints elsewhere on your network that the guest may reach,
|
||||
punched through the default RFC1918 drop. One entry per service.
|
||||
'';
|
||||
type = types.listOf (types.submodule {
|
||||
options = {
|
||||
address = mkOption { type = types.str; };
|
||||
port = mkOption { type = types.port; };
|
||||
proto = mkOption {
|
||||
type = types.enum [ "tcp" "udp" ];
|
||||
default = "tcp";
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
hostAccess = mkOption {
|
||||
default = [ ];
|
||||
example = [ { port = 8188; } ];
|
||||
description = ''
|
||||
Ports on this host itself that the guest may reach. The service must
|
||||
also be bound to an address the guest can see - see the README.
|
||||
'';
|
||||
type = types.listOf (types.submodule {
|
||||
options = {
|
||||
port = mkOption { type = types.port; };
|
||||
proto = mkOption {
|
||||
type = types.enum [ "tcp" "udp" ];
|
||||
default = "tcp";
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
publish = mkOption {
|
||||
default = [ ];
|
||||
example = [ { hostPort = 8080; guestPort = 8080; } ];
|
||||
description = ''
|
||||
Guest ports to publish on the host's uplink, reachable from the wider
|
||||
network. You do not need this to reach the guest from the host itself.
|
||||
'';
|
||||
type = types.listOf (types.submodule {
|
||||
options = {
|
||||
hostPort = mkOption { type = types.port; };
|
||||
guestPort = mkOption { type = types.port; };
|
||||
proto = mkOption {
|
||||
type = types.enum [ "tcp" "udp" ];
|
||||
default = "tcp";
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
network = {
|
||||
bridge = mkOption { type = types.str; default = "hermes-br"; };
|
||||
tapId = mkOption { type = types.str; default = "vm-hermes"; };
|
||||
mac = mkOption { type = types.str; default = "02:00:00:7e:00:01"; };
|
||||
hostAddress = mkOption { type = types.str; default = "10.77.0.1"; };
|
||||
guestAddress = mkOption { type = types.str; default = "10.77.0.2"; };
|
||||
prefixLength = mkOption { type = types.int; default = 24; };
|
||||
externalInterface = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
example = "eno1";
|
||||
description = "Host uplink. Required.";
|
||||
};
|
||||
};
|
||||
|
||||
passthrough = {
|
||||
iommu = mkOption {
|
||||
type = types.nullOr (types.enum [ "intel" "amd" ]);
|
||||
default = null;
|
||||
description = ''
|
||||
Enable IOMMU kernel parameters for your CPU vendor. Required for any
|
||||
PCI passthrough. The IOMMU is the only thing stopping a passed-through
|
||||
device from reading host memory, so this is not optional.
|
||||
'';
|
||||
};
|
||||
|
||||
pci = mkOption {
|
||||
default = [ ];
|
||||
example = [
|
||||
{ address = "0000:01:00.0"; id = "10de:2204"; }
|
||||
{ address = "0000:01:00.1"; id = "10de:1aef"; }
|
||||
];
|
||||
description = ''
|
||||
PCI devices handed to the guest via VFIO. The host loses them
|
||||
entirely while the VM runs.
|
||||
|
||||
You must pass every function of the device and, in practice, every
|
||||
device in its IOMMU group. A GPU is usually at least two entries:
|
||||
the graphics function and its HDMI audio function.
|
||||
|
||||
Set `id` (vendor:device) to have vfio-pci claim the device in the
|
||||
initrd, before a host driver can. Strongly recommended for GPUs;
|
||||
unbinding an active amdgpu or nvidia driver later is unreliable.
|
||||
'';
|
||||
type = types.listOf (types.submodule {
|
||||
options = {
|
||||
address = mkOption {
|
||||
type = types.str;
|
||||
example = "0000:01:00.0";
|
||||
description = "Full PCI address, as shown by lspci -D.";
|
||||
};
|
||||
id = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
example = "10de:2204";
|
||||
description = "vendor:device, from lspci -nn. Enables early binding.";
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
usb = mkOption {
|
||||
default = [ ];
|
||||
example = [ { vendorId = "8087"; productId = "0026"; } ];
|
||||
description = ''
|
||||
USB devices forwarded to the guest, matched by vendor and product
|
||||
ID from lsusb. Integrated Bluetooth radios are USB devices and go
|
||||
here too, though the host driver must be kept off them.
|
||||
|
||||
QEMU resolves these at start, so a device that is unplugged when the
|
||||
VM boots will not appear later without a restart.
|
||||
'';
|
||||
type = types.listOf (types.submodule {
|
||||
options = {
|
||||
vendorId = mkOption { type = types.str; example = "046d"; };
|
||||
productId = mkOption { type = types.str; example = "c52b"; };
|
||||
};
|
||||
});
|
||||
};
|
||||
};
|
||||
|
||||
cliName = mkOption {
|
||||
type = types.str;
|
||||
default = "hermes-ctl";
|
||||
description = ''
|
||||
Name of the host-side wrapper that drives Hermes inside the guest.
|
||||
Set to "hermes" if you want it to feel local and are sure nothing
|
||||
else on the host claims that name.
|
||||
'';
|
||||
};
|
||||
|
||||
gatewayUnit = mkOption {
|
||||
type = types.str;
|
||||
default = "hermes-agent";
|
||||
description = ''
|
||||
The systemd *user* unit that `hermes gateway install` created in the
|
||||
guest. Check with `hermes-ctl units` after installing; upstream has
|
||||
used more than one name for this.
|
||||
'';
|
||||
};
|
||||
|
||||
hermesHome = mkOption {
|
||||
type = types.str;
|
||||
default = "${stateDir}/.hermes";
|
||||
description = ''
|
||||
Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/,
|
||||
state.db, and so on) lives inside the guest. Defaults to the seeded
|
||||
layout under the state disk mount. Override this when carrying over
|
||||
an existing flat HERMES_HOME from elsewhere (e.g. a prior Docker
|
||||
deployment) that you're placing directly at the root of the state
|
||||
disk instead of under .hermes - see migrate-docker-hermes.sh.
|
||||
|
||||
This only changes where the host-side hermes-ctl wrapper (soul,
|
||||
memory, conf, secrets, reseed, pull) looks. It does not change what
|
||||
the gateway process itself uses - export HERMES_HOME to match inside
|
||||
the guest when you install it.
|
||||
'';
|
||||
};
|
||||
|
||||
vsockCid = mkOption {
|
||||
type = types.int;
|
||||
default = 77;
|
||||
description = "Host reaches the guest over vsock, so no inbound network path is needed.";
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = net.externalInterface != null;
|
||||
message = "services.hermesVm.network.externalInterface must be set.";
|
||||
}
|
||||
{
|
||||
assertion = cfg.backup.enable -> (cfg.backup.repository != "");
|
||||
message = "services.hermesVm.backup.repository must be set when backups are enabled.";
|
||||
}
|
||||
{
|
||||
assertion = anyPci -> (pt.iommu != null);
|
||||
message = ''
|
||||
services.hermesVm.passthrough.pci needs passthrough.iommu set to
|
||||
"intel" or "amd". Without the IOMMU, a passed-through device can
|
||||
read host memory directly and the VM boundary is meaningless.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
users.users.${vmUser} = {
|
||||
isSystemUser = true;
|
||||
group = vmUser;
|
||||
description = "Runs the Hermes guest VM";
|
||||
};
|
||||
users.groups.${vmUser} = { };
|
||||
|
||||
# =====================================================================
|
||||
# The VM itself
|
||||
#
|
||||
# No virtiofs, no shared filesystem of any kind, no vhost-net. The guest
|
||||
# gets a block device, a tap, a vsock and a serial port. That is all.
|
||||
# =====================================================================
|
||||
systemd.services.hermes-vm = {
|
||||
description = "Hermes Agent VM";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" "hermes-vm-tap.service" ];
|
||||
requires = [ "hermes-vm-tap.service" ];
|
||||
wants = [ "network-online.target" ];
|
||||
|
||||
preStart = ''
|
||||
for img in ${systemImage} ${stateImage}; do
|
||||
if [ ! -e "$img" ]; then
|
||||
echo "$img is missing. Run ./setup.sh first." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
# Per-VM writable UEFI variable store, seeded once.
|
||||
if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then
|
||||
install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd
|
||||
fi
|
||||
'';
|
||||
|
||||
serviceConfig = {
|
||||
User = vmUser;
|
||||
Group = vmUser;
|
||||
SupplementaryGroups = [ "kvm" ];
|
||||
RuntimeDirectory = "hermes-vm";
|
||||
# ProtectSystem=strict makes everything read-only; punch through for
|
||||
# the images and the UEFI variable store only.
|
||||
ReadWritePaths = [ cfg.disk.baseDir ];
|
||||
StateDirectory = "hermes-vm";
|
||||
RuntimeDirectoryMode = "0750";
|
||||
|
||||
ExecStart = lib.concatStringsSep " " ([
|
||||
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
|
||||
"-name ${cfg.name}"
|
||||
"-machine q35,accel=kvm,smm=on"
|
||||
"-cpu host"
|
||||
"-smp ${toString cfg.vcpu}"
|
||||
"-m ${toString cfg.mem}"
|
||||
"-nodefaults"
|
||||
"-no-user-config"
|
||||
"-display none"
|
||||
"-serial stdio"
|
||||
# QEMU's own seccomp sandbox, everything denied that we don't need.
|
||||
"-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"
|
||||
# UEFI: firmware read-only, variables per-VM and writable.
|
||||
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
|
||||
"-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd"
|
||||
# System disk: NixOS and the Nix store. Rebuildable, never snapshotted.
|
||||
# aio=threads and a writeback cache because these are files on a
|
||||
# filesystem, not block devices; aio=native wants the latter.
|
||||
"-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap"
|
||||
"-device virtio-blk-pci,drive=system"
|
||||
# State disk: everything irreplaceable. Mounted at /var/lib/hermes.
|
||||
"-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap"
|
||||
"-device virtio-blk-pci,drive=state"
|
||||
# Pre-created tap. vhost=off on purpose: vhost-net would move packet
|
||||
# handling into the host kernel, which is exactly the surface we are
|
||||
# trying not to expose. Costs throughput we do not need.
|
||||
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
|
||||
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
|
||||
# Host -> guest management, no network involved.
|
||||
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
|
||||
"-device virtio-rng-pci"
|
||||
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
|
||||
] ++ lib.optionals cfg.disk.quiesce [
|
||||
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
|
||||
"-device virtio-serial-pci"
|
||||
"-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
|
||||
]
|
||||
# Passed-through PCI devices. The host cannot use these while the VM
|
||||
# is running.
|
||||
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
|
||||
# A USB controller, only if something needs plugging into it.
|
||||
++ lib.optional anyUsb "-device qemu-xhci,id=xhci"
|
||||
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
|
||||
);
|
||||
|
||||
ExecStop = stopScript;
|
||||
TimeoutStopSec = 90;
|
||||
Restart = "always";
|
||||
RestartSec = 5;
|
||||
|
||||
# ---- hardening -------------------------------------------------
|
||||
DevicePolicy = "closed";
|
||||
DeviceAllow = [
|
||||
"/dev/kvm rw"
|
||||
"/dev/net/tun rw"
|
||||
"/dev/vhost-vsock rw"
|
||||
]
|
||||
++ lib.optionals anyPci [ "char-vfio rw" ]
|
||||
++ lib.optionals anyUsb [ "char-usb_device rw" ];
|
||||
|
||||
# VFIO pins the entire guest RAM allocation, so the default memlock
|
||||
# limit has to go or QEMU fails to start with an obscure error.
|
||||
LimitMEMLOCK = lib.mkIf anyPci "infinity";
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
ProtectProc = "invisible";
|
||||
ProcSubset = "pid";
|
||||
PrivateTmp = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHostname = true;
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
RestrictAddressFamilies = [ "AF_UNIX" "AF_NETLINK" "AF_VSOCK" ];
|
||||
LockPersonality = true;
|
||||
SystemCallArchitectures = "native";
|
||||
SystemCallFilter = [ "@system-service" "~@resources" "~@obsolete" ];
|
||||
# KVM means no JIT, so W^X is safe. If QEMU refuses to start, this is
|
||||
# the first thing to flip.
|
||||
MemoryDenyWriteExecute = true;
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
AmbientCapabilities = [ "" ];
|
||||
};
|
||||
};
|
||||
|
||||
# Tap device, created as root, owned by the VM user, bridged by networkd.
|
||||
systemd.services.hermes-vm-tap = {
|
||||
description = "TAP interface for the Hermes VM";
|
||||
after = [ "systemd-networkd.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
path = [ pkgs.iproute2 ];
|
||||
script = ''
|
||||
ip link show ${net.tapId} >/dev/null 2>&1 \
|
||||
|| ip tuntap add ${net.tapId} mode tap user ${vmUser} multi_queue
|
||||
ip link set ${net.tapId} up
|
||||
'';
|
||||
preStop = ''
|
||||
${pkgs.iproute2}/bin/ip link del ${net.tapId} || true
|
||||
'';
|
||||
};
|
||||
|
||||
# =====================================================================
|
||||
# Device passthrough
|
||||
# =====================================================================
|
||||
|
||||
# The IOMMU is what keeps a passed-through device from reading host
|
||||
# memory. Without it, PCI passthrough is not a boundary at all.
|
||||
boot.kernelParams = lib.optionals (pt.iommu != null) [
|
||||
(if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on")
|
||||
"iommu=pt"
|
||||
];
|
||||
|
||||
boot.initrd.kernelModules = lib.optionals anyPci [
|
||||
"vfio_pci"
|
||||
"vfio"
|
||||
"vfio_iommu_type1"
|
||||
];
|
||||
|
||||
# Claim the device before any host driver gets to it.
|
||||
boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) ''
|
||||
options vfio-pci ids=${lib.concatStringsSep "," earlyIds}
|
||||
'';
|
||||
|
||||
# Catch anything that was not early-bound, by address. Idempotent.
|
||||
systemd.services.hermes-vm-vfio = mkIf anyPci {
|
||||
description = "Bind passthrough PCI devices to vfio-pci";
|
||||
before = [ "hermes-vm.service" ];
|
||||
requiredBy = [ "hermes-vm.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
path = [ pkgs.kmod ];
|
||||
script = ''
|
||||
modprobe vfio-pci
|
||||
'' + lib.concatMapStrings (d: ''
|
||||
dev=/sys/bus/pci/devices/${d.address}
|
||||
if [ ! -e "$dev" ]; then
|
||||
echo "${d.address} is not present on this machine" >&2
|
||||
exit 1
|
||||
fi
|
||||
cur=none
|
||||
[ -e "$dev/driver" ] && cur=$(basename "$(readlink -f "$dev/driver")")
|
||||
if [ "$cur" != "vfio-pci" ]; then
|
||||
[ "$cur" != "none" ] && echo ${d.address} > "$dev/driver/unbind"
|
||||
echo vfio-pci > "$dev/driver_override"
|
||||
echo ${d.address} > /sys/bus/pci/drivers_probe
|
||||
fi
|
||||
'') pt.pci;
|
||||
};
|
||||
|
||||
services.udev.extraRules =
|
||||
lib.optionalString anyPci ''
|
||||
SUBSYSTEM=="vfio", OWNER="${vmUser}", GROUP="${vmUser}", MODE="0660"
|
||||
''
|
||||
+ lib.concatMapStrings (u: ''
|
||||
SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660"
|
||||
'') pt.usb;
|
||||
|
||||
# =====================================================================
|
||||
# Networking
|
||||
# =====================================================================
|
||||
networking.useNetworkd = true;
|
||||
|
||||
systemd.network.netdevs."10-${net.bridge}".netdevConfig = {
|
||||
Kind = "bridge";
|
||||
Name = net.bridge;
|
||||
};
|
||||
|
||||
systemd.network.networks."10-${net.bridge}" = {
|
||||
matchConfig.Name = net.bridge;
|
||||
address = [ "${net.hostAddress}/${toString net.prefixLength}" ];
|
||||
networkConfig = {
|
||||
ConfigureWithoutCarrier = true;
|
||||
IPv4Forwarding = true;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.network.networks."11-${net.tapId}" = {
|
||||
matchConfig.Name = net.tapId;
|
||||
networkConfig.Bridge = net.bridge;
|
||||
};
|
||||
|
||||
networking.nat = {
|
||||
enable = true;
|
||||
internalInterfaces = [ net.bridge ];
|
||||
externalInterface = net.externalInterface;
|
||||
forwardPorts = map (s: {
|
||||
sourcePort = s.hostPort;
|
||||
destination = "${net.guestAddress}:${toString s.guestPort}";
|
||||
inherit (s) proto;
|
||||
}) cfg.publish;
|
||||
};
|
||||
|
||||
# =====================================================================
|
||||
# Firewall
|
||||
#
|
||||
# The guest owns its own kernel and its own config now, so nothing inside
|
||||
# it constrains it. These rules are the boundary.
|
||||
# =====================================================================
|
||||
networking.nftables.enable = true;
|
||||
networking.firewall.enable = true;
|
||||
networking.firewall.filterForward = true;
|
||||
|
||||
# The deliberate guest -> host holes: the Nix cache, plus anything listed
|
||||
# in hostAccess.
|
||||
networking.firewall.extraInputRules =
|
||||
optionalString cfg.hostCache.enable ''
|
||||
iifname "${net.bridge}" ip saddr ${net.guestAddress} tcp dport ${toString cfg.hostCache.port} accept
|
||||
''
|
||||
+ lib.concatMapStrings (s: ''
|
||||
iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept
|
||||
'') cfg.hostAccess;
|
||||
|
||||
# Order matters here. Allowlists must precede the RFC1918 drop.
|
||||
networking.firewall.extraForwardRules =
|
||||
''
|
||||
ct state established,related accept
|
||||
''
|
||||
# Guest reaching named services elsewhere on your network.
|
||||
+ lib.concatMapStrings (s: ''
|
||||
iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
|
||||
'') cfg.lanAccess
|
||||
# Inbound to guest services published on the uplink.
|
||||
+ lib.concatMapStrings (s: ''
|
||||
oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept
|
||||
'') cfg.publish
|
||||
+ (if cfg.allowLan then ''
|
||||
iifname "${net.bridge}" accept
|
||||
'' else ''
|
||||
iifname "${net.bridge}" ip daddr {
|
||||
10.0.0.0/8,
|
||||
172.16.0.0/12,
|
||||
192.168.0.0/16,
|
||||
169.254.0.0/16,
|
||||
127.0.0.0/8,
|
||||
100.64.0.0/10
|
||||
} drop
|
||||
iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
|
||||
iifname "${net.bridge}" accept
|
||||
'');
|
||||
|
||||
# =====================================================================
|
||||
# Host store as a read-only cache for the guest
|
||||
# =====================================================================
|
||||
services.nix-serve = mkIf cfg.hostCache.enable {
|
||||
enable = true;
|
||||
package = pkgs.nix-serve-ng;
|
||||
bindAddress = net.hostAddress;
|
||||
inherit (cfg.hostCache) port secretKeyFile;
|
||||
openFirewall = false; # handled above, scoped to the guest address
|
||||
};
|
||||
|
||||
# =====================================================================
|
||||
# Snapshots of the state disk only.
|
||||
#
|
||||
# The system disk is not snapshotted on purpose. It is large, it is where
|
||||
# all the write churn happens, and setup.sh can rebuild it from the flake
|
||||
# in minutes. Snapshotting it would cost a lot and buy nothing.
|
||||
# =====================================================================
|
||||
systemd.services.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
|
||||
description = "Snapshot the Hermes VM state disk";
|
||||
path = with pkgs; [ btrfs-progs coreutils socat ];
|
||||
serviceConfig.Type = "oneshot";
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
stamp=$(date +%Y%m%d-%H%M%S)
|
||||
mkdir -p ${snapDir}
|
||||
|
||||
${optionalString cfg.disk.quiesce ''
|
||||
# Flush the guest's filesystems so the snapshot is consistent rather
|
||||
# than the equivalent of pulling the power. Takes milliseconds; the
|
||||
# VM keeps running throughout.
|
||||
${qga "freeze"}
|
||||
trap '${qga "thaw"}' EXIT
|
||||
''}
|
||||
|
||||
btrfs subvolume snapshot -r ${cfg.disk.baseDir}/state ${snapDir}/state-$stamp
|
||||
|
||||
${optionalString cfg.disk.quiesce ''
|
||||
${qga "thaw"}
|
||||
trap - EXIT
|
||||
''}
|
||||
|
||||
# Prune oldest first. Keep this list short: every live snapshot
|
||||
# reintroduces copy-on-write on a nodatacow image.
|
||||
ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \
|
||||
| while read -r old; do btrfs subvolume delete "$old"; done
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.snapshots.interval;
|
||||
Persistent = true;
|
||||
RandomizedDelaySec = "5m";
|
||||
};
|
||||
};
|
||||
|
||||
# =====================================================================
|
||||
# Backups: the guest's state, not its disk.
|
||||
#
|
||||
# Pulled file-by-file over the SSH channel that already exists, rather
|
||||
# than backing up the image. That keeps the backup portable - it restores
|
||||
# onto any filesystem, or a different machine entirely - and avoids
|
||||
# loop-mounting a guest-written filesystem on the host, which would mean
|
||||
# the host kernel parsing data the agent controls.
|
||||
# =====================================================================
|
||||
systemd.services.hermes-vm-backup = mkIf cfg.backup.enable {
|
||||
description = "Back up Hermes state";
|
||||
path = with pkgs; [ restic openssh socat gnutar ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
export RESTIC_REPOSITORY=${lib.escapeShellArg cfg.backup.repository}
|
||||
export RESTIC_PASSWORD_FILE=${cfg.backup.passwordFile}
|
||||
|
||||
restic snapshots >/dev/null 2>&1 || restic init
|
||||
|
||||
ssh -o ProxyCommand="socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22" \
|
||||
-o StrictHostKeyChecking=accept-new \
|
||||
hermes@${cfg.name} \
|
||||
'tar -C /var/lib/hermes \
|
||||
--exclude=./.cache \
|
||||
--exclude=./workspace/**/.direnv \
|
||||
--exclude=./workspace/**/result \
|
||||
-cf - .' \
|
||||
| restic backup --stdin --stdin-filename hermes-state.tar \
|
||||
--tag hermes --host ${cfg.name}
|
||||
|
||||
restic forget --tag hermes --host ${cfg.name} --prune ${lib.concatStringsSep " " cfg.backup.keep}
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.hermes-vm-backup = mkIf cfg.backup.enable {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.backup.interval;
|
||||
Persistent = true;
|
||||
RandomizedDelaySec = "30m";
|
||||
};
|
||||
};
|
||||
|
||||
# =====================================================================
|
||||
# Host GC is now safe: the guest has its own store and shares nothing.
|
||||
# =====================================================================
|
||||
nix.gc.automatic = lib.mkDefault true;
|
||||
nix.gc.options = lib.mkDefault "--delete-older-than 30d";
|
||||
|
||||
environment.systemPackages = [
|
||||
(pkgs.writeShellScriptBin "hermes-vm" ''
|
||||
set -euo pipefail
|
||||
SSH=(${pkgs.openssh}/bin/ssh
|
||||
-o ProxyCommand="${pkgs.socat}/bin/socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22"
|
||||
-o StrictHostKeyChecking=accept-new
|
||||
hermes@${cfg.name})
|
||||
cmd="''${1-}"; shift || true
|
||||
case "$cmd" in
|
||||
ssh) exec "''${SSH[@]}" "$@" ;;
|
||||
console) exec journalctl -u hermes-vm -f -o cat ;;
|
||||
start) exec systemctl start hermes-vm ;;
|
||||
stop) exec systemctl stop hermes-vm ;;
|
||||
restart) exec systemctl restart hermes-vm ;;
|
||||
|
||||
# What is the guest actually running right now?
|
||||
config)
|
||||
"''${SSH[@]}" 'echo "== current generation =="; \
|
||||
readlink -f /run/current-system; \
|
||||
echo; echo "== generations =="; \
|
||||
sudo nix-env --list-generations -p /nix/var/nix/profiles/system | tail -n 10; \
|
||||
echo; echo "== /etc/nixos history =="; \
|
||||
git -C /etc/nixos log --oneline -n 15; \
|
||||
echo; echo "== uncommitted =="; \
|
||||
git -C /etc/nixos status --short'
|
||||
;;
|
||||
# Copy the guest's live config here for reading or diffing.
|
||||
config-pull)
|
||||
dest="''${1:-./hermes-etc-nixos}"
|
||||
mkdir -p "$dest"
|
||||
"''${SSH[@]}" 'tar -C /etc/nixos -cf - .' | tar -C "$dest" -xf -
|
||||
echo "pulled to $dest"
|
||||
;;
|
||||
|
||||
snapshot) exec systemctl start hermes-vm-snapshot ;;
|
||||
snapshots)
|
||||
${pkgs.util-linux}/bin/column -t <<<"$(
|
||||
echo "SNAPSHOT SIZE CREATED"
|
||||
for d in ${snapDir}/state-*; do
|
||||
[ -e "$d" ] || continue
|
||||
echo "$(basename "$d") $(du -sh "$d" 2>/dev/null | cut -f1) $(stat -c %y "$d" | cut -d. -f1)"
|
||||
done
|
||||
)" ;;
|
||||
|
||||
# Restore the state disk from a snapshot. The system disk is left
|
||||
# alone - it holds nothing worth restoring.
|
||||
rollback)
|
||||
[ $# -ge 1 ] || { echo "usage: hermes-vm rollback <snapshot>" >&2; exit 1; }
|
||||
snap=${snapDir}/"$1"
|
||||
[ -d "$snap" ] || { echo "no such snapshot: $1" >&2; exit 1; }
|
||||
echo "Restores Hermes's identity, memories, workspace and NixOS" >&2
|
||||
echo "config to $1. Everything since is lost." >&2
|
||||
read -r -p "type 'rollback' to confirm: " c
|
||||
[ "$c" = rollback ] || { echo "aborted" >&2; exit 1; }
|
||||
systemctl stop hermes-vm
|
||||
mv ${cfg.disk.baseDir}/state ${cfg.disk.baseDir}/state.old-$(date +%s)
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot "$snap" ${cfg.disk.baseDir}/state
|
||||
systemctl start hermes-vm
|
||||
echo "restored; previous state kept as state.old-*" ;;
|
||||
|
||||
backup) exec systemctl start hermes-vm-backup ;;
|
||||
backups)
|
||||
exec ${pkgs.restic}/bin/restic \
|
||||
-r "$(systemctl show hermes-vm-backup -p Environment --value | tr ' ' '\n' | sed -n 's/^RESTIC_REPOSITORY=//p')" \
|
||||
snapshots --tag hermes ;;
|
||||
*)
|
||||
echo "usage: hermes-vm {ssh [cmd]|console|config|config-pull [dir]|start|stop|restart|snapshot|snapshots|rollback <snap>|backup|backups}" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
'')
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# Driving Hermes itself, as opposed to the VM around it.
|
||||
# ---------------------------------------------------------------
|
||||
(pkgs.writeShellScriptBin cfg.cliName ''
|
||||
set -euo pipefail
|
||||
SSH=(${pkgs.openssh}/bin/ssh
|
||||
-o ProxyCommand="${pkgs.socat}/bin/socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22"
|
||||
-o StrictHostKeyChecking=accept-new
|
||||
hermes@${cfg.name})
|
||||
HOME_DIR=${cfg.hermesHome}
|
||||
UNIT=${cfg.gatewayUnit}
|
||||
|
||||
# Pull a file out of the guest, open it in your editor, push it back.
|
||||
edit_remote() {
|
||||
remote="$1"; mode="''${2:-0644}"
|
||||
tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT
|
||||
"''${SSH[@]}" "cat $remote" > "$tmp" 2>/dev/null || true
|
||||
before=$(${pkgs.coreutils}/bin/sha256sum < "$tmp")
|
||||
"''${EDITOR:-${pkgs.vim}/bin/vim}" "$tmp"
|
||||
if [ "$(${pkgs.coreutils}/bin/sha256sum < "$tmp")" = "$before" ]; then
|
||||
echo "unchanged"; return
|
||||
fi
|
||||
"''${SSH[@]}" "install -m $mode /dev/stdin $remote" < "$tmp"
|
||||
echo "written to $remote"
|
||||
read -r -p "restart the gateway now? [y/N] " yn
|
||||
case "$yn" in [yY]*) "''${SSH[@]}" "systemctl --user restart $UNIT" ;; esac
|
||||
}
|
||||
|
||||
cmd="''${1-}"; shift || true
|
||||
case "$cmd" in
|
||||
soul)
|
||||
case "''${1-}" in
|
||||
edit) edit_remote "$HOME_DIR/SOUL.md" ;;
|
||||
*) "''${SSH[@]}" "cat $HOME_DIR/SOUL.md" ;;
|
||||
esac ;;
|
||||
|
||||
memory)
|
||||
case "''${1-}" in
|
||||
edit) edit_remote "$HOME_DIR/memories/USER.md" ;;
|
||||
*) "''${SSH[@]}" "cat $HOME_DIR/memories/USER.md" ;;
|
||||
esac ;;
|
||||
|
||||
conf)
|
||||
case "''${1-}" in
|
||||
edit) edit_remote "$HOME_DIR/config.yaml" ;;
|
||||
*) "''${SSH[@]}" "cat $HOME_DIR/config.yaml" ;;
|
||||
esac ;;
|
||||
|
||||
# API keys. Mode 0600, never enters the Nix store, and not printed
|
||||
# unless you ask for it by name.
|
||||
secrets)
|
||||
case "''${1-}" in
|
||||
edit) edit_remote "$HOME_DIR/.env" 0600 ;;
|
||||
show) "''${SSH[@]}" "cat $HOME_DIR/.env" ;;
|
||||
*) "''${SSH[@]}" "sed 's/=.*/=<set>/' $HOME_DIR/.env" ;;
|
||||
esac ;;
|
||||
|
||||
status) "''${SSH[@]}" "systemctl --user status $UNIT --no-pager" ;;
|
||||
logs) "''${SSH[@]}" -t "journalctl --user -u $UNIT -f" ;;
|
||||
restart) "''${SSH[@]}" "systemctl --user restart $UNIT" ;;
|
||||
units) "''${SSH[@]}" "systemctl --user list-units 'hermes*' --all --no-pager" ;;
|
||||
|
||||
# Overwrite the agent's identity from the seed. Destructive: this
|
||||
# is how you undo whatever it has decided about itself.
|
||||
reseed)
|
||||
echo "This overwrites SOUL.md, memories and config.yaml in the guest" >&2
|
||||
echo "with the seed from /etc/nixos/hermes. Anything Hermes has" >&2
|
||||
echo "learned and written there is lost." >&2
|
||||
read -r -p "type 'reseed' to confirm: " confirm
|
||||
[ "$confirm" = "reseed" ] || { echo "aborted" >&2; exit 1; }
|
||||
"''${SSH[@]}" "cp -rT /etc/nixos/hermes/home $HOME_DIR && \
|
||||
cp -rT /etc/nixos/hermes/workspace /var/lib/hermes/workspace && \
|
||||
systemctl --user restart $UNIT"
|
||||
echo "reseeded" ;;
|
||||
|
||||
# Copy the agent's live identity back here, so you can fold what it
|
||||
# has learned into the seed in guest/hermes/.
|
||||
pull)
|
||||
dest="''${1:-./hermes-home}"
|
||||
mkdir -p "$dest"
|
||||
"''${SSH[@]}" "tar -C $HOME_DIR --exclude=.env --exclude=sessions --exclude=state.db -cf - ." \
|
||||
| tar -C "$dest" -xf -
|
||||
echo "pulled to $dest (secrets and sessions excluded)" ;;
|
||||
|
||||
""|help|-h|--help)
|
||||
cat >&2 <<'USAGE'
|
||||
${cfg.cliName} - drive Hermes inside the guest
|
||||
|
||||
soul [edit] show or edit SOUL.md
|
||||
memory [edit] show or edit memories/USER.md
|
||||
conf [edit] show or edit config.yaml
|
||||
secrets [edit|show] manage .env, keys masked by default
|
||||
status | logs | restart | units
|
||||
reseed overwrite identity from /etc/nixos/hermes
|
||||
pull [dir] copy live identity back to the host
|
||||
|
||||
Anything else goes straight to the hermes CLI in the guest:
|
||||
${cfg.cliName} chat
|
||||
${cfg.cliName} update
|
||||
${cfg.cliName} sessions list
|
||||
USAGE
|
||||
exit 1 ;;
|
||||
|
||||
*) exec "''${SSH[@]}" -t hermes "$cmd" "$@" ;;
|
||||
esac
|
||||
'')
|
||||
];
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user