978 lines
33 KiB
Nix
978 lines
33 KiB
Nix
{
|
|
config,
|
|
lib,
|
|
pkgs,
|
|
...
|
|
}:
|
|
|
|
let
|
|
inherit (lib)
|
|
mkOption
|
|
mkEnableOption
|
|
mkIf
|
|
types
|
|
optionalString
|
|
;
|
|
cfg = config.services.hermenix;
|
|
net = cfg.network;
|
|
|
|
vmUser = "hermes-vm";
|
|
runDir = "/run/hermes-vm";
|
|
stateDir = "/var/lib/hermes-vm";
|
|
# Two disks, deliberately.
|
|
#
|
|
# The system disk is large, churny and entirely rebuildable - it is the
|
|
# NixOS install and the Nix store. We never snapshot it.
|
|
#
|
|
# The state disk holds the only things that cannot be reproduced: Hermes's
|
|
# identity and memories, its workspace, and the NixOS config it has written
|
|
# for itself. Small, so snapshots are cheap and backups are quick.
|
|
systemImage = "${cfg.disk.baseDir}/system/system.raw";
|
|
stateImage = "${cfg.disk.baseDir}/state/state.raw";
|
|
snapDir = "${cfg.disk.baseDir}/snapshots";
|
|
|
|
pt = cfg.passthrough;
|
|
anyPci = pt.pci != [ ];
|
|
anyUsb = pt.usb != [ ];
|
|
earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci);
|
|
|
|
# Graceful ACPI power button via QMP, then wait for QEMU to exit.
|
|
stopScript = pkgs.writeShellScript "hermes-vm-stop" ''
|
|
set -eu
|
|
printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \
|
|
| ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0
|
|
'';
|
|
|
|
# Freeze the guest filesystems around a snapshot so it captures a consistent
|
|
# state rather than the equivalent of pulling the power.
|
|
qga =
|
|
verb:
|
|
pkgs.writeShellScript "hermes-vm-fs${verb}" ''
|
|
set -eu
|
|
printf '%s' '{"execute":"guest-fs${verb}"}' \
|
|
| ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true
|
|
'';
|
|
in
|
|
{
|
|
options.services.hermenix = {
|
|
enable = mkEnableOption "the Hermes Agent VM";
|
|
|
|
name = mkOption {
|
|
type = types.str;
|
|
default = "hermes";
|
|
};
|
|
|
|
vcpu = mkOption {
|
|
type = types.ints.positive;
|
|
default = 4;
|
|
};
|
|
|
|
mem = mkOption {
|
|
type = types.ints.positive;
|
|
default = 8192;
|
|
description = "RAM in MiB.";
|
|
};
|
|
|
|
disk = {
|
|
baseDir = mkOption {
|
|
type = types.str;
|
|
default = "/var/lib/hermes-vm";
|
|
description = ''
|
|
Directory holding the guest's disk images. On btrfs this should be
|
|
laid out as three subvolumes - system/, state/ and snapshots/ - with
|
|
nodatacow set. `hermenix setup` does this for you.
|
|
'';
|
|
};
|
|
|
|
systemSize = mkOption {
|
|
type = types.str;
|
|
default = "80G";
|
|
description = ''
|
|
System disk: the NixOS install and the Nix store. Never snapshotted,
|
|
because it is entirely rebuildable from the flake. Size it for an
|
|
agent that builds things.
|
|
'';
|
|
};
|
|
|
|
stateSize = mkOption {
|
|
type = types.str;
|
|
default = "20G";
|
|
description = ''
|
|
State disk, mounted at /var/lib/hermes in the guest. Holds Hermes's
|
|
identity, memories, workspace, and the NixOS config it maintains for
|
|
itself. This is the only irreplaceable data, so it is the only thing
|
|
snapshotted and backed up.
|
|
'';
|
|
};
|
|
|
|
quiesce = mkOption {
|
|
type = types.bool;
|
|
default = true;
|
|
description = "fsfreeze the guest via qemu-guest-agent around each snapshot.";
|
|
};
|
|
};
|
|
|
|
snapshots = {
|
|
enable = mkOption {
|
|
type = types.bool;
|
|
default = true;
|
|
description = ''
|
|
Periodic btrfs snapshots of the state disk. The agent can persist
|
|
changes, which means it can persist a mistake, and these are the
|
|
fastest way back.
|
|
'';
|
|
};
|
|
|
|
interval = mkOption {
|
|
type = types.str;
|
|
default = "hourly";
|
|
description = "systemd calendar expression.";
|
|
};
|
|
|
|
keep = mkOption {
|
|
type = types.ints.positive;
|
|
default = 48;
|
|
description = ''
|
|
How many to retain. Keep this tighter than you would on ZFS: the
|
|
state image is nodatacow, and every live snapshot reintroduces
|
|
copy-on-write for the blocks it still references.
|
|
'';
|
|
};
|
|
};
|
|
|
|
backup = {
|
|
enable = mkEnableOption "restic backups of the guest's state";
|
|
|
|
repository = mkOption {
|
|
type = types.str;
|
|
default = "";
|
|
example = "sftp:backup@nas:/hermes";
|
|
description = "Any restic repository string.";
|
|
};
|
|
|
|
passwordFile = mkOption {
|
|
type = types.path;
|
|
default = "/var/lib/secrets/hermes-restic";
|
|
description = "File containing the restic repository password, mode 0400.";
|
|
};
|
|
|
|
environmentFile = mkOption {
|
|
type = types.nullOr types.path;
|
|
default = null;
|
|
description = "Optional file with credentials for the backend (S3 keys and so on).";
|
|
};
|
|
|
|
interval = mkOption {
|
|
type = types.str;
|
|
default = "daily";
|
|
};
|
|
|
|
keep = mkOption {
|
|
type = types.listOf types.str;
|
|
default = [
|
|
"--keep-daily"
|
|
"14"
|
|
"--keep-weekly"
|
|
"8"
|
|
"--keep-monthly"
|
|
"12"
|
|
];
|
|
description = "Arguments passed to restic forget.";
|
|
};
|
|
};
|
|
|
|
hostCache = {
|
|
enable = mkOption {
|
|
type = types.bool;
|
|
default = true;
|
|
description = ''
|
|
Serve the host's /nix/store to the guest as a read-only binary
|
|
cache, so the guest doesn't re-download from the internet what the
|
|
host already has.
|
|
|
|
This is the ONLY route from guest to host in this design. Turning it
|
|
off leaves the guest with literally no reachable host service, at
|
|
the cost of bandwidth on every guest build.
|
|
|
|
Note the guest can read every store path on the host through this.
|
|
Store paths are world-readable anyway, but check nothing sensitive
|
|
is baked into your host's store.
|
|
'';
|
|
};
|
|
port = mkOption {
|
|
type = types.port;
|
|
default = 5000;
|
|
};
|
|
secretKeyFile = mkOption {
|
|
type = types.path;
|
|
default = "/var/lib/secrets/hermes-cache.secret";
|
|
description = "Generate with nix-store --generate-binary-cache-key. See README.";
|
|
};
|
|
};
|
|
|
|
authorizedKeys = mkOption {
|
|
type = types.listOf types.str;
|
|
default = [ ];
|
|
};
|
|
|
|
allowLan = mkOption {
|
|
type = types.bool;
|
|
default = false;
|
|
description = ''
|
|
Blanket permission for the guest to reach RFC1918 space. Almost always
|
|
the wrong tool - use lanAccess to open single endpoints instead.
|
|
'';
|
|
};
|
|
|
|
lanAccess = mkOption {
|
|
default = [ ];
|
|
example = [
|
|
{
|
|
address = "192.168.1.50";
|
|
port = 11434;
|
|
}
|
|
];
|
|
description = ''
|
|
Specific endpoints elsewhere on your network that the guest may reach,
|
|
punched through the default RFC1918 drop. One entry per service.
|
|
'';
|
|
type = types.listOf (
|
|
types.submodule {
|
|
options = {
|
|
address = mkOption { type = types.str; };
|
|
port = mkOption { type = types.port; };
|
|
proto = mkOption {
|
|
type = types.enum [
|
|
"tcp"
|
|
"udp"
|
|
];
|
|
default = "tcp";
|
|
};
|
|
};
|
|
}
|
|
);
|
|
};
|
|
|
|
hostAccess = mkOption {
|
|
default = [ ];
|
|
example = [ { port = 8188; } ];
|
|
description = ''
|
|
Ports on this host itself that the guest may reach. The service must
|
|
also be bound to an address the guest can see - see the README.
|
|
'';
|
|
type = types.listOf (
|
|
types.submodule {
|
|
options = {
|
|
port = mkOption { type = types.port; };
|
|
proto = mkOption {
|
|
type = types.enum [
|
|
"tcp"
|
|
"udp"
|
|
];
|
|
default = "tcp";
|
|
};
|
|
};
|
|
}
|
|
);
|
|
};
|
|
|
|
publish = mkOption {
|
|
default = [ ];
|
|
example = [
|
|
{
|
|
hostPort = 8080;
|
|
guestPort = 8080;
|
|
}
|
|
];
|
|
description = ''
|
|
Guest ports to publish on the host's uplink, reachable from the wider
|
|
network. You do not need this to reach the guest from the host itself.
|
|
'';
|
|
type = types.listOf (
|
|
types.submodule {
|
|
options = {
|
|
hostPort = mkOption { type = types.port; };
|
|
guestPort = mkOption { type = types.port; };
|
|
proto = mkOption {
|
|
type = types.enum [
|
|
"tcp"
|
|
"udp"
|
|
];
|
|
default = "tcp";
|
|
};
|
|
};
|
|
}
|
|
);
|
|
};
|
|
|
|
network = {
|
|
bridge = mkOption {
|
|
type = types.str;
|
|
default = "hermes-br";
|
|
};
|
|
tapId = mkOption {
|
|
type = types.str;
|
|
default = "vm-hermes";
|
|
};
|
|
mac = mkOption {
|
|
type = types.str;
|
|
default = "02:00:00:7e:00:01";
|
|
};
|
|
hostAddress = mkOption {
|
|
type = types.str;
|
|
default = "10.77.0.1";
|
|
};
|
|
guestAddress = mkOption {
|
|
type = types.str;
|
|
default = "10.77.0.2";
|
|
};
|
|
prefixLength = mkOption {
|
|
type = types.int;
|
|
default = 24;
|
|
};
|
|
externalInterface = mkOption {
|
|
type = types.nullOr types.str;
|
|
default = null;
|
|
example = "eno1";
|
|
description = "Host uplink. Required.";
|
|
};
|
|
};
|
|
|
|
passthrough = {
|
|
iommu = mkOption {
|
|
type = types.nullOr (
|
|
types.enum [
|
|
"intel"
|
|
"amd"
|
|
]
|
|
);
|
|
default = null;
|
|
description = ''
|
|
Enable IOMMU kernel parameters for your CPU vendor. Required for any
|
|
PCI passthrough. The IOMMU is the only thing stopping a passed-through
|
|
device from reading host memory, so this is not optional.
|
|
'';
|
|
};
|
|
|
|
pci = mkOption {
|
|
default = [ ];
|
|
example = [
|
|
{
|
|
address = "0000:01:00.0";
|
|
id = "10de:2204";
|
|
}
|
|
{
|
|
address = "0000:01:00.1";
|
|
id = "10de:1aef";
|
|
}
|
|
];
|
|
description = ''
|
|
PCI devices handed to the guest via VFIO. The host loses them
|
|
entirely while the VM runs.
|
|
|
|
You must pass every function of the device and, in practice, every
|
|
device in its IOMMU group. A GPU is usually at least two entries:
|
|
the graphics function and its HDMI audio function.
|
|
|
|
Set `id` (vendor:device) to have vfio-pci claim the device in the
|
|
initrd, before a host driver can. Strongly recommended for GPUs;
|
|
unbinding an active amdgpu or nvidia driver later is unreliable.
|
|
'';
|
|
type = types.listOf (
|
|
types.submodule {
|
|
options = {
|
|
address = mkOption {
|
|
type = types.str;
|
|
example = "0000:01:00.0";
|
|
description = "Full PCI address, as shown by lspci -D.";
|
|
};
|
|
id = mkOption {
|
|
type = types.nullOr types.str;
|
|
default = null;
|
|
example = "10de:2204";
|
|
description = "vendor:device, from lspci -nn. Enables early binding.";
|
|
};
|
|
};
|
|
}
|
|
);
|
|
};
|
|
|
|
usb = mkOption {
|
|
default = [ ];
|
|
example = [
|
|
{
|
|
vendorId = "8087";
|
|
productId = "0026";
|
|
}
|
|
];
|
|
description = ''
|
|
USB devices forwarded to the guest, matched by vendor and product
|
|
ID from lsusb. Integrated Bluetooth radios are USB devices and go
|
|
here too, though the host driver must be kept off them.
|
|
|
|
QEMU resolves these at start, so a device that is unplugged when the
|
|
VM boots will not appear later without a restart.
|
|
'';
|
|
type = types.listOf (
|
|
types.submodule {
|
|
options = {
|
|
vendorId = mkOption {
|
|
type = types.str;
|
|
example = "046d";
|
|
};
|
|
productId = mkOption {
|
|
type = types.str;
|
|
example = "c52b";
|
|
};
|
|
};
|
|
}
|
|
);
|
|
};
|
|
};
|
|
|
|
gatewayUnit = mkOption {
|
|
type = types.str;
|
|
default = "hermes-agent";
|
|
description = ''
|
|
The systemd *user* unit that `hermes gateway install` created in the
|
|
guest. Check with `hermenix ctl units` after installing; upstream has
|
|
used more than one name for this.
|
|
'';
|
|
};
|
|
|
|
hermesHome = mkOption {
|
|
type = types.str;
|
|
default = "${stateDir}/.hermes";
|
|
description = ''
|
|
Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/,
|
|
state.db, and so on) lives inside the guest. Defaults to the seeded
|
|
layout under the state disk mount. Override this when carrying over
|
|
an existing flat HERMES_HOME from elsewhere (e.g. a prior Docker
|
|
deployment) that you're placing directly at the root of the state
|
|
disk instead of under .hermes - see `hermenix migrate --help`.
|
|
|
|
This only changes where the host-side `hermenix ctl` commands (soul,
|
|
memory, conf, secrets, reseed, pull) looks. It does not change what
|
|
the gateway process itself uses - export HERMES_HOME to match inside
|
|
the guest when you install it.
|
|
'';
|
|
};
|
|
|
|
vsockCid = mkOption {
|
|
type = types.int;
|
|
default = 77;
|
|
description = "Host reaches the guest over vsock, so no inbound network path is needed.";
|
|
};
|
|
|
|
version = mkOption {
|
|
type = types.str;
|
|
default = "dev";
|
|
description = ''
|
|
"YYYY.MM.DD.hh.mm-<commit-sha>" reported by `hermenix version`.
|
|
`version` subcommand. Normally supplied for free by this flake's own
|
|
`nixosModules.default` (from its git metadata) - the "dev" default
|
|
here only shows up if this module is imported some other way.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = net.externalInterface != null;
|
|
message = "services.hermenix.network.externalInterface must be set.";
|
|
}
|
|
{
|
|
assertion = cfg.backup.enable -> (cfg.backup.repository != "");
|
|
message = "services.hermenix.backup.repository must be set when backups are enabled.";
|
|
}
|
|
{
|
|
assertion = anyPci -> (pt.iommu != null);
|
|
message = ''
|
|
services.hermenix.passthrough.pci needs passthrough.iommu set to
|
|
"intel" or "amd". Without the IOMMU, a passed-through device can
|
|
read host memory directly and the VM boundary is meaningless.
|
|
'';
|
|
}
|
|
];
|
|
|
|
users.users.${vmUser} = {
|
|
isSystemUser = true;
|
|
group = vmUser;
|
|
description = "Runs the Hermes guest VM";
|
|
};
|
|
users.groups.${vmUser} = { };
|
|
|
|
# =====================================================================
|
|
# The VM itself
|
|
#
|
|
# No virtiofs, no shared filesystem of any kind, no vhost-net. The guest
|
|
# gets a block device, a tap, a vsock and a serial port. That is all.
|
|
# =====================================================================
|
|
systemd.services.hermes-vm = {
|
|
description = "Hermes Agent VM";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [
|
|
"network-online.target"
|
|
"hermes-vm-tap.service"
|
|
];
|
|
requires = [ "hermes-vm-tap.service" ];
|
|
wants = [ "network-online.target" ];
|
|
|
|
preStart = ''
|
|
for img in ${systemImage} ${stateImage}; do
|
|
if [ ! -e "$img" ]; then
|
|
echo "$img is missing. Run 'hermenix setup' first." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
# Per-VM writable UEFI variable store, seeded once.
|
|
if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then
|
|
install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd
|
|
fi
|
|
'';
|
|
|
|
serviceConfig = {
|
|
User = vmUser;
|
|
Group = vmUser;
|
|
SupplementaryGroups = [ "kvm" ];
|
|
RuntimeDirectory = "hermes-vm";
|
|
# ProtectSystem=strict makes everything read-only; punch through for
|
|
# the images and the UEFI variable store only.
|
|
ReadWritePaths = [ cfg.disk.baseDir ];
|
|
StateDirectory = "hermes-vm";
|
|
RuntimeDirectoryMode = "0750";
|
|
|
|
ExecStart = lib.concatStringsSep " " (
|
|
[
|
|
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
|
|
"-name ${cfg.name}"
|
|
"-machine q35,accel=kvm,smm=on"
|
|
"-cpu host"
|
|
"-smp ${toString cfg.vcpu}"
|
|
"-m ${toString cfg.mem}"
|
|
"-nodefaults"
|
|
"-no-user-config"
|
|
"-display none"
|
|
"-serial stdio"
|
|
# QEMU's own seccomp sandbox, everything denied that we don't need.
|
|
"-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"
|
|
# UEFI: firmware read-only, variables per-VM and writable.
|
|
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
|
|
"-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd"
|
|
# System disk: NixOS and the Nix store. Rebuildable, never snapshotted.
|
|
# aio=threads and a writeback cache because these are files on a
|
|
# filesystem, not block devices; aio=native wants the latter.
|
|
"-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap"
|
|
"-device virtio-blk-pci,drive=system"
|
|
# State disk: everything irreplaceable. Mounted at /var/lib/hermes.
|
|
"-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap"
|
|
"-device virtio-blk-pci,drive=state"
|
|
# Pre-created tap. vhost=off on purpose: vhost-net would move packet
|
|
# handling into the host kernel, which is exactly the surface we are
|
|
# trying not to expose. Costs throughput we do not need.
|
|
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
|
|
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
|
|
# Host -> guest management, no network involved.
|
|
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
|
|
"-device virtio-rng-pci"
|
|
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
|
|
]
|
|
++ lib.optionals cfg.disk.quiesce [
|
|
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
|
|
"-device virtio-serial-pci"
|
|
"-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
|
|
]
|
|
# Passed-through PCI devices. The host cannot use these while the VM
|
|
# is running.
|
|
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
|
|
# A USB controller, only if something needs plugging into it.
|
|
++ lib.optional anyUsb "-device qemu-xhci,id=xhci"
|
|
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
|
|
);
|
|
|
|
ExecStop = stopScript;
|
|
TimeoutStopSec = 90;
|
|
Restart = "always";
|
|
RestartSec = 5;
|
|
|
|
# ---- hardening -------------------------------------------------
|
|
DevicePolicy = "closed";
|
|
DeviceAllow = [
|
|
"/dev/kvm rw"
|
|
"/dev/net/tun rw"
|
|
"/dev/vhost-vsock rw"
|
|
]
|
|
++ lib.optionals anyPci [ "char-vfio rw" ]
|
|
++ lib.optionals anyUsb [ "char-usb_device rw" ];
|
|
|
|
# VFIO pins the entire guest RAM allocation, so the default memlock
|
|
# limit has to go or QEMU fails to start with an obscure error.
|
|
LimitMEMLOCK = lib.mkIf anyPci "infinity";
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
ProtectProc = "invisible";
|
|
ProcSubset = "pid";
|
|
PrivateTmp = true;
|
|
ProtectKernelTunables = true;
|
|
ProtectKernelModules = true;
|
|
ProtectKernelLogs = true;
|
|
ProtectClock = true;
|
|
ProtectControlGroups = true;
|
|
ProtectHostname = true;
|
|
RestrictNamespaces = true;
|
|
RestrictRealtime = true;
|
|
RestrictSUIDSGID = true;
|
|
RestrictAddressFamilies = [
|
|
"AF_UNIX"
|
|
"AF_NETLINK"
|
|
"AF_VSOCK"
|
|
];
|
|
LockPersonality = true;
|
|
SystemCallArchitectures = "native";
|
|
SystemCallFilter = [
|
|
"@system-service"
|
|
"~@resources"
|
|
"~@obsolete"
|
|
];
|
|
# KVM means no JIT, so W^X is safe. If QEMU refuses to start, this is
|
|
# the first thing to flip.
|
|
MemoryDenyWriteExecute = true;
|
|
CapabilityBoundingSet = [ "" ];
|
|
AmbientCapabilities = [ "" ];
|
|
};
|
|
};
|
|
|
|
# Tap device, created as root, owned by the VM user, bridged by networkd.
|
|
systemd.services.hermes-vm-tap = {
|
|
description = "TAP interface for the Hermes VM";
|
|
after = [ "systemd-networkd.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
};
|
|
path = [ pkgs.iproute2 ];
|
|
script = ''
|
|
ip link show ${net.tapId} >/dev/null 2>&1 \
|
|
|| ip tuntap add ${net.tapId} mode tap user ${vmUser}
|
|
ip link set ${net.tapId} up
|
|
'';
|
|
preStop = ''
|
|
${pkgs.iproute2}/bin/ip link del ${net.tapId} || true
|
|
'';
|
|
};
|
|
|
|
# =====================================================================
|
|
# Device passthrough
|
|
# =====================================================================
|
|
|
|
# The IOMMU is what keeps a passed-through device from reading host
|
|
# memory. Without it, PCI passthrough is not a boundary at all.
|
|
boot.kernelParams = lib.optionals (pt.iommu != null) [
|
|
(if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on")
|
|
"iommu=pt"
|
|
];
|
|
|
|
boot.initrd.kernelModules = lib.optionals anyPci [
|
|
"vfio_pci"
|
|
"vfio"
|
|
"vfio_iommu_type1"
|
|
];
|
|
|
|
# Claim the device before any host driver gets to it.
|
|
boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) ''
|
|
options vfio-pci ids=${lib.concatStringsSep "," earlyIds}
|
|
'';
|
|
|
|
# Catch anything that was not early-bound, by address. Idempotent.
|
|
systemd.services.hermes-vm-vfio = mkIf anyPci {
|
|
description = "Bind passthrough PCI devices to vfio-pci";
|
|
before = [ "hermes-vm.service" ];
|
|
requiredBy = [ "hermes-vm.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
};
|
|
path = [ pkgs.kmod ];
|
|
script = ''
|
|
modprobe vfio-pci
|
|
''
|
|
+ lib.concatMapStrings (d: ''
|
|
dev=/sys/bus/pci/devices/${d.address}
|
|
if [ ! -e "$dev" ]; then
|
|
echo "${d.address} is not present on this machine" >&2
|
|
exit 1
|
|
fi
|
|
cur=none
|
|
[ -e "$dev/driver" ] && cur=$(basename "$(readlink -f "$dev/driver")")
|
|
if [ "$cur" != "vfio-pci" ]; then
|
|
[ "$cur" != "none" ] && echo ${d.address} > "$dev/driver/unbind"
|
|
echo vfio-pci > "$dev/driver_override"
|
|
echo ${d.address} > /sys/bus/pci/drivers_probe
|
|
fi
|
|
'') pt.pci;
|
|
};
|
|
|
|
services.udev.extraRules =
|
|
lib.optionalString anyPci ''
|
|
SUBSYSTEM=="vfio", OWNER="${vmUser}", GROUP="${vmUser}", MODE="0660"
|
|
''
|
|
+ lib.concatMapStrings (u: ''
|
|
SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660"
|
|
'') pt.usb;
|
|
|
|
# =====================================================================
|
|
# Networking
|
|
# =====================================================================
|
|
networking.useNetworkd = true;
|
|
|
|
systemd.network.netdevs."10-${net.bridge}".netdevConfig = {
|
|
Kind = "bridge";
|
|
Name = net.bridge;
|
|
};
|
|
|
|
systemd.network.networks."10-${net.bridge}" = {
|
|
matchConfig.Name = net.bridge;
|
|
address = [ "${net.hostAddress}/${toString net.prefixLength}" ];
|
|
networkConfig = {
|
|
ConfigureWithoutCarrier = true;
|
|
IPv4Forwarding = true;
|
|
};
|
|
};
|
|
|
|
systemd.network.networks."11-${net.tapId}" = {
|
|
matchConfig.Name = net.tapId;
|
|
networkConfig.Bridge = net.bridge;
|
|
};
|
|
|
|
# ip_forward: use the same sysctl keys networking.nat used (mkOverride 99
|
|
# so they win over any lower-priority defaults elsewhere in the config).
|
|
boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true;
|
|
boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true;
|
|
|
|
# Masquerade outbound guest traffic and DNAT published ports inbound.
|
|
# We use an explicit nftables table instead of networking.nat so that the
|
|
# rules are certain to appear in the prerouting/postrouting hooks.
|
|
#
|
|
# NOTE: DNAT uses 'iifname != bridge' rather than 'iifname externalInterface'.
|
|
# Scoping to a single interface means traffic arriving on any other path
|
|
# (Tailscale, a second NIC, WiFi, loopback) silently skips DNAT: the
|
|
# packet stays addressed to the host's own IP, falls into the INPUT chain,
|
|
# and gets dropped because those ports are not in allowedTCPPorts.
|
|
# Excluding only the bridge itself prevents loops from inside the guest
|
|
# while covering every legitimate external path.
|
|
networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null) {
|
|
family = "ip";
|
|
content = ''
|
|
chain prerouting {
|
|
type nat hook prerouting priority dstnat;
|
|
''
|
|
+ lib.concatMapStrings (s: ''
|
|
iifname != "${net.bridge}" ${s.proto} dport ${toString s.hostPort} dnat to ${net.guestAddress}:${toString s.guestPort}
|
|
'') cfg.publish
|
|
+ ''
|
|
}
|
|
chain postrouting {
|
|
type nat hook postrouting priority srcnat;
|
|
iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade
|
|
}
|
|
'';
|
|
};
|
|
|
|
# =====================================================================
|
|
# Firewall
|
|
#
|
|
# The guest owns its own kernel and its own config now, so nothing inside
|
|
# it constrains it. These rules are the boundary.
|
|
# =====================================================================
|
|
networking.nftables.enable = true;
|
|
networking.firewall.enable = true;
|
|
networking.firewall.filterForward = true;
|
|
|
|
# The deliberate guest -> host holes: the Nix cache, plus anything listed
|
|
# in hostAccess.
|
|
networking.firewall.extraInputRules =
|
|
optionalString cfg.hostCache.enable ''
|
|
iifname "${net.bridge}" ip saddr ${net.guestAddress} tcp dport ${toString cfg.hostCache.port} accept
|
|
''
|
|
+ lib.concatMapStrings (s: ''
|
|
iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept
|
|
'') cfg.hostAccess;
|
|
|
|
# Order matters here. Allowlists must precede the RFC1918 drop.
|
|
networking.firewall.extraForwardRules = ''
|
|
iifname != "${net.bridge}" accept
|
|
ct state established,related accept
|
|
''
|
|
# Guest reaching named services elsewhere on your network.
|
|
+ lib.concatMapStrings (s: ''
|
|
iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
|
|
'') cfg.lanAccess
|
|
# Inbound to guest services published on the uplink.
|
|
+ lib.concatMapStrings (s: ''
|
|
oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept
|
|
'') cfg.publish
|
|
+ (
|
|
if cfg.allowLan then
|
|
''
|
|
iifname "${net.bridge}" accept
|
|
''
|
|
else
|
|
''
|
|
iifname "${net.bridge}" ip daddr {
|
|
10.0.0.0/8,
|
|
172.16.0.0/12,
|
|
192.168.0.0/16,
|
|
169.254.0.0/16,
|
|
127.0.0.0/8,
|
|
100.64.0.0/10
|
|
} drop
|
|
iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
|
|
iifname "${net.bridge}" accept
|
|
''
|
|
);
|
|
|
|
# =====================================================================
|
|
# Host store as a read-only cache for the guest
|
|
# =====================================================================
|
|
services.nix-serve = mkIf cfg.hostCache.enable {
|
|
enable = true;
|
|
package = pkgs.nix-serve-ng;
|
|
bindAddress = net.hostAddress;
|
|
inherit (cfg.hostCache) port secretKeyFile;
|
|
openFirewall = false; # handled above, scoped to the guest address
|
|
};
|
|
|
|
# =====================================================================
|
|
# Snapshots of the state disk only.
|
|
#
|
|
# The system disk is not snapshotted on purpose. It is large, it is where
|
|
# all the write churn happens, and `hermenix setup` can rebuild it from the flake
|
|
# in minutes. Snapshotting it would cost a lot and buy nothing.
|
|
# =====================================================================
|
|
systemd.services.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
|
|
description = "Snapshot the Hermes VM state disk";
|
|
path = with pkgs; [
|
|
btrfs-progs
|
|
coreutils
|
|
socat
|
|
];
|
|
serviceConfig.Type = "oneshot";
|
|
script = ''
|
|
set -euo pipefail
|
|
stamp=$(date +%Y%m%d-%H%M%S)
|
|
mkdir -p ${snapDir}
|
|
|
|
${optionalString cfg.disk.quiesce ''
|
|
# Flush the guest's filesystems so the snapshot is consistent rather
|
|
# than the equivalent of pulling the power. Takes milliseconds; the
|
|
# VM keeps running throughout.
|
|
${qga "freeze"}
|
|
trap '${qga "thaw"}' EXIT
|
|
''}
|
|
|
|
btrfs subvolume snapshot -r ${cfg.disk.baseDir}/state ${snapDir}/state-$stamp
|
|
|
|
${optionalString cfg.disk.quiesce ''
|
|
${qga "thaw"}
|
|
trap - EXIT
|
|
''}
|
|
|
|
# Prune oldest first. Keep this list short: every live snapshot
|
|
# reintroduces copy-on-write on a nodatacow image.
|
|
ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \
|
|
| while read -r old; do btrfs subvolume delete "$old"; done
|
|
'';
|
|
};
|
|
|
|
systemd.timers.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnCalendar = cfg.snapshots.interval;
|
|
Persistent = true;
|
|
RandomizedDelaySec = "5m";
|
|
};
|
|
};
|
|
|
|
# =====================================================================
|
|
# Backups: the guest's state, not its disk.
|
|
#
|
|
# Pulled file-by-file over the SSH channel that already exists, rather
|
|
# than backing up the image. That keeps the backup portable - it restores
|
|
# onto any filesystem, or a different machine entirely - and avoids
|
|
# loop-mounting a guest-written filesystem on the host, which would mean
|
|
# the host kernel parsing data the agent controls.
|
|
# =====================================================================
|
|
systemd.services.hermes-vm-backup = mkIf cfg.backup.enable {
|
|
description = "Back up Hermes state";
|
|
path = with pkgs; [
|
|
restic
|
|
openssh
|
|
socat
|
|
gnutar
|
|
];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile;
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
export RESTIC_REPOSITORY=${lib.escapeShellArg cfg.backup.repository}
|
|
export RESTIC_PASSWORD_FILE=${cfg.backup.passwordFile}
|
|
|
|
restic snapshots >/dev/null 2>&1 || restic init
|
|
|
|
ssh -o ProxyCommand="socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22" \
|
|
-o StrictHostKeyChecking=accept-new \
|
|
hermes@${cfg.name} \
|
|
'tar -C /var/lib/hermes \
|
|
--exclude=./.cache \
|
|
--exclude=./workspace/**/.direnv \
|
|
--exclude=./workspace/**/result \
|
|
-cf - .' \
|
|
| restic backup --stdin --stdin-filename hermes-state.tar \
|
|
--tag hermes --host ${cfg.name}
|
|
|
|
restic forget --tag hermes --host ${cfg.name} --prune ${lib.concatStringsSep " " cfg.backup.keep}
|
|
'';
|
|
};
|
|
|
|
systemd.timers.hermes-vm-backup = mkIf cfg.backup.enable {
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnCalendar = cfg.backup.interval;
|
|
Persistent = true;
|
|
RandomizedDelaySec = "30m";
|
|
};
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
# Nushell tab-completion definitions for hermenix. Installed to
|
|
# vendor/autoload so nushell picks them up automatically.
|
|
(pkgs.writeTextFile {
|
|
name = "hermenix-completions";
|
|
destination = "/share/nushell/vendor/autoload/hermenix.nu";
|
|
text = builtins.readFile ../nu/completions.nu;
|
|
})
|
|
|
|
# Single consolidated CLI: `hermenix vm *` controls the VM,
|
|
# `hermenix ctl *` drives Hermes inside it. Per-host values are
|
|
# added on top of the base package via makeWrapper.
|
|
(pkgs.symlinkJoin {
|
|
name = "hermenix";
|
|
paths = [ pkgs.hermenix ];
|
|
nativeBuildInputs = [ pkgs.makeWrapper ];
|
|
postBuild = ''
|
|
wrapProgram $out/bin/hermenix \
|
|
--set HERMES_VSOCK_CID ${toString cfg.vsockCid} \
|
|
--set HERMES_GUEST_NAME ${cfg.name} \
|
|
--set HERMES_DISK_BASE_DIR ${cfg.disk.baseDir} \
|
|
--set HERMES_HOME_DIR ${cfg.hermesHome} \
|
|
--set HERMES_GATEWAY_UNIT ${cfg.gatewayUnit}
|
|
'';
|
|
})
|
|
];
|
|
};
|
|
}
|