hooks + devenv

This commit is contained in:
ari committed 2026-09-17 14:14:14 -04:00
1 parent e07ae98c0b
commit d47c401013
19 files changed
+956 -280

No files matched your search

+290 -166
View File
@@ -1,7 +1,18 @@
{ config, lib, pkgs, ... }:
{
config,
lib,
pkgs,
...
}:
let
inherit (lib) mkOption mkEnableOption mkIf types optionalString;
inherit (lib)
mkOption
mkEnableOption
mkIf
types
optionalString
;
cfg = config.services.hermenix;
net = cfg.network;
@@ -34,11 +45,13 @@ let
# Freeze the guest filesystems around a snapshot so it captures a consistent
# state rather than the equivalent of pulling the power.
qga = verb: pkgs.writeShellScript "hermes-vm-fs${verb}" ''
set -eu
printf '%s' '{"execute":"guest-fs${verb}"}' \
| ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true
'';
qga =
verb:
pkgs.writeShellScript "hermes-vm-fs${verb}" ''
set -eu
printf '%s' '{"execute":"guest-fs${verb}"}' \
| ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true
'';
in
{
options.services.hermenix = {
@@ -156,7 +169,14 @@ in
keep = mkOption {
type = types.listOf types.str;
default = [ "--keep-daily" "14" "--keep-weekly" "8" "--keep-monthly" "12" ];
default = [
"--keep-daily"
"14"
"--keep-weekly"
"8"
"--keep-monthly"
"12"
];
description = "Arguments passed to restic forget.";
};
};
@@ -206,21 +226,31 @@ in
lanAccess = mkOption {
default = [ ];
example = [ { address = "192.168.1.50"; port = 11434; } ];
example = [
{
address = "192.168.1.50";
port = 11434;
}
];
description = ''
Specific endpoints elsewhere on your network that the guest may reach,
punched through the default RFC1918 drop. One entry per service.
'';
type = types.listOf (types.submodule {
options = {
address = mkOption { type = types.str; };
port = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [ "tcp" "udp" ];
default = "tcp";
type = types.listOf (
types.submodule {
options = {
address = mkOption { type = types.str; };
port = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
};
});
}
);
};
hostAccess = mkOption {
@@ -230,43 +260,76 @@ in
Ports on this host itself that the guest may reach. The service must
also be bound to an address the guest can see - see the README.
'';
type = types.listOf (types.submodule {
options = {
port = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [ "tcp" "udp" ];
default = "tcp";
type = types.listOf (
types.submodule {
options = {
port = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
};
});
}
);
};
publish = mkOption {
default = [ ];
example = [ { hostPort = 8080; guestPort = 8080; } ];
example = [
{
hostPort = 8080;
guestPort = 8080;
}
];
description = ''
Guest ports to publish on the host's uplink, reachable from the wider
network. You do not need this to reach the guest from the host itself.
'';
type = types.listOf (types.submodule {
options = {
hostPort = mkOption { type = types.port; };
guestPort = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [ "tcp" "udp" ];
default = "tcp";
type = types.listOf (
types.submodule {
options = {
hostPort = mkOption { type = types.port; };
guestPort = mkOption { type = types.port; };
proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
};
};
};
});
}
);
};
network = {
bridge = mkOption { type = types.str; default = "hermes-br"; };
tapId = mkOption { type = types.str; default = "vm-hermes"; };
mac = mkOption { type = types.str; default = "02:00:00:7e:00:01"; };
hostAddress = mkOption { type = types.str; default = "10.77.0.1"; };
guestAddress = mkOption { type = types.str; default = "10.77.0.2"; };
prefixLength = mkOption { type = types.int; default = 24; };
bridge = mkOption {
type = types.str;
default = "hermes-br";
};
tapId = mkOption {
type = types.str;
default = "vm-hermes";
};
mac = mkOption {
type = types.str;
default = "02:00:00:7e:00:01";
};
hostAddress = mkOption {
type = types.str;
default = "10.77.0.1";
};
guestAddress = mkOption {
type = types.str;
default = "10.77.0.2";
};
prefixLength = mkOption {
type = types.int;
default = 24;
};
externalInterface = mkOption {
type = types.nullOr types.str;
default = null;
@@ -277,7 +340,12 @@ in
passthrough = {
iommu = mkOption {
type = types.nullOr (types.enum [ "intel" "amd" ]);
type = types.nullOr (
types.enum [
"intel"
"amd"
]
);
default = null;
description = ''
Enable IOMMU kernel parameters for your CPU vendor. Required for any
@@ -289,8 +357,14 @@ in
pci = mkOption {
default = [ ];
example = [
{ address = "0000:01:00.0"; id = "10de:2204"; }
{ address = "0000:01:00.1"; id = "10de:1aef"; }
{
address = "0000:01:00.0";
id = "10de:2204";
}
{
address = "0000:01:00.1";
id = "10de:1aef";
}
];
description = ''
PCI devices handed to the guest via VFIO. The host loses them
@@ -304,26 +378,33 @@ in
initrd, before a host driver can. Strongly recommended for GPUs;
unbinding an active amdgpu or nvidia driver later is unreliable.
'';
type = types.listOf (types.submodule {
options = {
address = mkOption {
type = types.str;
example = "0000:01:00.0";
description = "Full PCI address, as shown by lspci -D.";
type = types.listOf (
types.submodule {
options = {
address = mkOption {
type = types.str;
example = "0000:01:00.0";
description = "Full PCI address, as shown by lspci -D.";
};
id = mkOption {
type = types.nullOr types.str;
default = null;
example = "10de:2204";
description = "vendor:device, from lspci -nn. Enables early binding.";
};
};
id = mkOption {
type = types.nullOr types.str;
default = null;
example = "10de:2204";
description = "vendor:device, from lspci -nn. Enables early binding.";
};
};
});
}
);
};
usb = mkOption {
default = [ ];
example = [ { vendorId = "8087"; productId = "0026"; } ];
example = [
{
vendorId = "8087";
productId = "0026";
}
];
description = ''
USB devices forwarded to the guest, matched by vendor and product
ID from lsusb. Integrated Bluetooth radios are USB devices and go
@@ -332,12 +413,20 @@ in
QEMU resolves these at start, so a device that is unplugged when the
VM boots will not appear later without a restart.
'';
type = types.listOf (types.submodule {
options = {
vendorId = mkOption { type = types.str; example = "046d"; };
productId = mkOption { type = types.str; example = "c52b"; };
};
});
type = types.listOf (
types.submodule {
options = {
vendorId = mkOption {
type = types.str;
example = "046d";
};
productId = mkOption {
type = types.str;
example = "c52b";
};
};
}
);
};
};
@@ -423,7 +512,10 @@ in
systemd.services.hermes-vm = {
description = "Hermes Agent VM";
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" "hermes-vm-tap.service" ];
after = [
"network-online.target"
"hermes-vm-tap.service"
];
requires = [ "hermes-vm-tap.service" ];
wants = [ "network-online.target" ];
@@ -451,50 +543,52 @@ in
StateDirectory = "hermes-vm";
RuntimeDirectoryMode = "0750";
ExecStart = lib.concatStringsSep " " ([
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
"-name ${cfg.name}"
"-machine q35,accel=kvm,smm=on"
"-cpu host"
"-smp ${toString cfg.vcpu}"
"-m ${toString cfg.mem}"
"-nodefaults"
"-no-user-config"
"-display none"
"-serial stdio"
# QEMU's own seccomp sandbox, everything denied that we don't need.
"-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"
# UEFI: firmware read-only, variables per-VM and writable.
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
"-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd"
# System disk: NixOS and the Nix store. Rebuildable, never snapshotted.
# aio=threads and a writeback cache because these are files on a
# filesystem, not block devices; aio=native wants the latter.
"-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=system"
# State disk: everything irreplaceable. Mounted at /var/lib/hermes.
"-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=state"
# Pre-created tap. vhost=off on purpose: vhost-net would move packet
# handling into the host kernel, which is exactly the surface we are
# trying not to expose. Costs throughput we do not need.
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
# Host -> guest management, no network involved.
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
"-device virtio-rng-pci"
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
] ++ lib.optionals cfg.disk.quiesce [
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
"-device virtio-serial-pci"
"-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
]
# Passed-through PCI devices. The host cannot use these while the VM
# is running.
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
# A USB controller, only if something needs plugging into it.
++ lib.optional anyUsb "-device qemu-xhci,id=xhci"
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
ExecStart = lib.concatStringsSep " " (
[
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
"-name ${cfg.name}"
"-machine q35,accel=kvm,smm=on"
"-cpu host"
"-smp ${toString cfg.vcpu}"
"-m ${toString cfg.mem}"
"-nodefaults"
"-no-user-config"
"-display none"
"-serial stdio"
# QEMU's own seccomp sandbox, everything denied that we don't need.
"-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"
# UEFI: firmware read-only, variables per-VM and writable.
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
"-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd"
# System disk: NixOS and the Nix store. Rebuildable, never snapshotted.
# aio=threads and a writeback cache because these are files on a
# filesystem, not block devices; aio=native wants the latter.
"-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=system"
# State disk: everything irreplaceable. Mounted at /var/lib/hermes.
"-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap"
"-device virtio-blk-pci,drive=state"
# Pre-created tap. vhost=off on purpose: vhost-net would move packet
# handling into the host kernel, which is exactly the surface we are
# trying not to expose. Costs throughput we do not need.
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
# Host -> guest management, no network involved.
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
"-device virtio-rng-pci"
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
]
++ lib.optionals cfg.disk.quiesce [
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
"-device virtio-serial-pci"
"-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
]
# Passed-through PCI devices. The host cannot use these while the VM
# is running.
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
# A USB controller, only if something needs plugging into it.
++ lib.optional anyUsb "-device qemu-xhci,id=xhci"
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
);
ExecStop = stopScript;
@@ -530,10 +624,18 @@ in
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
RestrictAddressFamilies = [ "AF_UNIX" "AF_NETLINK" "AF_VSOCK" ];
RestrictAddressFamilies = [
"AF_UNIX"
"AF_NETLINK"
"AF_VSOCK"
];
LockPersonality = true;
SystemCallArchitectures = "native";
SystemCallFilter = [ "@system-service" "~@resources" "~@obsolete" ];
SystemCallFilter = [
"@system-service"
"~@resources"
"~@obsolete"
];
# KVM means no JIT, so W^X is safe. If QEMU refuses to start, this is
# the first thing to flip.
MemoryDenyWriteExecute = true;
@@ -595,7 +697,8 @@ in
path = [ pkgs.kmod ];
script = ''
modprobe vfio-pci
'' + lib.concatMapStrings (d: ''
''
+ lib.concatMapStrings (d: ''
dev=/sys/bus/pci/devices/${d.address}
if [ ! -e "$dev" ]; then
echo "${d.address} is not present on this machine" >&2
@@ -646,7 +749,7 @@ in
networking.nat = {
enable = true;
internalInterfaces = [ net.bridge ];
externalInterface = net.externalInterface;
inherit (net) externalInterface;
forwardPorts = map (s: {
sourcePort = s.hostPort;
destination = "${net.guestAddress}:${toString s.guestPort}";
@@ -675,33 +778,37 @@ in
'') cfg.hostAccess;
# Order matters here. Allowlists must precede the RFC1918 drop.
networking.firewall.extraForwardRules =
''
iifname != "${net.bridge}" accept
ct state established,related accept
''
# Guest reaching named services elsewhere on your network.
+ lib.concatMapStrings (s: ''
iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
'') cfg.lanAccess
# Inbound to guest services published on the uplink.
+ lib.concatMapStrings (s: ''
oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept
'') cfg.publish
+ (if cfg.allowLan then ''
iifname "${net.bridge}" accept
'' else ''
iifname "${net.bridge}" ip daddr {
10.0.0.0/8,
172.16.0.0/12,
192.168.0.0/16,
169.254.0.0/16,
127.0.0.0/8,
100.64.0.0/10
} drop
iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
iifname "${net.bridge}" accept
'');
networking.firewall.extraForwardRules = ''
iifname != "${net.bridge}" accept
ct state established,related accept
''
# Guest reaching named services elsewhere on your network.
+ lib.concatMapStrings (s: ''
iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
'') cfg.lanAccess
# Inbound to guest services published on the uplink.
+ lib.concatMapStrings (s: ''
oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept
'') cfg.publish
+ (
if cfg.allowLan then
''
iifname "${net.bridge}" accept
''
else
''
iifname "${net.bridge}" ip daddr {
10.0.0.0/8,
172.16.0.0/12,
192.168.0.0/16,
169.254.0.0/16,
127.0.0.0/8,
100.64.0.0/10
} drop
iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
iifname "${net.bridge}" accept
''
);
# =====================================================================
# Host store as a read-only cache for the guest
@@ -723,7 +830,11 @@ in
# =====================================================================
systemd.services.hermes-vm-snapshot = mkIf cfg.snapshots.enable {
description = "Snapshot the Hermes VM state disk";
path = with pkgs; [ btrfs-progs coreutils socat ];
path = with pkgs; [
btrfs-progs
coreutils
socat
];
serviceConfig.Type = "oneshot";
script = ''
set -euo pipefail
@@ -772,7 +883,12 @@ in
# =====================================================================
systemd.services.hermes-vm-backup = mkIf cfg.backup.enable {
description = "Back up Hermes state";
path = with pkgs; [ restic openssh socat gnutar ];
path = with pkgs; [
restic
openssh
socat
gnutar
];
serviceConfig = {
Type = "oneshot";
EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile;
@@ -826,18 +942,21 @@ in
"--prefix"
"PATH"
":"
(lib.makeBinPath (with pkgs; [
openssh
socat
coreutils
gnutar
gnused
btrfs-progs
util-linux
restic
vim
nix
]))
(lib.makeBinPath (
with pkgs;
[
openssh
socat
coreutils
gnutar
gnused
btrfs-progs
util-linux
restic
vim
nix
]
))
"--set"
"HERMES_VSOCK_CID"
(toString cfg.vsockCid)
@@ -858,16 +977,21 @@ in
cfg.version
];
}
(builtins.concatStringsSep "\n" (map builtins.readFile [
../nu/lib/common.nu
../nu/lib/guest.nu
../nu/host/setup.nu
../nu/host/migrate.nu
../nu/host/remote.nu
../nu/vm/vm.nu
../nu/ctl/ctl.nu
../nu/hermenix.nu
])))
(
builtins.concatStringsSep "\n" (
map builtins.readFile [
../nu/lib/common.nu
../nu/lib/guest.nu
../nu/host/setup.nu
../nu/host/migrate.nu
../nu/host/remote.nu
../nu/vm/vm.nu
../nu/ctl/ctl.nu
../nu/hermenix.nu
]
)
)
)
];
};
}