splitting host.nix
This commit is contained in:
1 parent
053b9c4c95
commit
f1622fae84
16 files changed
+1706
-1471
No files matched your search
@@ -0,0 +1,71 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (import ./common.nix { inherit config lib pkgs; }) cfg;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.backup.enable -> (cfg.backup.repository != "");
|
||||
message = "services.hermenix.backup.repository must be set when backups are enabled.";
|
||||
}
|
||||
{
|
||||
assertion = cfg.devices.vsock.enable || !cfg.backup.enable;
|
||||
message = ''
|
||||
services.hermenix.backup.enable needs the vsock (the backup runs over
|
||||
SSH multiplexed on it). Leave devices.vsock.enable at its default, or
|
||||
turn backups off.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
systemd.services.hermes-vm-backup = lib.mkIf cfg.backup.enable {
|
||||
description = "Back up Hermes state";
|
||||
path = with pkgs; [
|
||||
restic
|
||||
openssh
|
||||
socat
|
||||
gnutar
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
EnvironmentFile = lib.mkIf (cfg.backup.environmentFile != null) cfg.backup.environmentFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
export RESTIC_REPOSITORY=${lib.escapeShellArg cfg.backup.repository}
|
||||
export RESTIC_PASSWORD_FILE=${cfg.backup.passwordFile}
|
||||
|
||||
restic snapshots >/dev/null 2>&1 || restic init
|
||||
|
||||
ssh -o ProxyCommand="socat - VSOCK-CONNECT:${toString cfg.vsockCid}:22" \
|
||||
-o StrictHostKeyChecking=accept-new \
|
||||
hermes@${cfg.name} \
|
||||
'tar -C /var/lib/hermes \
|
||||
--exclude=./.cache \
|
||||
--exclude=./workspace/**/.direnv \
|
||||
--exclude=./workspace/**/result \
|
||||
-cf - .' \
|
||||
| restic backup --stdin --stdin-filename hermes-state.tar \
|
||||
--tag hermes --host ${cfg.name}
|
||||
|
||||
restic forget --tag hermes --host ${cfg.name} --prune ${lib.concatStringsSep " " cfg.backup.keep}
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.hermes-vm-backup = lib.mkIf cfg.backup.enable {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.backup.interval;
|
||||
Persistent = true;
|
||||
RandomizedDelaySec = "30m";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
{ lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib)
|
||||
mkOption
|
||||
mkEnableOption
|
||||
types
|
||||
;
|
||||
in
|
||||
{
|
||||
options.services.hermenix.backup = {
|
||||
enable = mkEnableOption "restic backups of the guest's state";
|
||||
|
||||
repository = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
example = "sftp:backup@nas:/hermes";
|
||||
description = "Any restic repository string.";
|
||||
};
|
||||
|
||||
passwordFile = mkOption {
|
||||
type = types.path;
|
||||
default = "/var/lib/secrets/hermes-restic";
|
||||
description = "File containing the restic repository password, mode 0400.";
|
||||
};
|
||||
|
||||
environmentFile = mkOption {
|
||||
type = types.nullOr types.path;
|
||||
default = null;
|
||||
description = "Optional file with credentials for the backend (S3 keys and so on).";
|
||||
};
|
||||
|
||||
interval = mkOption {
|
||||
type = types.str;
|
||||
default = "daily";
|
||||
};
|
||||
|
||||
keep = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [
|
||||
"--keep-daily"
|
||||
"14"
|
||||
"--keep-weekly"
|
||||
"8"
|
||||
"--keep-monthly"
|
||||
"12"
|
||||
];
|
||||
description = "Arguments passed to restic forget.";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (import ./common.nix { inherit config lib pkgs; })
|
||||
cfg
|
||||
net
|
||||
bridgeMark
|
||||
netmaskFromPrefix
|
||||
;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf (cfg.enable && net.bridged) {
|
||||
assertions = [
|
||||
{
|
||||
assertion = net.lanAddress != "" && net.hostLanAddress != "" && net.lanGateway != "";
|
||||
message = ''
|
||||
services.hermenix.network.bridged = true requires a fully static
|
||||
LAN setup (DHCP is no longer supported in bridged mode). Set
|
||||
network.lanAddress (the guest's fixed LAN address),
|
||||
network.hostLanAddress (this host's own LAN address, which moves
|
||||
onto the bridge) and network.lanGateway (your router's address).
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# mkDefault: operator nameservers win if set.
|
||||
networking.nameservers = lib.mkDefault (
|
||||
if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns
|
||||
);
|
||||
|
||||
systemd.network.networks."12-${net.externalInterface}" = {
|
||||
matchConfig.Name = net.externalInterface;
|
||||
networkConfig.Bridge = net.bridge;
|
||||
};
|
||||
|
||||
# Bridge would inherit tap MAC (first port) and collide with guest — clone uplink MAC.
|
||||
systemd.services.hermes-bridge-mac = {
|
||||
description = "Clone the uplink MAC onto the hermenix bridge";
|
||||
after = [
|
||||
"systemd-networkd.service"
|
||||
"sys-subsystem-net-devices-${net.bridge}.device"
|
||||
"sys-subsystem-net-devices-${net.externalInterface}.device"
|
||||
];
|
||||
wants = [ "systemd-networkd.service" ];
|
||||
before = [ "network-online.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
unitConfig.ConditionPathExists = "/sys/class/net/${net.externalInterface}";
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
||||
[ -e "/sys/class/net/${net.bridge}" ] && break
|
||||
sleep 0.5
|
||||
done
|
||||
[ -e "/sys/class/net/${net.bridge}" ] || exit 1
|
||||
mac="$(${pkgs.coreutils}/bin/cat /sys/class/net/${net.externalInterface}/address)"
|
||||
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} down
|
||||
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} address "$mac"
|
||||
${pkgs.iproute2}/bin/ip link set dev ${net.bridge} up
|
||||
${pkgs.systemd}/bin/networkctl reconfigure ${net.bridge} || true
|
||||
'';
|
||||
};
|
||||
|
||||
# Single-address DHCP on the bridge (MAC-keyed). Broadcasts stay on-bridge.
|
||||
systemd.services.hermes-lan-dhcp = {
|
||||
description = "Static-lease DHCP for the bridged hermenix guest";
|
||||
after = [
|
||||
"systemd-networkd.service"
|
||||
"sys-subsystem-net-devices-${net.bridge}.device"
|
||||
];
|
||||
wants = [ "systemd-networkd.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
};
|
||||
script = ''
|
||||
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
||||
[ -e "/sys/class/net/${net.bridge}" ] && break
|
||||
sleep 0.5
|
||||
done
|
||||
exec ${pkgs.dnsmasq}/bin/dnsmasq --keep-in-foreground \
|
||||
--conf-file=/dev/null \
|
||||
--interface=${net.bridge} \
|
||||
--bind-dynamic \
|
||||
--port=0 \
|
||||
--dhcp-authoritative \
|
||||
--dhcp-range=${net.lanAddress},${net.lanAddress},${netmaskFromPrefix net.lanPrefixLength},12h \
|
||||
--dhcp-host=${net.mac},${net.lanAddress} \
|
||||
--dhcp-option=option:router,${net.lanGateway}
|
||||
'';
|
||||
};
|
||||
|
||||
# No DNAT when bridged. Scoped masquerade covers service-net fallback if DHCP lapses.
|
||||
networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null) {
|
||||
family = "ip";
|
||||
content = ''
|
||||
chain postrouting {
|
||||
type nat hook postrouting priority srcnat;
|
||||
iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
networking.nftables.tables."hermenix-bridge" = {
|
||||
family = "bridge";
|
||||
content = ''
|
||||
chain prerouting {
|
||||
type filter hook prerouting priority filter; policy accept;
|
||||
iifname "${net.tapId}" meta mark set ${toString bridgeMark}
|
||||
|
||||
# Guest DHCP stays on-bridge only (no race with LAN DHCP).
|
||||
iifname "${net.tapId}" oifname "${net.externalInterface}" ether type ip ip protocol udp udp sport 68 udp dport 67 drop
|
||||
iifname "${net.externalInterface}" oifname "${net.tapId}" ether type ip ip protocol udp udp sport 67 udp dport 68 drop
|
||||
}
|
||||
chain forward {
|
||||
type filter hook forward priority filter; policy accept;
|
||||
oifname "${net.tapId}" accept
|
||||
''
|
||||
+ lib.concatMapStrings (s: ''
|
||||
iifname "${net.tapId}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
|
||||
'') cfg.lanAccess
|
||||
+ (
|
||||
if cfg.allowLan then
|
||||
''
|
||||
iifname "${net.tapId}" accept
|
||||
''
|
||||
else
|
||||
''
|
||||
iifname "${net.tapId}" ip daddr {
|
||||
10.0.0.0/8,
|
||||
172.16.0.0/12,
|
||||
192.168.0.0/16,
|
||||
169.254.0.0/16,
|
||||
127.0.0.0/8,
|
||||
100.64.0.0/10
|
||||
} drop
|
||||
iifname "${net.tapId}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
|
||||
iifname "${net.tapId}" accept
|
||||
''
|
||||
)
|
||||
+ ''
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
# Guest→host holes (bridged: match the tap mark; the guest may use its LAN IP).
|
||||
networking.firewall.extraInputRules =
|
||||
lib.optionalString cfg.hostCache.enable ''
|
||||
meta mark ${toString bridgeMark} tcp dport ${toString cfg.hostCache.port} accept
|
||||
''
|
||||
+ lib.concatMapStrings (s: ''
|
||||
meta mark ${toString bridgeMark} ${s.proto} dport ${toString s.port} accept
|
||||
'') cfg.hostAccess;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
{ lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib)
|
||||
mkOption
|
||||
types
|
||||
;
|
||||
in
|
||||
{
|
||||
options.services.hermenix.network = {
|
||||
lanAddress = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
example = "10.0.12.50";
|
||||
description = "The guest's fixed address on your LAN. Required when `bridged = true`.";
|
||||
};
|
||||
hostLanAddress = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
example = "10.0.12.10";
|
||||
description = ''
|
||||
This host's own address on your LAN. When `bridged` is enabled the
|
||||
uplink becomes a bridge port and this address moves onto the bridge
|
||||
(which clones the uplink's MAC, so the host keeps its LAN identity).
|
||||
Required when `bridged = true`.
|
||||
'';
|
||||
};
|
||||
lanGateway = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
example = "10.0.12.1";
|
||||
description = ''
|
||||
Your LAN's router. Used as the default gateway for the guest and,
|
||||
while `bridged`, for this host's bridge too (DNS unless `lanDns` is
|
||||
set). Required when `bridged = true`.
|
||||
'';
|
||||
};
|
||||
lanPrefixLength = mkOption {
|
||||
type = types.int;
|
||||
default = 24;
|
||||
description = "Prefix length of your LAN subnet (netmask).";
|
||||
};
|
||||
lanDns = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ ];
|
||||
example = [ "1.1.1.1" ];
|
||||
description = ''
|
||||
DNS servers for this host while `bridged` (the bridge no longer
|
||||
takes a DHCP lease). Defaults to the LAN gateway.
|
||||
'';
|
||||
};
|
||||
bridged = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Put the guest directly on your LAN instead of behind the host's
|
||||
NAT. The bridge absorbs `externalInterface`, so any device on the
|
||||
LAN can reach any port the guest exposes.
|
||||
|
||||
Addressing is static from the operator's point of view: you pick
|
||||
fixed addresses for the guest (`lanAddress`), this host
|
||||
(`hostLanAddress` — it moves onto the bridge), and the LAN
|
||||
gateway (`lanGateway`). A DHCP server this module runs on the
|
||||
bridge hands `lanAddress` to the guest, keyed to `network.mac`,
|
||||
so the guest stays a plain DHCP client and never needs to know
|
||||
its own address. That lease never leaves the bridge; the LAN's
|
||||
own DHCP cannot race it.
|
||||
|
||||
Inbound is no longer limited to `publish` (which is ignored in
|
||||
this mode): the LAN may reach every guest port. Outbound is still
|
||||
governed by `lanAccess` / `allowLan`, and `hostAccess` plus the
|
||||
host cache still work over the host-guest service net
|
||||
(`hostAddress` stays configured on the bridge).
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
}:
|
||||
|
||||
let
|
||||
cfg = config.services.hermenix;
|
||||
net = cfg.network;
|
||||
pt = cfg.passthrough;
|
||||
|
||||
vmUser = "hermes-vm";
|
||||
runDir = "/run/hermes-vm";
|
||||
stateDir = "/var/lib/hermes-vm";
|
||||
systemImage = "${cfg.disk.baseDir}/system/system.raw";
|
||||
stateImage = "${cfg.disk.baseDir}/state/state.raw";
|
||||
snapDir = "${cfg.disk.baseDir}/snapshots";
|
||||
|
||||
anyPci = pt.pci != [ ];
|
||||
anyUsb = pt.usb != [ ];
|
||||
earlyIds = lib.filter (i: i != null) (map (d: d.id) pt.pci);
|
||||
|
||||
usbControllerOn =
|
||||
cfg.devices.usbController == "on" || (cfg.devices.usbController == "auto" && anyUsb);
|
||||
|
||||
# 18501 == 0x4845 ("HE"); marks guest traffic from the tap (bridged mode).
|
||||
bridgeMark = 18501;
|
||||
|
||||
# dnsmasq needs an explicit mask for a one-address DHCP range.
|
||||
netmaskFromPrefix =
|
||||
p:
|
||||
lib.concatStringsSep "." (
|
||||
lib.genList (
|
||||
b:
|
||||
toString (
|
||||
builtins.elemAt [
|
||||
0
|
||||
128
|
||||
192
|
||||
224
|
||||
240
|
||||
248
|
||||
252
|
||||
254
|
||||
255
|
||||
] (lib.max 0 (lib.min 8 (p - b * 8)))
|
||||
)
|
||||
) 4
|
||||
);
|
||||
|
||||
stopScript = pkgs.writeShellScript "hermes-vm-stop" ''
|
||||
set -eu
|
||||
printf '%s' '{"execute":"qmp_capabilities"}{"execute":"system_powerdown"}' \
|
||||
| ${pkgs.socat}/bin/socat - UNIX-CONNECT:${runDir}/qmp.sock >/dev/null 2>&1 || exit 0
|
||||
'';
|
||||
|
||||
qga =
|
||||
verb:
|
||||
pkgs.writeShellScript "hermes-vm-fs${verb}" ''
|
||||
set -eu
|
||||
printf '%s' '{"execute":"guest-fs${verb}"}' \
|
||||
| ${pkgs.socat}/bin/socat -t2 - UNIX-CONNECT:${runDir}/qga.sock >/dev/null 2>&1 || true
|
||||
'';
|
||||
in
|
||||
{
|
||||
inherit
|
||||
cfg
|
||||
net
|
||||
pt
|
||||
vmUser
|
||||
runDir
|
||||
stateDir
|
||||
systemImage
|
||||
stateImage
|
||||
snapDir
|
||||
anyPci
|
||||
anyUsb
|
||||
earlyIds
|
||||
usbControllerOn
|
||||
bridgeMark
|
||||
netmaskFromPrefix
|
||||
stopScript
|
||||
qga
|
||||
;
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (import ./common.nix { inherit config lib pkgs; }) cfg net;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = net.externalInterface != null;
|
||||
message = "services.hermenix.network.externalInterface must be set.";
|
||||
}
|
||||
];
|
||||
|
||||
networking.useNetworkd = true;
|
||||
|
||||
systemd.network.netdevs."10-${net.bridge}".netdevConfig = {
|
||||
Kind = "bridge";
|
||||
Name = net.bridge;
|
||||
};
|
||||
|
||||
systemd.network.networks."10-${net.bridge}" = {
|
||||
matchConfig.Name = net.bridge;
|
||||
# hostAddress = service net; bridged also adds hostLanAddress (static only).
|
||||
address = [
|
||||
"${net.hostAddress}/${toString net.prefixLength}"
|
||||
]
|
||||
++ lib.optional net.bridged "${net.hostLanAddress}/${toString net.lanPrefixLength}";
|
||||
networkConfig = {
|
||||
ConfigureWithoutCarrier = true;
|
||||
IPv4Forwarding = true;
|
||||
}
|
||||
// lib.optionalAttrs net.bridged {
|
||||
Gateway = net.lanGateway;
|
||||
DNS = if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.network.networks."11-${net.tapId}" = {
|
||||
matchConfig.Name = net.tapId;
|
||||
networkConfig.Bridge = net.bridge;
|
||||
};
|
||||
|
||||
# mkOverride 99: same keys as networking.nat, win over weaker defaults.
|
||||
boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true;
|
||||
boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true;
|
||||
|
||||
# Explicit nft nat (not networking.nat). DNAT iifname != bridge so extra paths still match.
|
||||
networking.nftables.tables.hermenix-nat = lib.mkIf (net.externalInterface != null && !net.bridged) {
|
||||
family = "ip";
|
||||
content = ''
|
||||
chain prerouting {
|
||||
type nat hook prerouting priority dstnat;
|
||||
''
|
||||
+ lib.concatMapStrings (s: ''
|
||||
iifname != "${net.bridge}" ${s.proto} dport ${toString s.hostPort} dnat to ${net.guestAddress}:${toString s.guestPort}
|
||||
'') cfg.publish
|
||||
+ ''
|
||||
}
|
||||
chain postrouting {
|
||||
type nat hook postrouting priority srcnat;
|
||||
iifname "${net.bridge}" oifname "${net.externalInterface}" masquerade
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
networking.nftables.enable = true;
|
||||
networking.firewall.enable = true;
|
||||
networking.firewall.filterForward = true;
|
||||
|
||||
# Guest→host holes (NAT mode: match the service-net address).
|
||||
networking.firewall.extraInputRules = lib.mkIf (!net.bridged) (
|
||||
lib.optionalString cfg.hostCache.enable ''
|
||||
iifname "${net.bridge}" ip saddr ${net.guestAddress} tcp dport ${toString cfg.hostCache.port} accept
|
||||
''
|
||||
+ lib.concatMapStrings (s: ''
|
||||
iifname "${net.bridge}" ip saddr ${net.guestAddress} ${s.proto} dport ${toString s.port} accept
|
||||
'') cfg.hostAccess
|
||||
);
|
||||
|
||||
# Shared by both modes; the bridged LAN is L2 (see bridge.nix).
|
||||
networking.firewall.extraForwardRules = ''
|
||||
iifname != "${net.bridge}" accept
|
||||
ct state established,related accept
|
||||
''
|
||||
+ lib.concatMapStrings (s: ''
|
||||
iifname "${net.bridge}" ip daddr ${s.address} ${s.proto} dport ${toString s.port} accept
|
||||
'') cfg.lanAccess
|
||||
+ lib.optionalString (!net.bridged) (
|
||||
lib.concatMapStrings (s: ''
|
||||
oifname "${net.bridge}" ip daddr ${net.guestAddress} ${s.proto} dport ${toString s.guestPort} accept
|
||||
'') cfg.publish
|
||||
)
|
||||
+ (
|
||||
if cfg.allowLan then
|
||||
''
|
||||
iifname "${net.bridge}" accept
|
||||
''
|
||||
else
|
||||
''
|
||||
iifname "${net.bridge}" ip daddr {
|
||||
10.0.0.0/8,
|
||||
172.16.0.0/12,
|
||||
192.168.0.0/16,
|
||||
169.254.0.0/16,
|
||||
127.0.0.0/8,
|
||||
100.64.0.0/10
|
||||
} drop
|
||||
iifname "${net.bridge}" ip6 daddr { fc00::/7, fe80::/10, ::1/128 } drop
|
||||
iifname "${net.bridge}" accept
|
||||
''
|
||||
);
|
||||
|
||||
services.nix-serve = lib.mkIf cfg.hostCache.enable {
|
||||
enable = true;
|
||||
package = pkgs.nix-serve-ng;
|
||||
bindAddress = net.hostAddress;
|
||||
inherit (cfg.hostCache) port secretKeyFile;
|
||||
openFirewall = false;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
{ lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib)
|
||||
mkOption
|
||||
types
|
||||
;
|
||||
in
|
||||
{
|
||||
options.services.hermenix = {
|
||||
hostCache = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Serve the host's /nix/store to the guest as a read-only binary
|
||||
cache, so the guest doesn't re-download from the internet what the
|
||||
host already has.
|
||||
|
||||
This is the ONLY route from guest to host in this design. Turning it
|
||||
off leaves the guest with literally no reachable host service, at
|
||||
the cost of bandwidth on every guest build.
|
||||
|
||||
Note the guest can read every store path on the host through this.
|
||||
Store paths are world-readable anyway, but check nothing sensitive
|
||||
is baked into your host's store.
|
||||
'';
|
||||
};
|
||||
port = mkOption {
|
||||
type = types.port;
|
||||
default = 5000;
|
||||
};
|
||||
secretKeyFile = mkOption {
|
||||
type = types.path;
|
||||
default = "/var/lib/secrets/hermes-cache.secret";
|
||||
description = "Generate with nix-store --generate-binary-cache-key. See README.";
|
||||
};
|
||||
};
|
||||
|
||||
allowLan = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Blanket permission for the guest to reach RFC1918 space. Almost always
|
||||
the wrong tool - use lanAccess to open single endpoints instead.
|
||||
'';
|
||||
};
|
||||
|
||||
lanAccess = mkOption {
|
||||
default = [ ];
|
||||
example = [
|
||||
{
|
||||
address = "192.168.1.50";
|
||||
port = 11434;
|
||||
}
|
||||
];
|
||||
description = ''
|
||||
Specific endpoints elsewhere on your network that the guest may reach,
|
||||
punched through the default RFC1918 drop. One entry per service.
|
||||
'';
|
||||
type = types.listOf (
|
||||
types.submodule {
|
||||
options = {
|
||||
address = mkOption { type = types.str; };
|
||||
port = mkOption { type = types.port; };
|
||||
proto = mkOption {
|
||||
type = types.enum [
|
||||
"tcp"
|
||||
"udp"
|
||||
];
|
||||
default = "tcp";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
hostAccess = mkOption {
|
||||
default = [ ];
|
||||
example = [ { port = 8188; } ];
|
||||
description = ''
|
||||
Ports on this host itself that the guest may reach. The service must
|
||||
also be bound to an address the guest can see - see the README.
|
||||
'';
|
||||
type = types.listOf (
|
||||
types.submodule {
|
||||
options = {
|
||||
port = mkOption { type = types.port; };
|
||||
proto = mkOption {
|
||||
type = types.enum [
|
||||
"tcp"
|
||||
"udp"
|
||||
];
|
||||
default = "tcp";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
publish = mkOption {
|
||||
default = [ ];
|
||||
example = [
|
||||
{
|
||||
hostPort = 8080;
|
||||
guestPort = 8080;
|
||||
}
|
||||
];
|
||||
description = ''
|
||||
Guest ports to publish on the host's uplink, reachable from the wider
|
||||
network. You do not need this to reach the guest from the host itself.
|
||||
'';
|
||||
type = types.listOf (
|
||||
types.submodule {
|
||||
options = {
|
||||
hostPort = mkOption { type = types.port; };
|
||||
guestPort = mkOption { type = types.port; };
|
||||
proto = mkOption {
|
||||
type = types.enum [
|
||||
"tcp"
|
||||
"udp"
|
||||
];
|
||||
default = "tcp";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
network = {
|
||||
bridge = mkOption {
|
||||
type = types.str;
|
||||
default = "hermes-br";
|
||||
};
|
||||
tapId = mkOption {
|
||||
type = types.str;
|
||||
default = "vm-hermes";
|
||||
};
|
||||
mac = mkOption {
|
||||
type = types.str;
|
||||
default = "02:00:00:7e:00:01";
|
||||
};
|
||||
hostAddress = mkOption {
|
||||
type = types.str;
|
||||
default = "10.77.0.1";
|
||||
};
|
||||
guestAddress = mkOption {
|
||||
type = types.str;
|
||||
default = "10.77.0.2";
|
||||
};
|
||||
prefixLength = mkOption {
|
||||
type = types.int;
|
||||
default = 24;
|
||||
};
|
||||
externalInterface = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
example = "eno1";
|
||||
description = "Host uplink. Required.";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (import ./common.nix { inherit config lib pkgs; }) cfg;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
environment.systemPackages = [
|
||||
(pkgs.writeTextFile {
|
||||
name = "hermenix-completions";
|
||||
destination = "/share/nushell/vendor/autoload/hermenix.nu";
|
||||
text = builtins.readFile ../nu/completions.nu;
|
||||
})
|
||||
|
||||
(pkgs.symlinkJoin {
|
||||
name = "hermenix";
|
||||
paths = [ pkgs.hermenix ];
|
||||
nativeBuildInputs = [ pkgs.makeWrapper ];
|
||||
postBuild = ''
|
||||
wrapProgram $out/bin/hermenix \
|
||||
--set HERMES_VSOCK_CID ${toString cfg.vsockCid} \
|
||||
--set HERMES_GUEST_NAME ${cfg.name} \
|
||||
--set HERMES_DISK_BASE_DIR ${cfg.disk.baseDir} \
|
||||
--set HERMES_HOME_DIR ${cfg.hermesHome} \
|
||||
--set HERMES_GATEWAY_UNIT ${cfg.gatewayUnit}
|
||||
'';
|
||||
})
|
||||
];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (import ./common.nix { inherit config lib pkgs; })
|
||||
cfg
|
||||
pt
|
||||
anyPci
|
||||
anyUsb
|
||||
earlyIds
|
||||
vmUser
|
||||
;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = anyPci -> (pt.iommu != null);
|
||||
message = ''
|
||||
services.hermenix.passthrough.pci needs passthrough.iommu set to
|
||||
"intel" or "amd". Without the IOMMU, a passed-through device can
|
||||
read host memory directly and the VM boundary is meaningless.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = cfg.devices.usbController != "off" || !anyUsb;
|
||||
message = ''
|
||||
services.hermenix.passthrough.usb is set but
|
||||
services.hermenix.devices.usbController is "off", so those devices
|
||||
have no controller to attach to.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
boot.kernelParams = lib.optionals (pt.iommu != null) [
|
||||
(if pt.iommu == "intel" then "intel_iommu=on" else "amd_iommu=on")
|
||||
"iommu=pt"
|
||||
];
|
||||
|
||||
boot.initrd.kernelModules = lib.optionals anyPci [
|
||||
"vfio_pci"
|
||||
"vfio"
|
||||
"vfio_iommu_type1"
|
||||
];
|
||||
|
||||
boot.extraModprobeConfig = lib.optionalString (earlyIds != [ ]) ''
|
||||
options vfio-pci ids=${lib.concatStringsSep "," earlyIds}
|
||||
'';
|
||||
|
||||
systemd.services.hermes-vm-vfio = lib.mkIf anyPci {
|
||||
description = "Bind passthrough PCI devices to vfio-pci";
|
||||
before = [ "hermes-vm.service" ];
|
||||
requiredBy = [ "hermes-vm.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
path = [ pkgs.kmod ];
|
||||
script = ''
|
||||
modprobe vfio-pci
|
||||
''
|
||||
+ lib.concatMapStrings (d: ''
|
||||
dev=/sys/bus/pci/devices/${d.address}
|
||||
if [ ! -e "$dev" ]; then
|
||||
echo "${d.address} is not present on this machine" >&2
|
||||
exit 1
|
||||
fi
|
||||
cur=none
|
||||
[ -e "$dev/driver" ] && cur=$(basename "$(readlink -f "$dev/driver")")
|
||||
if [ "$cur" != "vfio-pci" ]; then
|
||||
[ "$cur" != "none" ] && echo ${d.address} > "$dev/driver/unbind"
|
||||
echo vfio-pci > "$dev/driver_override"
|
||||
echo ${d.address} > /sys/bus/pci/drivers_probe
|
||||
fi
|
||||
'') pt.pci;
|
||||
};
|
||||
|
||||
services.udev.extraRules =
|
||||
lib.optionalString anyPci ''
|
||||
SUBSYSTEM=="vfio", OWNER="${vmUser}", GROUP="${vmUser}", MODE="0660"
|
||||
''
|
||||
+ lib.concatMapStrings (u: ''
|
||||
SUBSYSTEM=="usb", ATTR{idVendor}=="${u.vendorId}", ATTR{idProduct}=="${u.productId}", OWNER="${vmUser}", MODE="0660"
|
||||
'') pt.usb;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
{ lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib)
|
||||
mkOption
|
||||
types
|
||||
;
|
||||
in
|
||||
{
|
||||
options.services.hermenix.passthrough = {
|
||||
iommu = mkOption {
|
||||
type = types.nullOr (
|
||||
types.enum [
|
||||
"intel"
|
||||
"amd"
|
||||
]
|
||||
);
|
||||
default = null;
|
||||
description = ''
|
||||
Enable IOMMU kernel parameters for your CPU vendor. Required for any
|
||||
PCI passthrough. The IOMMU is the only thing stopping a passed-through
|
||||
device from reading host memory, so this is not optional.
|
||||
'';
|
||||
};
|
||||
|
||||
pci = mkOption {
|
||||
default = [ ];
|
||||
example = [
|
||||
{
|
||||
address = "0000:01:00.0";
|
||||
id = "10de:2204";
|
||||
}
|
||||
{
|
||||
address = "0000:01:00.1";
|
||||
id = "10de:1aef";
|
||||
}
|
||||
];
|
||||
description = ''
|
||||
PCI devices handed to the guest via VFIO. The host loses them
|
||||
entirely while the VM runs.
|
||||
|
||||
You must pass every function of the device and, in practice, every
|
||||
device in its IOMMU group. A GPU is usually at least two entries:
|
||||
the graphics function and its HDMI audio function.
|
||||
|
||||
Set `id` (vendor:device) to have vfio-pci claim the device in the
|
||||
initrd, before a host driver can. Strongly recommended for GPUs;
|
||||
unbinding an active amdgpu or nvidia driver later is unreliable.
|
||||
'';
|
||||
type = types.listOf (
|
||||
types.submodule {
|
||||
options = {
|
||||
address = mkOption {
|
||||
type = types.str;
|
||||
example = "0000:01:00.0";
|
||||
description = "Full PCI address, as shown by lspci -D.";
|
||||
};
|
||||
id = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
example = "10de:2204";
|
||||
description = "vendor:device, from lspci -nn. Enables early binding.";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
usb = mkOption {
|
||||
default = [ ];
|
||||
example = [
|
||||
{
|
||||
vendorId = "8087";
|
||||
productId = "0026";
|
||||
}
|
||||
];
|
||||
description = ''
|
||||
USB devices forwarded to the guest, matched by vendor and product
|
||||
ID from lsusb. Integrated Bluetooth radios are USB devices and go
|
||||
here too, though the host driver must be kept off them.
|
||||
|
||||
QEMU resolves these at start, so a device that is unplugged when the
|
||||
VM boots will not appear later without a restart.
|
||||
'';
|
||||
type = types.listOf (
|
||||
types.submodule {
|
||||
options = {
|
||||
vendorId = mkOption {
|
||||
type = types.str;
|
||||
example = "046d";
|
||||
};
|
||||
productId = mkOption {
|
||||
type = types.str;
|
||||
example = "c52b";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
{ lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib)
|
||||
mkOption
|
||||
mkEnableOption
|
||||
types
|
||||
;
|
||||
in
|
||||
{
|
||||
options.services.hermenix = {
|
||||
enable = mkEnableOption "the Hermes Agent VM";
|
||||
|
||||
name = mkOption {
|
||||
type = types.str;
|
||||
default = "hermes";
|
||||
};
|
||||
|
||||
authorizedKeys = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ ];
|
||||
};
|
||||
|
||||
gatewayUnit = mkOption {
|
||||
type = types.str;
|
||||
default = "hermes-agent";
|
||||
description = ''
|
||||
The systemd *user* unit that `hermes gateway install` created in the
|
||||
guest. Check with `hermenix ctl units` after installing; upstream has
|
||||
used more than one name for this.
|
||||
'';
|
||||
};
|
||||
|
||||
hermesHome = mkOption {
|
||||
type = types.str;
|
||||
# Guest path (not host disk.baseDir).
|
||||
default = "/var/lib/hermes/.hermes";
|
||||
description = ''
|
||||
Where Hermes's own data (SOUL.md, config.yaml, memories/, sessions/,
|
||||
state.db, and so on) lives inside the guest. Defaults to the seeded
|
||||
layout under the state disk mount (`/var/lib/hermes/.hermes`).
|
||||
Override when carrying over an existing flat HERMES_HOME from
|
||||
elsewhere (e.g. a prior Docker deployment) that you're placing
|
||||
directly at the root of the state disk instead of under .hermes —
|
||||
see `hermenix migrate --help`.
|
||||
|
||||
This only changes where the host-side `hermenix ctl` commands (soul,
|
||||
memory, conf, secrets, reseed, pull) look. It does not change what
|
||||
the gateway process itself uses — export HERMES_HOME to match inside
|
||||
the guest when you install it.
|
||||
'';
|
||||
};
|
||||
|
||||
vsockCid = mkOption {
|
||||
type = types.int;
|
||||
default = 77;
|
||||
description = "Host reaches the guest over vsock, so no inbound network path is needed.";
|
||||
};
|
||||
|
||||
version = mkOption {
|
||||
type = types.str;
|
||||
default = "dev";
|
||||
description = ''
|
||||
"YYYY.MM.DD.hh.mm-<commit-sha>" reported by `hermenix version`.
|
||||
`version` subcommand. Normally supplied for free by this flake's own
|
||||
`nixosModules.default` (from its git metadata) - the "dev" default
|
||||
here only shows up if this module is imported some other way.
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (import ./common.nix { inherit config lib pkgs; })
|
||||
cfg
|
||||
snapDir
|
||||
qga
|
||||
;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
systemd.services.hermes-vm-snapshot = lib.mkIf cfg.snapshots.enable {
|
||||
description = "Snapshot the Hermes VM state disk";
|
||||
path = with pkgs; [
|
||||
btrfs-progs
|
||||
coreutils
|
||||
socat
|
||||
];
|
||||
serviceConfig.Type = "oneshot";
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
stamp=$(date +%Y%m%d-%H%M%S)
|
||||
mkdir -p ${snapDir}
|
||||
|
||||
${lib.optionalString cfg.disk.quiesce ''
|
||||
${qga "freeze"}
|
||||
trap '${qga "thaw"}' EXIT
|
||||
''}
|
||||
|
||||
btrfs subvolume snapshot -r ${cfg.disk.baseDir}/state ${snapDir}/state-$stamp
|
||||
|
||||
${lib.optionalString cfg.disk.quiesce ''
|
||||
${qga "thaw"}
|
||||
trap - EXIT
|
||||
''}
|
||||
|
||||
# Live snapshots reintroduce CoW on nodatacow images — keep retention tight.
|
||||
ls -1d ${snapDir}/state-* 2>/dev/null | sort | head -n -${toString cfg.snapshots.keep} \
|
||||
| while read -r old; do btrfs subvolume delete "$old"; done
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.hermes-vm-snapshot = lib.mkIf cfg.snapshots.enable {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.snapshots.interval;
|
||||
Persistent = true;
|
||||
RandomizedDelaySec = "5m";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
{ lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib)
|
||||
mkOption
|
||||
types
|
||||
;
|
||||
in
|
||||
{
|
||||
options.services.hermenix.snapshots = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Periodic btrfs snapshots of the state disk. The agent can persist
|
||||
changes, which means it can persist a mistake, and these are the
|
||||
fastest way back.
|
||||
'';
|
||||
};
|
||||
|
||||
interval = mkOption {
|
||||
type = types.str;
|
||||
default = "hourly";
|
||||
description = "systemd calendar expression.";
|
||||
};
|
||||
|
||||
keep = mkOption {
|
||||
type = types.ints.positive;
|
||||
default = 48;
|
||||
description = ''
|
||||
How many to retain. Keep this tighter than you would on ZFS: the
|
||||
state image is nodatacow, and every live snapshot reintroduces
|
||||
copy-on-write for the blocks it still references.
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (import ./common.nix { inherit config lib pkgs; })
|
||||
cfg
|
||||
net
|
||||
pt
|
||||
vmUser
|
||||
runDir
|
||||
stateDir
|
||||
systemImage
|
||||
stateImage
|
||||
anyPci
|
||||
anyUsb
|
||||
usbControllerOn
|
||||
stopScript
|
||||
;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = !cfg.devices.tpm.enable || lib.hasPrefix "/var/lib/" cfg.devices.tpm.stateDir;
|
||||
message = ''
|
||||
services.hermenix.devices.tpm.stateDir must live under /var/lib/ so
|
||||
the swtpm unit's StateDirectory can create and own it.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
users.users.${vmUser} = {
|
||||
isSystemUser = true;
|
||||
group = vmUser;
|
||||
description = "Runs the Hermes guest VM";
|
||||
};
|
||||
users.groups.${vmUser} = { };
|
||||
|
||||
systemd.services.hermes-vm = {
|
||||
description = "Hermes Agent VM";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [
|
||||
"network-online.target"
|
||||
"hermes-vm-tap.service"
|
||||
]
|
||||
++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service";
|
||||
requires = [
|
||||
"hermes-vm-tap.service"
|
||||
]
|
||||
++ lib.optional cfg.devices.tpm.enable "hermes-vm-swtpm.service";
|
||||
wants = [ "network-online.target" ];
|
||||
|
||||
preStart = ''
|
||||
for img in ${systemImage} ${stateImage}; do
|
||||
if [ ! -e "$img" ]; then
|
||||
echo "$img is missing. Run 'hermenix setup' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [ ! -e ${stateDir}/OVMF_VARS.fd ]; then
|
||||
install -m 0600 ${pkgs.OVMF.variables} ${stateDir}/OVMF_VARS.fd
|
||||
fi
|
||||
'';
|
||||
|
||||
serviceConfig = {
|
||||
User = vmUser;
|
||||
Group = vmUser;
|
||||
SupplementaryGroups = [ "kvm" ];
|
||||
RuntimeDirectory = "hermes-vm";
|
||||
ReadWritePaths = [ cfg.disk.baseDir ];
|
||||
StateDirectory = "hermes-vm";
|
||||
RuntimeDirectoryMode = "0750";
|
||||
|
||||
ExecStart = lib.concatStringsSep " " (
|
||||
[
|
||||
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
|
||||
"-name ${cfg.name}"
|
||||
"-machine q35,accel=kvm,smm=on"
|
||||
"-cpu host"
|
||||
"-smp ${toString cfg.vcpu}"
|
||||
"-m ${toString cfg.mem}"
|
||||
"-nodefaults"
|
||||
"-no-user-config"
|
||||
"-display none"
|
||||
"-serial stdio"
|
||||
"-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"
|
||||
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
|
||||
"-drive if=pflash,format=raw,unit=1,file=${stateDir}/OVMF_VARS.fd"
|
||||
# aio=threads+writeback: images are files, not blockdevs.
|
||||
"-drive if=none,id=system,format=raw,file=${systemImage},cache=writeback,aio=threads,discard=unmap"
|
||||
"-device virtio-blk-pci,drive=system"
|
||||
"-drive if=none,id=state,format=raw,file=${stateImage},cache=writeback,aio=threads,discard=unmap"
|
||||
"-device virtio-blk-pci,drive=state"
|
||||
# vhost=off: keep packet handling out of the host kernel.
|
||||
"-netdev tap,id=net0,ifname=${net.tapId},script=no,downscript=no,vhost=off"
|
||||
"-device virtio-net-pci,netdev=net0,mac=${net.mac}"
|
||||
"-qmp unix:${runDir}/qmp.sock,server=on,wait=off"
|
||||
]
|
||||
++ lib.optionals cfg.devices.rng.enable [
|
||||
"-device virtio-rng-pci"
|
||||
]
|
||||
++ lib.optionals cfg.devices.vsock.enable [
|
||||
"-device vhost-vsock-pci,guest-cid=${toString cfg.vsockCid}"
|
||||
]
|
||||
++ lib.optionals cfg.devices.video.enable [
|
||||
"-vga ${cfg.devices.video.model}"
|
||||
]
|
||||
++ lib.optionals cfg.devices.balloon.enable [
|
||||
"-device virtio-balloon-pci"
|
||||
]
|
||||
++ lib.optionals cfg.devices.watchdog.enable [
|
||||
"-device ${cfg.devices.watchdog.model}"
|
||||
"-watchdog-action ${cfg.devices.watchdog.action}"
|
||||
]
|
||||
++ lib.optionals cfg.devices.input.enable (
|
||||
lib.optional cfg.devices.input.keyboard "-device virtio-keyboard-pci"
|
||||
++ lib.optional cfg.devices.input.tablet "-device virtio-tablet-pci"
|
||||
)
|
||||
++ lib.optionals cfg.devices.audio.enable (
|
||||
[ "-audiodev ${cfg.devices.audio.backend},id=audio0" ]
|
||||
++ (
|
||||
if cfg.devices.audio.model == "virtio-sound" then
|
||||
[ "-device virtio-sound-pci,audiodev=audio0" ]
|
||||
else
|
||||
[
|
||||
"-device intel-hda"
|
||||
"-device hda-duplex,audiodev=audio0"
|
||||
]
|
||||
)
|
||||
)
|
||||
++ lib.optionals cfg.devices.tpm.enable [
|
||||
"-chardev socket,id=chrtpm,path=${runDir}/swtpm.sock"
|
||||
"-tpmdev emulator,id=tpm0,chardev=chrtpm"
|
||||
"-device tpm-crb,tpmdev=tpm0"
|
||||
]
|
||||
++ lib.optionals (cfg.devices.cdroms != [ ]) [
|
||||
"-device virtio-scsi-pci,id=scsi0"
|
||||
]
|
||||
++ lib.concatLists (
|
||||
lib.imap0 (i: c: [
|
||||
"-drive if=none,id=cd${toString i},media=cdrom,readonly=on,file=${c.path}"
|
||||
"-device scsi-cd,drive=cd${toString i},bus=scsi0.0"
|
||||
]) cfg.devices.cdroms
|
||||
)
|
||||
++ lib.concatMap (d: [
|
||||
"-drive if=none,id=${d.name},format=raw,file=${d.path},cache=${d.cache},aio=${d.aio}${lib.optionalString d.readonly ",readonly=on"}"
|
||||
"-device virtio-blk-pci,drive=${d.name}"
|
||||
]) cfg.devices.extraDisks
|
||||
++ lib.optionals usbControllerOn [
|
||||
"-device qemu-xhci,id=xhci"
|
||||
]
|
||||
++ lib.optionals cfg.disk.quiesce [
|
||||
"-chardev socket,path=${runDir}/qga.sock,server=on,wait=off,id=qga0"
|
||||
"-device virtio-serial-pci"
|
||||
"-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
|
||||
]
|
||||
++ map (d: "-device vfio-pci,host=${d.address}") pt.pci
|
||||
++ map (u: "-device usb-host,vendorid=0x${u.vendorId},productid=0x${u.productId}") pt.usb
|
||||
++ cfg.extraQemuArgs
|
||||
);
|
||||
|
||||
ExecStop = stopScript;
|
||||
TimeoutStopSec = 90;
|
||||
Restart = "always";
|
||||
RestartSec = 5;
|
||||
|
||||
DevicePolicy = "closed";
|
||||
DeviceAllow = [
|
||||
"/dev/kvm rw"
|
||||
"/dev/net/tun rw"
|
||||
"/dev/vhost-vsock rw"
|
||||
]
|
||||
++ lib.optionals anyPci [ "char-vfio rw" ]
|
||||
++ lib.optionals anyUsb [ "char-usb_device rw" ];
|
||||
|
||||
# VFIO pins guest RAM; default memlock is too low.
|
||||
LimitMEMLOCK = lib.mkIf anyPci "infinity";
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
ProtectProc = "invisible";
|
||||
ProcSubset = "pid";
|
||||
PrivateTmp = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHostname = true;
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_UNIX"
|
||||
"AF_NETLINK"
|
||||
"AF_VSOCK"
|
||||
];
|
||||
LockPersonality = true;
|
||||
SystemCallArchitectures = "native";
|
||||
SystemCallFilter = [
|
||||
"@system-service"
|
||||
"~@resources"
|
||||
"~@obsolete"
|
||||
];
|
||||
# Safe under KVM (no JIT). First flip if QEMU won't start.
|
||||
MemoryDenyWriteExecute = true;
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
AmbientCapabilities = [ "" ];
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.hermes-vm-swtpm = lib.mkIf cfg.devices.tpm.enable {
|
||||
description = "swtpm TPM emulator for the Hermes VM";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [ "hermes-vm.service" ];
|
||||
requiredBy = [ "hermes-vm.service" ];
|
||||
serviceConfig = {
|
||||
User = vmUser;
|
||||
Group = vmUser;
|
||||
RuntimeDirectory = "hermes-vm";
|
||||
RuntimeDirectoryMode = "0750";
|
||||
StateDirectory = baseNameOf cfg.devices.tpm.stateDir;
|
||||
Restart = "always";
|
||||
RestartSec = 5;
|
||||
|
||||
DevicePolicy = "closed";
|
||||
NoNewPrivileges = true;
|
||||
PrivateTmp = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
ProtectProc = "invisible";
|
||||
ProcSubset = "pid";
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHostname = true;
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
RestrictAddressFamilies = [ "AF_UNIX" ];
|
||||
LockPersonality = true;
|
||||
SystemCallArchitectures = "native";
|
||||
SystemCallFilter = [ "@system-service" ];
|
||||
MemoryDenyWriteExecute = true;
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
AmbientCapabilities = [ "" ];
|
||||
};
|
||||
script = ''
|
||||
exec ${pkgs.swtpm}/bin/swtpm socket \
|
||||
--tpm2 \
|
||||
--tpmstate dir=${cfg.devices.tpm.stateDir} \
|
||||
--ctrl type=unixio,path=${runDir}/swtpm.sock \
|
||||
--flags not-need-init,startup-clear
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.services.hermes-vm-tap = {
|
||||
description = "TAP interface for the Hermes VM";
|
||||
after = [ "systemd-networkd.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
path = [ pkgs.iproute2 ];
|
||||
script = ''
|
||||
ip link show ${net.tapId} >/dev/null 2>&1 \
|
||||
|| ip tuntap add ${net.tapId} mode tap user ${vmUser}
|
||||
ip link set ${net.tapId} up
|
||||
'';
|
||||
preStop = ''
|
||||
${pkgs.iproute2}/bin/ip link del ${net.tapId} || true
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
{ lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib)
|
||||
mkOption
|
||||
types
|
||||
;
|
||||
|
||||
tpmStateDir = "/var/lib/hermes-vm-tpm";
|
||||
in
|
||||
{
|
||||
options.services.hermenix = {
|
||||
vcpu = mkOption {
|
||||
type = types.ints.positive;
|
||||
default = 4;
|
||||
};
|
||||
|
||||
mem = mkOption {
|
||||
type = types.ints.positive;
|
||||
default = 8192;
|
||||
description = "RAM in MiB.";
|
||||
};
|
||||
|
||||
disk = {
|
||||
baseDir = mkOption {
|
||||
type = types.str;
|
||||
default = "/var/lib/hermes-vm";
|
||||
description = ''
|
||||
Directory holding the guest's disk images. On btrfs this should be
|
||||
laid out as three subvolumes - system/, state/ and snapshots/ - with
|
||||
nodatacow set. `hermenix setup` does this for you.
|
||||
'';
|
||||
};
|
||||
|
||||
systemSize = mkOption {
|
||||
type = types.str;
|
||||
default = "80G";
|
||||
description = ''
|
||||
System disk: the NixOS install and the Nix store. Never snapshotted,
|
||||
because it is entirely rebuildable from the flake. Size it for an
|
||||
agent that builds things.
|
||||
'';
|
||||
};
|
||||
|
||||
stateSize = mkOption {
|
||||
type = types.str;
|
||||
default = "20G";
|
||||
description = ''
|
||||
State disk, mounted at /var/lib/hermes in the guest. Holds Hermes's
|
||||
identity, memories, workspace, and the NixOS config it maintains for
|
||||
itself. This is the only irreplaceable data, so it is the only thing
|
||||
snapshotted and backed up.
|
||||
'';
|
||||
};
|
||||
|
||||
quiesce = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "fsfreeze the guest via qemu-guest-agent around each snapshot.";
|
||||
};
|
||||
};
|
||||
|
||||
devices = {
|
||||
rng.enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Attach a virtio entropy source (-device virtio-rng-pci).";
|
||||
};
|
||||
|
||||
vsock.enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Attach the host<->guest vsock (-device vhost-vsock-pci). `hermenix
|
||||
ctl` and the SSH-based backup both reach the guest over it, so
|
||||
turning this off breaks them.
|
||||
'';
|
||||
};
|
||||
|
||||
video = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Attach a virtual display adapter, giving the guest a DRM device
|
||||
(/dev/dri). Needed by a Wayland compositor or anything that wants
|
||||
a render node. The console stays serial either way.
|
||||
'';
|
||||
};
|
||||
model = mkOption {
|
||||
type = types.enum [
|
||||
"virtio"
|
||||
"qxl"
|
||||
"cirrus"
|
||||
"vmware"
|
||||
"std"
|
||||
];
|
||||
default = "virtio";
|
||||
description = ''
|
||||
Adapter model passed to -vga. Only "virtio" exposes a render node
|
||||
(/dev/dri/renderD128); the legacy models give card0 without one.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
balloon.enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Attach a virtio memory balloon (-device virtio-balloon-pci).";
|
||||
};
|
||||
|
||||
watchdog = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Attach a hardware watchdog. The guest must also arm it (systemd's
|
||||
RuntimeWatchdogSec) for it to do anything.
|
||||
'';
|
||||
};
|
||||
model = mkOption {
|
||||
type = types.enum [
|
||||
"i6300esb"
|
||||
"ib700"
|
||||
];
|
||||
default = "i6300esb";
|
||||
};
|
||||
action = mkOption {
|
||||
type = types.enum [
|
||||
"reset"
|
||||
"shutdown"
|
||||
"poweroff"
|
||||
"pause"
|
||||
"debug"
|
||||
"none"
|
||||
];
|
||||
default = "reset";
|
||||
description = "What QEMU does when the watchdog fires.";
|
||||
};
|
||||
};
|
||||
|
||||
input = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Attach virtio input devices. Only useful with devices.video.enable.";
|
||||
};
|
||||
keyboard = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
};
|
||||
tablet = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
};
|
||||
|
||||
audio = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Attach an audio device. NOTE: the VM unit is hardened (ProtectHome,
|
||||
PrivateTmp, a dedicated system user) and cannot reach the operator's
|
||||
audio daemon without loosening that; leave this off to keep the
|
||||
hardening intact.
|
||||
'';
|
||||
};
|
||||
backend = mkOption {
|
||||
type = types.enum [
|
||||
"none"
|
||||
"pipewire"
|
||||
"alsa"
|
||||
"pa"
|
||||
"sdl"
|
||||
];
|
||||
default = "pipewire";
|
||||
};
|
||||
model = mkOption {
|
||||
type = types.enum [
|
||||
"intel-hda"
|
||||
"virtio-sound"
|
||||
];
|
||||
default = "intel-hda";
|
||||
};
|
||||
};
|
||||
|
||||
tpm = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Attach a TPM 2.0 via swtpm. A hermes-vm-swtpm unit starts first and
|
||||
QEMU connects to it over a unix socket. Useful for measured boot or
|
||||
sealing secrets to the VM.
|
||||
'';
|
||||
};
|
||||
stateDir = mkOption {
|
||||
type = types.str;
|
||||
default = tpmStateDir;
|
||||
description = ''
|
||||
Where swtpm keeps its state. Must live under /var/lib/ so it can be
|
||||
created and owned by the unit's StateDirectory.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
cdroms = mkOption {
|
||||
default = [ ];
|
||||
description = "Read-only CD-ROM images to attach, on a virtio-scsi controller.";
|
||||
type = types.listOf (
|
||||
types.submodule {
|
||||
options.path = mkOption { type = types.str; };
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
extraDisks = mkOption {
|
||||
default = [ ];
|
||||
description = "Extra raw disk images, each on its own virtio-blk device.";
|
||||
type = types.listOf (
|
||||
types.submodule {
|
||||
options = {
|
||||
name = mkOption {
|
||||
type = types.str;
|
||||
description = "Unique QEMU drive id, e.g. scratch.";
|
||||
};
|
||||
path = mkOption { type = types.str; };
|
||||
readonly = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
aio = mkOption {
|
||||
type = types.enum [
|
||||
"threads"
|
||||
"native"
|
||||
"io_uring"
|
||||
];
|
||||
default = "threads";
|
||||
};
|
||||
cache = mkOption {
|
||||
type = types.enum [
|
||||
"writeback"
|
||||
"none"
|
||||
"directsync"
|
||||
"unsafe"
|
||||
];
|
||||
default = "writeback";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
usbController = mkOption {
|
||||
type = types.enum [
|
||||
"auto"
|
||||
"on"
|
||||
"off"
|
||||
];
|
||||
default = "auto";
|
||||
description = ''
|
||||
Whether to attach a qemu-xhci USB controller. "auto" adds one only
|
||||
when passthrough.usb is non-empty; "on"/"off" force the decision.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
extraQemuArgs = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ ];
|
||||
description = ''
|
||||
Raw additional QEMU arguments, appended to the VM command line.
|
||||
Unvalidated escape hatch - prefer the typed devices.* options.
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
+17
-1471
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user