281 lines
11 KiB
Nix
281 lines
11 KiB
Nix
# Static evaluation check for the host module's two networking modes (NAT
|
|
# and bridged), its QEMU command line and unit sandbox, and the guest's
|
|
# mode-agnostic NIC configuration.
|
|
#
|
|
# Nothing here is built or booted: the check evaluates services.hermenix
|
|
# (modules/host.nix) in both modes plus guest/common.nix, and asserts on the
|
|
# networking configuration the modules produce - the nftables rulesets, the
|
|
# ip-family input/forward rules, the systemd-networkd units, and the guest's
|
|
# DHCP / route-metric setup. Any failing assertion fails the check listing
|
|
# what broke.
|
|
#
|
|
# Run by the operator/CI, never by an agent:
|
|
# nix build .#checks.x86_64-linux.network-modes
|
|
#
|
|
# The host evaluations use minimal stub options for the NixOS modules the
|
|
# host module's config feeds (networking.nftables, networking.firewall,
|
|
# systemd.network) because this check only reads those outputs back; every
|
|
# other option host.nix writes is out of scope and passed through with
|
|
# _module.check = false. The guest evaluation is different: guest/common.nix
|
|
# runs against the full NixOS option set (module-list.nix) because its real
|
|
# imports (qemu-guest profile and friends) need it.
|
|
{ pkgs }:
|
|
|
|
let
|
|
inherit (pkgs) lib;
|
|
nixpkgsPath = pkgs.path;
|
|
has = lib.strings.hasInfix;
|
|
|
|
# Host module, evaluated with stub options for what this check reads. #
|
|
hostOptions = { lib, ... }: {
|
|
config._module.check = false;
|
|
options = {
|
|
networking = {
|
|
useNetworkd = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
};
|
|
nameservers = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
};
|
|
nftables = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
};
|
|
tables = lib.mkOption {
|
|
type = lib.types.attrsOf lib.types.anything;
|
|
default = { };
|
|
};
|
|
};
|
|
firewall = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
};
|
|
filterForward = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
};
|
|
extraInputRules = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "";
|
|
};
|
|
extraForwardRules = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "";
|
|
};
|
|
};
|
|
};
|
|
systemd.services = lib.mkOption {
|
|
type = lib.types.attrsOf lib.types.anything;
|
|
default = { };
|
|
};
|
|
systemd.network = {
|
|
networks = lib.mkOption {
|
|
type = lib.types.attrsOf lib.types.anything;
|
|
default = { };
|
|
};
|
|
netdevs = lib.mkOption {
|
|
type = lib.types.attrsOf lib.types.anything;
|
|
default = { };
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
# The one scenario every assertion is made against.
|
|
evalHost =
|
|
bridged:
|
|
lib.evalModules {
|
|
modules = [
|
|
hostOptions
|
|
../modules/host.nix
|
|
{
|
|
services.hermenix = {
|
|
enable = true;
|
|
network = {
|
|
externalInterface = "eno1";
|
|
inherit bridged;
|
|
}
|
|
// lib.optionalAttrs bridged {
|
|
# Bridged mode is static-only: guest address, host address on
|
|
# the bridge, and the LAN router.
|
|
lanAddress = "10.7.7.50";
|
|
hostLanAddress = "10.7.7.2";
|
|
lanGateway = "10.7.7.1";
|
|
};
|
|
allowLan = false;
|
|
publish = [
|
|
{
|
|
hostPort = 8080;
|
|
guestPort = 8080;
|
|
}
|
|
];
|
|
lanAccess = [
|
|
{
|
|
address = "192.168.1.50";
|
|
port = 11434;
|
|
}
|
|
];
|
|
hostAccess = [ { port = 8188; } ];
|
|
};
|
|
}
|
|
];
|
|
specialArgs = { inherit pkgs; };
|
|
};
|
|
|
|
nat = evalHost false;
|
|
br = evalHost true;
|
|
|
|
natNft = nat.config.networking.nftables.tables;
|
|
brNft = br.config.networking.nftables.tables;
|
|
natNat = natNft."hermenix-nat" or { };
|
|
brNat = brNft."hermenix-nat" or { };
|
|
brBr = brNft."hermenix-bridge" or { };
|
|
|
|
natNetworks = nat.config.systemd.network.networks;
|
|
brNetworks = br.config.systemd.network.networks;
|
|
|
|
natInput = nat.config.networking.firewall.extraInputRules;
|
|
natForward = nat.config.networking.firewall.extraForwardRules;
|
|
brInput = br.config.networking.firewall.extraInputRules;
|
|
brForward = br.config.networking.firewall.extraForwardRules;
|
|
|
|
vm = nat.config.systemd.services."hermes-vm".serviceConfig;
|
|
|
|
hostResults = {
|
|
# --- NAT mode: the original behaviour has to be unchanged ----------
|
|
"nat: no bridged-mode bridge table" = !(natNft ? "hermenix-bridge");
|
|
"nat: publish is DNATed to the guest" = has "dnat to 10.77.0.2:8080" natNat.content;
|
|
"nat: outbound masquerades on the uplink" = has "masquerade" natNat.content;
|
|
"nat: input rules match the fixed service-net address" =
|
|
has "ip saddr 10.77.0.2 tcp dport 5000 accept" natInput && !(has "meta mark" natInput);
|
|
"nat: published port has a forward accept" =
|
|
has "ip daddr 10.77.0.2 tcp dport 8080 accept" natForward;
|
|
"nat: bridge keeps the static service-net address, no LAN DHCP" =
|
|
natNetworks."10-hermes-br".address == [ "10.77.0.1/24" ]
|
|
&& (natNetworks."10-hermes-br".networkConfig.DHCP or null) == null;
|
|
"nat: uplink is not a bridge port" = !(natNetworks ? "12-eno1");
|
|
"nat: tap is a bridge port" = natNetworks."11-vm-hermes".networkConfig.Bridge == "hermes-br";
|
|
"nat: bridge netdev exists" =
|
|
nat.config.systemd.network.netdevs."10-hermes-br".netdevConfig.Kind == "bridge";
|
|
|
|
# --- bridged mode: the guest is directly on the LAN, static-only -----
|
|
"bridged: bridge-family table exists" = brNft ? "hermenix-bridge" && brBr.family == "bridge";
|
|
"bridged: guest traffic is marked as it enters the tap" = has "meta mark set 18501" brBr.content;
|
|
"bridged: inbound from the LAN is open on every port" =
|
|
has "oifname \"vm-hermes\" accept" brBr.content;
|
|
"bridged: lanAccess still gates guest egress" =
|
|
has "ip daddr 192.168.1.50 tcp dport 11434 accept" brBr.content;
|
|
"bridged: private-space egress is dropped unless allowlisted" =
|
|
has "192.168.0.0/16" brBr.content && has "fc00::/7" brBr.content;
|
|
"bridged: guest DHCP is confined to the bridge, can't race the LAN router" =
|
|
has "udp sport 68 udp dport 67 drop" brBr.content
|
|
&& has "udp sport 67 udp dport 68 drop" brBr.content;
|
|
"bridged: no DNAT for publish" =
|
|
!(has "dnat to" brNat.content) && !(has "chain prerouting" brNat.content);
|
|
"bridged: scoped fallback masquerade remains" = has "masquerade" brNat.content;
|
|
"bridged: input rules match the mark, not an address" =
|
|
has "meta mark 18501 tcp dport 5000 accept" brInput
|
|
&& has "meta mark 18501 tcp dport 8188 accept" brInput
|
|
&& !(has "ip saddr 10.77.0.2" brInput);
|
|
"bridged: publish forward rules are dropped" =
|
|
!(has "ip daddr 10.77.0.2 tcp dport 8080 accept" brForward);
|
|
"bridged: bridge carries only static addresses, never a DHCP lease" =
|
|
brNetworks."10-hermes-br".address == [
|
|
"10.77.0.1/24"
|
|
"10.7.7.2/24"
|
|
]
|
|
&& (brNetworks."10-hermes-br".networkConfig.DHCP or null) == null
|
|
&& !(has "DHCP" (builtins.toJSON brNetworks."10-hermes-br"));
|
|
"bridged: host gateway and DNS are static on the bridge" =
|
|
brNetworks."10-hermes-br".networkConfig.Gateway == "10.7.7.1"
|
|
&& brNetworks."10-hermes-br".networkConfig.DNS == [ "10.7.7.1" ];
|
|
"bridged: guest is handed lanAddress by the bridge DHCP server" =
|
|
let
|
|
dhcp = br.config.systemd.services."hermes-lan-dhcp".script;
|
|
in
|
|
has "--interface=hermes-br" dhcp
|
|
&& has "--dhcp-range=10.7.7.50,10.7.7.50,255.255.255.0,12h" dhcp
|
|
&& has "--dhcp-host=02:00:00:7e:00:01,10.7.7.50" dhcp
|
|
&& has "--dhcp-option=option:router,10.7.7.1" dhcp
|
|
&& has "--port=0" dhcp;
|
|
"bridged: uplink MAC is cloned onto the bridge, host keeps its identity" =
|
|
let
|
|
mac = br.config.systemd.services."hermes-bridge-mac".script;
|
|
in
|
|
has "ip link set dev hermes-br address" mac && has "/sys/class/net/eno1/address" mac;
|
|
"bridged: uplink is a bridge port" = brNetworks."12-eno1".networkConfig.Bridge == "hermes-br";
|
|
"bridged: tap is still a bridge port" =
|
|
brNetworks."11-vm-hermes".networkConfig.Bridge == "hermes-br";
|
|
};
|
|
|
|
# The QEMU command line and the unit's sandbox, which the host VM test
|
|
# (tests/host-vm-test.nix) cannot exercise without nested virtualization.
|
|
vmResults = {
|
|
"vm: qemu runs under kvm with smm" = has "-machine q35,accel=kvm,smm=on" vm.ExecStart;
|
|
"vm: tap netdev wired to the guest NIC" =
|
|
has "-netdev tap,id=net0,ifname=vm-hermes" vm.ExecStart
|
|
&& has "-device virtio-net-pci,netdev=net0,mac=02:00:00:7e:00:01" vm.ExecStart;
|
|
"vm: system and state disks attached" =
|
|
has "id=system,format=raw" vm.ExecStart && has "id=state,format=raw" vm.ExecStart;
|
|
"vm: qmp socket exposed for hermenix vm control" =
|
|
has "qmp unix:/run/hermes-vm/qmp.sock" vm.ExecStart;
|
|
"vm: default devices (rng, vsock, qga) present" =
|
|
has "-device virtio-rng-pci" vm.ExecStart
|
|
&& has "-device vhost-vsock-pci,guest-cid=77" vm.ExecStart
|
|
&& has "name=org.qemu.guest_agent.0" vm.ExecStart;
|
|
"vm: optional devices off by default" =
|
|
!(has "-vga" vm.ExecStart) && !(has "virtio-balloon" vm.ExecStart);
|
|
"vm: sandbox hardening intact" =
|
|
vm.ProtectSystem == "strict" && vm.NoNewPrivileges == true && vm.DevicePolicy == "closed";
|
|
"vm: device allowlist covers kvm, tun, vsock" =
|
|
vm.DeviceAllow == [
|
|
"/dev/kvm rw"
|
|
"/dev/net/tun rw"
|
|
"/dev/vhost-vsock rw"
|
|
];
|
|
};
|
|
|
|
# Guest module: must produce one NIC config that works in both modes. #
|
|
# Evaluated against the full NixOS option set. #
|
|
guest = lib.evalModules {
|
|
modules = (import "${nixpkgsPath}/nixos/modules/module-list.nix") ++ [
|
|
{ config._module.check = false; }
|
|
../guest/common.nix
|
|
];
|
|
specialArgs = {
|
|
inherit pkgs;
|
|
modulesPath = "${nixpkgsPath}/nixos/modules";
|
|
};
|
|
};
|
|
gEth = guest.config.systemd.network.networks."10-eth";
|
|
gRoute = builtins.elemAt gEth.routes 0;
|
|
|
|
guestResults = {
|
|
"guest: NIC asks for a DHCP lease (bridged mode)" = gEth.networkConfig.DHCP == "ipv4";
|
|
"guest: service-net address stays static (both modes)" = gEth.address == [ "10.77.0.2/24" ];
|
|
"guest: static default route via the host is the NAT fallback" =
|
|
gRoute.Gateway == "10.77.0.1" && gRoute.Metric == 200;
|
|
"guest: DHCP default route wins (metric 100)" = gEth.dhcpV4Config.RouteMetric == 100;
|
|
"guest: resolvers stay QUAD9, DHCP DNS is ignored" =
|
|
gEth.dhcpV4Config.UseDNS == false
|
|
&&
|
|
gEth.networkConfig.DNS == [
|
|
"9.9.9.9"
|
|
"149.112.112.112"
|
|
];
|
|
};
|
|
|
|
results = hostResults // vmResults // guestResults;
|
|
|
|
failing = builtins.filter (n: !results.${n}) (builtins.attrNames results);
|
|
in
|
|
assert lib.assertMsg (failing == [ ]) ''
|
|
hermenix network-modes check FAILED:
|
|
${lib.concatMapStringsSep "\n" (n: " - " + n) failing}
|
|
'';
|
|
pkgs.runCommand "hermenix-network-modes-check" { } "touch $out"
|