From d5afd6510f95650bf07cd89b8af77cd75a3114d5 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Wed, 23 Sep 2026 19:29:28 -0400 Subject: [PATCH] safer bridge mode --- README.md | 133 ++++++++++--------- guest/common.nix | 8 +- guest/hermes/home/SOUL.md | 7 +- guest/vm-tests/common.nix | 2 +- guest/vm-tests/test_bridge.nix | 33 ++--- modules/host.nix | 225 +++++++++++++++++++++++++++++---- tests/network-modes-check.nix | 49 ++++++- 7 files changed, 337 insertions(+), 120 deletions(-) diff --git a/README.md b/README.md index f43fae8..93def99 100644 --- a/README.md +++ b/README.md @@ -483,89 +483,86 @@ auth in front of anything that isn't a toy, and keep the list short. ## Putting the guest directly on your LAN -The default setup puts the guest on an isolated bridge behind the host's NAT, -reachable from outside only through `publish`. This section is for when you -want the VM to be a first-class device on your LAN instead: its own address -from the LAN's DHCP, reachable on any port from any device on the network. +The default setup keeps the guest on an isolated bridge behind the host's +NAT, reachable from outside only through `publish`. This section is for +when you want the VM to be a first-class device on your LAN instead: its +own fixed address on the network, reachable on any port from any device, +while keeping everything it can see outbound locked down. -Enable it: +Enable it and tell it your LAN's numbers: ```nix -services.hermenix.network.bridged = true; -``` +services.hermenix.network = { + bridged = true; + externalInterface = "enp191s0"; # your uplink (required) -In one sentence, what that does: the bridge absorbs your uplink -(`network.externalInterface`), the guest and the host share the LAN, the -guest takes a DHCP lease, and inbound traffic to the guest is open on every -port. Outbound is still gated by `lanAccess` / `allowLan`, and `hostAccess` -plus the host Nix cache still work over the service net at `10.77.0.1`. - -### 1. Ready the guest first (one-time, if the image is older than this repo) - -The guest keeps a static service-net address (`10.77.0.2/24`) for talking to -the host and adds DHCP on the same NIC, so one config works in both modes. -That is already what `guest/common.nix` does; if your guest image predates -it, apply the equivalent change to `/etc/nixos/common.nix` inside the VM: - -```nix -systemd.network.networks."10-eth" = { - matchConfig.Type = "ether"; - address = [ "10.77.0.2/24" ]; # service net to the host - routes = [ { Gateway = "10.77.0.1"; Metric = 200; } ]; # NAT-mode fallback - networkConfig.DHCP = "ipv4"; # LAN lease in bridged mode - dhcpV4Config = { # DHCP default route wins (100) - UseDNS = false; - RouteMetric = 100; - }; + # Static LAN addressing: bridged mode does not use your LAN's DHCP. + lanAddress = "10.0.12.50"; # the guest's fixed LAN address + hostLanAddress = "10.0.12.10"; # this host's own LAN address + lanGateway = "10.0.12.1"; # your router + lanPrefixLength = 24; # default 24 + # lanDns = [ "1.1.1.1" ]; # optional; defaults to lanGateway }; ``` -`cd /etc/nixos && git apply -p1 < patch`, commit, and -`sudo nixos-rebuild switch --flake /etc/nixos#hermes`. In NAT mode the DHCP -attempt fails quietly (there is no server on the isolated bridge), so check -nothing broke: `ip -4 addr` still shows `10.77.0.2` and `ip route` still -goes via `10.77.0.1`. +What that gives you: the bridge absorbs the uplink, the host and the guest +share the LAN, and each side has a fixed address. The guest never needs to +know its own address - the module runs a small DHCP server on the bridge +that reserves `lanAddress` for the guest's MAC, so the VM stays a plain +DHCP client and just receives its address. This host keeps its LAN identity +too: the bridge clones your uplink's MAC, so the LAN sees this host under +its usual MAC (no duplicate-address surprises), and the guest's DHCP +broadcast never leaves the bridge, so nothing on your LAN can race the +reservation. -### 2. Rebuild the host with the toggle on +### What is and isn't reachable -Set `services.hermenix.network.bridged = true` alongside -`network.externalInterface`, then `sudo nixos-rebuild switch`. On rebuild -the host's uplink becomes a bridge port, the host takes a fresh DHCP lease -on the bridge, and the guest's DHCP is answered by the LAN router. The -network drops briefly while the host reconfigures - plan for it. +- **Inbound to the guest, from anywhere on the LAN: every port** is open. + The guest's own firewall decides what answers - by default it only opens + port 22, so the agent opens whatever it wants LAN-reachable. +- **Guest egress is still gated**, exactly as in NAT mode: `lanAccess` + entries and `allowLan` are unchanged. The VM sees the LAN, but only the + parts you have allowed. +- **Guest to host still works**: `hostAccess` and the host Nix cache are + reachable over the service net at `10.77.0.1` in both modes. +- **`publish` is ignored** in this mode - every inbound port is already + reachable. Drop the entries if you come back to NAT mode. -### 3. Verify +### Switching over -From the host, `hermenix vm ssh`. Inside the guest, `ip -4 addr` shows the -service net _and_ a LAN address, and `ip route` shows a default route via -your LAN gateway. From any other device on the LAN: +1. Pick three free addresses: the guest (`lanAddress`), this host + (`hostLanAddress` - use the address it already has on that interface, + e.g. a router reservation, so nothing else on the LAN breaks), and the + router itself (`lanGateway`). Keep `hostLanAddress` and `lanAddress` + outside any DHCP pool, or reserve them in the router first. +2. Set `network.bridged = true` with the options above, then + `sudo nixos-rebuild switch`. The network drops briefly while the host + reconfigures - plan for it. +3. Verify from the host: `hermenix vm ssh`; inside the guest, + `ip -4 addr` shows the service net (`10.77.0.2`) **and** its fixed LAN + address. From any other device on the LAN: -```bash -ssh hermes@ # and any other port the guest opens -``` - -The guest's own firewall decides what answers - by default it only opens -port 22, so the agent must open whatever it wants LAN-reachable. Egress is -still gated too: from the guest, `192.168.1.50:11434` only works if it is -in `lanAccess` (or `allowLan = true`). - -### Gotchas - -- The LAN needs a working DHCP server. -- `publish` does nothing in this mode - every port is reachable, so its - entries are ignored. Drop them if you come back to NAT mode. -- The host's LAN IP comes from a fresh DHCP lease on the bridge, and the - bridge MAC can differ from the uplink's - the host's address may change - after the switch. Reserve it in the DHCP server if that matters. -- Any network config you had for the uplink interface is replaced by its - bridge-port role while bridged (this module's networkd unit for it takes - priority over most others). + ```bash + ssh hermes@ # and any other port the guest opens + ``` ### Switching back -Set `bridged = false` and rebuild the host. The guest's DHCP client finds -no server on the isolated bridge and settles onto its static route via the -host again; no guest-side change needed. +Set `bridged = false` (the `lan*` options are ignored) and rebuild the +host. The guest's DHCP client finds no server on the isolated bridge and +settles onto its static route via the host again; no guest-side change +needed. + +### Gotchas + +- `hostLanAddress` is a static claim on the LAN - pick one your router + won't hand out, or reserve it. +- Any network config you had for the uplink interface is replaced by its + bridge-port role while bridged (this module's networkd unit for it takes + priority over most others). +- An older guest image that predates the DHCP-on-NIC setup in the current + `guest/common.nix` won't take an address in bridged mode; upgrade the + image or apply that config inside the VM first. --- diff --git a/guest/common.nix b/guest/common.nix index 4359881..9c7e3a2 100644 --- a/guest/common.nix +++ b/guest/common.nix @@ -182,9 +182,11 @@ in # onto the LAN (services.hermenix.network.bridged = true on the host). # - NAT mode: no DHCP server on the host bridge; the static default # route via the host is the only one and carries all traffic. - # - bridged mode: the LAN's DHCP leases this NIC a real LAN address; - # that lease's lower-metric default route (100) wins over the static - # fallback (200), and LAN devices reach the VM at the lease address. + # - bridged mode: the host runs a DHCP server on its bridge that hands + # this NIC its fixed LAN address (services.hermenix.network.lanAddress, + # keyed to this NIC's MAC); that lease's lower-metric default route + # (100) wins over the static fallback (200), and LAN devices reach + # the VM at the fixed address. The VM never needs to know the address. # Resolvers stay QUAD9 (UseDNS = false) in both modes. # ===================================================================== networking.useNetworkd = true; diff --git a/guest/hermes/home/SOUL.md b/guest/hermes/home/SOUL.md index b3c04f6..ecbf654 100644 --- a/guest/hermes/home/SOUL.md +++ b/guest/hermes/home/SOUL.md @@ -25,9 +25,10 @@ changes persist across reboots. the cheapest undo available to you, and future-you will want it. - If you break the system badly enough that you cannot fix it, the operator has snapshots of the state disk. Say so plainly rather than thrashing. -- You have no route to the host or the local network unless the operator has - opened a specific port. If something is unreachable, that is probably - deliberate — ask rather than working around it. +- Reachability is gated on the host: what you can reach from here is + allowlisted (lanAccess/allowLan), in both network modes. If something is + unreachable, that is probably deliberate — ask rather than working around + it. ## What I care about diff --git a/guest/vm-tests/common.nix b/guest/vm-tests/common.nix index 8b321ca..a04659e 100644 --- a/guest/vm-tests/common.nix +++ b/guest/vm-tests/common.nix @@ -4,7 +4,7 @@ # The guest node's configuration and every assertion that must hold in both # networking modes are defined here, once; each mode test imports this file, # uses guestNode as its machine, and appends only the checks that apply to -# its mode (the isolated-bridge NAT setup vs. the DHCP-served LAN bridge). +# its mode (the isolated-bridge NAT setup vs. the fixed-address LAN bridge). # # Both tests boot guest/common.nix — the same module set hermes-guest-image # is built from — in an ephemeral, throwaway NixOS test VM. No real disk diff --git a/guest/vm-tests/test_bridge.nix b/guest/vm-tests/test_bridge.nix index 77febce..8cf3482 100644 --- a/guest/vm-tests/test_bridge.nix +++ b/guest/vm-tests/test_bridge.nix @@ -1,9 +1,11 @@ -# Guest-config VM test, bridged mode: the guest on a LAN served by DHCP, -# with a second node acting as the LAN (a DHCP server, a throwaway HTTP -# service, and a plain device that connects back to the guest). This is the -# guest-facing half of services.hermenix.network.bridged = true: the module's -# own bridge/enforcement rules are covered statically by -# tests/network-modes-check.nix. +# Guest-config VM test, bridged mode: the guest on a LAN that hands it one +# fixed address (the production shape of services.hermenix.network.bridged +# = true: the host-side DHCP server reserves lanAddress for the guest's MAC, +# so the guest is effectively static without knowing it). A second node acts +# as the LAN: the reservation-style DHCP server, a throwaway HTTP service, +# and a plain device that connects back to the guest. This is the +# guest-facing half of the module; the module's own bridge/enforcement rules +# are covered statically by tests/network-modes-check.nix. # # The guest node and the mode-independent assertions (seeding, idempotency, # account, SSH) are shared with guest-vm-unbridged via @@ -27,10 +29,10 @@ pkgs.testers.runNixOSTest { nodes = { machine = common.guestNode; - # The "LAN": serves DHCP to the guest and hosts a throwaway HTTP - # service, standing in for the LAN router and another LAN device. Its - # firewall is off so the guest<->LAN checks exercise the network path - # itself, not this throwaway node's own firewall. + # The "LAN": hands the guest its single fixed address and hosts a + # throwaway HTTP service, standing in for the router and another LAN + # device. Its firewall is off so the guest<->LAN checks exercise the + # network path itself, not this throwaway node's own firewall. lan = { pkgs, lib, ... }: { @@ -63,7 +65,9 @@ pkgs.testers.runNixOSTest { networkConfig.LinkLocalAddressing = "no"; }; - # DHCP only (port = 0 disables dnsmasq's DNS half; the guest keeps + # DHCP only, and reservation-style: a single-address range, exactly + # how the host-side hermes-lan-dhcp hands the real guest its fixed + # lanAddress (port = 0 disables dnsmasq's DNS half; the guest keeps # its own QUAD9 resolvers regardless). Bound to the node's static # vlan address rather than an interface name - the framework's # interface naming is not something this test should depend on. @@ -73,10 +77,9 @@ pkgs.testers.runNixOSTest { port = 0; listen-address = "192.168.1.2"; bind-interfaces = true; - dhcp-range = "192.168.1.100,192.168.1.200,12h"; + dhcp-range = "192.168.1.100,192.168.1.100,12h"; dhcp-option = [ "3,192.168.1.2" # default gateway -> this node - "6,192.168.1.2" # DNS -> this node ]; }; }; @@ -93,7 +96,7 @@ pkgs.testers.runNixOSTest { + common.commonScript + '' # --- bridged-mode networking --------------------------------------- - # The LAN's DHCP gave the guest a lease alongside its static + # The LAN handed the guest its fixed address alongside its static # service-net address. (Both test nodes share one vlan, standing in for # the bridged uplink on a real host.) machine.wait_until_succeeds("ip -4 route show default | grep -q 'metric 100'") @@ -105,7 +108,7 @@ pkgs.testers.runNixOSTest { if not ip.startswith("10.77."): guest_lan_ip = ip assert any("10.77.0.2/24" in line for line in addrs), f"static service-net address missing in bridged mode: {addrs}" - assert guest_lan_ip is not None, f"guest never took a LAN DHCP lease: {addrs}" + assert guest_lan_ip == "192.168.1.100", f"guest should have been handed its fixed LAN address, got: {addrs}" # Both default routes are in place: the LAN's (metric 100, in use) and # the host service-net fallback (metric 200) that NAT mode relies on. diff --git a/modules/host.nix b/modules/host.nix index a59ff4b..ed81b05 100644 --- a/modules/host.nix +++ b/modules/host.nix @@ -38,11 +38,36 @@ let # Netfilter mark tagging guest-originated traffic in bridged mode. The # guest's only way in is the tap interface, so marking at the bridge - # prerouting hook is as specific as an address match - and it is the only - # way to identify the guest at the ip-family layer, where its DHCP-issued - # LAN address is not statically known. 18501 == 0x4845 ("HE"), for hermenix. + # prerouting hook identifies the guest regardless of which address it + # currently presents - its static service-net address, or the fixed LAN + # address served to it by the host-side DHCP on the bridge. 18501 == + # 0x4845 ("HE"), for hermenix. bridgeMark = 18501; + # Dotted-quad netmask for a prefix length, used by the bridged-mode DHCP + # server's single-address lease (dnsmasq needs an explicit mask for a + # one-address range). + netmaskFromPrefix = + p: + lib.concatStringsSep "." ( + lib.genList ( + b: + toString ( + builtins.elemAt [ + 0 + 128 + 192 + 224 + 240 + 248 + 252 + 254 + 255 + ] (lib.max 0 (lib.min 8 (p - b * 8))) + ) + ) 4 + ); + # Graceful ACPI power button via QMP, then wait for QEMU to exit. stopScript = pkgs.writeShellScript "hermes-vm-stop" '' set -eu @@ -337,24 +362,68 @@ in type = types.int; default = 24; }; + lanAddress = mkOption { + type = types.str; + default = ""; + example = "10.0.12.50"; + description = "The guest's fixed address on your LAN. Required when `bridged = true`."; + }; + hostLanAddress = mkOption { + type = types.str; + default = ""; + example = "10.0.12.10"; + description = '' + This host's own address on your LAN. When `bridged` is enabled the + uplink becomes a bridge port and this address moves onto the bridge + (which clones the uplink's MAC, so the host keeps its LAN identity). + Required when `bridged = true`. + ''; + }; + lanGateway = mkOption { + type = types.str; + default = ""; + example = "10.0.12.1"; + description = '' + Your LAN's router. Used as the default gateway for the guest and, + while `bridged`, for this host's bridge too (DNS unless `lanDns` is + set). Required when `bridged = true`. + ''; + }; + lanPrefixLength = mkOption { + type = types.int; + default = 24; + description = "Prefix length of your LAN subnet (netmask)."; + }; + lanDns = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "1.1.1.1" ]; + description = '' + DNS servers for this host while `bridged` (the bridge no longer + takes a DHCP lease). Defaults to the LAN gateway. + ''; + }; bridged = mkOption { type = types.bool; default = false; description = '' Put the guest directly on your LAN instead of behind the host's - NAT. The bridge absorbs `externalInterface`, and the guest takes - an address from your LAN's DHCP, so any device on the LAN can - reach any port the guest exposes. + NAT. The bridge absorbs `externalInterface`, so any device on the + LAN can reach any port the guest exposes. + + Addressing is fully static - no DHCP is involved. You must give + the guest a fixed LAN address (`lanAddress`), the host's own LAN + address (`hostLanAddress` - it moves onto the bridge), and the + LAN gateway (`lanGateway`). The guest is handed `lanAddress` by a + DHCP server this module runs on the bridge, keyed to the guest's + MAC (`network.mac`), so the guest itself stays a plain DHCP + client and never needs to know its own address. Inbound is no longer limited to `publish` (which is ignored in this mode): the LAN may reach every guest port. Outbound is still governed by `lanAccess` / `allowLan`, and `hostAccess` plus the host cache still work over the host-guest service net (`hostAddress` stays configured on the bridge). - - Requires a working DHCP server on the LAN, and a guest that runs - a DHCP client on its NIC (see guest/common.nix - this repo's - guest does). ''; }; externalInterface = mkOption { @@ -509,6 +578,17 @@ in assertion = net.externalInterface != null; message = "services.hermenix.network.externalInterface must be set."; } + { + assertion = + !net.bridged || (net.lanAddress != "" && net.hostLanAddress != "" && net.lanGateway != ""); + message = '' + services.hermenix.network.bridged = true requires a fully static + LAN setup (DHCP is no longer supported in bridged mode). Set + network.lanAddress (the guest's fixed LAN address), + network.hostLanAddress (this host's own LAN address, which moves + onto the bridge) and network.lanGateway (your router's address). + ''; + } { assertion = cfg.backup.enable -> (cfg.backup.repository != ""); message = "services.hermenix.backup.repository must be set when backups are enabled."; @@ -762,32 +842,120 @@ in systemd.network.networks."10-${net.bridge}" = { matchConfig.Name = net.bridge; # hostAddress stays on the bridge in both modes: it is the guest's - # service net (host cache, hostAccess). In bridged mode the LAN DHCP - # lease is added on top of it. - address = [ "${net.hostAddress}/${toString net.prefixLength}" ]; + # service net (host cache, hostAccess). In bridged mode the host's + # static LAN address is added on top of it - the bridge only ever has + # static addresses, never a DHCP lease. + address = [ + "${net.hostAddress}/${toString net.prefixLength}" + ] + ++ lib.optional net.bridged "${net.hostLanAddress}/${toString net.lanPrefixLength}"; networkConfig = { ConfigureWithoutCarrier = true; IPv4Forwarding = true; } - // lib.optionalAttrs net.bridged { DHCP = "ipv4"; }; + // lib.optionalAttrs net.bridged { + # Static LAN config for the host: the bridge no longer takes a DHCP + # lease (bridged mode is static-only), so its gateway and DNS come + # from the module options instead. + Gateway = net.lanGateway; + DNS = if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns; + }; }; + # The host's DNS, now that the bridge cannot learn it from a DHCP lease. + # mkDefault so the operator's own nameservers win if they set any. + networking.nameservers = lib.mkIf net.bridged ( + lib.mkDefault (if net.lanDns == [ ] then [ net.lanGateway ] else net.lanDns) + ); + systemd.network.networks."11-${net.tapId}" = { matchConfig.Name = net.tapId; networkConfig.Bridge = net.bridge; }; # Bridged mode: the physical uplink becomes a bridge port so the LAN - # shares an L2 domain with the guest and the guest's DHCP is answered - # by the LAN itself. The port must not carry its own address config - - # the bridge owns the LAN address now. Note the bridge takes a fresh - # DHCP lease, and because its MAC can differ from the uplink's, the - # host's LAN IP may change after the first switch to bridged mode. + # shares an L2 domain with the guest. The port must not carry its own + # address config - the bridge owns the LAN address now (static, via + # hostLanAddress; there is no DHCP on the bridge). systemd.network.networks."12-${net.externalInterface}" = lib.mkIf net.bridged { matchConfig.Name = net.externalInterface; networkConfig.Bridge = net.bridge; }; + # The bridge is created with no MAC of its own, so Linux makes it adopt + # the lowest-numbered port's MAC - and the tap is always the first port. + # That is how the host once ended up presenting the VM's MAC on the LAN + # (a duplicate address, since the guest uses the same MAC). Clone the + # uplink's MAC onto the bridge so the host keeps its LAN identity; the + # guest keeps its own tap MAC (net.mac). + systemd.services.hermes-bridge-mac = lib.mkIf net.bridged { + description = "Clone the uplink MAC onto the hermenix bridge"; + after = [ + "systemd-networkd.service" + "sys-subsystem-net-devices-${net.bridge}.device" + "sys-subsystem-net-devices-${net.externalInterface}.device" + ]; + wants = [ "systemd-networkd.service" ]; + before = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + unitConfig.ConditionPathExists = "/sys/class/net/${net.externalInterface}"; + script = '' + set -euo pipefail + # networkd creates the bridge netdev asynchronously; give it a moment. + for _ in 1 2 3 4 5 6 7 8 9 10; do + [ -e "/sys/class/net/${net.bridge}" ] && break + sleep 0.5 + done + [ -e "/sys/class/net/${net.bridge}" ] || exit 1 + mac="$(${pkgs.coreutils}/bin/cat /sys/class/net/${net.externalInterface}/address)" + ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} down + ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} address "$mac" + ${pkgs.iproute2}/bin/ip link set dev ${net.bridge} up + # Re-apply the bridge's static address config under the new MAC. + ${pkgs.systemd}/bin/networkctl reconfigure ${net.bridge} || true + ''; + }; + + # The guest is handed its fixed LAN address (lanAddress) by a dedicated + # DHCP server bound to the bridge, keyed to the guest's MAC. This is the + # guest's ONLY DHCP: its broadcast is blocked from leaving the bridge + # (see the hermenix-bridge table), so the LAN's own DHCP server can + # never race the reservation. DNS is disabled (port=0) - the guest runs + # static QUAD9 resolvers, and this host gets its DNS from lanDns. + systemd.services.hermes-lan-dhcp = lib.mkIf net.bridged { + description = "Static-lease DHCP for the bridged hermenix guest"; + after = [ + "systemd-networkd.service" + "sys-subsystem-net-devices-${net.bridge}.device" + ]; + wants = [ "systemd-networkd.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "simple"; + Restart = "on-failure"; + RestartSec = 5; + }; + script = '' + for _ in 1 2 3 4 5 6 7 8 9 10; do + [ -e "/sys/class/net/${net.bridge}" ] && break + sleep 0.5 + done + exec ${pkgs.dnsmasq}/bin/dnsmasq --keep-in-foreground \ + --conf-file=/dev/null \ + --interface=${net.bridge} \ + --bind-dynamic \ + --port=0 \ + --dhcp-authoritative \ + --dhcp-range=${net.lanAddress},${net.lanAddress},${netmaskFromPrefix net.lanPrefixLength},12h \ + --dhcp-host=${net.mac},${net.lanAddress} \ + --dhcp-option=option:router,${net.lanGateway} + ''; + }; + # ip_forward: use the same sysctl keys networking.nat used (mkOverride 99 # so they win over any lower-priority defaults elsewhere in the config). boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true; @@ -856,6 +1024,13 @@ in # Tag anything entering from the guest. ip-family input rules # (host cache, hostAccess) match this mark. iifname "${net.tapId}" meta mark set ${toString bridgeMark} + + # The guest's address is static - served by hermes-lan-dhcp on + # this bridge - so keep its DHCP traffic off the LAN and keep the + # LAN's DHCP answers out of the guest. Only the host-side server + # may talk DHCP to it. + iifname "${net.tapId}" oifname "${net.externalInterface}" ether type ip ip protocol udp udp sport 68 udp dport 67 drop + iifname "${net.externalInterface}" oifname "${net.tapId}" ether type ip ip protocol udp udp sport 67 udp dport 68 drop } chain forward { type filter hook forward priority filter; policy accept; @@ -903,12 +1078,12 @@ in # in hostAccess. # # NAT mode: the guest always arrives from its fixed service-net address, - # so match address + bridge. Bridged mode: the guest's LAN address comes - # from DHCP and is not known here, so the bridge table above tags guest - # traffic with a netfilter mark as it enters the tap - the rules match - # that mark instead. The tap is exclusive to the guest, so the mark is - # as specific as an address match: LAN devices can never inherit these - # holes. + # so match address + bridge. Bridged mode: the guest presents either the + # service-net address or its LAN address (served by the bridge's DHCP), + # so the bridge table above tags guest traffic with a netfilter mark as + # it enters the tap - the rules match that mark instead. The tap is + # exclusive to the guest, so the mark is as specific as an address + # match: LAN devices can never inherit these holes. networking.firewall.extraInputRules = if net.bridged then optionalString cfg.hostCache.enable '' diff --git a/tests/network-modes-check.nix b/tests/network-modes-check.nix index 722dcea..52d845f 100644 --- a/tests/network-modes-check.nix +++ b/tests/network-modes-check.nix @@ -36,6 +36,10 @@ let type = lib.types.bool; default = false; }; + nameservers = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + }; nftables = { enable = lib.mkOption { type = lib.types.bool; @@ -65,6 +69,10 @@ let }; }; }; + systemd.services = lib.mkOption { + type = lib.types.attrsOf lib.types.anything; + default = { }; + }; systemd.network = { networks = lib.mkOption { type = lib.types.attrsOf lib.types.anything; @@ -91,6 +99,13 @@ let network = { externalInterface = "eno1"; inherit bridged; + } + // lib.optionalAttrs bridged { + # Bridged mode is static-only: guest address, host address on + # the bridge, and the LAN router. + lanAddress = "10.7.7.50"; + hostLanAddress = "10.7.7.2"; + lanGateway = "10.7.7.1"; }; allowLan = false; publish = [ @@ -146,7 +161,7 @@ let "nat: bridge netdev exists" = nat.config.systemd.network.netdevs."10-hermes-br".netdevConfig.Kind == "bridge"; - # --- bridged mode: the guest is directly on the LAN ---------------- + # --- bridged mode: the guest is directly on the LAN, static-only ----- "bridged: bridge-family table exists" = brNft ? "hermenix-bridge" && brBr.family == "bridge"; "bridged: guest traffic is marked as it enters the tap" = has "meta mark set 18501" brBr.content; "bridged: inbound from the LAN is open on every port" = @@ -155,18 +170,42 @@ let has "ip daddr 192.168.1.50 tcp dport 11434 accept" brBr.content; "bridged: private-space egress is dropped unless allowlisted" = has "192.168.0.0/16" brBr.content && has "fc00::/7" brBr.content; + "bridged: guest DHCP is confined to the bridge, can't race the LAN router" = + has "udp sport 68 udp dport 67 drop" brBr.content + && has "udp sport 67 udp dport 68 drop" brBr.content; "bridged: no DNAT for publish" = !(has "dnat to" brNat.content) && !(has "chain prerouting" brNat.content); "bridged: scoped fallback masquerade remains" = has "masquerade" brNat.content; - "bridged: input rules match the mark, not the unknown DHCP address" = + "bridged: input rules match the mark, not an address" = has "meta mark 18501 tcp dport 5000 accept" brInput && has "meta mark 18501 tcp dport 8188 accept" brInput && !(has "ip saddr 10.77.0.2" brInput); "bridged: publish forward rules are dropped" = !(has "ip daddr 10.77.0.2 tcp dport 8080 accept" brForward); - "bridged: bridge keeps the service net and takes LAN DHCP" = - brNetworks."10-hermes-br".address == [ "10.77.0.1/24" ] - && brNetworks."10-hermes-br".networkConfig.DHCP == "ipv4"; + "bridged: bridge carries only static addresses, never a DHCP lease" = + brNetworks."10-hermes-br".address == [ + "10.77.0.1/24" + "10.7.7.2/24" + ] + && (brNetworks."10-hermes-br".networkConfig.DHCP or null) == null + && !(has "DHCP" (builtins.toJSON brNetworks."10-hermes-br")); + "bridged: host gateway and DNS are static on the bridge" = + brNetworks."10-hermes-br".networkConfig.Gateway == "10.7.7.1" + && brNetworks."10-hermes-br".networkConfig.DNS == [ "10.7.7.1" ]; + "bridged: guest is handed lanAddress by the bridge DHCP server" = + let + dhcp = br.config.systemd.services."hermes-lan-dhcp".script; + in + has "--interface=hermes-br" dhcp + && has "--dhcp-range=10.7.7.50,10.7.7.50,255.255.255.0,12h" dhcp + && has "--dhcp-host=02:00:00:7e:00:01,10.7.7.50" dhcp + && has "--dhcp-option=option:router,10.7.7.1" dhcp + && has "--port=0" dhcp; + "bridged: uplink MAC is cloned onto the bridge, host keeps its identity" = + let + mac = br.config.systemd.services."hermes-bridge-mac".script; + in + has "ip link set dev hermes-br address" mac && has "/sys/class/net/eno1/address" mac; "bridged: uplink is a bridge port" = brNetworks."12-eno1".networkConfig.Bridge == "hermes-br"; "bridged: tap is still a bridge port" = brNetworks."11-vm-hermes".networkConfig.Bridge == "hermes-br";