Files
nixos-setup/modules/nixos/server/robotnix/network.nix
T
2026-10-26 14:36:03 -04:00

305 lines
8.9 KiB
Nix

# robotnix network: one routed virtual switch, per-robot firewall and DNS.
#
# Robots plug into the switch bridge and hold their LAN IP as a /32. The host
# routes to them and answers ARP for their IPs on the uplink (proxy ARP), so
# every packet passes the host firewall. Robots never see the host's netns.
{ config
, lib
, pkgs
, ...
}:
let
cfg = config.skyg.nixos.server.robotnix;
h = import ./helpers.nix { inherit config lib; };
rs = h.robotList;
br = cfg.switch.name;
sw = cfg.switch.address;
set = xs: "{ ${lib.concatStringsSep ", " xs} }";
# Never reachable, whatever the tier says.
neverRanges = [
"0.0.0.0/8"
"127.0.0.0/8"
"224.0.0.0/4"
"240.0.0.0/4"
];
# Private ranges: only via lanAccess / allowLan.
lanRanges = [
"10.0.0.0/8"
"172.16.0.0/12"
"192.168.0.0/16"
"169.254.0.0/16"
"100.64.0.0/10"
];
# Outbound (forwarded) traffic from one robot. Order matters.
outChain =
r:
let
a = r.access;
in
''
chain out_${r.nft} {
# DNS only via the host (port 53 is redirected in prerouting).
meta l4proto { tcp, udp } th dport 853 drop
meta l4proto { tcp, udp } th dport 53 drop
# The redirected DNS request (now dport = the robot's dnsmasq port)
# must pass before the private-range drop.
ip daddr ${sw} meta l4proto { tcp, udp } th dport ${toString r.dnsPort} accept
ip daddr ${set neverRanges} drop
''
+ lib.optionalString (a.blockedCidrs != [ ]) ''
ip daddr ${set a.blockedCidrs} drop
''
+ lib.concatMapStrings
(e: ''
ip daddr ${e.address} ${e.proto} dport ${toString e.port} accept
'')
a.lanAccess
+ (
if a.allowLan then
''
ip daddr ${set lanRanges} accept
''
else
''
ip daddr ${set lanRanges} drop
''
)
+ (
if a.internet == "all" then
''
accept
''
else
lib.optionalString (a.allowedCidrs != [ ]) ''
ip daddr ${set a.allowedCidrs} accept
''
+ ''
ip daddr @allow_${r.nft} accept
drop
''
)
+ ''
}
'';
# Traffic from one robot to the host itself.
inChain = r: ''
chain in_${r.nft} {
meta l4proto { tcp, udp } th dport ${toString r.dnsPort} accept
icmp type echo-request accept
''
+ lib.concatMapStrings
(p: ''
${p.proto} dport ${toString p.port} accept
'')
r.access.hostAccess
+ ''
drop
}
'';
vmap = prefix: set (map (r: "${r.address} : jump ${prefix}_${r.nft}") rs);
# Every robot's own address.
robotIps = set (map (r: r.address) rs);
# Host-side dnsmasq per robot. It is the robot's only resolver.
dnsConf =
r:
let
a = r.access;
servers = d: lib.concatMapStrings (u: "server=/${d}/${u}\n") cfg.dnsUpstreams;
in
pkgs.writeText "robotnix-${r.name}-dnsmasq.conf" (
''
listen-address=${sw}
port=${toString r.dnsPort}
bind-dynamic
no-resolv
no-hosts
no-poll
domain-needed
bogus-priv
user=root
cache-size=1000
# Short TTLs so allowlist IP sets stay fresh after a host reload.
max-ttl=300
max-cache-ttl=300
''
+ lib.concatMapStrings servers cfg.localDomains
+ (
if a.internet == "all" then
lib.concatMapStrings (u: "server=${u}\n") cfg.dnsUpstreams
else
''
# Allowlist: everything else is NXDOMAIN.
address=/#/
''
+ lib.concatMapStrings
(
d: servers d + "nftset=/${d}/4#inet#robotnix#allow_${r.nft}\n"
)
a.allowedDomains
)
+ lib.concatMapStrings (d: "address=/${d}/\n") a.blockedDomains
);
in
lib.mkIf cfg.enable {
# Same keys and priority networking.nat (and hermenix) use.
boot.kernel.sysctl = {
"net.ipv4.conf.all.forwarding" = lib.mkOverride 99 true;
"net.ipv4.conf.default.forwarding" = lib.mkOverride 99 true;
};
# ---------------------------------------------------------------- switch
systemd.network.enable = true;
systemd.network.netdevs."40-${br}".netdevConfig = {
Kind = "bridge";
Name = br;
};
systemd.network.networks."40-${br}" = {
matchConfig.Name = br;
address = [ "${sw}/${toString cfg.switch.prefixLength}" ];
routes = map
(r: {
Destination = "${r.address}/32";
Scope = "link";
})
rs;
networkConfig = {
ConfigureWithoutCarrier = true;
LinkLocalAddressing = "no";
IPv6AcceptRA = false;
};
linkConfig.RequiredForOnline = "no";
};
# Robot ports (nspawn names them vb-<robot>). Isolated: robots can't talk
# to each other at L2, only through the host's rules.
systemd.network.networks."41-${br}-ports" = lib.mkIf (rs != [ ]) {
matchConfig.Name = lib.concatMapStringsSep " " (r: r.port) rs;
networkConfig = {
Bridge = br;
LinkLocalAddressing = "no";
};
bridgeConfig.Isolated = true;
linkConfig.RequiredForOnline = "no";
};
networking.networkmanager.unmanaged = [ "interface-name:${br}" ] ++ map (r: "interface-name:${r.port}") rs;
# -------------------------------------------------------------- firewall
networking.nftables.enable = true;
# L2: each port may only speak IPv4/ARP from its own address.
networking.nftables.tables.robotnix-bridge = lib.mkIf (rs != [ ]) {
family = "bridge";
content = ''
chain prerouting {
type filter hook prerouting priority filter; policy accept;
''
+ lib.concatMapStrings
(r: ''
iifname "${r.port}" ether type != { ip, arp } drop
iifname "${r.port}" ip saddr != ${r.address} drop
iifname "${r.port}" arp saddr ip != ${r.address} drop
'')
rs
+ ''
}
'';
};
# L3. Runs before nixos-fw, so a drop here is final no matter what other
# modules accept. Source addresses are trusted thanks to the bridge table.
networking.nftables.tables.robotnix = lib.mkIf (rs != [ ]) {
family = "inet";
content = ''
${lib.concatMapStrings (r: ''
set allow_${r.nft} { type ipv4_addr; }
'') rs}
chain prerouting {
type nat hook prerouting priority dstnat; policy accept;
${lib.concatMapStrings (r: ''
iifname "${br}" ip saddr ${r.address} meta l4proto { tcp, udp } th dport 53 dnat ip to ${sw}:${toString r.dnsPort}
'') rs}
}
chain forward {
type filter hook forward priority filter - 10; policy accept;
# Traffic meant for a robot goes to that robot, nowhere else (e.g.
# another module's DNAT). Replies are not touched.
ct direction original ct original ip daddr ${robotIps} oifname != "${br}" drop
iifname "${br}" ct state established,related accept
iifname "${br}" meta nfproto ipv6 drop
oifname "${br}" meta nfproto ipv6 drop
# A robot's own connection that someone redirected elsewhere.
iifname "${br}" ct status dnat drop
iifname "${br}" ip saddr vmap ${vmap "out"}
iifname "${br}" drop
# Inbound from the LAN to robots: open.
}
chain input {
type filter hook input priority filter - 10; policy accept;
# Traffic meant for a robot, redirected to the host itself.
ct direction original ct original ip daddr ${robotIps} drop
iifname != "${br}" accept
ct state established,related accept
meta nfproto ipv6 drop
ip saddr vmap ${vmap "in"}
drop
}
${lib.concatMapStrings outChain rs}
${lib.concatMapStrings inChain rs}
'';
};
# The robotnix table already filtered; let nixos-fw pass the rest.
networking.firewall.extraInputRules = ''
iifname "${br}" accept
'';
networking.firewall.extraForwardRules = ''
iifname "${br}" accept
oifname "${br}" accept
'';
# ------------------------------------------------------------------- DNS
systemd.services = lib.listToAttrs (
map
(
r:
lib.nameValuePair "${r.unit}-dns" {
description = "robotnix DNS for ${r.name}";
wantedBy = [ "multi-user.target" ];
after = [
"network.target"
"nftables.service"
];
serviceConfig = {
ExecStart = "${pkgs.dnsmasq}/bin/dnsmasq --keep-in-foreground --pid-file --conf-file=${dnsConf r}";
Restart = "always";
RestartSec = 2;
# Root only for nftset; everything else is locked down.
CapabilityBoundingSet = [
"CAP_NET_ADMIN"
"CAP_NET_BIND_SERVICE"
"CAP_NET_RAW"
"CAP_SETUID"
"CAP_SETGID"
];
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
NoNewPrivileges = true;
};
}
)
rs
);
}