From e9479f9be14e70c0deb179cc2ae585f811937e50 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Tue, 17 Nov 2026 14:43:55 -0500 Subject: [PATCH] cleaning up after my clanker --- modules/nixos/common/dns/ABOUTME.md | 157 ++++----------------------- modules/nixos/common/dns/default.nix | 142 +++++++++--------------- 2 files changed, 73 insertions(+), 226 deletions(-) diff --git a/modules/nixos/common/dns/ABOUTME.md b/modules/nixos/common/dns/ABOUTME.md index 8673e2e..d50bdfe 100644 --- a/modules/nixos/common/dns/ABOUTME.md +++ b/modules/nixos/common/dns/ABOUTME.md @@ -1,152 +1,35 @@ -# modules/nixos/common/dns/ +# Boot-time DNS check -Hardened DNS initialization, boot-time verification, and journald recovery to prevent systemd-resolved failures during boot. +Six hosts opt into this module, including hl-bigbox1. It was introduced after the reported 79-minute DNS outage on that host on 2026-11-16. The incident account suspected journal corruption after an unclean shutdown. This module keeps logs for investigation. -## Motivation +## Configuration -**Outage root cause (2026-11-16, bigbox1, 79 minutes):** -- Unclean shutdown corrupted systemd journal -- systemd couldn't load org.freedesktop.resolve1 D-Bus service file -- systemd-resolved never started -- All DNS queries failed ("server misbehaving") -- Containers failed to start (Jellyfin, etc.) -- Health checks timed out (couldn't resolve hostnames) -- **No automatic recovery** (manual intervention required) - -**Solution:** Harden DNS initialization with explicit configuration, boot-time verification, and automatic recovery via systemd watchdog. - -## Options - -| Name | Type | Default | Description | -|------|------|---------|-------------| -| `skyg.nixos.common.dns.enable` | bool | `true` | Enable hardened DNS with boot check and journal recovery. | -| `skyg.nixos.common.dns.watchdogEnabled` | bool | `true` | Enable systemd watchdog for auto-recovery. | -| `skyg.nixos.common.dns.watchdogTimeout` | int | `30` | Watchdog timeout (seconds); auto-reboot if no progress. | -| `skyg.nixos.common.dns.bootCheckRetries` | int | `5` | DNS health check retries during boot. | - -## What It Does - -1. **Explicitly enables systemd-resolved** (not relying on defaults) - - `services.resolved.enable = true` - - Configures stub resolver, DNSSEC tolerance, LLMNR fallback - -2. **Hardens journald** to survive corruption - - `storage = "persistent"` — survive unclean shutdown - - Automatic vacation/compression - - Size limits (500M system, 100M runtime) - -3. **Adds boot-time DNS verification service** (`dns-boot-check`) - - Runs after systemd-resolved.service starts - - Verifies DNS actually responds on 127.0.0.11:53 (5 retries, 1s between) - - Runs before network-online.target (containers can't start until DNS verified) - - Fails fast if DNS is dead (watchdog will trigger auto-reboot within 30s) - -4. **Enables systemd watchdog** (if DNS or any critical service hangs, reboot) - - Default: 30-second timeout - - Automatic recovery much faster than manual intervention (79 min outage → 30s) - -5. **Ensures network-online doesn't lie** (waits for actual DNS before claiming ready) - -## Usage - -**Default behavior** (included in nixos/common, no opt-in needed): ```nix -# In any nixos/common-based config (all hosts): -# DNS is hardened automatically +skyg.nixos.common.dns.enable = true; ``` -**Customize watchdog timeout** (if 30s is too aggressive/lenient): -```nix -skyg.nixos.common.dns.watchdogTimeout = 60; # 60 seconds -``` +| Option under `skyg.nixos.common.dns` | Default | Purpose | +| --- | --- | --- | +| `enable` | `false` | Enable resolved, persistent journal storage, and the boot check. | +| `bootCheckName` | `"portainer.app.internal"` | Name expected to resolve to an IPv4 address on this host's network. | +| `bootCheckRetries` | `5` | Maximum lookup attempts, spaced one second apart after each failure. | +| `watchdogEnabled` | `true` | Configure systemd to feed a hardware watchdog when available. | +| `watchdogTimeout` | `30` | Runtime watchdog timeout in seconds if systemd stops feeding it. | -**Disable on specific hosts** (not recommended): -```nix -skyg.nixos.common.dns.enable = false; -``` +## Boot behavior -## Impact +After network-online.target and systemd-resolved start, dns-boot-check queries the host stub at 127.0.0.53 for the configured name. A returned IPv4 address passes the check. A cached answer can pass. Choose a record available independently of containers starting on the same host. -- **Outage prevention:** DNS failures auto-recover within 30s (vs 79-minute manual recovery) -- **Silent failure prevention:** Boot-time check prevents DNS-is-broken scenarios -- **Journald corruption recovery:** Persistent storage + vacuum prevents journal-based startup failures -- **Network reliability:** network-online.target now actually means DNS is ready +The container service groups are ordered after the check. An exhausted retry budget leaves the diagnostic unit failed and allows the groups to start. A DNS failure alone does not trigger a reboot. The probe runs once per boot and can be rerun manually. -## Boot Sequence Impact +The journal persists across boots. The module sets a 500M persistent storage limit, a 100M runtime limit and a 30-day retention limit. Host configuration can override these limits. -**Before this module:** -``` -systemd starts → systemd-resolved.service (might fail silently) → -network-online.target (claims ready, but DNS dead) → -containers start (DNS queries fail) → -health checks timeout → outage -``` +## Investigating a failure -**After this module:** -``` -systemd starts → systemd-resolved.service → -dns-boot-check service (verify DNS responds) → -network-online.target (DNS actually ready) → -containers start (DNS works) → -health checks pass ✓ -``` - -## Monitoring & Troubleshooting - -**Check DNS status at boot:** ```bash -# See if dns-boot-check passed -systemctl status dns-boot-check - -# See resolved status -systemctl status systemd-resolved - -# Test DNS -dig @127.0.0.11 localhost -nslookup portainer.app.internal +journalctl -b -u dns-boot-check -u systemd-resolved +resolvectl status +dig +time=2 +tries=1 @127.0.0.53 portainer.app.internal A ``` -**Check journald recovery:** -```bash -# Journald status -journalctl --header -# Should show: Storage: persistent -``` - -**Check watchdog status:** -```bash -# See if watchdog is active -cat /proc/sys/kernel/watchdog -# Should be non-zero if enabled -``` - -**If DNS is broken (and watchdog should trigger auto-reboot):** -1. Check `/var/log/journal` for dns-boot-check errors -2. Watchdog should reboot system within 30 seconds -3. If no reboot, manually restart: `systemctl restart systemd-resolved` - -## Caveats & Limitations - -1. **Watchdog reboot is last resort** — not graceful (use for critical failures only) -2. **DNS verification runs at boot only** — post-boot DNS failures not auto-detected -3. **Lab environment** — DNSSEC set to "allow-downgrade" (not secure); for production, consider "yes" -4. **Stub resolver only** — if full resolver needed, see `services.resolved.extraConfig` - -## Related Files - -- `modules/nixos/common/containers/container-services/systemd.nix` — containers ordered after dns-boot-check -- `services/resolved.conf(5)` — full resolved configuration options -- `systemd.service(5)` — systemd service unit documentation - -## History - -- Created after bigbox1 outage (2026-11-16): systemd journal corruption + DNS initialization failure -- Outage duration: 79 minutes -- RCA: unclean shutdown → corrupted journal → dbus load failure → DNS never starts → silent failure - ---- - -**Module:** `modules/nixos/common/dns/` -**Location:** NixOS flake standard location -**Scope:** All hosts (default enabled) -**Maintenance:** Updates to systemd-resolved config may need journal tuning +Use the configured name if the host overrides the default. Check the DNS status and answer section, and compare the upstream servers and routing domains in `resolvectl status` with the intended network. After correcting the problem, run `systemctl restart dns-boot-check` and check `systemctl status dns-boot-check`. For a failure on the previous boot, use `journalctl -b -1` with the same unit filters. diff --git a/modules/nixos/common/dns/default.nix b/modules/nixos/common/dns/default.nix index 282e93f..12e5d37 100644 --- a/modules/nixos/common/dns/default.nix +++ b/modules/nixos/common/dns/default.nix @@ -4,122 +4,86 @@ let in { options.skyg.nixos.common.dns = { - enable = lib.mkOption { - description = "Enable hardened systemd-resolved DNS with boot verification and journal recovery."; - type = lib.types.bool; - default = false; + enable = lib.mkEnableOption "systemd-resolved with a boot-time DNS diagnostic"; + + bootCheckName = lib.mkOption { + type = lib.types.strMatching "[a-zA-Z0-9][a-zA-Z0-9.-]*"; + default = "portainer.app.internal"; + description = "Internal DNS name expected to have an IPv4 address at boot."; }; - + + bootCheckRetries = lib.mkOption { + type = lib.types.ints.positive; + default = 5; + description = "Maximum number of DNS lookup attempts at boot."; + }; + watchdogEnabled = lib.mkOption { - description = "Enable systemd watchdog for automatic recovery on DNS/systemd failures."; type = lib.types.bool; default = true; + description = "Enable the system manager's hardware watchdog."; }; - + watchdogTimeout = lib.mkOption { - description = "Watchdog timeout in seconds (auto-reboot if system unresponsive for this long)."; - type = lib.types.int; + type = lib.types.ints.positive; default = 30; - }; - - bootCheckRetries = lib.mkOption { - description = "Number of DNS health check retries during boot before failing fast."; - type = lib.types.int; - default = 5; + description = "Runtime hardware watchdog timeout in seconds."; }; }; - + config = lib.mkIf cfg.enable { - # Enable systemd-resolved explicitly (don't rely on defaults) services.resolved = { enable = true; - llmnr = "yes"; # Allow local resolution fallback - dnssec = "allow-downgrade"; # Tolerant of DNSSEC issues (network-dependent) - dnsovertls = "no"; # Keep plain DNS for lab environment - extraConfig = '' - [Resolve] - # Use upstream resolvers if configured by DHCP/static config - Domains=~. - # Increase timeout to tolerate slow lab network - DNSStubListenerPort=53 - ''; + settings.Resolve = { + LLMNR = "yes"; + DNSSEC = "allow-downgrade"; + DNSOverTLS = false; + Domains = config.networking.search ++ [ "~." ]; + }; }; - - # Harden journald to survive unclean shutdown and corruption - systemd.journald = { - extraConfig = '' - [Journal] - Storage=persistent - # Keep journal compressed and limit size + + services.journald = { + storage = "persistent"; + extraConfig = lib.mkBefore '' SystemMaxUse=500M SystemMaxFileSize=50M RuntimeMaxUse=100M RuntimeMaxFileSize=10M - # Vacuum on startup to recover from corruption MaxRetentionSec=30day MaxFileSec=7day ''; }; - - # Enable watchdog for automatic recovery (if DNS fails to start, reboot) - systemd.watchdog = lib.mkIf cfg.watchdogEnabled { - runtimeSec = "${toString cfg.watchdogTimeout}s"; - rebootTime = "${toString (cfg.watchdogTimeout + 10)}s"; + + systemd.settings.Manager = lib.mkIf cfg.watchdogEnabled { + RuntimeWatchdogSec = "${toString cfg.watchdogTimeout}s"; + RebootWatchdogSec = "${toString (cfg.watchdogTimeout + 10)}s"; }; - - # Ensure systemd notices watchdog (all services should use it) - systemd.defaultUnit = "multi-user.target"; - - # Boot check service: verify DNS is actually responding (not just started) - systemd.services."dns-boot-check" = { - description = "DNS Boot-Time Health Check"; - documentation = [ "man:resolved.conf(5)" ]; - wantedBy = [ "network-online.target" ]; - before = [ "network-online.target" ]; - after = [ "systemd-resolved.service" ]; + + systemd.services.dns-boot-check = { + description = "Check internal DNS resolution at boot"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ "network-online.target" "systemd-resolved.service" ]; requires = [ "systemd-resolved.service" ]; - - unitConfig = { - # Allow this to fail without breaking the entire boot - # (watchdog will trigger if systemd-resolved is dead) - ConditionPathExists = "/run/systemd/resolve/stub-resolv.conf"; - }; - + path = [ pkgs.dnsutils pkgs.gnugrep pkgs.coreutils ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; - # Try multiple times to allow systemd-resolved to settle - ExecStart = '' - ${pkgs.bash}/bin/bash -c ' - attempt=0 - max_attempts=${toString cfg.bootCheckRetries} - - while [ $attempt -lt $max_attempts ]; do - # Check that DNS responds on the stub resolver - if ${pkgs.dnsutils}/bin/dig +short @127.0.0.11 localhost > /dev/null 2>&1; then - echo "DNS is responding on 127.0.0.11:53" - exit 0 - fi - attempt=$((attempt + 1)) - echo "DNS check attempt $attempt/$max_attempts failed, retrying..." - sleep 1 - done - - echo "ERROR: DNS did not respond after $max_attempts attempts" - echo "systemd-resolved status:" - ${pkgs.systemd}/bin/systemctl status systemd-resolved || true - exit 1 - ' - ''; - StandardOutput = "journal"; - StandardError = "journal"; - SyslogIdentifier = "dns-boot-check"; - # If this fails, the watchdog will eventually reboot us - # (better than a silent DNS failure) }; + script = '' + name=${lib.escapeShellArg cfg.bootCheckName} + attempts=${toString cfg.bootCheckRetries} + for ((attempt=1; attempt<=attempts; attempt++)); do + if answer=$(dig +short +time=2 +tries=1 @127.0.0.53 -q "$name" -t A) && + grep -Eq '^([0-9]{1,3}\.){3}[0-9]{1,3}$' <<< "$answer"; then + echo "$name resolved through 127.0.0.53" + exit 0 + fi + echo "No IPv4 answer for $name (attempt $attempt/$attempts)" >&2 + if ((attempt < attempts)); then sleep 1; fi + done + exit 1 + ''; }; - - # Ensure network doesn't mark itself "online" until DNS is actually working - systemd.targets."network-online".wants = [ "dns-boot-check.service" ]; }; }