diff --git a/README.md b/README.md index 60182d3..bdfd2bd 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,8 @@ Supported remote commands: `test`, `dry-activate`, `run`, `switch`, `build` Example: `skyg remote switch bigbox1` deploys using the `hl-bigbox1` profile to `root@bigbox1.lab.internal` +- **`skyg remote reboot-pending`** - Find every remote with a pending reboot (its booted system differs from its current system) and reboot them. Lists the affected hosts and asks for confirmation, then reboots them one at a time, waiting for each to come back online and sending a desktop notification when it does. + #### Home Manager - **`skyg hm switch [profile]`** - Apply home-manager configuration (default: `ari`) diff --git a/bin/ABOUTME.md b/bin/ABOUTME.md index 7a1025c..8c86af6 100644 --- a/bin/ABOUTME.md +++ b/bin/ABOUTME.md @@ -34,6 +34,7 @@ bin/ | `skyg check-flake-versions` | Verify NixOS release version consistency in flake.nix | | `skyg remote boot-all` | Boot all remote hosts (builds on `fwdesk`) | | `skyg remote status` | Check status of all remote hosts | +| `skyg remote reboot-pending` | Reboot every remote host with a pending reboot (`booted-system` != `current-system`). Lists them, asks for confirmation, then reboots one at a time, waiting for each to come back online and sending a desktop notification when it does. | | `skyg secrets` | List known secret names | | `skyg decrypt ` | Decrypt a secret to `.tmp/unencrypted-` | | `skyg encrypt [--yaml]` | Encrypt a working copy back into the secret | diff --git a/bin/lib/cmds.nu b/bin/lib/cmds.nu index a36002d..7c05d76 100644 --- a/bin/lib/cmds.nu +++ b/bin/lib/cmds.nu @@ -348,6 +348,160 @@ Report written to ($report_file)" print $report } +# Poll SSH until the host reaches the desired reachability, or the timeout elapses. +def wait-for-host [target: string, want_online: bool, timeout: int] { + let deadline = (date now) + ($timeout * 1sec) + while (date now) < $deadline { + let online = (try { + bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1" + true + } catch { false }) + if $online == $want_online { + return true + } + sleep 3sec + } + false +} + +# Send a desktop notification if notify-send is available; never fail the caller. +def notify [title: string, body: string] { + try { + ^notify-send $title $body + } catch { } +} + +# Reboot every remote host that is pending a reboot (booted-system != current-system). +# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a +# time, waiting for it to come back online and sending a desktop notification when it does. +export def "skyg remote reboot-pending" [] { + cd $REPO_ROOT + mkdir .tmp + let current_host = "fwbook" + let report_file = ".tmp/reboot-pending-report.txt" + + let all_hosts = remote_targets | where {|h| $h != $current_host } + print $"Checking ($all_hosts | length) remotes for pending reboots..." + + # Phase 1: find hosts that are online and pending a reboot + mut pending = [] + mut offline = [] + mut up_to_date = [] + + for host in $all_hosts { + let target = $"root@($host).lab.internal" + let online = (try { + bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1" + true + } catch { false }) + + if not $online { + print $"(ansi yellow)OFFLINE(ansi reset): ($host)" + $offline = ($offline | append $host) + continue + } + + let info = (try { + let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim + let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim + {booted: $booted, current: $current, pending: ($booted != $current)} + } catch { + {booted: "", current: "", pending: false} + }) + + if $info.pending { + print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)" + $pending = ($pending | append $host) + } else { + print $"(ansi green)UP TO DATE(ansi reset): ($host)" + $up_to_date = ($up_to_date | append $host) + } + } + + if ($pending | length) == 0 { + print "" + print "No remotes are pending a reboot." + return + } + + # Phase 2: confirm before rebooting anything + print "" + print $"(ansi yellow_bold)($pending | length) remote\(s\) pending a reboot:(ansi reset)" + for host in $pending { + print $" - ($host)" + } + print "" + let confirm = (input $"Reboot these ($pending | length) host\(s\)? [y/N]: " | str trim | str downcase) + if not ($confirm in ["y" "yes"]) { + print "Aborted — no hosts were rebooted." + return + } + + # Phase 3: reboot each pending host one at a time, waiting for it to come back + mut rebooted = [] + mut failed = [] + for host in $pending { + let target = $"root@($host).lab.internal" + print $"\n=== Rebooting ($host) ===" + + # Capture the boot id so we can confirm the host actually rebooted. + let boot_id_before = (try { + bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim + } catch { "" }) + + let runcmd = $"ssh -o BatchMode=yes -o ConnectTimeout=5 ($target) 'systemctl reboot'" + print $runcmd + # The SSH connection drops as the host shuts down, so a non-zero exit is + # expected here and does not mean the reboot failed. + try { + bash -c $"($runcmd) 2>/dev/null" + } catch { } + + print $"Waiting for ($host) to shut down..." + let went_down = (wait-for-host $target false 90) + if not $went_down { + print $"(ansi yellow)($host) did not go offline within 90s — continuing to wait for it to come back.(ansi reset)" + } + + print $"Waiting for ($host) to come back online..." + let came_up = (wait-for-host $target true 300) + if $came_up { + let boot_id_after = (try { + bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim + } catch { "" }) + let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after) + if $confirmed { + print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)" + } else { + print $"(ansi green)($host) is back online(ansi reset)" + } + notify $"✅ ($host) rebooted" "Back online and reachable" + $rebooted = ($rebooted | append $host) + } else { + print $"(ansi red)($host) did not come back online within 300s(ansi reset)" + notify $"❌ ($host) did not come back" "Still unreachable after 300s" + $failed = ($failed | append $host) + } + } + + # Write report + let timestamp = date now | format date "%Y-%m-%d %H:%M:%S" + let report = $"Reboot-Pending Report - ($timestamp) + +Total hosts considered: ($all_hosts | length) +Pending reboot: ($pending | length) - ($pending | str join ', ') +Rebooted: ($rebooted | length) - ($rebooted | str join ', ') +Failed to return: ($failed | length) - ($failed | str join ', ') +Offline: ($offline | length) - ($offline | str join ', ') +Up to date: ($up_to_date | length) - ($up_to_date | str join ', ') +" + + $report | save -f $report_file + print $" +Report written to ($report_file)" + print $report +} + # Deploy OpenWrt router configuration export def "skyg openwrt" [ router: string = "glmain",