diff --git a/bin/ABOUTME.md b/bin/ABOUTME.md index 6bf8189..89678b6 100644 --- a/bin/ABOUTME.md +++ b/bin/ABOUTME.md @@ -37,10 +37,10 @@ bin/ | `skyg os rollback` | Roll back to previous generation | | `skyg remote switch ` | Deploy to a remote host | | `skyg remote install ` | Fresh-install a remote host via nixos-anywhere, using `hosts/.disko.nix`. Always wipes the disks in that file. Aborts if the file is missing, if the host already boots NixOS from disk (unless `--force-wipe`), or if you do not type the short hostname. | -| `skyg remote status ` | Check status of a single remote host | +| `skyg remote status ` | Check status of a single remote host: booted / current / next-boot generation. `reboot_pending` = booted != next boot (`/nix/var/nix/profiles/system`), `needs_build` = local build != next boot. So hosts deployed with `boot` show up correctly. | | `skyg remote status --all` | Check status of all remote hosts | | `skyg remote boot-all` | Boot all remote hosts | -| `skyg remote reboot-pending` | Reboot every remote host with a pending reboot (`booted-system` != `current-system`). Lists them, asks for confirmation, then reboots one at a time, waiting for each to come back online and sending a desktop notification when it does. | +| `skyg remote reboot-pending` | Reboot every remote host with a pending reboot (`booted-system` != next-boot generation, so `boot` deploys count). Lists them, asks for confirmation, then reboots one at a time. Each host must prove it rebooted (boot id changed) and come back on its next-boot generation, otherwise it is reported as failed. Sends a desktop notification per host. | | `skyg hm switch` | Apply Home Manager config | | `skyg flake update [input]` | Update flake inputs | | `skyg flake check` | Run `nix flake check --no-build` | diff --git a/bin/lib/remote.nu b/bin/lib/remote.nu index 655a4e1..2d384aa 100644 --- a/bin/lib/remote.nu +++ b/bin/lib/remote.nu @@ -212,9 +212,10 @@ export def "skyg remote status-all" [] { # Print each host's status as one block, as soon as it finishes. let booted = (if ($info.booted | is-empty) { "-" } else { $info.booted | path basename }) let current = (if ($info.current | is-empty) { "-" } else { $info.current | path basename }) + let next_boot = (if ($info.next_boot | is-empty) { "-" } else { $info.next_boot | path basename }) let reboot_line = if $info.reboot_pending { $"(ansi yellow)REBOOT PENDING(ansi reset)\n" } else { "" } let build_line = if $info.needs_build { $"(ansi red)NEEDS NEW BUILD(ansi reset)" } else { $"(ansi green)UP TO DATE(ansi reset)" } - print $"($host) | Booted: ($booted) | Current: ($current)" + print $"($host) | Booted: ($booted) | Current: ($current) | Next boot: ($next_boot)" $info }) @@ -222,7 +223,7 @@ export def "skyg remote status-all" [] { return ($results | select host status reboot_pending needs_build) } -# Reboot every remote host that is pending a reboot (booted-system != current-system). +# Reboot every remote host that is pending a reboot (booted-system != next-boot generation). # Detects pending hosts, lists them, asks for confirmation, then reboots each one at a # time, waiting for it to come back online and sending a desktop notification when it does. export def "skyg remote reboot-pending" [] { @@ -253,15 +254,14 @@ export def "skyg remote reboot-pending" [] { } let info = (try { - let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim - let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim - {booted: $booted, current: $current, pending: ($booted != $current)} + let s = (get-boot-state $target) + {booted: $s.booted, next_boot: $s.next_boot, pending: ($s.booted != "" and $s.booted != $s.next_boot)} } catch { - {booted: "", current: "", pending: false} + {booted: "", next_boot: "", pending: false} }) if $info.pending { - print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)" + print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) next=($info.next_boot | path basename)" $pending = ($pending | append $host) } else { print $"(ansi green)UP TO DATE(ansi reset): ($host)" @@ -320,12 +320,31 @@ export def "skyg remote reboot-pending" [] { let boot_id_after = (try { bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim } catch { "" }) - let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after) - if $confirmed { - print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)" + # Proof of reboot: boot id changed. If we can't read it, fall back to "it went offline". + let did_reboot = if $boot_id_before != "" and $boot_id_after != "" { + $boot_id_before != $boot_id_after } else { - print $"(ansi green)($host) is back online(ansi reset)" + $went_down } + if not $did_reboot { + print $"(ansi red)($host) answered but did NOT reboot (same boot id, never went offline)(ansi reset)" + notify $"❌ ($host) did not reboot" "Reboot command had no effect" + $failed = ($failed | append $host) + continue + } + + # Check the host really came up on the generation we wanted. + let after = (try { get-boot-state $target } catch { null }) + if $after != null and $after.booted != $after.next_boot { + print $"(ansi red)($host) rebooted but is NOT on its latest generation(ansi reset)" + print $" booted=($after.booted | path basename)" + print $" wanted=($after.next_boot | path basename)" + notify $"⚠️ ($host) wrong generation" "Rebooted but not on the latest generation" + $failed = ($failed | append $host) + continue + } + + print $"(ansi green)($host) rebooted and is back online(ansi reset)" notify $"✅ ($host) rebooted" "Back online and reachable" $rebooted = ($rebooted | append $host) } else { @@ -342,7 +361,7 @@ export def "skyg remote reboot-pending" [] { Total hosts considered: ($all_hosts | length) Pending reboot: ($pending | length) - ($pending | str join ', ') Rebooted: ($rebooted | length) - ($rebooted | str join ', ') -Failed to return: ($failed | length) - ($failed | str join ', ') +Failed: ($failed | length) - ($failed | str join ', ') Offline: ($offline | length) - ($offline | str join ', ') Up to date: ($up_to_date | length) - ($up_to_date | str join ', ') " diff --git a/bin/lib/remote.utils.nu b/bin/lib/remote.utils.nu index 45bc1d3..cfb2e3c 100644 --- a/bin/lib/remote.utils.nu +++ b/bin/lib/remote.utils.nu @@ -1,6 +1,19 @@ use types.nu * use profile.nu * +# Read the 3 system paths on a remote host: +# booted = what it booted from +# current = what is running now (changed by switch/test) +# next_boot = what the next reboot will use (changed by switch/boot) +export def get-boot-state [target: string] { + let lines = (bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system; readlink /run/current-system; readlink -f /nix/var/nix/profiles/system' 2>/dev/null" | lines) + let booted = ($lines | get 0? | default "") + let current = ($lines | get 1? | default "") + let profile = ($lines | get 2? | default "") + let next_boot = if $profile == "" { $current } else { $profile } + {booted: $booted, current: $current, next_boot: $next_boot} +} + export def get-remote-boot-info [ host: string@remote_targets ] { let target = $"root@($host).lab.internal" let is_online = (try { @@ -8,13 +21,13 @@ export def get-remote-boot-info [ host: string@remote_targets ] { true } catch { false }) if (not $is_online) { - return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""} + return {host: $host, status: "offline", booted: "", current: "", next_boot: "", reboot_pending: false, needs_build: false, local_toplevel: ""} } return (try { # Fetch remote system paths and the local toplevel concurrently. let fetched = (["remote" "local"] | par-each {|what| if $what == "remote" { - bash -c $"ssh ($target) 'readlink /run/booted-system; readlink /run/current-system' 2>/dev/null" | lines + get-boot-state $target } else { try { bash -c $"nix eval --raw '.#nixosConfigurations.hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true" | str trim @@ -22,16 +35,19 @@ export def get-remote-boot-info [ host: string@remote_targets ] { } }) - let booted = ($fetched.0 | get 0? | default "") - let current = ($fetched.0 | get 1? | default "") + let state = $fetched.0 + let booted = $state.booted + let current = $state.current + let next_boot = $state.next_boot let local_toplevel = $fetched.1 - let reboot_pending = ($booted != $current) - let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $current } + # A reboot only matters if it would land on a different system than we booted. + let reboot_pending = ($booted != "" and $booted != $next_boot) + let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $next_boot } - {host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel} + {host: $host, status: "online", booted: $booted, current: $current, next_boot: $next_boot, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel} } catch { - {host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""} + {host: $host, status: "error", booted: "", current: "", next_boot: "", reboot_pending: false, needs_build: false, local_toplevel: ""} }) }