fixing boot tracking

This commit is contained in:
ari committed 2026-10-19 15:49:19 -04:00
1 parent 4f3a95b42c
commit 53c41cf826
3 files changed
+57 -22

No files matched your search

+2 -2
View File
@@ -37,10 +37,10 @@ bin/
| `skyg os rollback` | Roll back to previous generation |
| `skyg remote switch <host>` | Deploy to a remote host |
| `skyg remote install <host>` | Fresh-install a remote host via nixos-anywhere, using `hosts/<profile>.disko.nix`. Always wipes the disks in that file. Aborts if the file is missing, if the host already boots NixOS from disk (unless `--force-wipe`), or if you do not type the short hostname. |
| `skyg remote status <host>` | Check status of a single remote host |
| `skyg remote status <host>` | Check status of a single remote host: booted / current / next-boot generation. `reboot_pending` = booted != next boot (`/nix/var/nix/profiles/system`), `needs_build` = local build != next boot. So hosts deployed with `boot` show up correctly. |
| `skyg remote status --all` | Check status of all remote hosts |
| `skyg remote boot-all` | Boot all remote hosts |
| `skyg remote reboot-pending` | Reboot every remote host with a pending reboot (`booted-system` != `current-system`). Lists them, asks for confirmation, then reboots one at a time, waiting for each to come back online and sending a desktop notification when it does. |
| `skyg remote reboot-pending` | Reboot every remote host with a pending reboot (`booted-system` != next-boot generation, so `boot` deploys count). Lists them, asks for confirmation, then reboots one at a time. Each host must prove it rebooted (boot id changed) and come back on its next-boot generation, otherwise it is reported as failed. Sends a desktop notification per host. |
| `skyg hm switch` | Apply Home Manager config |
| `skyg flake update [input]` | Update flake inputs |
| `skyg flake check` | Run `nix flake check --no-build` |
+31 -12
View File
@@ -212,9 +212,10 @@ export def "skyg remote status-all" [] {
# Print each host's status as one block, as soon as it finishes.
let booted = (if ($info.booted | is-empty) { "-" } else { $info.booted | path basename })
let current = (if ($info.current | is-empty) { "-" } else { $info.current | path basename })
let next_boot = (if ($info.next_boot | is-empty) { "-" } else { $info.next_boot | path basename })
let reboot_line = if $info.reboot_pending { $"(ansi yellow)REBOOT PENDING(ansi reset)\n" } else { "" }
let build_line = if $info.needs_build { $"(ansi red)NEEDS NEW BUILD(ansi reset)" } else { $"(ansi green)UP TO DATE(ansi reset)" }
print $"($host) | Booted: ($booted) | Current: ($current)"
print $"($host) | Booted: ($booted) | Current: ($current) | Next boot: ($next_boot)"
$info
})
@@ -222,7 +223,7 @@ export def "skyg remote status-all" [] {
return ($results | select host status reboot_pending needs_build)
}
# Reboot every remote host that is pending a reboot (booted-system != current-system).
# Reboot every remote host that is pending a reboot (booted-system != next-boot generation).
# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a
# time, waiting for it to come back online and sending a desktop notification when it does.
export def "skyg remote reboot-pending" [] {
@@ -253,15 +254,14 @@ export def "skyg remote reboot-pending" [] {
}
let info = (try {
let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
{booted: $booted, current: $current, pending: ($booted != $current)}
let s = (get-boot-state $target)
{booted: $s.booted, next_boot: $s.next_boot, pending: ($s.booted != "" and $s.booted != $s.next_boot)}
} catch {
{booted: "", current: "", pending: false}
{booted: "", next_boot: "", pending: false}
})
if $info.pending {
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)"
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) next=($info.next_boot | path basename)"
$pending = ($pending | append $host)
} else {
print $"(ansi green)UP TO DATE(ansi reset): ($host)"
@@ -320,12 +320,31 @@ export def "skyg remote reboot-pending" [] {
let boot_id_after = (try {
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
} catch { "" })
let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after)
if $confirmed {
print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)"
# Proof of reboot: boot id changed. If we can't read it, fall back to "it went offline".
let did_reboot = if $boot_id_before != "" and $boot_id_after != "" {
$boot_id_before != $boot_id_after
} else {
print $"(ansi green)($host) is back online(ansi reset)"
$went_down
}
if not $did_reboot {
print $"(ansi red)($host) answered but did NOT reboot (same boot id, never went offline)(ansi reset)"
notify $"❌ ($host) did not reboot" "Reboot command had no effect"
$failed = ($failed | append $host)
continue
}
# Check the host really came up on the generation we wanted.
let after = (try { get-boot-state $target } catch { null })
if $after != null and $after.booted != $after.next_boot {
print $"(ansi red)($host) rebooted but is NOT on its latest generation(ansi reset)"
print $" booted=($after.booted | path basename)"
print $" wanted=($after.next_boot | path basename)"
notify $"⚠️ ($host) wrong generation" "Rebooted but not on the latest generation"
$failed = ($failed | append $host)
continue
}
print $"(ansi green)($host) rebooted and is back online(ansi reset)"
notify $"✅ ($host) rebooted" "Back online and reachable"
$rebooted = ($rebooted | append $host)
} else {
@@ -342,7 +361,7 @@ export def "skyg remote reboot-pending" [] {
Total hosts considered: ($all_hosts | length)
Pending reboot: ($pending | length) - ($pending | str join ', ')
Rebooted: ($rebooted | length) - ($rebooted | str join ', ')
Failed to return: ($failed | length) - ($failed | str join ', ')
Failed: ($failed | length) - ($failed | str join ', ')
Offline: ($offline | length) - ($offline | str join ', ')
Up to date: ($up_to_date | length) - ($up_to_date | str join ', ')
"
+24 -8
View File
@@ -1,6 +1,19 @@
use types.nu *
use profile.nu *
# Read the 3 system paths on a remote host:
# booted = what it booted from
# current = what is running now (changed by switch/test)
# next_boot = what the next reboot will use (changed by switch/boot)
export def get-boot-state [target: string] {
let lines = (bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system; readlink /run/current-system; readlink -f /nix/var/nix/profiles/system' 2>/dev/null" | lines)
let booted = ($lines | get 0? | default "")
let current = ($lines | get 1? | default "")
let profile = ($lines | get 2? | default "")
let next_boot = if $profile == "" { $current } else { $profile }
{booted: $booted, current: $current, next_boot: $next_boot}
}
export def get-remote-boot-info [ host: string@remote_targets ] {
let target = $"root@($host).lab.internal"
let is_online = (try {
@@ -8,13 +21,13 @@ export def get-remote-boot-info [ host: string@remote_targets ] {
true
} catch { false })
if (not $is_online) {
return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
return {host: $host, status: "offline", booted: "", current: "", next_boot: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
}
return (try {
# Fetch remote system paths and the local toplevel concurrently.
let fetched = (["remote" "local"] | par-each {|what|
if $what == "remote" {
bash -c $"ssh ($target) 'readlink /run/booted-system; readlink /run/current-system' 2>/dev/null" | lines
get-boot-state $target
} else {
try {
bash -c $"nix eval --raw '.#nixosConfigurations.hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true" | str trim
@@ -22,16 +35,19 @@ export def get-remote-boot-info [ host: string@remote_targets ] {
}
})
let booted = ($fetched.0 | get 0? | default "")
let current = ($fetched.0 | get 1? | default "")
let state = $fetched.0
let booted = $state.booted
let current = $state.current
let next_boot = $state.next_boot
let local_toplevel = $fetched.1
let reboot_pending = ($booted != $current)
let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $current }
# A reboot only matters if it would land on a different system than we booted.
let reboot_pending = ($booted != "" and $booted != $next_boot)
let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $next_boot }
{host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel}
{host: $host, status: "online", booted: $booted, current: $current, next_boot: $next_boot, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel}
} catch {
{host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
{host: $host, status: "error", booted: "", current: "", next_boot: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
})
}