fixing boot tracking
This commit is contained in:
1 parent
4f3a95b42c
commit
53c41cf826
3 files changed
+57
-22
No files matched your search
+2
-2
@@ -37,10 +37,10 @@ bin/
|
||||
| `skyg os rollback` | Roll back to previous generation |
|
||||
| `skyg remote switch <host>` | Deploy to a remote host |
|
||||
| `skyg remote install <host>` | Fresh-install a remote host via nixos-anywhere, using `hosts/<profile>.disko.nix`. Always wipes the disks in that file. Aborts if the file is missing, if the host already boots NixOS from disk (unless `--force-wipe`), or if you do not type the short hostname. |
|
||||
| `skyg remote status <host>` | Check status of a single remote host |
|
||||
| `skyg remote status <host>` | Check status of a single remote host: booted / current / next-boot generation. `reboot_pending` = booted != next boot (`/nix/var/nix/profiles/system`), `needs_build` = local build != next boot. So hosts deployed with `boot` show up correctly. |
|
||||
| `skyg remote status --all` | Check status of all remote hosts |
|
||||
| `skyg remote boot-all` | Boot all remote hosts |
|
||||
| `skyg remote reboot-pending` | Reboot every remote host with a pending reboot (`booted-system` != `current-system`). Lists them, asks for confirmation, then reboots one at a time, waiting for each to come back online and sending a desktop notification when it does. |
|
||||
| `skyg remote reboot-pending` | Reboot every remote host with a pending reboot (`booted-system` != next-boot generation, so `boot` deploys count). Lists them, asks for confirmation, then reboots one at a time. Each host must prove it rebooted (boot id changed) and come back on its next-boot generation, otherwise it is reported as failed. Sends a desktop notification per host. |
|
||||
| `skyg hm switch` | Apply Home Manager config |
|
||||
| `skyg flake update [input]` | Update flake inputs |
|
||||
| `skyg flake check` | Run `nix flake check --no-build` |
|
||||
|
||||
+31
-12
@@ -212,9 +212,10 @@ export def "skyg remote status-all" [] {
|
||||
# Print each host's status as one block, as soon as it finishes.
|
||||
let booted = (if ($info.booted | is-empty) { "-" } else { $info.booted | path basename })
|
||||
let current = (if ($info.current | is-empty) { "-" } else { $info.current | path basename })
|
||||
let next_boot = (if ($info.next_boot | is-empty) { "-" } else { $info.next_boot | path basename })
|
||||
let reboot_line = if $info.reboot_pending { $"(ansi yellow)REBOOT PENDING(ansi reset)\n" } else { "" }
|
||||
let build_line = if $info.needs_build { $"(ansi red)NEEDS NEW BUILD(ansi reset)" } else { $"(ansi green)UP TO DATE(ansi reset)" }
|
||||
print $"($host) | Booted: ($booted) | Current: ($current)"
|
||||
print $"($host) | Booted: ($booted) | Current: ($current) | Next boot: ($next_boot)"
|
||||
|
||||
$info
|
||||
})
|
||||
@@ -222,7 +223,7 @@ export def "skyg remote status-all" [] {
|
||||
return ($results | select host status reboot_pending needs_build)
|
||||
}
|
||||
|
||||
# Reboot every remote host that is pending a reboot (booted-system != current-system).
|
||||
# Reboot every remote host that is pending a reboot (booted-system != next-boot generation).
|
||||
# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a
|
||||
# time, waiting for it to come back online and sending a desktop notification when it does.
|
||||
export def "skyg remote reboot-pending" [] {
|
||||
@@ -253,15 +254,14 @@ export def "skyg remote reboot-pending" [] {
|
||||
}
|
||||
|
||||
let info = (try {
|
||||
let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
|
||||
let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
|
||||
{booted: $booted, current: $current, pending: ($booted != $current)}
|
||||
let s = (get-boot-state $target)
|
||||
{booted: $s.booted, next_boot: $s.next_boot, pending: ($s.booted != "" and $s.booted != $s.next_boot)}
|
||||
} catch {
|
||||
{booted: "", current: "", pending: false}
|
||||
{booted: "", next_boot: "", pending: false}
|
||||
})
|
||||
|
||||
if $info.pending {
|
||||
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)"
|
||||
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) next=($info.next_boot | path basename)"
|
||||
$pending = ($pending | append $host)
|
||||
} else {
|
||||
print $"(ansi green)UP TO DATE(ansi reset): ($host)"
|
||||
@@ -320,12 +320,31 @@ export def "skyg remote reboot-pending" [] {
|
||||
let boot_id_after = (try {
|
||||
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
|
||||
} catch { "" })
|
||||
let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after)
|
||||
if $confirmed {
|
||||
print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)"
|
||||
# Proof of reboot: boot id changed. If we can't read it, fall back to "it went offline".
|
||||
let did_reboot = if $boot_id_before != "" and $boot_id_after != "" {
|
||||
$boot_id_before != $boot_id_after
|
||||
} else {
|
||||
print $"(ansi green)($host) is back online(ansi reset)"
|
||||
$went_down
|
||||
}
|
||||
if not $did_reboot {
|
||||
print $"(ansi red)($host) answered but did NOT reboot (same boot id, never went offline)(ansi reset)"
|
||||
notify $"❌ ($host) did not reboot" "Reboot command had no effect"
|
||||
$failed = ($failed | append $host)
|
||||
continue
|
||||
}
|
||||
|
||||
# Check the host really came up on the generation we wanted.
|
||||
let after = (try { get-boot-state $target } catch { null })
|
||||
if $after != null and $after.booted != $after.next_boot {
|
||||
print $"(ansi red)($host) rebooted but is NOT on its latest generation(ansi reset)"
|
||||
print $" booted=($after.booted | path basename)"
|
||||
print $" wanted=($after.next_boot | path basename)"
|
||||
notify $"⚠️ ($host) wrong generation" "Rebooted but not on the latest generation"
|
||||
$failed = ($failed | append $host)
|
||||
continue
|
||||
}
|
||||
|
||||
print $"(ansi green)($host) rebooted and is back online(ansi reset)"
|
||||
notify $"✅ ($host) rebooted" "Back online and reachable"
|
||||
$rebooted = ($rebooted | append $host)
|
||||
} else {
|
||||
@@ -342,7 +361,7 @@ export def "skyg remote reboot-pending" [] {
|
||||
Total hosts considered: ($all_hosts | length)
|
||||
Pending reboot: ($pending | length) - ($pending | str join ', ')
|
||||
Rebooted: ($rebooted | length) - ($rebooted | str join ', ')
|
||||
Failed to return: ($failed | length) - ($failed | str join ', ')
|
||||
Failed: ($failed | length) - ($failed | str join ', ')
|
||||
Offline: ($offline | length) - ($offline | str join ', ')
|
||||
Up to date: ($up_to_date | length) - ($up_to_date | str join ', ')
|
||||
"
|
||||
|
||||
+24
-8
@@ -1,6 +1,19 @@
|
||||
use types.nu *
|
||||
use profile.nu *
|
||||
|
||||
# Read the 3 system paths on a remote host:
|
||||
# booted = what it booted from
|
||||
# current = what is running now (changed by switch/test)
|
||||
# next_boot = what the next reboot will use (changed by switch/boot)
|
||||
export def get-boot-state [target: string] {
|
||||
let lines = (bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system; readlink /run/current-system; readlink -f /nix/var/nix/profiles/system' 2>/dev/null" | lines)
|
||||
let booted = ($lines | get 0? | default "")
|
||||
let current = ($lines | get 1? | default "")
|
||||
let profile = ($lines | get 2? | default "")
|
||||
let next_boot = if $profile == "" { $current } else { $profile }
|
||||
{booted: $booted, current: $current, next_boot: $next_boot}
|
||||
}
|
||||
|
||||
export def get-remote-boot-info [ host: string@remote_targets ] {
|
||||
let target = $"root@($host).lab.internal"
|
||||
let is_online = (try {
|
||||
@@ -8,13 +21,13 @@ export def get-remote-boot-info [ host: string@remote_targets ] {
|
||||
true
|
||||
} catch { false })
|
||||
if (not $is_online) {
|
||||
return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||
return {host: $host, status: "offline", booted: "", current: "", next_boot: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||
}
|
||||
return (try {
|
||||
# Fetch remote system paths and the local toplevel concurrently.
|
||||
let fetched = (["remote" "local"] | par-each {|what|
|
||||
if $what == "remote" {
|
||||
bash -c $"ssh ($target) 'readlink /run/booted-system; readlink /run/current-system' 2>/dev/null" | lines
|
||||
get-boot-state $target
|
||||
} else {
|
||||
try {
|
||||
bash -c $"nix eval --raw '.#nixosConfigurations.hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true" | str trim
|
||||
@@ -22,16 +35,19 @@ export def get-remote-boot-info [ host: string@remote_targets ] {
|
||||
}
|
||||
})
|
||||
|
||||
let booted = ($fetched.0 | get 0? | default "")
|
||||
let current = ($fetched.0 | get 1? | default "")
|
||||
let state = $fetched.0
|
||||
let booted = $state.booted
|
||||
let current = $state.current
|
||||
let next_boot = $state.next_boot
|
||||
let local_toplevel = $fetched.1
|
||||
|
||||
let reboot_pending = ($booted != $current)
|
||||
let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $current }
|
||||
# A reboot only matters if it would land on a different system than we booted.
|
||||
let reboot_pending = ($booted != "" and $booted != $next_boot)
|
||||
let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $next_boot }
|
||||
|
||||
{host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel}
|
||||
{host: $host, status: "online", booted: $booted, current: $current, next_boot: $next_boot, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel}
|
||||
} catch {
|
||||
{host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||
{host: $host, status: "error", booted: "", current: "", next_boot: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user