fixing up remote

This commit is contained in:
ari committed 2026-10-11 01:53:43 -04:00
1 parent 6094b9dc93
commit 8ca671c89b
4 files changed
+408 -414

No files matched your search

+3 -2
View File
@@ -11,7 +11,8 @@ bin/
├── temp-monitor # NuShell script: collect CPU/GPU temps to CSV + optional graph
├── TEMP_MONITOR_README.md # Usage docs for temp-monitor
└── lib/
├── cmds.nu # All subcommand implementations (NuShell — source of truth); commands only
├── cmds.nu # Core subcommand implementations (NuShell — source of truth); commands only
├── remote.nu # `skyg remote*` commands (deploy, install, boot-all, status, reboot-pending)
├── helpers.nu # Shared helpers: REPO_ROOT const + internal functions used by commands
├── profile.nu # Profile helpers and tab-completion data
└── types.nu # Type definitions + tab-completion data
@@ -46,4 +47,4 @@ bin/
## Agent Note
> **Never invoke `skyg` commands that trigger builds or switches.**
> Only the user runs these. Edits to `cmds.nu` are safe if the user asks for CLI changes.
> Only the user runs these. Edits to `cmds.nu` / `remote.nu` are safe if the user asks for CLI changes.
+1 -412
View File
@@ -1,6 +1,7 @@
use types.nu *
use profile.nu *
use helpers.nu *
export use remote.nu *
export def skyg [] {
print $REPO_ROOT
@@ -90,418 +91,6 @@ export def "skyg os" [cmd: string, --build-host: string = ""] {
bash -c $runcmd
}
# Deploy to remote machine
export def "skyg remote" [
cmd: string@remote-cmds,
target: string@remote_targets = "",
--build-host: string@remote_targets = "", # Optional remote builder host (e.g. builder.lab.internal)
] {
cd $REPO_ROOT
mut target_host = $"root@($target).lab.internal"
mut profile = $"hl-($target)"
if not (is-valid-profile $profile true) {
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
return (remote_targets)
}
$"Using [(ansi green_bold)($profile)(ansi reset)]" | print
$"($cmd) on ($target_host) with ($profile)" | print
mut runcmd = $"nh os ($cmd) --no-nom --target-host ($target_host) --ask .#($profile)"
if $build_host != "" {
$runcmd = $"($runcmd) --build-host ($build_host).lab.internal"
}
$runcmd | print
bash -c $runcmd
}
# Fresh-install a remote machine via nixos-anywhere, using its disko config.
# Always reformats every disk in the disko file — a second run is as destructive as the first.
# Missing disko file, live disk OS (without --force-wipe), or mistyped hostname all abort.
export def "skyg remote install" [
target: string@remote_targets,
--force-wipe, # wipe even if the host already runs NixOS from disk
] {
cd $REPO_ROOT
let target_host = $"root@($target).lab.internal"
let profile = $"hl-($target)"
if not (is-valid-profile $profile true) {
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
return (remote_targets)
}
let disko_file = $"hosts/($profile).disko.nix"
if not ($disko_file | path exists) {
error make {msg: $"Missing disko config: ($disko_file). Refusing to install ($profile) without a disko layout."}
}
print ""
print $"(ansi red_bold)WARNING: this WIPES every disk listed in ($disko_file).(ansi reset)"
print "This is not a switch. Use `skyg remote switch` / `skyg remote boot` to update an installed host."
print ""
print "Disks in the disko file:"
let disks = (
open $disko_file
| lines
| where {|l| ($l | str contains "device") and ($l | str contains "=") }
| each {|l| $l | str trim }
)
for line in $disks {
print $" ($line)"
}
print ""
let ssh = $"ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ($target_host)"
let root_fstype = (try {
bash -c $"($ssh) 'findmnt -n -o FSTYPE /' 2>/dev/null" | str trim
} catch { "" })
if $root_fstype == "" {
print $"(ansi yellow)Could not SSH to ($target_host) to check for a live OS.(ansi reset)"
} else {
print $"Remote / fstype: ($root_fstype)"
let installer_fs = ["overlay" "tmpfs" "squashfs"]
if not ($root_fstype in $installer_fs) {
if not $force_wipe {
error make {msg: $"($target_host) already boots from disk (root fstype ($root_fstype)). Refusing to wipe. Use `skyg remote switch ($target)` to update, or pass --force-wipe to reinstall."}
}
print $"(ansi yellow)--force-wipe set: proceeding against a live disk OS.(ansi reset)"
} else {
print "Remote looks like an installer live image (ok to install)."
}
}
let confirm = input $"Type ($target) to continue: " | str trim
if $confirm != $target {
error make {msg: $"Aborted: typed '($confirm)', expected '($target)'."}
}
$"Installing [(ansi green_bold)($profile)(ansi reset)] on ($target_host) via nixos-anywhere, using ($disko_file)" | print
let runcmd = $"nix run github:nix-community/nixos-anywhere -- --flake .#($profile) ($target_host)"
$runcmd | print
let bash_path = which bash | get 0.path
with-env { SHELL: $bash_path } {
bash -c $runcmd
}
}
# Boot all remote hosts except the current machine (fwbook)
# Checks online status first, runs `skyg remote boot --build-host fwdesk <host>` for each online host,
# and writes a summary report to .tmp/boot-report.txt
export def "skyg remote boot-all" [] {
cd $REPO_ROOT
mkdir .tmp
let current_host = "fwbook"
let report_file = ".tmp/boot-report.txt"
# Discover remote targets from the flake (same logic as skyg remote)
let all_hosts = remote_targets | where {|h| $h != $current_host }
print $"Found ($all_hosts | length) candidate hosts excluding ($current_host)"
mut successes = []
mut failures = []
mut skipped = []
# Phase 1: sequential SSH reachability checks (fast)
mut online_hosts = []
for host in $all_hosts {
print $"\n=== Checking ($host) ==="
let target = $"root@($host).lab.internal"
let is_online = (try {
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
true
} catch { false })
if not $is_online {
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
$skipped = ($skipped | append $host)
} else {
print $"(ansi green)ONLINE(ansi reset): ($host)"
$online_hosts = ($online_hosts | append $host)
}
}
# Phase 2: run boots in parallel (2 concurrent) without interactive prompts
if ($online_hosts | length) > 0 {
print "
Starting parallel boots (concurrency: 2)..."
let boot_results = ($online_hosts | par-each --threads 1 { |host|
let target_host = $"root@($host).lab.internal"
let profile = $"hl-($host)"
let runcmd = $"nh os boot --target-host ($target_host) .#($profile)"
print $runcmd
let ok = (try {
bash -c $runcmd
true
} catch {
false
})
{host: $host, success: $ok}
})
for r in $boot_results {
if $r.success {
$successes = ($successes | append $r.host)
print $"(ansi green)SUCCESS(ansi reset): ($r.host)"
} else {
$failures = ($failures | append $r.host)
print $"(ansi red)FAILED(ansi reset): ($r.host)"
}
}
}
# Write report
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
let report = $"Boot Report - ($timestamp)
Total hosts considered: ($all_hosts | length)
Skipped offline: ($skipped | length) - ($skipped | str join ', ')
Successful: ($successes | length) - ($successes | str join ', ')
Failed: ($failures | length) - ($failures | str join ', ')
Details:
Online & Booted: ($successes | str join ', ')
Offline: ($skipped | str join ', ')
Failed: ($failures | str join ', ')
"
$report | save -f $report_file
print $"
Report written to ($report_file)"
print $report
}
# Check status of all remotes: generation, pending reboot, needs rebuild
export def "skyg remote status" [] {
cd $REPO_ROOT
mkdir .tmp
let current_host = "fwbook"
let report_file = ".tmp/status-report.txt"
let all_hosts = remote_targets | where {|h| $h != $current_host }
print $"Checking status for ($all_hosts | length) remotes..."
let results = ($all_hosts | each { |host|
print $"
=== ($host) ==="
let target = $"root@($host).lab.internal"
let online = (try {
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
true
} catch { false })
if not $online {
print $"(ansi yellow)OFFLINE(ansi reset)"
return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false}
}
let info = (try {
let booted = bash -c $"ssh ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
let current = bash -c $"ssh ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
let reboot_pending = ($booted != $current)
# Compare what the local flake currently evaluates to vs remote's active system
let local_toplevel = (try {
(bash -c $"nix eval --raw '.#hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true") | str trim
} catch { "" })
let needs_build = if $local_toplevel == "" { false } else { $local_toplevel != $current }
{host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build}
} catch {
{host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false}
})
print $"Booted: ($info.booted | path basename)"
print $"Current: ($info.current | path basename)"
if $info.reboot_pending {
print $"(ansi yellow)REBOOT PENDING(ansi reset)"
}
if $info.needs_build {
print $"(ansi red)NEEDS NEW BUILD(ansi reset)"
} else {
print $"(ansi green)UP TO DATE(ansi reset)"
}
$info
})
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
let lines = ($results | each { |r|
if $r.status == "offline" {
$"- ($r.host): (ansi yellow)OFFLINE(ansi reset)"
} else if $r.status == "error" {
$"- ($r.host): (ansi red)ERROR querying status(ansi reset)"
} else {
let pending = if $r.reboot_pending { $" [(ansi blue)REBOOT PENDING(ansi reset)]" } else { "" }
let build = if $r.needs_build { $" [(ansi red)NEEDS REBUILD(ansi reset)]" } else { $" [(ansi green)current(ansi reset)]" }
$"- ($r.host): booted=($r.booted | path basename) current=($r.current | path basename)($pending)($build)"
}
})
let report = $"Remote Status Report - ($timestamp)
" + ($lines | str join "
") + "
"
$report | save -f $report_file
print $"
Report written to ($report_file)"
print $report
}
# Poll SSH until the host reaches the desired reachability, or the timeout elapses.
def wait-for-host [target: string, want_online: bool, timeout: int] {
let deadline = (date now) + ($timeout * 1sec)
while (date now) < $deadline {
let online = (try {
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
true
} catch { false })
if $online == $want_online {
return true
}
sleep 3sec
}
false
}
# Send a desktop notification if notify-send is available; never fail the caller.
def notify [title: string, body: string] {
try {
^notify-send $title $body
} catch { }
}
# Reboot every remote host that is pending a reboot (booted-system != current-system).
# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a
# time, waiting for it to come back online and sending a desktop notification when it does.
export def "skyg remote reboot-pending" [] {
cd $REPO_ROOT
mkdir .tmp
let current_host = "fwbook"
let report_file = ".tmp/reboot-pending-report.txt"
let all_hosts = remote_targets | where {|h| $h != $current_host }
print $"Checking ($all_hosts | length) remotes for pending reboots..."
# Phase 1: find hosts that are online and pending a reboot
mut pending = []
mut offline = []
mut up_to_date = []
for host in $all_hosts {
let target = $"root@($host).lab.internal"
let online = (try {
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
true
} catch { false })
if not $online {
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
$offline = ($offline | append $host)
continue
}
let info = (try {
let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
{booted: $booted, current: $current, pending: ($booted != $current)}
} catch {
{booted: "", current: "", pending: false}
})
if $info.pending {
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)"
$pending = ($pending | append $host)
} else {
print $"(ansi green)UP TO DATE(ansi reset): ($host)"
$up_to_date = ($up_to_date | append $host)
}
}
if ($pending | length) == 0 {
print ""
print "No remotes are pending a reboot."
return
}
# Phase 2: confirm before rebooting anything
print ""
print $"(ansi yellow_bold)($pending | length) remote\(s\) pending a reboot:(ansi reset)"
for host in $pending {
print $" - ($host)"
}
print ""
let confirm = (input $"Reboot these ($pending | length) host\(s\)? [y/N]: " | str trim | str downcase)
if not ($confirm in ["y" "yes"]) {
print "Aborted — no hosts were rebooted."
return
}
# Phase 3: reboot each pending host one at a time, waiting for it to come back
mut rebooted = []
mut failed = []
for host in $pending {
let target = $"root@($host).lab.internal"
print $"\n=== Rebooting ($host) ==="
# Capture the boot id so we can confirm the host actually rebooted.
let boot_id_before = (try {
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
} catch { "" })
let runcmd = $"ssh -o BatchMode=yes -o ConnectTimeout=5 ($target) 'systemctl reboot'"
print $runcmd
# The SSH connection drops as the host shuts down, so a non-zero exit is
# expected here and does not mean the reboot failed.
try {
bash -c $"($runcmd) 2>/dev/null"
} catch { }
print $"Waiting for ($host) to shut down..."
let went_down = (wait-for-host $target false 90)
if not $went_down {
print $"(ansi yellow)($host) did not go offline within 90s — continuing to wait for it to come back.(ansi reset)"
}
print $"Waiting for ($host) to come back online..."
let came_up = (wait-for-host $target true 300)
if $came_up {
let boot_id_after = (try {
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
} catch { "" })
let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after)
if $confirmed {
print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)"
} else {
print $"(ansi green)($host) is back online(ansi reset)"
}
notify $"✅ ($host) rebooted" "Back online and reachable"
$rebooted = ($rebooted | append $host)
} else {
print $"(ansi red)($host) did not come back online within 300s(ansi reset)"
notify $"❌ ($host) did not come back" "Still unreachable after 300s"
$failed = ($failed | append $host)
}
}
# Write report
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
let report = $"Reboot-Pending Report - ($timestamp)
Total hosts considered: ($all_hosts | length)
Pending reboot: ($pending | length) - ($pending | str join ', ')
Rebooted: ($rebooted | length) - ($rebooted | str join ', ')
Failed to return: ($failed | length) - ($failed | str join ', ')
Offline: ($offline | length) - ($offline | str join ', ')
Up to date: ($up_to_date | length) - ($up_to_date | str join ', ')
"
$report | save -f $report_file
print $"
Report written to ($report_file)"
print $report
}
# Deploy OpenWrt router configuration
export def "skyg openwrt" [
router: string = "glmain",
+368
View File
@@ -0,0 +1,368 @@
use types.nu *
use profile.nu *
use helpers.nu *
use remote.utils.nu *
# Deploy to remote machine
export def "skyg remote" [
cmd: string@remote-cmds,
target: string@remote_targets = "",
--build-host: string@remote_targets = "", # Optional remote builder host (e.g. builder.lab.internal)
] {
cd $REPO_ROOT
mut target_host = $"root@($target).lab.internal"
mut profile = $"hl-($target)"
if not (is-valid-profile $profile true) {
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
return (remote_targets)
}
$"Using [(ansi green_bold)($profile)(ansi reset)]" | print
$"($cmd) on ($target_host) with ($profile)" | print
mut runcmd = $"nh os ($cmd) --no-nom --target-host ($target_host) --ask .#($profile)"
if $build_host != "" {
$runcmd = $"($runcmd) --build-host ($build_host).lab.internal"
}
$runcmd | print
bash -c $runcmd
}
# Fresh-install a remote machine via nixos-anywhere, using its disko config.
# Always reformats every disk in the disko file — a second run is as destructive as the first.
# Missing disko file, live disk OS (without --force-wipe), or mistyped hostname all abort.
export def "skyg remote install" [
target: string@remote_targets,
--force-wipe, # wipe even if the host already runs NixOS from disk
] {
cd $REPO_ROOT
let target_host = $"root@($target).lab.internal"
let profile = $"hl-($target)"
if not (is-valid-profile $profile true) {
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
return (remote_targets)
}
let disko_file = $"hosts/($profile).disko.nix"
if not ($disko_file | path exists) {
error make {msg: $"Missing disko config: ($disko_file). Refusing to install ($profile) without a disko layout."}
}
print ""
print $"(ansi red_bold)WARNING: this WIPES every disk listed in ($disko_file).(ansi reset)"
print "This is not a switch. Use `skyg remote switch` / `skyg remote boot` to update an installed host."
print ""
print "Disks in the disko file:"
let disks = (
open $disko_file
| lines
| where {|l| ($l | str contains "device") and ($l | str contains "=") }
| each {|l| $l | str trim }
)
for line in $disks {
print $" ($line)"
}
print ""
let ssh = $"ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ($target_host)"
let root_fstype = (try {
bash -c $"($ssh) 'findmnt -n -o FSTYPE /' 2>/dev/null" | str trim
} catch { "" })
if $root_fstype == "" {
print $"(ansi yellow)Could not SSH to ($target_host) to check for a live OS.(ansi reset)"
} else {
print $"Remote / fstype: ($root_fstype)"
let installer_fs = ["overlay" "tmpfs" "squashfs"]
if not ($root_fstype in $installer_fs) {
if not $force_wipe {
error make {msg: $"($target_host) already boots from disk (root fstype ($root_fstype)). Refusing to wipe. Use `skyg remote switch ($target)` to update, or pass --force-wipe to reinstall."}
}
print $"(ansi yellow)--force-wipe set: proceeding against a live disk OS.(ansi reset)"
} else {
print "Remote looks like an installer live image (ok to install)."
}
}
let confirm = input $"Type ($target) to continue: " | str trim
if $confirm != $target {
error make {msg: $"Aborted: typed '($confirm)', expected '($target)'."}
}
$"Installing [(ansi green_bold)($profile)(ansi reset)] on ($target_host) via nixos-anywhere, using ($disko_file)" | print
let runcmd = $"nix run github:nix-community/nixos-anywhere -- --flake .#($profile) ($target_host)"
$runcmd | print
let bash_path = which bash | get 0.path
with-env { SHELL: $bash_path } {
bash -c $runcmd
}
}
# Boot all remote hosts except the current machine (fwbook)
# Checks online status first, runs `skyg remote boot --build-host fwdesk <host>` for each online host,
# and writes a summary report to .tmp/boot-report.txt
export def "skyg remote boot-all" [] {
cd $REPO_ROOT
mkdir .tmp
let current_host = "fwbook"
let report_file = ".tmp/boot-report.txt"
# Discover remote targets from the flake (same logic as skyg remote)
let all_hosts = remote_targets | where {|h| $h != $current_host }
print $"Found ($all_hosts | length) candidate hosts excluding ($current_host)"
mut successes = []
mut failures = []
mut skipped = []
# Phase 1: sequential SSH reachability checks (fast)
mut online_hosts = []
for host in $all_hosts {
print $"\n=== Checking ($host) ==="
let target = $"root@($host).lab.internal"
let is_online = (try {
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
true
} catch { false })
if not $is_online {
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
$skipped = ($skipped | append $host)
} else {
print $"(ansi green)ONLINE(ansi reset): ($host)"
$online_hosts = ($online_hosts | append $host)
}
}
# Phase 2: run boots in parallel (2 concurrent) without interactive prompts
if ($online_hosts | length) > 0 {
print "
Starting parallel boots (concurrency: 2)..."
let boot_results = ($online_hosts | par-each --threads 1 { |host|
let target_host = $"root@($host).lab.internal"
let profile = $"hl-($host)"
let runcmd = $"nh os boot --target-host ($target_host) .#($profile)"
print $runcmd
let ok = (try {
bash -c $runcmd
true
} catch {
false
})
{host: $host, success: $ok}
})
for r in $boot_results {
if $r.success {
$successes = ($successes | append $r.host)
print $"(ansi green)SUCCESS(ansi reset): ($r.host)"
} else {
$failures = ($failures | append $r.host)
print $"(ansi red)FAILED(ansi reset): ($r.host)"
}
}
}
# Write report
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
let report = $"Boot Report - ($timestamp)
Total hosts considered: ($all_hosts | length)
Skipped offline: ($skipped | length) - ($skipped | str join ', ')
Successful: ($successes | length) - ($successes | str join ', ')
Failed: ($failures | length) - ($failures | str join ', ')
Details:
Online & Booted: ($successes | str join ', ')
Offline: ($skipped | str join ', ')
Failed: ($failures | str join ', ')
"
$report | save -f $report_file
print $"
Report written to ($report_file)"
print $report
}
export def "skyg remote status" [ target: string@remote_targets ] {
get-remote-boot-info $target
}
# Check status of all remotes: generation, pending reboot, needs rebuild
export def "skyg remote status all" [] {
cd $REPO_ROOT
mkdir .tmp
let current_host = "fwbook"
let report_file = ".tmp/status-report.txt"
let all_hosts = remote_targets | where {|h| $h != $current_host }
print $"Checking status for ($all_hosts | length) remotes..."
let results = ($all_hosts | par-each { |host|
let info = get-remote-boot-info $host
# Print each host's status as one block, as soon as it finishes.
let booted = (if ($info.booted | is-empty) { "-" } else { $info.booted | path basename })
let current = (if ($info.current | is-empty) { "-" } else { $info.current | path basename })
let reboot_line = if $info.reboot_pending { $"(ansi yellow)REBOOT PENDING(ansi reset)\n" } else { "" }
let build_line = if $info.needs_build { $"(ansi red)NEEDS NEW BUILD(ansi reset)" } else { $"(ansi green)UP TO DATE(ansi reset)" }
print $"($host) | Booted: ($booted) | Current: ($current)"
$info
})
return ($results | select host status reboot_pending needs_build)
}
# Poll SSH until the host reaches the desired reachability, or the timeout elapses.
def wait-for-host [target: string, want_online: bool, timeout: int] {
let deadline = (date now) + ($timeout * 1sec)
while (date now) < $deadline {
let online = (try {
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
true
} catch { false })
if $online == $want_online {
return true
}
sleep 3sec
}
false
}
# Send a desktop notification if notify-send is available; never fail the caller.
def notify [title: string, body: string] {
try {
^notify-send $title $body
} catch { }
}
# Reboot every remote host that is pending a reboot (booted-system != current-system).
# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a
# time, waiting for it to come back online and sending a desktop notification when it does.
export def "skyg remote reboot-pending" [] {
cd $REPO_ROOT
mkdir .tmp
let current_host = "fwbook"
let report_file = ".tmp/reboot-pending-report.txt"
let all_hosts = remote_targets | where {|h| $h != $current_host }
print $"Checking ($all_hosts | length) remotes for pending reboots..."
# Phase 1: find hosts that are online and pending a reboot
mut pending = []
mut offline = []
mut up_to_date = []
for host in $all_hosts {
let target = $"root@($host).lab.internal"
let online = (try {
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
true
} catch { false })
if not $online {
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
$offline = ($offline | append $host)
continue
}
let info = (try {
let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
{booted: $booted, current: $current, pending: ($booted != $current)}
} catch {
{booted: "", current: "", pending: false}
})
if $info.pending {
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)"
$pending = ($pending | append $host)
} else {
print $"(ansi green)UP TO DATE(ansi reset): ($host)"
$up_to_date = ($up_to_date | append $host)
}
}
if ($pending | length) == 0 {
print ""
print "No remotes are pending a reboot."
return
}
# Phase 2: confirm before rebooting anything
print ""
print $"(ansi yellow_bold)($pending | length) remote\(s\) pending a reboot:(ansi reset)"
for host in $pending {
print $" - ($host)"
}
print ""
let confirm = (input $"Reboot these ($pending | length) host\(s\)? [y/N]: " | str trim | str downcase)
if not ($confirm in ["y" "yes"]) {
print "Aborted — no hosts were rebooted."
return
}
# Phase 3: reboot each pending host one at a time, waiting for it to come back
mut rebooted = []
mut failed = []
for host in $pending {
let target = $"root@($host).lab.internal"
print $"\n=== Rebooting ($host) ==="
# Capture the boot id so we can confirm the host actually rebooted.
let boot_id_before = (try {
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
} catch { "" })
let runcmd = $"ssh -o BatchMode=yes -o ConnectTimeout=5 ($target) 'systemctl reboot'"
print $runcmd
# The SSH connection drops as the host shuts down, so a non-zero exit is
# expected here and does not mean the reboot failed.
try {
bash -c $"($runcmd) 2>/dev/null"
} catch { }
print $"Waiting for ($host) to shut down..."
let went_down = (wait-for-host $target false 90)
if not $went_down {
print $"(ansi yellow)($host) did not go offline within 90s — continuing to wait for it to come back.(ansi reset)"
}
print $"Waiting for ($host) to come back online..."
let came_up = (wait-for-host $target true 300)
if $came_up {
let boot_id_after = (try {
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
} catch { "" })
let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after)
if $confirmed {
print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)"
} else {
print $"(ansi green)($host) is back online(ansi reset)"
}
notify $"✅ ($host) rebooted" "Back online and reachable"
$rebooted = ($rebooted | append $host)
} else {
print $"(ansi red)($host) did not come back online within 300s(ansi reset)"
notify $"❌ ($host) did not come back" "Still unreachable after 300s"
$failed = ($failed | append $host)
}
}
# Write report
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
let report = $"Reboot-Pending Report - ($timestamp)
Total hosts considered: ($all_hosts | length)
Pending reboot: ($pending | length) - ($pending | str join ', ')
Rebooted: ($rebooted | length) - ($rebooted | str join ', ')
Failed to return: ($failed | length) - ($failed | str join ', ')
Offline: ($offline | length) - ($offline | str join ', ')
Up to date: ($up_to_date | length) - ($up_to_date | str join ', ')
"
$report | save -f $report_file
print $"
Report written to ($report_file)"
print $report
}
+36
View File
@@ -0,0 +1,36 @@
use types.nu *
use profile.nu *
export def get-remote-boot-info [ host: string@remote_targets ] {
let target = $"root@($host).lab.internal"
let is_online = (try {
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
true
} catch { false })
if (not $is_online) {
return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
}
return (try {
# Fetch remote system paths and the local toplevel concurrently.
let fetched = (["remote" "local"] | par-each {|what|
if $what == "remote" {
bash -c $"ssh ($target) 'readlink /run/booted-system; readlink /run/current-system' 2>/dev/null" | lines
} else {
try {
bash -c $"nix eval --raw '.#nixosConfigurations.hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true" | str trim
} catch { "" }
}
})
let booted = ($fetched.0 | get 0? | default "")
let current = ($fetched.0 | get 1? | default "")
let local_toplevel = $fetched.1
let reboot_pending = ($booted != $current)
let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $current }
{host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel}
} catch {
{host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
})
}