fixing up remote
This commit is contained in:
1 parent
6094b9dc93
commit
8ca671c89b
4 files changed
+408
-414
No files matched your search
+3
-2
@@ -11,7 +11,8 @@ bin/
|
||||
├── temp-monitor # NuShell script: collect CPU/GPU temps to CSV + optional graph
|
||||
├── TEMP_MONITOR_README.md # Usage docs for temp-monitor
|
||||
└── lib/
|
||||
├── cmds.nu # All subcommand implementations (NuShell — source of truth); commands only
|
||||
├── cmds.nu # Core subcommand implementations (NuShell — source of truth); commands only
|
||||
├── remote.nu # `skyg remote*` commands (deploy, install, boot-all, status, reboot-pending)
|
||||
├── helpers.nu # Shared helpers: REPO_ROOT const + internal functions used by commands
|
||||
├── profile.nu # Profile helpers and tab-completion data
|
||||
└── types.nu # Type definitions + tab-completion data
|
||||
@@ -46,4 +47,4 @@ bin/
|
||||
## Agent Note
|
||||
|
||||
> **Never invoke `skyg` commands that trigger builds or switches.**
|
||||
> Only the user runs these. Edits to `cmds.nu` are safe if the user asks for CLI changes.
|
||||
> Only the user runs these. Edits to `cmds.nu` / `remote.nu` are safe if the user asks for CLI changes.
|
||||
+1
-412
@@ -1,6 +1,7 @@
|
||||
use types.nu *
|
||||
use profile.nu *
|
||||
use helpers.nu *
|
||||
export use remote.nu *
|
||||
|
||||
export def skyg [] {
|
||||
print $REPO_ROOT
|
||||
@@ -90,418 +91,6 @@ export def "skyg os" [cmd: string, --build-host: string = ""] {
|
||||
bash -c $runcmd
|
||||
}
|
||||
|
||||
# Deploy to remote machine
|
||||
export def "skyg remote" [
|
||||
cmd: string@remote-cmds,
|
||||
target: string@remote_targets = "",
|
||||
--build-host: string@remote_targets = "", # Optional remote builder host (e.g. builder.lab.internal)
|
||||
] {
|
||||
cd $REPO_ROOT
|
||||
mut target_host = $"root@($target).lab.internal"
|
||||
mut profile = $"hl-($target)"
|
||||
if not (is-valid-profile $profile true) {
|
||||
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
|
||||
return (remote_targets)
|
||||
}
|
||||
$"Using [(ansi green_bold)($profile)(ansi reset)]" | print
|
||||
$"($cmd) on ($target_host) with ($profile)" | print
|
||||
mut runcmd = $"nh os ($cmd) --no-nom --target-host ($target_host) --ask .#($profile)"
|
||||
if $build_host != "" {
|
||||
$runcmd = $"($runcmd) --build-host ($build_host).lab.internal"
|
||||
}
|
||||
$runcmd | print
|
||||
bash -c $runcmd
|
||||
}
|
||||
|
||||
# Fresh-install a remote machine via nixos-anywhere, using its disko config.
|
||||
# Always reformats every disk in the disko file — a second run is as destructive as the first.
|
||||
# Missing disko file, live disk OS (without --force-wipe), or mistyped hostname all abort.
|
||||
export def "skyg remote install" [
|
||||
target: string@remote_targets,
|
||||
--force-wipe, # wipe even if the host already runs NixOS from disk
|
||||
] {
|
||||
cd $REPO_ROOT
|
||||
let target_host = $"root@($target).lab.internal"
|
||||
let profile = $"hl-($target)"
|
||||
if not (is-valid-profile $profile true) {
|
||||
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
|
||||
return (remote_targets)
|
||||
}
|
||||
let disko_file = $"hosts/($profile).disko.nix"
|
||||
if not ($disko_file | path exists) {
|
||||
error make {msg: $"Missing disko config: ($disko_file). Refusing to install ($profile) without a disko layout."}
|
||||
}
|
||||
|
||||
print ""
|
||||
print $"(ansi red_bold)WARNING: this WIPES every disk listed in ($disko_file).(ansi reset)"
|
||||
print "This is not a switch. Use `skyg remote switch` / `skyg remote boot` to update an installed host."
|
||||
print ""
|
||||
print "Disks in the disko file:"
|
||||
let disks = (
|
||||
open $disko_file
|
||||
| lines
|
||||
| where {|l| ($l | str contains "device") and ($l | str contains "=") }
|
||||
| each {|l| $l | str trim }
|
||||
)
|
||||
for line in $disks {
|
||||
print $" ($line)"
|
||||
}
|
||||
print ""
|
||||
|
||||
let ssh = $"ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ($target_host)"
|
||||
let root_fstype = (try {
|
||||
bash -c $"($ssh) 'findmnt -n -o FSTYPE /' 2>/dev/null" | str trim
|
||||
} catch { "" })
|
||||
|
||||
if $root_fstype == "" {
|
||||
print $"(ansi yellow)Could not SSH to ($target_host) to check for a live OS.(ansi reset)"
|
||||
} else {
|
||||
print $"Remote / fstype: ($root_fstype)"
|
||||
let installer_fs = ["overlay" "tmpfs" "squashfs"]
|
||||
if not ($root_fstype in $installer_fs) {
|
||||
if not $force_wipe {
|
||||
error make {msg: $"($target_host) already boots from disk (root fstype ($root_fstype)). Refusing to wipe. Use `skyg remote switch ($target)` to update, or pass --force-wipe to reinstall."}
|
||||
}
|
||||
print $"(ansi yellow)--force-wipe set: proceeding against a live disk OS.(ansi reset)"
|
||||
} else {
|
||||
print "Remote looks like an installer live image (ok to install)."
|
||||
}
|
||||
}
|
||||
|
||||
let confirm = input $"Type ($target) to continue: " | str trim
|
||||
if $confirm != $target {
|
||||
error make {msg: $"Aborted: typed '($confirm)', expected '($target)'."}
|
||||
}
|
||||
|
||||
$"Installing [(ansi green_bold)($profile)(ansi reset)] on ($target_host) via nixos-anywhere, using ($disko_file)" | print
|
||||
let runcmd = $"nix run github:nix-community/nixos-anywhere -- --flake .#($profile) ($target_host)"
|
||||
$runcmd | print
|
||||
let bash_path = which bash | get 0.path
|
||||
with-env { SHELL: $bash_path } {
|
||||
bash -c $runcmd
|
||||
}
|
||||
}
|
||||
|
||||
# Boot all remote hosts except the current machine (fwbook)
|
||||
# Checks online status first, runs `skyg remote boot --build-host fwdesk <host>` for each online host,
|
||||
# and writes a summary report to .tmp/boot-report.txt
|
||||
|
||||
export def "skyg remote boot-all" [] {
|
||||
cd $REPO_ROOT
|
||||
mkdir .tmp
|
||||
let current_host = "fwbook"
|
||||
let report_file = ".tmp/boot-report.txt"
|
||||
|
||||
# Discover remote targets from the flake (same logic as skyg remote)
|
||||
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
||||
|
||||
print $"Found ($all_hosts | length) candidate hosts excluding ($current_host)"
|
||||
|
||||
mut successes = []
|
||||
mut failures = []
|
||||
mut skipped = []
|
||||
|
||||
# Phase 1: sequential SSH reachability checks (fast)
|
||||
mut online_hosts = []
|
||||
for host in $all_hosts {
|
||||
print $"\n=== Checking ($host) ==="
|
||||
let target = $"root@($host).lab.internal"
|
||||
let is_online = (try {
|
||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
|
||||
true
|
||||
} catch { false })
|
||||
|
||||
if not $is_online {
|
||||
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
|
||||
$skipped = ($skipped | append $host)
|
||||
} else {
|
||||
print $"(ansi green)ONLINE(ansi reset): ($host)"
|
||||
$online_hosts = ($online_hosts | append $host)
|
||||
}
|
||||
}
|
||||
|
||||
# Phase 2: run boots in parallel (2 concurrent) without interactive prompts
|
||||
if ($online_hosts | length) > 0 {
|
||||
print "
|
||||
Starting parallel boots (concurrency: 2)..."
|
||||
let boot_results = ($online_hosts | par-each --threads 1 { |host|
|
||||
let target_host = $"root@($host).lab.internal"
|
||||
let profile = $"hl-($host)"
|
||||
let runcmd = $"nh os boot --target-host ($target_host) .#($profile)"
|
||||
print $runcmd
|
||||
let ok = (try {
|
||||
bash -c $runcmd
|
||||
true
|
||||
} catch {
|
||||
false
|
||||
})
|
||||
{host: $host, success: $ok}
|
||||
})
|
||||
|
||||
for r in $boot_results {
|
||||
if $r.success {
|
||||
$successes = ($successes | append $r.host)
|
||||
print $"(ansi green)SUCCESS(ansi reset): ($r.host)"
|
||||
} else {
|
||||
$failures = ($failures | append $r.host)
|
||||
print $"(ansi red)FAILED(ansi reset): ($r.host)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Write report
|
||||
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
|
||||
let report = $"Boot Report - ($timestamp)
|
||||
|
||||
Total hosts considered: ($all_hosts | length)
|
||||
Skipped offline: ($skipped | length) - ($skipped | str join ', ')
|
||||
Successful: ($successes | length) - ($successes | str join ', ')
|
||||
Failed: ($failures | length) - ($failures | str join ', ')
|
||||
|
||||
Details:
|
||||
Online & Booted: ($successes | str join ', ')
|
||||
Offline: ($skipped | str join ', ')
|
||||
Failed: ($failures | str join ', ')
|
||||
"
|
||||
|
||||
$report | save -f $report_file
|
||||
print $"
|
||||
Report written to ($report_file)"
|
||||
print $report
|
||||
}
|
||||
|
||||
# Check status of all remotes: generation, pending reboot, needs rebuild
|
||||
export def "skyg remote status" [] {
|
||||
cd $REPO_ROOT
|
||||
mkdir .tmp
|
||||
let current_host = "fwbook"
|
||||
let report_file = ".tmp/status-report.txt"
|
||||
|
||||
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
||||
print $"Checking status for ($all_hosts | length) remotes..."
|
||||
|
||||
let results = ($all_hosts | each { |host|
|
||||
print $"
|
||||
=== ($host) ==="
|
||||
let target = $"root@($host).lab.internal"
|
||||
let online = (try {
|
||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
|
||||
true
|
||||
} catch { false })
|
||||
|
||||
if not $online {
|
||||
print $"(ansi yellow)OFFLINE(ansi reset)"
|
||||
return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false}
|
||||
}
|
||||
|
||||
let info = (try {
|
||||
let booted = bash -c $"ssh ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
|
||||
let current = bash -c $"ssh ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
|
||||
let reboot_pending = ($booted != $current)
|
||||
|
||||
# Compare what the local flake currently evaluates to vs remote's active system
|
||||
let local_toplevel = (try {
|
||||
(bash -c $"nix eval --raw '.#hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true") | str trim
|
||||
} catch { "" })
|
||||
let needs_build = if $local_toplevel == "" { false } else { $local_toplevel != $current }
|
||||
|
||||
{host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build}
|
||||
} catch {
|
||||
{host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false}
|
||||
})
|
||||
|
||||
print $"Booted: ($info.booted | path basename)"
|
||||
print $"Current: ($info.current | path basename)"
|
||||
if $info.reboot_pending {
|
||||
print $"(ansi yellow)REBOOT PENDING(ansi reset)"
|
||||
}
|
||||
if $info.needs_build {
|
||||
print $"(ansi red)NEEDS NEW BUILD(ansi reset)"
|
||||
} else {
|
||||
print $"(ansi green)UP TO DATE(ansi reset)"
|
||||
}
|
||||
|
||||
$info
|
||||
})
|
||||
|
||||
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
|
||||
let lines = ($results | each { |r|
|
||||
if $r.status == "offline" {
|
||||
$"- ($r.host): (ansi yellow)OFFLINE(ansi reset)"
|
||||
} else if $r.status == "error" {
|
||||
$"- ($r.host): (ansi red)ERROR querying status(ansi reset)"
|
||||
} else {
|
||||
let pending = if $r.reboot_pending { $" [(ansi blue)REBOOT PENDING(ansi reset)]" } else { "" }
|
||||
let build = if $r.needs_build { $" [(ansi red)NEEDS REBUILD(ansi reset)]" } else { $" [(ansi green)current(ansi reset)]" }
|
||||
$"- ($r.host): booted=($r.booted | path basename) current=($r.current | path basename)($pending)($build)"
|
||||
}
|
||||
})
|
||||
let report = $"Remote Status Report - ($timestamp)
|
||||
|
||||
" + ($lines | str join "
|
||||
") + "
|
||||
"
|
||||
|
||||
$report | save -f $report_file
|
||||
print $"
|
||||
Report written to ($report_file)"
|
||||
print $report
|
||||
}
|
||||
|
||||
# Poll SSH until the host reaches the desired reachability, or the timeout elapses.
|
||||
def wait-for-host [target: string, want_online: bool, timeout: int] {
|
||||
let deadline = (date now) + ($timeout * 1sec)
|
||||
while (date now) < $deadline {
|
||||
let online = (try {
|
||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
|
||||
true
|
||||
} catch { false })
|
||||
if $online == $want_online {
|
||||
return true
|
||||
}
|
||||
sleep 3sec
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
# Send a desktop notification if notify-send is available; never fail the caller.
|
||||
def notify [title: string, body: string] {
|
||||
try {
|
||||
^notify-send $title $body
|
||||
} catch { }
|
||||
}
|
||||
|
||||
# Reboot every remote host that is pending a reboot (booted-system != current-system).
|
||||
# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a
|
||||
# time, waiting for it to come back online and sending a desktop notification when it does.
|
||||
export def "skyg remote reboot-pending" [] {
|
||||
cd $REPO_ROOT
|
||||
mkdir .tmp
|
||||
let current_host = "fwbook"
|
||||
let report_file = ".tmp/reboot-pending-report.txt"
|
||||
|
||||
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
||||
print $"Checking ($all_hosts | length) remotes for pending reboots..."
|
||||
|
||||
# Phase 1: find hosts that are online and pending a reboot
|
||||
mut pending = []
|
||||
mut offline = []
|
||||
mut up_to_date = []
|
||||
|
||||
for host in $all_hosts {
|
||||
let target = $"root@($host).lab.internal"
|
||||
let online = (try {
|
||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
|
||||
true
|
||||
} catch { false })
|
||||
|
||||
if not $online {
|
||||
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
|
||||
$offline = ($offline | append $host)
|
||||
continue
|
||||
}
|
||||
|
||||
let info = (try {
|
||||
let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
|
||||
let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
|
||||
{booted: $booted, current: $current, pending: ($booted != $current)}
|
||||
} catch {
|
||||
{booted: "", current: "", pending: false}
|
||||
})
|
||||
|
||||
if $info.pending {
|
||||
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)"
|
||||
$pending = ($pending | append $host)
|
||||
} else {
|
||||
print $"(ansi green)UP TO DATE(ansi reset): ($host)"
|
||||
$up_to_date = ($up_to_date | append $host)
|
||||
}
|
||||
}
|
||||
|
||||
if ($pending | length) == 0 {
|
||||
print ""
|
||||
print "No remotes are pending a reboot."
|
||||
return
|
||||
}
|
||||
|
||||
# Phase 2: confirm before rebooting anything
|
||||
print ""
|
||||
print $"(ansi yellow_bold)($pending | length) remote\(s\) pending a reboot:(ansi reset)"
|
||||
for host in $pending {
|
||||
print $" - ($host)"
|
||||
}
|
||||
print ""
|
||||
let confirm = (input $"Reboot these ($pending | length) host\(s\)? [y/N]: " | str trim | str downcase)
|
||||
if not ($confirm in ["y" "yes"]) {
|
||||
print "Aborted — no hosts were rebooted."
|
||||
return
|
||||
}
|
||||
|
||||
# Phase 3: reboot each pending host one at a time, waiting for it to come back
|
||||
mut rebooted = []
|
||||
mut failed = []
|
||||
for host in $pending {
|
||||
let target = $"root@($host).lab.internal"
|
||||
print $"\n=== Rebooting ($host) ==="
|
||||
|
||||
# Capture the boot id so we can confirm the host actually rebooted.
|
||||
let boot_id_before = (try {
|
||||
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
|
||||
} catch { "" })
|
||||
|
||||
let runcmd = $"ssh -o BatchMode=yes -o ConnectTimeout=5 ($target) 'systemctl reboot'"
|
||||
print $runcmd
|
||||
# The SSH connection drops as the host shuts down, so a non-zero exit is
|
||||
# expected here and does not mean the reboot failed.
|
||||
try {
|
||||
bash -c $"($runcmd) 2>/dev/null"
|
||||
} catch { }
|
||||
|
||||
print $"Waiting for ($host) to shut down..."
|
||||
let went_down = (wait-for-host $target false 90)
|
||||
if not $went_down {
|
||||
print $"(ansi yellow)($host) did not go offline within 90s — continuing to wait for it to come back.(ansi reset)"
|
||||
}
|
||||
|
||||
print $"Waiting for ($host) to come back online..."
|
||||
let came_up = (wait-for-host $target true 300)
|
||||
if $came_up {
|
||||
let boot_id_after = (try {
|
||||
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
|
||||
} catch { "" })
|
||||
let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after)
|
||||
if $confirmed {
|
||||
print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)"
|
||||
} else {
|
||||
print $"(ansi green)($host) is back online(ansi reset)"
|
||||
}
|
||||
notify $"✅ ($host) rebooted" "Back online and reachable"
|
||||
$rebooted = ($rebooted | append $host)
|
||||
} else {
|
||||
print $"(ansi red)($host) did not come back online within 300s(ansi reset)"
|
||||
notify $"❌ ($host) did not come back" "Still unreachable after 300s"
|
||||
$failed = ($failed | append $host)
|
||||
}
|
||||
}
|
||||
|
||||
# Write report
|
||||
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
|
||||
let report = $"Reboot-Pending Report - ($timestamp)
|
||||
|
||||
Total hosts considered: ($all_hosts | length)
|
||||
Pending reboot: ($pending | length) - ($pending | str join ', ')
|
||||
Rebooted: ($rebooted | length) - ($rebooted | str join ', ')
|
||||
Failed to return: ($failed | length) - ($failed | str join ', ')
|
||||
Offline: ($offline | length) - ($offline | str join ', ')
|
||||
Up to date: ($up_to_date | length) - ($up_to_date | str join ', ')
|
||||
"
|
||||
|
||||
$report | save -f $report_file
|
||||
print $"
|
||||
Report written to ($report_file)"
|
||||
print $report
|
||||
}
|
||||
|
||||
# Deploy OpenWrt router configuration
|
||||
export def "skyg openwrt" [
|
||||
router: string = "glmain",
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
use types.nu *
|
||||
use profile.nu *
|
||||
use helpers.nu *
|
||||
use remote.utils.nu *
|
||||
|
||||
# Deploy to remote machine
|
||||
export def "skyg remote" [
|
||||
cmd: string@remote-cmds,
|
||||
target: string@remote_targets = "",
|
||||
--build-host: string@remote_targets = "", # Optional remote builder host (e.g. builder.lab.internal)
|
||||
] {
|
||||
cd $REPO_ROOT
|
||||
mut target_host = $"root@($target).lab.internal"
|
||||
mut profile = $"hl-($target)"
|
||||
if not (is-valid-profile $profile true) {
|
||||
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
|
||||
return (remote_targets)
|
||||
}
|
||||
$"Using [(ansi green_bold)($profile)(ansi reset)]" | print
|
||||
$"($cmd) on ($target_host) with ($profile)" | print
|
||||
mut runcmd = $"nh os ($cmd) --no-nom --target-host ($target_host) --ask .#($profile)"
|
||||
if $build_host != "" {
|
||||
$runcmd = $"($runcmd) --build-host ($build_host).lab.internal"
|
||||
}
|
||||
$runcmd | print
|
||||
bash -c $runcmd
|
||||
}
|
||||
|
||||
# Fresh-install a remote machine via nixos-anywhere, using its disko config.
|
||||
# Always reformats every disk in the disko file — a second run is as destructive as the first.
|
||||
# Missing disko file, live disk OS (without --force-wipe), or mistyped hostname all abort.
|
||||
export def "skyg remote install" [
|
||||
target: string@remote_targets,
|
||||
--force-wipe, # wipe even if the host already runs NixOS from disk
|
||||
] {
|
||||
cd $REPO_ROOT
|
||||
let target_host = $"root@($target).lab.internal"
|
||||
let profile = $"hl-($target)"
|
||||
if not (is-valid-profile $profile true) {
|
||||
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
|
||||
return (remote_targets)
|
||||
}
|
||||
let disko_file = $"hosts/($profile).disko.nix"
|
||||
if not ($disko_file | path exists) {
|
||||
error make {msg: $"Missing disko config: ($disko_file). Refusing to install ($profile) without a disko layout."}
|
||||
}
|
||||
|
||||
print ""
|
||||
print $"(ansi red_bold)WARNING: this WIPES every disk listed in ($disko_file).(ansi reset)"
|
||||
print "This is not a switch. Use `skyg remote switch` / `skyg remote boot` to update an installed host."
|
||||
print ""
|
||||
print "Disks in the disko file:"
|
||||
let disks = (
|
||||
open $disko_file
|
||||
| lines
|
||||
| where {|l| ($l | str contains "device") and ($l | str contains "=") }
|
||||
| each {|l| $l | str trim }
|
||||
)
|
||||
for line in $disks {
|
||||
print $" ($line)"
|
||||
}
|
||||
print ""
|
||||
|
||||
let ssh = $"ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ($target_host)"
|
||||
let root_fstype = (try {
|
||||
bash -c $"($ssh) 'findmnt -n -o FSTYPE /' 2>/dev/null" | str trim
|
||||
} catch { "" })
|
||||
|
||||
if $root_fstype == "" {
|
||||
print $"(ansi yellow)Could not SSH to ($target_host) to check for a live OS.(ansi reset)"
|
||||
} else {
|
||||
print $"Remote / fstype: ($root_fstype)"
|
||||
let installer_fs = ["overlay" "tmpfs" "squashfs"]
|
||||
if not ($root_fstype in $installer_fs) {
|
||||
if not $force_wipe {
|
||||
error make {msg: $"($target_host) already boots from disk (root fstype ($root_fstype)). Refusing to wipe. Use `skyg remote switch ($target)` to update, or pass --force-wipe to reinstall."}
|
||||
}
|
||||
print $"(ansi yellow)--force-wipe set: proceeding against a live disk OS.(ansi reset)"
|
||||
} else {
|
||||
print "Remote looks like an installer live image (ok to install)."
|
||||
}
|
||||
}
|
||||
|
||||
let confirm = input $"Type ($target) to continue: " | str trim
|
||||
if $confirm != $target {
|
||||
error make {msg: $"Aborted: typed '($confirm)', expected '($target)'."}
|
||||
}
|
||||
|
||||
$"Installing [(ansi green_bold)($profile)(ansi reset)] on ($target_host) via nixos-anywhere, using ($disko_file)" | print
|
||||
let runcmd = $"nix run github:nix-community/nixos-anywhere -- --flake .#($profile) ($target_host)"
|
||||
$runcmd | print
|
||||
let bash_path = which bash | get 0.path
|
||||
with-env { SHELL: $bash_path } {
|
||||
bash -c $runcmd
|
||||
}
|
||||
}
|
||||
|
||||
# Boot all remote hosts except the current machine (fwbook)
|
||||
# Checks online status first, runs `skyg remote boot --build-host fwdesk <host>` for each online host,
|
||||
# and writes a summary report to .tmp/boot-report.txt
|
||||
|
||||
export def "skyg remote boot-all" [] {
|
||||
cd $REPO_ROOT
|
||||
mkdir .tmp
|
||||
let current_host = "fwbook"
|
||||
let report_file = ".tmp/boot-report.txt"
|
||||
|
||||
# Discover remote targets from the flake (same logic as skyg remote)
|
||||
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
||||
|
||||
print $"Found ($all_hosts | length) candidate hosts excluding ($current_host)"
|
||||
|
||||
mut successes = []
|
||||
mut failures = []
|
||||
mut skipped = []
|
||||
|
||||
# Phase 1: sequential SSH reachability checks (fast)
|
||||
mut online_hosts = []
|
||||
for host in $all_hosts {
|
||||
print $"\n=== Checking ($host) ==="
|
||||
let target = $"root@($host).lab.internal"
|
||||
let is_online = (try {
|
||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
|
||||
true
|
||||
} catch { false })
|
||||
|
||||
if not $is_online {
|
||||
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
|
||||
$skipped = ($skipped | append $host)
|
||||
} else {
|
||||
print $"(ansi green)ONLINE(ansi reset): ($host)"
|
||||
$online_hosts = ($online_hosts | append $host)
|
||||
}
|
||||
}
|
||||
|
||||
# Phase 2: run boots in parallel (2 concurrent) without interactive prompts
|
||||
if ($online_hosts | length) > 0 {
|
||||
print "
|
||||
Starting parallel boots (concurrency: 2)..."
|
||||
let boot_results = ($online_hosts | par-each --threads 1 { |host|
|
||||
let target_host = $"root@($host).lab.internal"
|
||||
let profile = $"hl-($host)"
|
||||
let runcmd = $"nh os boot --target-host ($target_host) .#($profile)"
|
||||
print $runcmd
|
||||
let ok = (try {
|
||||
bash -c $runcmd
|
||||
true
|
||||
} catch {
|
||||
false
|
||||
})
|
||||
{host: $host, success: $ok}
|
||||
})
|
||||
|
||||
for r in $boot_results {
|
||||
if $r.success {
|
||||
$successes = ($successes | append $r.host)
|
||||
print $"(ansi green)SUCCESS(ansi reset): ($r.host)"
|
||||
} else {
|
||||
$failures = ($failures | append $r.host)
|
||||
print $"(ansi red)FAILED(ansi reset): ($r.host)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Write report
|
||||
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
|
||||
let report = $"Boot Report - ($timestamp)
|
||||
|
||||
Total hosts considered: ($all_hosts | length)
|
||||
Skipped offline: ($skipped | length) - ($skipped | str join ', ')
|
||||
Successful: ($successes | length) - ($successes | str join ', ')
|
||||
Failed: ($failures | length) - ($failures | str join ', ')
|
||||
|
||||
Details:
|
||||
Online & Booted: ($successes | str join ', ')
|
||||
Offline: ($skipped | str join ', ')
|
||||
Failed: ($failures | str join ', ')
|
||||
"
|
||||
|
||||
$report | save -f $report_file
|
||||
print $"
|
||||
Report written to ($report_file)"
|
||||
print $report
|
||||
}
|
||||
|
||||
export def "skyg remote status" [ target: string@remote_targets ] {
|
||||
get-remote-boot-info $target
|
||||
}
|
||||
|
||||
# Check status of all remotes: generation, pending reboot, needs rebuild
|
||||
export def "skyg remote status all" [] {
|
||||
cd $REPO_ROOT
|
||||
mkdir .tmp
|
||||
let current_host = "fwbook"
|
||||
let report_file = ".tmp/status-report.txt"
|
||||
|
||||
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
||||
print $"Checking status for ($all_hosts | length) remotes..."
|
||||
|
||||
let results = ($all_hosts | par-each { |host|
|
||||
let info = get-remote-boot-info $host
|
||||
|
||||
# Print each host's status as one block, as soon as it finishes.
|
||||
let booted = (if ($info.booted | is-empty) { "-" } else { $info.booted | path basename })
|
||||
let current = (if ($info.current | is-empty) { "-" } else { $info.current | path basename })
|
||||
let reboot_line = if $info.reboot_pending { $"(ansi yellow)REBOOT PENDING(ansi reset)\n" } else { "" }
|
||||
let build_line = if $info.needs_build { $"(ansi red)NEEDS NEW BUILD(ansi reset)" } else { $"(ansi green)UP TO DATE(ansi reset)" }
|
||||
print $"($host) | Booted: ($booted) | Current: ($current)"
|
||||
|
||||
$info
|
||||
})
|
||||
|
||||
return ($results | select host status reboot_pending needs_build)
|
||||
}
|
||||
|
||||
# Poll SSH until the host reaches the desired reachability, or the timeout elapses.
|
||||
def wait-for-host [target: string, want_online: bool, timeout: int] {
|
||||
let deadline = (date now) + ($timeout * 1sec)
|
||||
while (date now) < $deadline {
|
||||
let online = (try {
|
||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
|
||||
true
|
||||
} catch { false })
|
||||
if $online == $want_online {
|
||||
return true
|
||||
}
|
||||
sleep 3sec
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
# Send a desktop notification if notify-send is available; never fail the caller.
|
||||
def notify [title: string, body: string] {
|
||||
try {
|
||||
^notify-send $title $body
|
||||
} catch { }
|
||||
}
|
||||
|
||||
# Reboot every remote host that is pending a reboot (booted-system != current-system).
|
||||
# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a
|
||||
# time, waiting for it to come back online and sending a desktop notification when it does.
|
||||
export def "skyg remote reboot-pending" [] {
|
||||
cd $REPO_ROOT
|
||||
mkdir .tmp
|
||||
let current_host = "fwbook"
|
||||
let report_file = ".tmp/reboot-pending-report.txt"
|
||||
|
||||
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
||||
print $"Checking ($all_hosts | length) remotes for pending reboots..."
|
||||
|
||||
# Phase 1: find hosts that are online and pending a reboot
|
||||
mut pending = []
|
||||
mut offline = []
|
||||
mut up_to_date = []
|
||||
|
||||
for host in $all_hosts {
|
||||
let target = $"root@($host).lab.internal"
|
||||
let online = (try {
|
||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
|
||||
true
|
||||
} catch { false })
|
||||
|
||||
if not $online {
|
||||
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
|
||||
$offline = ($offline | append $host)
|
||||
continue
|
||||
}
|
||||
|
||||
let info = (try {
|
||||
let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
|
||||
let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
|
||||
{booted: $booted, current: $current, pending: ($booted != $current)}
|
||||
} catch {
|
||||
{booted: "", current: "", pending: false}
|
||||
})
|
||||
|
||||
if $info.pending {
|
||||
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)"
|
||||
$pending = ($pending | append $host)
|
||||
} else {
|
||||
print $"(ansi green)UP TO DATE(ansi reset): ($host)"
|
||||
$up_to_date = ($up_to_date | append $host)
|
||||
}
|
||||
}
|
||||
|
||||
if ($pending | length) == 0 {
|
||||
print ""
|
||||
print "No remotes are pending a reboot."
|
||||
return
|
||||
}
|
||||
|
||||
# Phase 2: confirm before rebooting anything
|
||||
print ""
|
||||
print $"(ansi yellow_bold)($pending | length) remote\(s\) pending a reboot:(ansi reset)"
|
||||
for host in $pending {
|
||||
print $" - ($host)"
|
||||
}
|
||||
print ""
|
||||
let confirm = (input $"Reboot these ($pending | length) host\(s\)? [y/N]: " | str trim | str downcase)
|
||||
if not ($confirm in ["y" "yes"]) {
|
||||
print "Aborted — no hosts were rebooted."
|
||||
return
|
||||
}
|
||||
|
||||
# Phase 3: reboot each pending host one at a time, waiting for it to come back
|
||||
mut rebooted = []
|
||||
mut failed = []
|
||||
for host in $pending {
|
||||
let target = $"root@($host).lab.internal"
|
||||
print $"\n=== Rebooting ($host) ==="
|
||||
|
||||
# Capture the boot id so we can confirm the host actually rebooted.
|
||||
let boot_id_before = (try {
|
||||
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
|
||||
} catch { "" })
|
||||
|
||||
let runcmd = $"ssh -o BatchMode=yes -o ConnectTimeout=5 ($target) 'systemctl reboot'"
|
||||
print $runcmd
|
||||
# The SSH connection drops as the host shuts down, so a non-zero exit is
|
||||
# expected here and does not mean the reboot failed.
|
||||
try {
|
||||
bash -c $"($runcmd) 2>/dev/null"
|
||||
} catch { }
|
||||
|
||||
print $"Waiting for ($host) to shut down..."
|
||||
let went_down = (wait-for-host $target false 90)
|
||||
if not $went_down {
|
||||
print $"(ansi yellow)($host) did not go offline within 90s — continuing to wait for it to come back.(ansi reset)"
|
||||
}
|
||||
|
||||
print $"Waiting for ($host) to come back online..."
|
||||
let came_up = (wait-for-host $target true 300)
|
||||
if $came_up {
|
||||
let boot_id_after = (try {
|
||||
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
|
||||
} catch { "" })
|
||||
let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after)
|
||||
if $confirmed {
|
||||
print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)"
|
||||
} else {
|
||||
print $"(ansi green)($host) is back online(ansi reset)"
|
||||
}
|
||||
notify $"✅ ($host) rebooted" "Back online and reachable"
|
||||
$rebooted = ($rebooted | append $host)
|
||||
} else {
|
||||
print $"(ansi red)($host) did not come back online within 300s(ansi reset)"
|
||||
notify $"❌ ($host) did not come back" "Still unreachable after 300s"
|
||||
$failed = ($failed | append $host)
|
||||
}
|
||||
}
|
||||
|
||||
# Write report
|
||||
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
|
||||
let report = $"Reboot-Pending Report - ($timestamp)
|
||||
|
||||
Total hosts considered: ($all_hosts | length)
|
||||
Pending reboot: ($pending | length) - ($pending | str join ', ')
|
||||
Rebooted: ($rebooted | length) - ($rebooted | str join ', ')
|
||||
Failed to return: ($failed | length) - ($failed | str join ', ')
|
||||
Offline: ($offline | length) - ($offline | str join ', ')
|
||||
Up to date: ($up_to_date | length) - ($up_to_date | str join ', ')
|
||||
"
|
||||
|
||||
$report | save -f $report_file
|
||||
print $"
|
||||
Report written to ($report_file)"
|
||||
print $report
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
use types.nu *
|
||||
use profile.nu *
|
||||
|
||||
export def get-remote-boot-info [ host: string@remote_targets ] {
|
||||
let target = $"root@($host).lab.internal"
|
||||
let is_online = (try {
|
||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
|
||||
true
|
||||
} catch { false })
|
||||
if (not $is_online) {
|
||||
return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||
}
|
||||
return (try {
|
||||
# Fetch remote system paths and the local toplevel concurrently.
|
||||
let fetched = (["remote" "local"] | par-each {|what|
|
||||
if $what == "remote" {
|
||||
bash -c $"ssh ($target) 'readlink /run/booted-system; readlink /run/current-system' 2>/dev/null" | lines
|
||||
} else {
|
||||
try {
|
||||
bash -c $"nix eval --raw '.#nixosConfigurations.hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true" | str trim
|
||||
} catch { "" }
|
||||
}
|
||||
})
|
||||
|
||||
let booted = ($fetched.0 | get 0? | default "")
|
||||
let current = ($fetched.0 | get 1? | default "")
|
||||
let local_toplevel = $fetched.1
|
||||
|
||||
let reboot_pending = ($booted != $current)
|
||||
let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $current }
|
||||
|
||||
{host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel}
|
||||
} catch {
|
||||
{host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user