fixing up remote
This commit is contained in:
1 parent
6094b9dc93
commit
8ca671c89b
4 files changed
+408
-414
No files matched your search
+3
-2
@@ -11,7 +11,8 @@ bin/
|
|||||||
├── temp-monitor # NuShell script: collect CPU/GPU temps to CSV + optional graph
|
├── temp-monitor # NuShell script: collect CPU/GPU temps to CSV + optional graph
|
||||||
├── TEMP_MONITOR_README.md # Usage docs for temp-monitor
|
├── TEMP_MONITOR_README.md # Usage docs for temp-monitor
|
||||||
└── lib/
|
└── lib/
|
||||||
├── cmds.nu # All subcommand implementations (NuShell — source of truth); commands only
|
├── cmds.nu # Core subcommand implementations (NuShell — source of truth); commands only
|
||||||
|
├── remote.nu # `skyg remote*` commands (deploy, install, boot-all, status, reboot-pending)
|
||||||
├── helpers.nu # Shared helpers: REPO_ROOT const + internal functions used by commands
|
├── helpers.nu # Shared helpers: REPO_ROOT const + internal functions used by commands
|
||||||
├── profile.nu # Profile helpers and tab-completion data
|
├── profile.nu # Profile helpers and tab-completion data
|
||||||
└── types.nu # Type definitions + tab-completion data
|
└── types.nu # Type definitions + tab-completion data
|
||||||
@@ -46,4 +47,4 @@ bin/
|
|||||||
## Agent Note
|
## Agent Note
|
||||||
|
|
||||||
> **Never invoke `skyg` commands that trigger builds or switches.**
|
> **Never invoke `skyg` commands that trigger builds or switches.**
|
||||||
> Only the user runs these. Edits to `cmds.nu` are safe if the user asks for CLI changes.
|
> Only the user runs these. Edits to `cmds.nu` / `remote.nu` are safe if the user asks for CLI changes.
|
||||||
+1
-412
@@ -1,6 +1,7 @@
|
|||||||
use types.nu *
|
use types.nu *
|
||||||
use profile.nu *
|
use profile.nu *
|
||||||
use helpers.nu *
|
use helpers.nu *
|
||||||
|
export use remote.nu *
|
||||||
|
|
||||||
export def skyg [] {
|
export def skyg [] {
|
||||||
print $REPO_ROOT
|
print $REPO_ROOT
|
||||||
@@ -90,418 +91,6 @@ export def "skyg os" [cmd: string, --build-host: string = ""] {
|
|||||||
bash -c $runcmd
|
bash -c $runcmd
|
||||||
}
|
}
|
||||||
|
|
||||||
# Deploy to remote machine
|
|
||||||
export def "skyg remote" [
|
|
||||||
cmd: string@remote-cmds,
|
|
||||||
target: string@remote_targets = "",
|
|
||||||
--build-host: string@remote_targets = "", # Optional remote builder host (e.g. builder.lab.internal)
|
|
||||||
] {
|
|
||||||
cd $REPO_ROOT
|
|
||||||
mut target_host = $"root@($target).lab.internal"
|
|
||||||
mut profile = $"hl-($target)"
|
|
||||||
if not (is-valid-profile $profile true) {
|
|
||||||
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
|
|
||||||
return (remote_targets)
|
|
||||||
}
|
|
||||||
$"Using [(ansi green_bold)($profile)(ansi reset)]" | print
|
|
||||||
$"($cmd) on ($target_host) with ($profile)" | print
|
|
||||||
mut runcmd = $"nh os ($cmd) --no-nom --target-host ($target_host) --ask .#($profile)"
|
|
||||||
if $build_host != "" {
|
|
||||||
$runcmd = $"($runcmd) --build-host ($build_host).lab.internal"
|
|
||||||
}
|
|
||||||
$runcmd | print
|
|
||||||
bash -c $runcmd
|
|
||||||
}
|
|
||||||
|
|
||||||
# Fresh-install a remote machine via nixos-anywhere, using its disko config.
|
|
||||||
# Always reformats every disk in the disko file — a second run is as destructive as the first.
|
|
||||||
# Missing disko file, live disk OS (without --force-wipe), or mistyped hostname all abort.
|
|
||||||
export def "skyg remote install" [
|
|
||||||
target: string@remote_targets,
|
|
||||||
--force-wipe, # wipe even if the host already runs NixOS from disk
|
|
||||||
] {
|
|
||||||
cd $REPO_ROOT
|
|
||||||
let target_host = $"root@($target).lab.internal"
|
|
||||||
let profile = $"hl-($target)"
|
|
||||||
if not (is-valid-profile $profile true) {
|
|
||||||
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
|
|
||||||
return (remote_targets)
|
|
||||||
}
|
|
||||||
let disko_file = $"hosts/($profile).disko.nix"
|
|
||||||
if not ($disko_file | path exists) {
|
|
||||||
error make {msg: $"Missing disko config: ($disko_file). Refusing to install ($profile) without a disko layout."}
|
|
||||||
}
|
|
||||||
|
|
||||||
print ""
|
|
||||||
print $"(ansi red_bold)WARNING: this WIPES every disk listed in ($disko_file).(ansi reset)"
|
|
||||||
print "This is not a switch. Use `skyg remote switch` / `skyg remote boot` to update an installed host."
|
|
||||||
print ""
|
|
||||||
print "Disks in the disko file:"
|
|
||||||
let disks = (
|
|
||||||
open $disko_file
|
|
||||||
| lines
|
|
||||||
| where {|l| ($l | str contains "device") and ($l | str contains "=") }
|
|
||||||
| each {|l| $l | str trim }
|
|
||||||
)
|
|
||||||
for line in $disks {
|
|
||||||
print $" ($line)"
|
|
||||||
}
|
|
||||||
print ""
|
|
||||||
|
|
||||||
let ssh = $"ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ($target_host)"
|
|
||||||
let root_fstype = (try {
|
|
||||||
bash -c $"($ssh) 'findmnt -n -o FSTYPE /' 2>/dev/null" | str trim
|
|
||||||
} catch { "" })
|
|
||||||
|
|
||||||
if $root_fstype == "" {
|
|
||||||
print $"(ansi yellow)Could not SSH to ($target_host) to check for a live OS.(ansi reset)"
|
|
||||||
} else {
|
|
||||||
print $"Remote / fstype: ($root_fstype)"
|
|
||||||
let installer_fs = ["overlay" "tmpfs" "squashfs"]
|
|
||||||
if not ($root_fstype in $installer_fs) {
|
|
||||||
if not $force_wipe {
|
|
||||||
error make {msg: $"($target_host) already boots from disk (root fstype ($root_fstype)). Refusing to wipe. Use `skyg remote switch ($target)` to update, or pass --force-wipe to reinstall."}
|
|
||||||
}
|
|
||||||
print $"(ansi yellow)--force-wipe set: proceeding against a live disk OS.(ansi reset)"
|
|
||||||
} else {
|
|
||||||
print "Remote looks like an installer live image (ok to install)."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let confirm = input $"Type ($target) to continue: " | str trim
|
|
||||||
if $confirm != $target {
|
|
||||||
error make {msg: $"Aborted: typed '($confirm)', expected '($target)'."}
|
|
||||||
}
|
|
||||||
|
|
||||||
$"Installing [(ansi green_bold)($profile)(ansi reset)] on ($target_host) via nixos-anywhere, using ($disko_file)" | print
|
|
||||||
let runcmd = $"nix run github:nix-community/nixos-anywhere -- --flake .#($profile) ($target_host)"
|
|
||||||
$runcmd | print
|
|
||||||
let bash_path = which bash | get 0.path
|
|
||||||
with-env { SHELL: $bash_path } {
|
|
||||||
bash -c $runcmd
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Boot all remote hosts except the current machine (fwbook)
|
|
||||||
# Checks online status first, runs `skyg remote boot --build-host fwdesk <host>` for each online host,
|
|
||||||
# and writes a summary report to .tmp/boot-report.txt
|
|
||||||
|
|
||||||
export def "skyg remote boot-all" [] {
|
|
||||||
cd $REPO_ROOT
|
|
||||||
mkdir .tmp
|
|
||||||
let current_host = "fwbook"
|
|
||||||
let report_file = ".tmp/boot-report.txt"
|
|
||||||
|
|
||||||
# Discover remote targets from the flake (same logic as skyg remote)
|
|
||||||
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
|
||||||
|
|
||||||
print $"Found ($all_hosts | length) candidate hosts excluding ($current_host)"
|
|
||||||
|
|
||||||
mut successes = []
|
|
||||||
mut failures = []
|
|
||||||
mut skipped = []
|
|
||||||
|
|
||||||
# Phase 1: sequential SSH reachability checks (fast)
|
|
||||||
mut online_hosts = []
|
|
||||||
for host in $all_hosts {
|
|
||||||
print $"\n=== Checking ($host) ==="
|
|
||||||
let target = $"root@($host).lab.internal"
|
|
||||||
let is_online = (try {
|
|
||||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
|
|
||||||
true
|
|
||||||
} catch { false })
|
|
||||||
|
|
||||||
if not $is_online {
|
|
||||||
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
|
|
||||||
$skipped = ($skipped | append $host)
|
|
||||||
} else {
|
|
||||||
print $"(ansi green)ONLINE(ansi reset): ($host)"
|
|
||||||
$online_hosts = ($online_hosts | append $host)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Phase 2: run boots in parallel (2 concurrent) without interactive prompts
|
|
||||||
if ($online_hosts | length) > 0 {
|
|
||||||
print "
|
|
||||||
Starting parallel boots (concurrency: 2)..."
|
|
||||||
let boot_results = ($online_hosts | par-each --threads 1 { |host|
|
|
||||||
let target_host = $"root@($host).lab.internal"
|
|
||||||
let profile = $"hl-($host)"
|
|
||||||
let runcmd = $"nh os boot --target-host ($target_host) .#($profile)"
|
|
||||||
print $runcmd
|
|
||||||
let ok = (try {
|
|
||||||
bash -c $runcmd
|
|
||||||
true
|
|
||||||
} catch {
|
|
||||||
false
|
|
||||||
})
|
|
||||||
{host: $host, success: $ok}
|
|
||||||
})
|
|
||||||
|
|
||||||
for r in $boot_results {
|
|
||||||
if $r.success {
|
|
||||||
$successes = ($successes | append $r.host)
|
|
||||||
print $"(ansi green)SUCCESS(ansi reset): ($r.host)"
|
|
||||||
} else {
|
|
||||||
$failures = ($failures | append $r.host)
|
|
||||||
print $"(ansi red)FAILED(ansi reset): ($r.host)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Write report
|
|
||||||
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
|
|
||||||
let report = $"Boot Report - ($timestamp)
|
|
||||||
|
|
||||||
Total hosts considered: ($all_hosts | length)
|
|
||||||
Skipped offline: ($skipped | length) - ($skipped | str join ', ')
|
|
||||||
Successful: ($successes | length) - ($successes | str join ', ')
|
|
||||||
Failed: ($failures | length) - ($failures | str join ', ')
|
|
||||||
|
|
||||||
Details:
|
|
||||||
Online & Booted: ($successes | str join ', ')
|
|
||||||
Offline: ($skipped | str join ', ')
|
|
||||||
Failed: ($failures | str join ', ')
|
|
||||||
"
|
|
||||||
|
|
||||||
$report | save -f $report_file
|
|
||||||
print $"
|
|
||||||
Report written to ($report_file)"
|
|
||||||
print $report
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check status of all remotes: generation, pending reboot, needs rebuild
|
|
||||||
export def "skyg remote status" [] {
|
|
||||||
cd $REPO_ROOT
|
|
||||||
mkdir .tmp
|
|
||||||
let current_host = "fwbook"
|
|
||||||
let report_file = ".tmp/status-report.txt"
|
|
||||||
|
|
||||||
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
|
||||||
print $"Checking status for ($all_hosts | length) remotes..."
|
|
||||||
|
|
||||||
let results = ($all_hosts | each { |host|
|
|
||||||
print $"
|
|
||||||
=== ($host) ==="
|
|
||||||
let target = $"root@($host).lab.internal"
|
|
||||||
let online = (try {
|
|
||||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
|
|
||||||
true
|
|
||||||
} catch { false })
|
|
||||||
|
|
||||||
if not $online {
|
|
||||||
print $"(ansi yellow)OFFLINE(ansi reset)"
|
|
||||||
return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false}
|
|
||||||
}
|
|
||||||
|
|
||||||
let info = (try {
|
|
||||||
let booted = bash -c $"ssh ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
|
|
||||||
let current = bash -c $"ssh ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
|
|
||||||
let reboot_pending = ($booted != $current)
|
|
||||||
|
|
||||||
# Compare what the local flake currently evaluates to vs remote's active system
|
|
||||||
let local_toplevel = (try {
|
|
||||||
(bash -c $"nix eval --raw '.#hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true") | str trim
|
|
||||||
} catch { "" })
|
|
||||||
let needs_build = if $local_toplevel == "" { false } else { $local_toplevel != $current }
|
|
||||||
|
|
||||||
{host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build}
|
|
||||||
} catch {
|
|
||||||
{host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false}
|
|
||||||
})
|
|
||||||
|
|
||||||
print $"Booted: ($info.booted | path basename)"
|
|
||||||
print $"Current: ($info.current | path basename)"
|
|
||||||
if $info.reboot_pending {
|
|
||||||
print $"(ansi yellow)REBOOT PENDING(ansi reset)"
|
|
||||||
}
|
|
||||||
if $info.needs_build {
|
|
||||||
print $"(ansi red)NEEDS NEW BUILD(ansi reset)"
|
|
||||||
} else {
|
|
||||||
print $"(ansi green)UP TO DATE(ansi reset)"
|
|
||||||
}
|
|
||||||
|
|
||||||
$info
|
|
||||||
})
|
|
||||||
|
|
||||||
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
|
|
||||||
let lines = ($results | each { |r|
|
|
||||||
if $r.status == "offline" {
|
|
||||||
$"- ($r.host): (ansi yellow)OFFLINE(ansi reset)"
|
|
||||||
} else if $r.status == "error" {
|
|
||||||
$"- ($r.host): (ansi red)ERROR querying status(ansi reset)"
|
|
||||||
} else {
|
|
||||||
let pending = if $r.reboot_pending { $" [(ansi blue)REBOOT PENDING(ansi reset)]" } else { "" }
|
|
||||||
let build = if $r.needs_build { $" [(ansi red)NEEDS REBUILD(ansi reset)]" } else { $" [(ansi green)current(ansi reset)]" }
|
|
||||||
$"- ($r.host): booted=($r.booted | path basename) current=($r.current | path basename)($pending)($build)"
|
|
||||||
}
|
|
||||||
})
|
|
||||||
let report = $"Remote Status Report - ($timestamp)
|
|
||||||
|
|
||||||
" + ($lines | str join "
|
|
||||||
") + "
|
|
||||||
"
|
|
||||||
|
|
||||||
$report | save -f $report_file
|
|
||||||
print $"
|
|
||||||
Report written to ($report_file)"
|
|
||||||
print $report
|
|
||||||
}
|
|
||||||
|
|
||||||
# Poll SSH until the host reaches the desired reachability, or the timeout elapses.
|
|
||||||
def wait-for-host [target: string, want_online: bool, timeout: int] {
|
|
||||||
let deadline = (date now) + ($timeout * 1sec)
|
|
||||||
while (date now) < $deadline {
|
|
||||||
let online = (try {
|
|
||||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
|
|
||||||
true
|
|
||||||
} catch { false })
|
|
||||||
if $online == $want_online {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
sleep 3sec
|
|
||||||
}
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
# Send a desktop notification if notify-send is available; never fail the caller.
|
|
||||||
def notify [title: string, body: string] {
|
|
||||||
try {
|
|
||||||
^notify-send $title $body
|
|
||||||
} catch { }
|
|
||||||
}
|
|
||||||
|
|
||||||
# Reboot every remote host that is pending a reboot (booted-system != current-system).
|
|
||||||
# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a
|
|
||||||
# time, waiting for it to come back online and sending a desktop notification when it does.
|
|
||||||
export def "skyg remote reboot-pending" [] {
|
|
||||||
cd $REPO_ROOT
|
|
||||||
mkdir .tmp
|
|
||||||
let current_host = "fwbook"
|
|
||||||
let report_file = ".tmp/reboot-pending-report.txt"
|
|
||||||
|
|
||||||
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
|
||||||
print $"Checking ($all_hosts | length) remotes for pending reboots..."
|
|
||||||
|
|
||||||
# Phase 1: find hosts that are online and pending a reboot
|
|
||||||
mut pending = []
|
|
||||||
mut offline = []
|
|
||||||
mut up_to_date = []
|
|
||||||
|
|
||||||
for host in $all_hosts {
|
|
||||||
let target = $"root@($host).lab.internal"
|
|
||||||
let online = (try {
|
|
||||||
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
|
|
||||||
true
|
|
||||||
} catch { false })
|
|
||||||
|
|
||||||
if not $online {
|
|
||||||
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
|
|
||||||
$offline = ($offline | append $host)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
let info = (try {
|
|
||||||
let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
|
|
||||||
let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
|
|
||||||
{booted: $booted, current: $current, pending: ($booted != $current)}
|
|
||||||
} catch {
|
|
||||||
{booted: "", current: "", pending: false}
|
|
||||||
})
|
|
||||||
|
|
||||||
if $info.pending {
|
|
||||||
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)"
|
|
||||||
$pending = ($pending | append $host)
|
|
||||||
} else {
|
|
||||||
print $"(ansi green)UP TO DATE(ansi reset): ($host)"
|
|
||||||
$up_to_date = ($up_to_date | append $host)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($pending | length) == 0 {
|
|
||||||
print ""
|
|
||||||
print "No remotes are pending a reboot."
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
# Phase 2: confirm before rebooting anything
|
|
||||||
print ""
|
|
||||||
print $"(ansi yellow_bold)($pending | length) remote\(s\) pending a reboot:(ansi reset)"
|
|
||||||
for host in $pending {
|
|
||||||
print $" - ($host)"
|
|
||||||
}
|
|
||||||
print ""
|
|
||||||
let confirm = (input $"Reboot these ($pending | length) host\(s\)? [y/N]: " | str trim | str downcase)
|
|
||||||
if not ($confirm in ["y" "yes"]) {
|
|
||||||
print "Aborted — no hosts were rebooted."
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
# Phase 3: reboot each pending host one at a time, waiting for it to come back
|
|
||||||
mut rebooted = []
|
|
||||||
mut failed = []
|
|
||||||
for host in $pending {
|
|
||||||
let target = $"root@($host).lab.internal"
|
|
||||||
print $"\n=== Rebooting ($host) ==="
|
|
||||||
|
|
||||||
# Capture the boot id so we can confirm the host actually rebooted.
|
|
||||||
let boot_id_before = (try {
|
|
||||||
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
|
|
||||||
} catch { "" })
|
|
||||||
|
|
||||||
let runcmd = $"ssh -o BatchMode=yes -o ConnectTimeout=5 ($target) 'systemctl reboot'"
|
|
||||||
print $runcmd
|
|
||||||
# The SSH connection drops as the host shuts down, so a non-zero exit is
|
|
||||||
# expected here and does not mean the reboot failed.
|
|
||||||
try {
|
|
||||||
bash -c $"($runcmd) 2>/dev/null"
|
|
||||||
} catch { }
|
|
||||||
|
|
||||||
print $"Waiting for ($host) to shut down..."
|
|
||||||
let went_down = (wait-for-host $target false 90)
|
|
||||||
if not $went_down {
|
|
||||||
print $"(ansi yellow)($host) did not go offline within 90s — continuing to wait for it to come back.(ansi reset)"
|
|
||||||
}
|
|
||||||
|
|
||||||
print $"Waiting for ($host) to come back online..."
|
|
||||||
let came_up = (wait-for-host $target true 300)
|
|
||||||
if $came_up {
|
|
||||||
let boot_id_after = (try {
|
|
||||||
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
|
|
||||||
} catch { "" })
|
|
||||||
let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after)
|
|
||||||
if $confirmed {
|
|
||||||
print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)"
|
|
||||||
} else {
|
|
||||||
print $"(ansi green)($host) is back online(ansi reset)"
|
|
||||||
}
|
|
||||||
notify $"✅ ($host) rebooted" "Back online and reachable"
|
|
||||||
$rebooted = ($rebooted | append $host)
|
|
||||||
} else {
|
|
||||||
print $"(ansi red)($host) did not come back online within 300s(ansi reset)"
|
|
||||||
notify $"❌ ($host) did not come back" "Still unreachable after 300s"
|
|
||||||
$failed = ($failed | append $host)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Write report
|
|
||||||
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
|
|
||||||
let report = $"Reboot-Pending Report - ($timestamp)
|
|
||||||
|
|
||||||
Total hosts considered: ($all_hosts | length)
|
|
||||||
Pending reboot: ($pending | length) - ($pending | str join ', ')
|
|
||||||
Rebooted: ($rebooted | length) - ($rebooted | str join ', ')
|
|
||||||
Failed to return: ($failed | length) - ($failed | str join ', ')
|
|
||||||
Offline: ($offline | length) - ($offline | str join ', ')
|
|
||||||
Up to date: ($up_to_date | length) - ($up_to_date | str join ', ')
|
|
||||||
"
|
|
||||||
|
|
||||||
$report | save -f $report_file
|
|
||||||
print $"
|
|
||||||
Report written to ($report_file)"
|
|
||||||
print $report
|
|
||||||
}
|
|
||||||
|
|
||||||
# Deploy OpenWrt router configuration
|
# Deploy OpenWrt router configuration
|
||||||
export def "skyg openwrt" [
|
export def "skyg openwrt" [
|
||||||
router: string = "glmain",
|
router: string = "glmain",
|
||||||
|
|||||||
@@ -0,0 +1,368 @@
|
|||||||
|
use types.nu *
|
||||||
|
use profile.nu *
|
||||||
|
use helpers.nu *
|
||||||
|
use remote.utils.nu *
|
||||||
|
|
||||||
|
# Deploy to remote machine
|
||||||
|
export def "skyg remote" [
|
||||||
|
cmd: string@remote-cmds,
|
||||||
|
target: string@remote_targets = "",
|
||||||
|
--build-host: string@remote_targets = "", # Optional remote builder host (e.g. builder.lab.internal)
|
||||||
|
] {
|
||||||
|
cd $REPO_ROOT
|
||||||
|
mut target_host = $"root@($target).lab.internal"
|
||||||
|
mut profile = $"hl-($target)"
|
||||||
|
if not (is-valid-profile $profile true) {
|
||||||
|
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
|
||||||
|
return (remote_targets)
|
||||||
|
}
|
||||||
|
$"Using [(ansi green_bold)($profile)(ansi reset)]" | print
|
||||||
|
$"($cmd) on ($target_host) with ($profile)" | print
|
||||||
|
mut runcmd = $"nh os ($cmd) --no-nom --target-host ($target_host) --ask .#($profile)"
|
||||||
|
if $build_host != "" {
|
||||||
|
$runcmd = $"($runcmd) --build-host ($build_host).lab.internal"
|
||||||
|
}
|
||||||
|
$runcmd | print
|
||||||
|
bash -c $runcmd
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fresh-install a remote machine via nixos-anywhere, using its disko config.
|
||||||
|
# Always reformats every disk in the disko file — a second run is as destructive as the first.
|
||||||
|
# Missing disko file, live disk OS (without --force-wipe), or mistyped hostname all abort.
|
||||||
|
export def "skyg remote install" [
|
||||||
|
target: string@remote_targets,
|
||||||
|
--force-wipe, # wipe even if the host already runs NixOS from disk
|
||||||
|
] {
|
||||||
|
cd $REPO_ROOT
|
||||||
|
let target_host = $"root@($target).lab.internal"
|
||||||
|
let profile = $"hl-($target)"
|
||||||
|
if not (is-valid-profile $profile true) {
|
||||||
|
print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:"
|
||||||
|
return (remote_targets)
|
||||||
|
}
|
||||||
|
let disko_file = $"hosts/($profile).disko.nix"
|
||||||
|
if not ($disko_file | path exists) {
|
||||||
|
error make {msg: $"Missing disko config: ($disko_file). Refusing to install ($profile) without a disko layout."}
|
||||||
|
}
|
||||||
|
|
||||||
|
print ""
|
||||||
|
print $"(ansi red_bold)WARNING: this WIPES every disk listed in ($disko_file).(ansi reset)"
|
||||||
|
print "This is not a switch. Use `skyg remote switch` / `skyg remote boot` to update an installed host."
|
||||||
|
print ""
|
||||||
|
print "Disks in the disko file:"
|
||||||
|
let disks = (
|
||||||
|
open $disko_file
|
||||||
|
| lines
|
||||||
|
| where {|l| ($l | str contains "device") and ($l | str contains "=") }
|
||||||
|
| each {|l| $l | str trim }
|
||||||
|
)
|
||||||
|
for line in $disks {
|
||||||
|
print $" ($line)"
|
||||||
|
}
|
||||||
|
print ""
|
||||||
|
|
||||||
|
let ssh = $"ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ($target_host)"
|
||||||
|
let root_fstype = (try {
|
||||||
|
bash -c $"($ssh) 'findmnt -n -o FSTYPE /' 2>/dev/null" | str trim
|
||||||
|
} catch { "" })
|
||||||
|
|
||||||
|
if $root_fstype == "" {
|
||||||
|
print $"(ansi yellow)Could not SSH to ($target_host) to check for a live OS.(ansi reset)"
|
||||||
|
} else {
|
||||||
|
print $"Remote / fstype: ($root_fstype)"
|
||||||
|
let installer_fs = ["overlay" "tmpfs" "squashfs"]
|
||||||
|
if not ($root_fstype in $installer_fs) {
|
||||||
|
if not $force_wipe {
|
||||||
|
error make {msg: $"($target_host) already boots from disk (root fstype ($root_fstype)). Refusing to wipe. Use `skyg remote switch ($target)` to update, or pass --force-wipe to reinstall."}
|
||||||
|
}
|
||||||
|
print $"(ansi yellow)--force-wipe set: proceeding against a live disk OS.(ansi reset)"
|
||||||
|
} else {
|
||||||
|
print "Remote looks like an installer live image (ok to install)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let confirm = input $"Type ($target) to continue: " | str trim
|
||||||
|
if $confirm != $target {
|
||||||
|
error make {msg: $"Aborted: typed '($confirm)', expected '($target)'."}
|
||||||
|
}
|
||||||
|
|
||||||
|
$"Installing [(ansi green_bold)($profile)(ansi reset)] on ($target_host) via nixos-anywhere, using ($disko_file)" | print
|
||||||
|
let runcmd = $"nix run github:nix-community/nixos-anywhere -- --flake .#($profile) ($target_host)"
|
||||||
|
$runcmd | print
|
||||||
|
let bash_path = which bash | get 0.path
|
||||||
|
with-env { SHELL: $bash_path } {
|
||||||
|
bash -c $runcmd
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Boot all remote hosts except the current machine (fwbook)
|
||||||
|
# Checks online status first, runs `skyg remote boot --build-host fwdesk <host>` for each online host,
|
||||||
|
# and writes a summary report to .tmp/boot-report.txt
|
||||||
|
|
||||||
|
export def "skyg remote boot-all" [] {
|
||||||
|
cd $REPO_ROOT
|
||||||
|
mkdir .tmp
|
||||||
|
let current_host = "fwbook"
|
||||||
|
let report_file = ".tmp/boot-report.txt"
|
||||||
|
|
||||||
|
# Discover remote targets from the flake (same logic as skyg remote)
|
||||||
|
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
||||||
|
|
||||||
|
print $"Found ($all_hosts | length) candidate hosts excluding ($current_host)"
|
||||||
|
|
||||||
|
mut successes = []
|
||||||
|
mut failures = []
|
||||||
|
mut skipped = []
|
||||||
|
|
||||||
|
# Phase 1: sequential SSH reachability checks (fast)
|
||||||
|
mut online_hosts = []
|
||||||
|
for host in $all_hosts {
|
||||||
|
print $"\n=== Checking ($host) ==="
|
||||||
|
let target = $"root@($host).lab.internal"
|
||||||
|
let is_online = (try {
|
||||||
|
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
|
||||||
|
true
|
||||||
|
} catch { false })
|
||||||
|
|
||||||
|
if not $is_online {
|
||||||
|
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
|
||||||
|
$skipped = ($skipped | append $host)
|
||||||
|
} else {
|
||||||
|
print $"(ansi green)ONLINE(ansi reset): ($host)"
|
||||||
|
$online_hosts = ($online_hosts | append $host)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Phase 2: run boots in parallel (2 concurrent) without interactive prompts
|
||||||
|
if ($online_hosts | length) > 0 {
|
||||||
|
print "
|
||||||
|
Starting parallel boots (concurrency: 2)..."
|
||||||
|
let boot_results = ($online_hosts | par-each --threads 1 { |host|
|
||||||
|
let target_host = $"root@($host).lab.internal"
|
||||||
|
let profile = $"hl-($host)"
|
||||||
|
let runcmd = $"nh os boot --target-host ($target_host) .#($profile)"
|
||||||
|
print $runcmd
|
||||||
|
let ok = (try {
|
||||||
|
bash -c $runcmd
|
||||||
|
true
|
||||||
|
} catch {
|
||||||
|
false
|
||||||
|
})
|
||||||
|
{host: $host, success: $ok}
|
||||||
|
})
|
||||||
|
|
||||||
|
for r in $boot_results {
|
||||||
|
if $r.success {
|
||||||
|
$successes = ($successes | append $r.host)
|
||||||
|
print $"(ansi green)SUCCESS(ansi reset): ($r.host)"
|
||||||
|
} else {
|
||||||
|
$failures = ($failures | append $r.host)
|
||||||
|
print $"(ansi red)FAILED(ansi reset): ($r.host)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Write report
|
||||||
|
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
|
||||||
|
let report = $"Boot Report - ($timestamp)
|
||||||
|
|
||||||
|
Total hosts considered: ($all_hosts | length)
|
||||||
|
Skipped offline: ($skipped | length) - ($skipped | str join ', ')
|
||||||
|
Successful: ($successes | length) - ($successes | str join ', ')
|
||||||
|
Failed: ($failures | length) - ($failures | str join ', ')
|
||||||
|
|
||||||
|
Details:
|
||||||
|
Online & Booted: ($successes | str join ', ')
|
||||||
|
Offline: ($skipped | str join ', ')
|
||||||
|
Failed: ($failures | str join ', ')
|
||||||
|
"
|
||||||
|
|
||||||
|
$report | save -f $report_file
|
||||||
|
print $"
|
||||||
|
Report written to ($report_file)"
|
||||||
|
print $report
|
||||||
|
}
|
||||||
|
|
||||||
|
export def "skyg remote status" [ target: string@remote_targets ] {
|
||||||
|
get-remote-boot-info $target
|
||||||
|
}
|
||||||
|
|
||||||
|
# Check status of all remotes: generation, pending reboot, needs rebuild
|
||||||
|
export def "skyg remote status all" [] {
|
||||||
|
cd $REPO_ROOT
|
||||||
|
mkdir .tmp
|
||||||
|
let current_host = "fwbook"
|
||||||
|
let report_file = ".tmp/status-report.txt"
|
||||||
|
|
||||||
|
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
||||||
|
print $"Checking status for ($all_hosts | length) remotes..."
|
||||||
|
|
||||||
|
let results = ($all_hosts | par-each { |host|
|
||||||
|
let info = get-remote-boot-info $host
|
||||||
|
|
||||||
|
# Print each host's status as one block, as soon as it finishes.
|
||||||
|
let booted = (if ($info.booted | is-empty) { "-" } else { $info.booted | path basename })
|
||||||
|
let current = (if ($info.current | is-empty) { "-" } else { $info.current | path basename })
|
||||||
|
let reboot_line = if $info.reboot_pending { $"(ansi yellow)REBOOT PENDING(ansi reset)\n" } else { "" }
|
||||||
|
let build_line = if $info.needs_build { $"(ansi red)NEEDS NEW BUILD(ansi reset)" } else { $"(ansi green)UP TO DATE(ansi reset)" }
|
||||||
|
print $"($host) | Booted: ($booted) | Current: ($current)"
|
||||||
|
|
||||||
|
$info
|
||||||
|
})
|
||||||
|
|
||||||
|
return ($results | select host status reboot_pending needs_build)
|
||||||
|
}
|
||||||
|
|
||||||
|
# Poll SSH until the host reaches the desired reachability, or the timeout elapses.
|
||||||
|
def wait-for-host [target: string, want_online: bool, timeout: int] {
|
||||||
|
let deadline = (date now) + ($timeout * 1sec)
|
||||||
|
while (date now) < $deadline {
|
||||||
|
let online = (try {
|
||||||
|
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
|
||||||
|
true
|
||||||
|
} catch { false })
|
||||||
|
if $online == $want_online {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
sleep 3sec
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
# Send a desktop notification if notify-send is available; never fail the caller.
|
||||||
|
def notify [title: string, body: string] {
|
||||||
|
try {
|
||||||
|
^notify-send $title $body
|
||||||
|
} catch { }
|
||||||
|
}
|
||||||
|
|
||||||
|
# Reboot every remote host that is pending a reboot (booted-system != current-system).
|
||||||
|
# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a
|
||||||
|
# time, waiting for it to come back online and sending a desktop notification when it does.
|
||||||
|
export def "skyg remote reboot-pending" [] {
|
||||||
|
cd $REPO_ROOT
|
||||||
|
mkdir .tmp
|
||||||
|
let current_host = "fwbook"
|
||||||
|
let report_file = ".tmp/reboot-pending-report.txt"
|
||||||
|
|
||||||
|
let all_hosts = remote_targets | where {|h| $h != $current_host }
|
||||||
|
print $"Checking ($all_hosts | length) remotes for pending reboots..."
|
||||||
|
|
||||||
|
# Phase 1: find hosts that are online and pending a reboot
|
||||||
|
mut pending = []
|
||||||
|
mut offline = []
|
||||||
|
mut up_to_date = []
|
||||||
|
|
||||||
|
for host in $all_hosts {
|
||||||
|
let target = $"root@($host).lab.internal"
|
||||||
|
let online = (try {
|
||||||
|
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1"
|
||||||
|
true
|
||||||
|
} catch { false })
|
||||||
|
|
||||||
|
if not $online {
|
||||||
|
print $"(ansi yellow)OFFLINE(ansi reset): ($host)"
|
||||||
|
$offline = ($offline | append $host)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
let info = (try {
|
||||||
|
let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
|
||||||
|
let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
|
||||||
|
{booted: $booted, current: $current, pending: ($booted != $current)}
|
||||||
|
} catch {
|
||||||
|
{booted: "", current: "", pending: false}
|
||||||
|
})
|
||||||
|
|
||||||
|
if $info.pending {
|
||||||
|
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)"
|
||||||
|
$pending = ($pending | append $host)
|
||||||
|
} else {
|
||||||
|
print $"(ansi green)UP TO DATE(ansi reset): ($host)"
|
||||||
|
$up_to_date = ($up_to_date | append $host)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($pending | length) == 0 {
|
||||||
|
print ""
|
||||||
|
print "No remotes are pending a reboot."
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
# Phase 2: confirm before rebooting anything
|
||||||
|
print ""
|
||||||
|
print $"(ansi yellow_bold)($pending | length) remote\(s\) pending a reboot:(ansi reset)"
|
||||||
|
for host in $pending {
|
||||||
|
print $" - ($host)"
|
||||||
|
}
|
||||||
|
print ""
|
||||||
|
let confirm = (input $"Reboot these ($pending | length) host\(s\)? [y/N]: " | str trim | str downcase)
|
||||||
|
if not ($confirm in ["y" "yes"]) {
|
||||||
|
print "Aborted — no hosts were rebooted."
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
# Phase 3: reboot each pending host one at a time, waiting for it to come back
|
||||||
|
mut rebooted = []
|
||||||
|
mut failed = []
|
||||||
|
for host in $pending {
|
||||||
|
let target = $"root@($host).lab.internal"
|
||||||
|
print $"\n=== Rebooting ($host) ==="
|
||||||
|
|
||||||
|
# Capture the boot id so we can confirm the host actually rebooted.
|
||||||
|
let boot_id_before = (try {
|
||||||
|
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
|
||||||
|
} catch { "" })
|
||||||
|
|
||||||
|
let runcmd = $"ssh -o BatchMode=yes -o ConnectTimeout=5 ($target) 'systemctl reboot'"
|
||||||
|
print $runcmd
|
||||||
|
# The SSH connection drops as the host shuts down, so a non-zero exit is
|
||||||
|
# expected here and does not mean the reboot failed.
|
||||||
|
try {
|
||||||
|
bash -c $"($runcmd) 2>/dev/null"
|
||||||
|
} catch { }
|
||||||
|
|
||||||
|
print $"Waiting for ($host) to shut down..."
|
||||||
|
let went_down = (wait-for-host $target false 90)
|
||||||
|
if not $went_down {
|
||||||
|
print $"(ansi yellow)($host) did not go offline within 90s — continuing to wait for it to come back.(ansi reset)"
|
||||||
|
}
|
||||||
|
|
||||||
|
print $"Waiting for ($host) to come back online..."
|
||||||
|
let came_up = (wait-for-host $target true 300)
|
||||||
|
if $came_up {
|
||||||
|
let boot_id_after = (try {
|
||||||
|
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
|
||||||
|
} catch { "" })
|
||||||
|
let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after)
|
||||||
|
if $confirmed {
|
||||||
|
print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)"
|
||||||
|
} else {
|
||||||
|
print $"(ansi green)($host) is back online(ansi reset)"
|
||||||
|
}
|
||||||
|
notify $"✅ ($host) rebooted" "Back online and reachable"
|
||||||
|
$rebooted = ($rebooted | append $host)
|
||||||
|
} else {
|
||||||
|
print $"(ansi red)($host) did not come back online within 300s(ansi reset)"
|
||||||
|
notify $"❌ ($host) did not come back" "Still unreachable after 300s"
|
||||||
|
$failed = ($failed | append $host)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Write report
|
||||||
|
let timestamp = date now | format date "%Y-%m-%d %H:%M:%S"
|
||||||
|
let report = $"Reboot-Pending Report - ($timestamp)
|
||||||
|
|
||||||
|
Total hosts considered: ($all_hosts | length)
|
||||||
|
Pending reboot: ($pending | length) - ($pending | str join ', ')
|
||||||
|
Rebooted: ($rebooted | length) - ($rebooted | str join ', ')
|
||||||
|
Failed to return: ($failed | length) - ($failed | str join ', ')
|
||||||
|
Offline: ($offline | length) - ($offline | str join ', ')
|
||||||
|
Up to date: ($up_to_date | length) - ($up_to_date | str join ', ')
|
||||||
|
"
|
||||||
|
|
||||||
|
$report | save -f $report_file
|
||||||
|
print $"
|
||||||
|
Report written to ($report_file)"
|
||||||
|
print $report
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
use types.nu *
|
||||||
|
use profile.nu *
|
||||||
|
|
||||||
|
export def get-remote-boot-info [ host: string@remote_targets ] {
|
||||||
|
let target = $"root@($host).lab.internal"
|
||||||
|
let is_online = (try {
|
||||||
|
bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1"
|
||||||
|
true
|
||||||
|
} catch { false })
|
||||||
|
if (not $is_online) {
|
||||||
|
return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||||
|
}
|
||||||
|
return (try {
|
||||||
|
# Fetch remote system paths and the local toplevel concurrently.
|
||||||
|
let fetched = (["remote" "local"] | par-each {|what|
|
||||||
|
if $what == "remote" {
|
||||||
|
bash -c $"ssh ($target) 'readlink /run/booted-system; readlink /run/current-system' 2>/dev/null" | lines
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
bash -c $"nix eval --raw '.#nixosConfigurations.hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true" | str trim
|
||||||
|
} catch { "" }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
let booted = ($fetched.0 | get 0? | default "")
|
||||||
|
let current = ($fetched.0 | get 1? | default "")
|
||||||
|
let local_toplevel = $fetched.1
|
||||||
|
|
||||||
|
let reboot_pending = ($booted != $current)
|
||||||
|
let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $current }
|
||||||
|
|
||||||
|
{host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel}
|
||||||
|
} catch {
|
||||||
|
{host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||||
|
})
|
||||||
|
}
|
||||||
Reference in new issue
Block a user