From 8ca671c89b8de3881ebe25ca4f367ffebd9af906 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Sun, 11 Oct 2026 01:53:43 -0400 Subject: [PATCH] fixing up remote --- bin/ABOUTME.md | 5 +- bin/lib/cmds.nu | 413 +--------------------------------------- bin/lib/remote.nu | 368 +++++++++++++++++++++++++++++++++++ bin/lib/remote.utils.nu | 36 ++++ 4 files changed, 408 insertions(+), 414 deletions(-) create mode 100644 bin/lib/remote.nu create mode 100644 bin/lib/remote.utils.nu diff --git a/bin/ABOUTME.md b/bin/ABOUTME.md index 8c86af6..0e4326f 100644 --- a/bin/ABOUTME.md +++ b/bin/ABOUTME.md @@ -11,7 +11,8 @@ bin/ ├── temp-monitor # NuShell script: collect CPU/GPU temps to CSV + optional graph ├── TEMP_MONITOR_README.md # Usage docs for temp-monitor └── lib/ - ├── cmds.nu # All subcommand implementations (NuShell — source of truth); commands only + ├── cmds.nu # Core subcommand implementations (NuShell — source of truth); commands only + ├── remote.nu # `skyg remote*` commands (deploy, install, boot-all, status, reboot-pending) ├── helpers.nu # Shared helpers: REPO_ROOT const + internal functions used by commands ├── profile.nu # Profile helpers and tab-completion data └── types.nu # Type definitions + tab-completion data @@ -46,4 +47,4 @@ bin/ ## Agent Note > **Never invoke `skyg` commands that trigger builds or switches.** -> Only the user runs these. Edits to `cmds.nu` are safe if the user asks for CLI changes. +> Only the user runs these. Edits to `cmds.nu` / `remote.nu` are safe if the user asks for CLI changes. diff --git a/bin/lib/cmds.nu b/bin/lib/cmds.nu index 7c05d76..4a3431a 100644 --- a/bin/lib/cmds.nu +++ b/bin/lib/cmds.nu @@ -1,6 +1,7 @@ use types.nu * use profile.nu * use helpers.nu * +export use remote.nu * export def skyg [] { print $REPO_ROOT @@ -90,418 +91,6 @@ export def "skyg os" [cmd: string, --build-host: string = ""] { bash -c $runcmd } -# Deploy to remote machine -export def "skyg remote" [ - cmd: string@remote-cmds, - target: string@remote_targets = "", - --build-host: string@remote_targets = "", # Optional remote builder host (e.g. builder.lab.internal) -] { - cd $REPO_ROOT - mut target_host = $"root@($target).lab.internal" - mut profile = $"hl-($target)" - if not (is-valid-profile $profile true) { - print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:" - return (remote_targets) - } - $"Using [(ansi green_bold)($profile)(ansi reset)]" | print - $"($cmd) on ($target_host) with ($profile)" | print - mut runcmd = $"nh os ($cmd) --no-nom --target-host ($target_host) --ask .#($profile)" - if $build_host != "" { - $runcmd = $"($runcmd) --build-host ($build_host).lab.internal" - } - $runcmd | print - bash -c $runcmd -} - -# Fresh-install a remote machine via nixos-anywhere, using its disko config. -# Always reformats every disk in the disko file — a second run is as destructive as the first. -# Missing disko file, live disk OS (without --force-wipe), or mistyped hostname all abort. -export def "skyg remote install" [ - target: string@remote_targets, - --force-wipe, # wipe even if the host already runs NixOS from disk -] { - cd $REPO_ROOT - let target_host = $"root@($target).lab.internal" - let profile = $"hl-($target)" - if not (is-valid-profile $profile true) { - print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:" - return (remote_targets) - } - let disko_file = $"hosts/($profile).disko.nix" - if not ($disko_file | path exists) { - error make {msg: $"Missing disko config: ($disko_file). Refusing to install ($profile) without a disko layout."} - } - - print "" - print $"(ansi red_bold)WARNING: this WIPES every disk listed in ($disko_file).(ansi reset)" - print "This is not a switch. Use `skyg remote switch` / `skyg remote boot` to update an installed host." - print "" - print "Disks in the disko file:" - let disks = ( - open $disko_file - | lines - | where {|l| ($l | str contains "device") and ($l | str contains "=") } - | each {|l| $l | str trim } - ) - for line in $disks { - print $" ($line)" - } - print "" - - let ssh = $"ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ($target_host)" - let root_fstype = (try { - bash -c $"($ssh) 'findmnt -n -o FSTYPE /' 2>/dev/null" | str trim - } catch { "" }) - - if $root_fstype == "" { - print $"(ansi yellow)Could not SSH to ($target_host) to check for a live OS.(ansi reset)" - } else { - print $"Remote / fstype: ($root_fstype)" - let installer_fs = ["overlay" "tmpfs" "squashfs"] - if not ($root_fstype in $installer_fs) { - if not $force_wipe { - error make {msg: $"($target_host) already boots from disk (root fstype ($root_fstype)). Refusing to wipe. Use `skyg remote switch ($target)` to update, or pass --force-wipe to reinstall."} - } - print $"(ansi yellow)--force-wipe set: proceeding against a live disk OS.(ansi reset)" - } else { - print "Remote looks like an installer live image (ok to install)." - } - } - - let confirm = input $"Type ($target) to continue: " | str trim - if $confirm != $target { - error make {msg: $"Aborted: typed '($confirm)', expected '($target)'."} - } - - $"Installing [(ansi green_bold)($profile)(ansi reset)] on ($target_host) via nixos-anywhere, using ($disko_file)" | print - let runcmd = $"nix run github:nix-community/nixos-anywhere -- --flake .#($profile) ($target_host)" - $runcmd | print - let bash_path = which bash | get 0.path - with-env { SHELL: $bash_path } { - bash -c $runcmd - } -} - -# Boot all remote hosts except the current machine (fwbook) -# Checks online status first, runs `skyg remote boot --build-host fwdesk ` for each online host, -# and writes a summary report to .tmp/boot-report.txt - -export def "skyg remote boot-all" [] { - cd $REPO_ROOT - mkdir .tmp - let current_host = "fwbook" - let report_file = ".tmp/boot-report.txt" - - # Discover remote targets from the flake (same logic as skyg remote) - let all_hosts = remote_targets | where {|h| $h != $current_host } - - print $"Found ($all_hosts | length) candidate hosts excluding ($current_host)" - - mut successes = [] - mut failures = [] - mut skipped = [] - - # Phase 1: sequential SSH reachability checks (fast) - mut online_hosts = [] - for host in $all_hosts { - print $"\n=== Checking ($host) ===" - let target = $"root@($host).lab.internal" - let is_online = (try { - bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1" - true - } catch { false }) - - if not $is_online { - print $"(ansi yellow)OFFLINE(ansi reset): ($host)" - $skipped = ($skipped | append $host) - } else { - print $"(ansi green)ONLINE(ansi reset): ($host)" - $online_hosts = ($online_hosts | append $host) - } - } - - # Phase 2: run boots in parallel (2 concurrent) without interactive prompts - if ($online_hosts | length) > 0 { - print " -Starting parallel boots (concurrency: 2)..." - let boot_results = ($online_hosts | par-each --threads 1 { |host| - let target_host = $"root@($host).lab.internal" - let profile = $"hl-($host)" - let runcmd = $"nh os boot --target-host ($target_host) .#($profile)" - print $runcmd - let ok = (try { - bash -c $runcmd - true - } catch { - false - }) - {host: $host, success: $ok} - }) - - for r in $boot_results { - if $r.success { - $successes = ($successes | append $r.host) - print $"(ansi green)SUCCESS(ansi reset): ($r.host)" - } else { - $failures = ($failures | append $r.host) - print $"(ansi red)FAILED(ansi reset): ($r.host)" - } - } - } - - # Write report - let timestamp = date now | format date "%Y-%m-%d %H:%M:%S" - let report = $"Boot Report - ($timestamp) - -Total hosts considered: ($all_hosts | length) -Skipped offline: ($skipped | length) - ($skipped | str join ', ') -Successful: ($successes | length) - ($successes | str join ', ') -Failed: ($failures | length) - ($failures | str join ', ') - -Details: - Online & Booted: ($successes | str join ', ') - Offline: ($skipped | str join ', ') - Failed: ($failures | str join ', ') -" - - $report | save -f $report_file - print $" -Report written to ($report_file)" - print $report -} - -# Check status of all remotes: generation, pending reboot, needs rebuild -export def "skyg remote status" [] { - cd $REPO_ROOT - mkdir .tmp - let current_host = "fwbook" - let report_file = ".tmp/status-report.txt" - - let all_hosts = remote_targets | where {|h| $h != $current_host } - print $"Checking status for ($all_hosts | length) remotes..." - - let results = ($all_hosts | each { |host| - print $" -=== ($host) ===" - let target = $"root@($host).lab.internal" - let online = (try { - bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1" - true - } catch { false }) - - if not $online { - print $"(ansi yellow)OFFLINE(ansi reset)" - return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false} - } - - let info = (try { - let booted = bash -c $"ssh ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim - let current = bash -c $"ssh ($target) 'readlink /run/current-system' 2>/dev/null" | str trim - let reboot_pending = ($booted != $current) - - # Compare what the local flake currently evaluates to vs remote's active system - let local_toplevel = (try { - (bash -c $"nix eval --raw '.#hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true") | str trim - } catch { "" }) - let needs_build = if $local_toplevel == "" { false } else { $local_toplevel != $current } - - {host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build} - } catch { - {host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false} - }) - - print $"Booted: ($info.booted | path basename)" - print $"Current: ($info.current | path basename)" - if $info.reboot_pending { - print $"(ansi yellow)REBOOT PENDING(ansi reset)" - } - if $info.needs_build { - print $"(ansi red)NEEDS NEW BUILD(ansi reset)" - } else { - print $"(ansi green)UP TO DATE(ansi reset)" - } - - $info - }) - - let timestamp = date now | format date "%Y-%m-%d %H:%M:%S" - let lines = ($results | each { |r| - if $r.status == "offline" { - $"- ($r.host): (ansi yellow)OFFLINE(ansi reset)" - } else if $r.status == "error" { - $"- ($r.host): (ansi red)ERROR querying status(ansi reset)" - } else { - let pending = if $r.reboot_pending { $" [(ansi blue)REBOOT PENDING(ansi reset)]" } else { "" } - let build = if $r.needs_build { $" [(ansi red)NEEDS REBUILD(ansi reset)]" } else { $" [(ansi green)current(ansi reset)]" } - $"- ($r.host): booted=($r.booted | path basename) current=($r.current | path basename)($pending)($build)" - } - }) - let report = $"Remote Status Report - ($timestamp) - -" + ($lines | str join " -") + " -" - - $report | save -f $report_file - print $" -Report written to ($report_file)" - print $report -} - -# Poll SSH until the host reaches the desired reachability, or the timeout elapses. -def wait-for-host [target: string, want_online: bool, timeout: int] { - let deadline = (date now) + ($timeout * 1sec) - while (date now) < $deadline { - let online = (try { - bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1" - true - } catch { false }) - if $online == $want_online { - return true - } - sleep 3sec - } - false -} - -# Send a desktop notification if notify-send is available; never fail the caller. -def notify [title: string, body: string] { - try { - ^notify-send $title $body - } catch { } -} - -# Reboot every remote host that is pending a reboot (booted-system != current-system). -# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a -# time, waiting for it to come back online and sending a desktop notification when it does. -export def "skyg remote reboot-pending" [] { - cd $REPO_ROOT - mkdir .tmp - let current_host = "fwbook" - let report_file = ".tmp/reboot-pending-report.txt" - - let all_hosts = remote_targets | where {|h| $h != $current_host } - print $"Checking ($all_hosts | length) remotes for pending reboots..." - - # Phase 1: find hosts that are online and pending a reboot - mut pending = [] - mut offline = [] - mut up_to_date = [] - - for host in $all_hosts { - let target = $"root@($host).lab.internal" - let online = (try { - bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1" - true - } catch { false }) - - if not $online { - print $"(ansi yellow)OFFLINE(ansi reset): ($host)" - $offline = ($offline | append $host) - continue - } - - let info = (try { - let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim - let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim - {booted: $booted, current: $current, pending: ($booted != $current)} - } catch { - {booted: "", current: "", pending: false} - }) - - if $info.pending { - print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)" - $pending = ($pending | append $host) - } else { - print $"(ansi green)UP TO DATE(ansi reset): ($host)" - $up_to_date = ($up_to_date | append $host) - } - } - - if ($pending | length) == 0 { - print "" - print "No remotes are pending a reboot." - return - } - - # Phase 2: confirm before rebooting anything - print "" - print $"(ansi yellow_bold)($pending | length) remote\(s\) pending a reboot:(ansi reset)" - for host in $pending { - print $" - ($host)" - } - print "" - let confirm = (input $"Reboot these ($pending | length) host\(s\)? [y/N]: " | str trim | str downcase) - if not ($confirm in ["y" "yes"]) { - print "Aborted — no hosts were rebooted." - return - } - - # Phase 3: reboot each pending host one at a time, waiting for it to come back - mut rebooted = [] - mut failed = [] - for host in $pending { - let target = $"root@($host).lab.internal" - print $"\n=== Rebooting ($host) ===" - - # Capture the boot id so we can confirm the host actually rebooted. - let boot_id_before = (try { - bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim - } catch { "" }) - - let runcmd = $"ssh -o BatchMode=yes -o ConnectTimeout=5 ($target) 'systemctl reboot'" - print $runcmd - # The SSH connection drops as the host shuts down, so a non-zero exit is - # expected here and does not mean the reboot failed. - try { - bash -c $"($runcmd) 2>/dev/null" - } catch { } - - print $"Waiting for ($host) to shut down..." - let went_down = (wait-for-host $target false 90) - if not $went_down { - print $"(ansi yellow)($host) did not go offline within 90s — continuing to wait for it to come back.(ansi reset)" - } - - print $"Waiting for ($host) to come back online..." - let came_up = (wait-for-host $target true 300) - if $came_up { - let boot_id_after = (try { - bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim - } catch { "" }) - let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after) - if $confirmed { - print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)" - } else { - print $"(ansi green)($host) is back online(ansi reset)" - } - notify $"✅ ($host) rebooted" "Back online and reachable" - $rebooted = ($rebooted | append $host) - } else { - print $"(ansi red)($host) did not come back online within 300s(ansi reset)" - notify $"❌ ($host) did not come back" "Still unreachable after 300s" - $failed = ($failed | append $host) - } - } - - # Write report - let timestamp = date now | format date "%Y-%m-%d %H:%M:%S" - let report = $"Reboot-Pending Report - ($timestamp) - -Total hosts considered: ($all_hosts | length) -Pending reboot: ($pending | length) - ($pending | str join ', ') -Rebooted: ($rebooted | length) - ($rebooted | str join ', ') -Failed to return: ($failed | length) - ($failed | str join ', ') -Offline: ($offline | length) - ($offline | str join ', ') -Up to date: ($up_to_date | length) - ($up_to_date | str join ', ') -" - - $report | save -f $report_file - print $" -Report written to ($report_file)" - print $report -} - # Deploy OpenWrt router configuration export def "skyg openwrt" [ router: string = "glmain", diff --git a/bin/lib/remote.nu b/bin/lib/remote.nu new file mode 100644 index 0000000..e5184e5 --- /dev/null +++ b/bin/lib/remote.nu @@ -0,0 +1,368 @@ +use types.nu * +use profile.nu * +use helpers.nu * +use remote.utils.nu * + +# Deploy to remote machine +export def "skyg remote" [ + cmd: string@remote-cmds, + target: string@remote_targets = "", + --build-host: string@remote_targets = "", # Optional remote builder host (e.g. builder.lab.internal) +] { + cd $REPO_ROOT + mut target_host = $"root@($target).lab.internal" + mut profile = $"hl-($target)" + if not (is-valid-profile $profile true) { + print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:" + return (remote_targets) + } + $"Using [(ansi green_bold)($profile)(ansi reset)]" | print + $"($cmd) on ($target_host) with ($profile)" | print + mut runcmd = $"nh os ($cmd) --no-nom --target-host ($target_host) --ask .#($profile)" + if $build_host != "" { + $runcmd = $"($runcmd) --build-host ($build_host).lab.internal" + } + $runcmd | print + bash -c $runcmd +} + +# Fresh-install a remote machine via nixos-anywhere, using its disko config. +# Always reformats every disk in the disko file — a second run is as destructive as the first. +# Missing disko file, live disk OS (without --force-wipe), or mistyped hostname all abort. +export def "skyg remote install" [ + target: string@remote_targets, + --force-wipe, # wipe even if the host already runs NixOS from disk +] { + cd $REPO_ROOT + let target_host = $"root@($target).lab.internal" + let profile = $"hl-($target)" + if not (is-valid-profile $profile true) { + print $"Invalid Remote Profile '(ansi green_bold)($target)(ansi reset)', please select one of:" + return (remote_targets) + } + let disko_file = $"hosts/($profile).disko.nix" + if not ($disko_file | path exists) { + error make {msg: $"Missing disko config: ($disko_file). Refusing to install ($profile) without a disko layout."} + } + + print "" + print $"(ansi red_bold)WARNING: this WIPES every disk listed in ($disko_file).(ansi reset)" + print "This is not a switch. Use `skyg remote switch` / `skyg remote boot` to update an installed host." + print "" + print "Disks in the disko file:" + let disks = ( + open $disko_file + | lines + | where {|l| ($l | str contains "device") and ($l | str contains "=") } + | each {|l| $l | str trim } + ) + for line in $disks { + print $" ($line)" + } + print "" + + let ssh = $"ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ($target_host)" + let root_fstype = (try { + bash -c $"($ssh) 'findmnt -n -o FSTYPE /' 2>/dev/null" | str trim + } catch { "" }) + + if $root_fstype == "" { + print $"(ansi yellow)Could not SSH to ($target_host) to check for a live OS.(ansi reset)" + } else { + print $"Remote / fstype: ($root_fstype)" + let installer_fs = ["overlay" "tmpfs" "squashfs"] + if not ($root_fstype in $installer_fs) { + if not $force_wipe { + error make {msg: $"($target_host) already boots from disk (root fstype ($root_fstype)). Refusing to wipe. Use `skyg remote switch ($target)` to update, or pass --force-wipe to reinstall."} + } + print $"(ansi yellow)--force-wipe set: proceeding against a live disk OS.(ansi reset)" + } else { + print "Remote looks like an installer live image (ok to install)." + } + } + + let confirm = input $"Type ($target) to continue: " | str trim + if $confirm != $target { + error make {msg: $"Aborted: typed '($confirm)', expected '($target)'."} + } + + $"Installing [(ansi green_bold)($profile)(ansi reset)] on ($target_host) via nixos-anywhere, using ($disko_file)" | print + let runcmd = $"nix run github:nix-community/nixos-anywhere -- --flake .#($profile) ($target_host)" + $runcmd | print + let bash_path = which bash | get 0.path + with-env { SHELL: $bash_path } { + bash -c $runcmd + } +} + +# Boot all remote hosts except the current machine (fwbook) +# Checks online status first, runs `skyg remote boot --build-host fwdesk ` for each online host, +# and writes a summary report to .tmp/boot-report.txt + +export def "skyg remote boot-all" [] { + cd $REPO_ROOT + mkdir .tmp + let current_host = "fwbook" + let report_file = ".tmp/boot-report.txt" + + # Discover remote targets from the flake (same logic as skyg remote) + let all_hosts = remote_targets | where {|h| $h != $current_host } + + print $"Found ($all_hosts | length) candidate hosts excluding ($current_host)" + + mut successes = [] + mut failures = [] + mut skipped = [] + + # Phase 1: sequential SSH reachability checks (fast) + mut online_hosts = [] + for host in $all_hosts { + print $"\n=== Checking ($host) ===" + let target = $"root@($host).lab.internal" + let is_online = (try { + bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1" + true + } catch { false }) + + if not $is_online { + print $"(ansi yellow)OFFLINE(ansi reset): ($host)" + $skipped = ($skipped | append $host) + } else { + print $"(ansi green)ONLINE(ansi reset): ($host)" + $online_hosts = ($online_hosts | append $host) + } + } + + # Phase 2: run boots in parallel (2 concurrent) without interactive prompts + if ($online_hosts | length) > 0 { + print " +Starting parallel boots (concurrency: 2)..." + let boot_results = ($online_hosts | par-each --threads 1 { |host| + let target_host = $"root@($host).lab.internal" + let profile = $"hl-($host)" + let runcmd = $"nh os boot --target-host ($target_host) .#($profile)" + print $runcmd + let ok = (try { + bash -c $runcmd + true + } catch { + false + }) + {host: $host, success: $ok} + }) + + for r in $boot_results { + if $r.success { + $successes = ($successes | append $r.host) + print $"(ansi green)SUCCESS(ansi reset): ($r.host)" + } else { + $failures = ($failures | append $r.host) + print $"(ansi red)FAILED(ansi reset): ($r.host)" + } + } + } + + # Write report + let timestamp = date now | format date "%Y-%m-%d %H:%M:%S" + let report = $"Boot Report - ($timestamp) + +Total hosts considered: ($all_hosts | length) +Skipped offline: ($skipped | length) - ($skipped | str join ', ') +Successful: ($successes | length) - ($successes | str join ', ') +Failed: ($failures | length) - ($failures | str join ', ') + +Details: + Online & Booted: ($successes | str join ', ') + Offline: ($skipped | str join ', ') + Failed: ($failures | str join ', ') +" + + $report | save -f $report_file + print $" +Report written to ($report_file)" + print $report +} + +export def "skyg remote status" [ target: string@remote_targets ] { + get-remote-boot-info $target +} + +# Check status of all remotes: generation, pending reboot, needs rebuild +export def "skyg remote status all" [] { + cd $REPO_ROOT + mkdir .tmp + let current_host = "fwbook" + let report_file = ".tmp/status-report.txt" + + let all_hosts = remote_targets | where {|h| $h != $current_host } + print $"Checking status for ($all_hosts | length) remotes..." + + let results = ($all_hosts | par-each { |host| + let info = get-remote-boot-info $host + + # Print each host's status as one block, as soon as it finishes. + let booted = (if ($info.booted | is-empty) { "-" } else { $info.booted | path basename }) + let current = (if ($info.current | is-empty) { "-" } else { $info.current | path basename }) + let reboot_line = if $info.reboot_pending { $"(ansi yellow)REBOOT PENDING(ansi reset)\n" } else { "" } + let build_line = if $info.needs_build { $"(ansi red)NEEDS NEW BUILD(ansi reset)" } else { $"(ansi green)UP TO DATE(ansi reset)" } + print $"($host) | Booted: ($booted) | Current: ($current)" + + $info + }) + + return ($results | select host status reboot_pending needs_build) +} + +# Poll SSH until the host reaches the desired reachability, or the timeout elapses. +def wait-for-host [target: string, want_online: bool, timeout: int] { + let deadline = (date now) + ($timeout * 1sec) + while (date now) < $deadline { + let online = (try { + bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1" + true + } catch { false }) + if $online == $want_online { + return true + } + sleep 3sec + } + false +} + +# Send a desktop notification if notify-send is available; never fail the caller. +def notify [title: string, body: string] { + try { + ^notify-send $title $body + } catch { } +} + +# Reboot every remote host that is pending a reboot (booted-system != current-system). +# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a +# time, waiting for it to come back online and sending a desktop notification when it does. +export def "skyg remote reboot-pending" [] { + cd $REPO_ROOT + mkdir .tmp + let current_host = "fwbook" + let report_file = ".tmp/reboot-pending-report.txt" + + let all_hosts = remote_targets | where {|h| $h != $current_host } + print $"Checking ($all_hosts | length) remotes for pending reboots..." + + # Phase 1: find hosts that are online and pending a reboot + mut pending = [] + mut offline = [] + mut up_to_date = [] + + for host in $all_hosts { + let target = $"root@($host).lab.internal" + let online = (try { + bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new ($target) 'true' >/dev/null 2>&1" + true + } catch { false }) + + if not $online { + print $"(ansi yellow)OFFLINE(ansi reset): ($host)" + $offline = ($offline | append $host) + continue + } + + let info = (try { + let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim + let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim + {booted: $booted, current: $current, pending: ($booted != $current)} + } catch { + {booted: "", current: "", pending: false} + }) + + if $info.pending { + print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)" + $pending = ($pending | append $host) + } else { + print $"(ansi green)UP TO DATE(ansi reset): ($host)" + $up_to_date = ($up_to_date | append $host) + } + } + + if ($pending | length) == 0 { + print "" + print "No remotes are pending a reboot." + return + } + + # Phase 2: confirm before rebooting anything + print "" + print $"(ansi yellow_bold)($pending | length) remote\(s\) pending a reboot:(ansi reset)" + for host in $pending { + print $" - ($host)" + } + print "" + let confirm = (input $"Reboot these ($pending | length) host\(s\)? [y/N]: " | str trim | str downcase) + if not ($confirm in ["y" "yes"]) { + print "Aborted — no hosts were rebooted." + return + } + + # Phase 3: reboot each pending host one at a time, waiting for it to come back + mut rebooted = [] + mut failed = [] + for host in $pending { + let target = $"root@($host).lab.internal" + print $"\n=== Rebooting ($host) ===" + + # Capture the boot id so we can confirm the host actually rebooted. + let boot_id_before = (try { + bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim + } catch { "" }) + + let runcmd = $"ssh -o BatchMode=yes -o ConnectTimeout=5 ($target) 'systemctl reboot'" + print $runcmd + # The SSH connection drops as the host shuts down, so a non-zero exit is + # expected here and does not mean the reboot failed. + try { + bash -c $"($runcmd) 2>/dev/null" + } catch { } + + print $"Waiting for ($host) to shut down..." + let went_down = (wait-for-host $target false 90) + if not $went_down { + print $"(ansi yellow)($host) did not go offline within 90s — continuing to wait for it to come back.(ansi reset)" + } + + print $"Waiting for ($host) to come back online..." + let came_up = (wait-for-host $target true 300) + if $came_up { + let boot_id_after = (try { + bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim + } catch { "" }) + let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after) + if $confirmed { + print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)" + } else { + print $"(ansi green)($host) is back online(ansi reset)" + } + notify $"✅ ($host) rebooted" "Back online and reachable" + $rebooted = ($rebooted | append $host) + } else { + print $"(ansi red)($host) did not come back online within 300s(ansi reset)" + notify $"❌ ($host) did not come back" "Still unreachable after 300s" + $failed = ($failed | append $host) + } + } + + # Write report + let timestamp = date now | format date "%Y-%m-%d %H:%M:%S" + let report = $"Reboot-Pending Report - ($timestamp) + +Total hosts considered: ($all_hosts | length) +Pending reboot: ($pending | length) - ($pending | str join ', ') +Rebooted: ($rebooted | length) - ($rebooted | str join ', ') +Failed to return: ($failed | length) - ($failed | str join ', ') +Offline: ($offline | length) - ($offline | str join ', ') +Up to date: ($up_to_date | length) - ($up_to_date | str join ', ') +" + + $report | save -f $report_file + print $" +Report written to ($report_file)" + print $report +} diff --git a/bin/lib/remote.utils.nu b/bin/lib/remote.utils.nu new file mode 100644 index 0000000..c80cf0d --- /dev/null +++ b/bin/lib/remote.utils.nu @@ -0,0 +1,36 @@ +use types.nu * +use profile.nu * + +export def get-remote-boot-info [ host: string@remote_targets ] { + let target = $"root@($host).lab.internal" + let is_online = (try { + bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1" + true + } catch { false }) + if (not $is_online) { + return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""} + } + return (try { + # Fetch remote system paths and the local toplevel concurrently. + let fetched = (["remote" "local"] | par-each {|what| + if $what == "remote" { + bash -c $"ssh ($target) 'readlink /run/booted-system; readlink /run/current-system' 2>/dev/null" | lines + } else { + try { + bash -c $"nix eval --raw '.#nixosConfigurations.hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true" | str trim + } catch { "" } + } + }) + + let booted = ($fetched.0 | get 0? | default "") + let current = ($fetched.0 | get 1? | default "") + let local_toplevel = $fetched.1 + + let reboot_pending = ($booted != $current) + let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $current } + + {host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel} + } catch { + {host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""} + }) +}