diff --git a/bin/lib/cmds.nu b/bin/lib/cmds.nu index ab2d693..dcbfa52 100644 --- a/bin/lib/cmds.nu +++ b/bin/lib/cmds.nu @@ -166,15 +166,17 @@ export def "skyg remote boot-all" [ } } - # Phase 2: run boots in parallel (2 concurrent) + # Phase 2: run boots in parallel (2 concurrent) without interactive prompts if ($online_hosts | length) > 0 { print " Starting parallel boots (concurrency: 2)..." - let boot_results = ($online_hosts | par-each --threads 2 { |host| - let cmd = $"skyg remote boot --build-host ($build_host) ($host)" - print $cmd + let boot_results = ($online_hosts | par-each --threads 1 { |host| + let target_host = $"root@($host).lab.internal" + let profile = $"hl-($host)" + let runcmd = $"nh os boot --target-host ($target_host) --build-host ($build_host).lab.internal .#($profile)" + print $runcmd let ok = (try { - bash -c $cmd + bash -c $runcmd true } catch { false @@ -214,6 +216,84 @@ Report written to ($report_file)" print $report } +# Check status of all remotes: generation, pending reboot, needs rebuild +export def "skyg remote status" [] { + cd $REPO_ROOT + mkdir .tmp + let current_host = "fwbook" + let report_file = ".tmp/status-report.txt" + + let all_hosts = (remote_targets | where { |h| $h != $current_host }) + print $"Checking status for ($all_hosts | length) remotes..." + + let results = ($all_hosts | each { |host| + print $" +=== ($host) ===" + let target = $"root@($host).lab.internal" + let online = (try { + bash -c $"ssh -o BatchMode=yes -o ConnectTimeout=3 ($target) 'true' >/dev/null 2>&1" + true + } catch { false }) + + if not $online { + print $"(ansi yellow)OFFLINE(ansi reset)" + return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false} + } + + let info = (try { + let booted = (bash -c $"ssh ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim) + let current = (bash -c $"ssh ($target) 'readlink /run/current-system' 2>/dev/null" | str trim) + let reboot_pending = ($booted != $current) + + # Compare what the local flake currently evaluates to vs remote's active system + let local_toplevel = (try { + (bash -c $"nix eval --raw '.#hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true") | str trim + } catch { "" }) + let needs_build = if $local_toplevel == "" { false } else { $local_toplevel != $current } + + {host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build} + } catch { + {host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false} + }) + + print $"Booted: ($info.booted | path basename)" + print $"Current: ($info.current | path basename)" + if $info.reboot_pending { + print $"(ansi yellow)REBOOT PENDING(ansi reset)" + } + if $info.needs_build { + print $"(ansi red)NEEDS NEW BUILD(ansi reset)" + } else { + print $"(ansi green)UP TO DATE(ansi reset)" + } + + $info + }) + + let timestamp = (date now | format date "%Y-%m-%d %H:%M:%S") + let lines = ($results | each { |r| + if $r.status == "offline" { + $"- ($r.host): (ansi yellow)OFFLINE(ansi reset)" + } else if $r.status == "error" { + $"- ($r.host): (ansi red)ERROR querying status(ansi reset)" + } else { + let pending = if $r.reboot_pending { $" [(ansi blue)REBOOT PENDING(ansi reset)]" } else { "" } + let build = if $r.needs_build { $" [(ansi red)NEEDS REBUILD(ansi reset)]" } else { $" [(ansi green)current(ansi reset)]" } + $"- ($r.host): booted=($r.booted | path basename) current=($r.current | path basename)($pending)($build)" + } + }) + let report = $"Remote Status Report - ($timestamp) + +" + ($lines | str join " +") + " +" + + $report | save -f $report_file + print $" +Report written to ($report_file)" + print $report +} + # Deploy OpenWrt router configuration export def "skyg openwrt" [router: string = "glmain"] { cd $REPO_ROOT diff --git a/hosts/hl-fwdesk.nix b/hosts/hl-fwdesk.nix index d211987..572d0f4 100644 --- a/hosts/hl-fwdesk.nix +++ b/hosts/hl-fwdesk.nix @@ -13,7 +13,6 @@ let wyoming = 10400; comfyui = 8188; libretranslate = 5000; - signal = 8080; ds4 = 8000; hermes = 8642; hermesDashboard = 9119; @@ -185,14 +184,9 @@ in package = pkgs.ollama-vulkan; user = "ollama"; home = "/var/lib/ollama"; - - environmentVariables = { - # Stability fix — SDMA is buggy on Strix Halo unified memory HSA_ENABLE_SDMA = "0"; - # Flash attention for better performance OLLAMA_FLASH_ATTENTION = "1"; - # Keep models loaded OLLAMA_KEEP_ALIVE = "24h"; }; }; @@ -210,33 +204,33 @@ in skyg.nixos.common.container-services.hermes-agent = { enable = true; timeoutStopSec = 210; - services.hermes = { - image = "nousresearch/hermes-agent"; - command = [ "gateway" "run" ]; - ports = [ - "${toString ports.hermes}:8642" - "${toString ports.hermesDashboard}:9119" - ]; - volumes = [ - "/var/lib/hermes:/opt/data" - ]; - environmentFiles = [ config.age.secrets.hermes-env.path ]; - environment = { - PUID = "1000"; - PGID = "100"; + services = { + hermes = { + image = "nousresearch/hermes-agent"; + command = [ "gateway" "run" ]; + ports = [ + "${toString ports.hermes}:8642" + "${toString ports.hermesDashboard}:9119" + ]; + volumes = [ + "/var/lib/hermes:/opt/data" + ]; + environmentFiles = [ config.age.secrets.hermes-env.path ]; + environment = { + PUID = "1000"; + PGID = "100"; + }; + shm_size = "1g"; + extra_hosts = [ "host.docker.internal:host-gateway" ]; + networks = [ + "main" + ]; }; - shm_size = "1g"; - extra_hosts = [ "host.docker.internal:host-gateway" ]; + }; + networks = { + "main" = { }; }; }; - # Signal CLI - skyg.nixos.server.services.signal-cli = { - enable = false; - host = "0.0.0.0"; - port = ports.signal; - environmentFile = config.age.secrets.hermes-env.path; - }; - hardware.enableAllFirmware = true; hardware.amdgpu = { opencl.enable = true; diff --git a/secrets/hermes-env.age b/secrets/hermes-env.age index 6ed659d..f2e5545 100644 Binary files a/secrets/hermes-env.age and b/secrets/hermes-env.age differ