Compare commits
3
Commits
da93290953
...
7dd5f9e17b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7dd5f9e17b | ||
|
|
53c41cf826 | ||
|
|
4f3a95b42c |
No files matched your search
+2
-2
@@ -37,10 +37,10 @@ bin/
|
||||
| `skyg os rollback` | Roll back to previous generation |
|
||||
| `skyg remote switch <host>` | Deploy to a remote host |
|
||||
| `skyg remote install <host>` | Fresh-install a remote host via nixos-anywhere, using `hosts/<profile>.disko.nix`. Always wipes the disks in that file. Aborts if the file is missing, if the host already boots NixOS from disk (unless `--force-wipe`), or if you do not type the short hostname. |
|
||||
| `skyg remote status <host>` | Check status of a single remote host |
|
||||
| `skyg remote status <host>` | Check status of a single remote host: booted / current / next-boot generation. `reboot_pending` = booted != next boot (`/nix/var/nix/profiles/system`), `needs_build` = local build != next boot. So hosts deployed with `boot` show up correctly. |
|
||||
| `skyg remote status --all` | Check status of all remote hosts |
|
||||
| `skyg remote boot-all` | Boot all remote hosts |
|
||||
| `skyg remote reboot-pending` | Reboot every remote host with a pending reboot (`booted-system` != `current-system`). Lists them, asks for confirmation, then reboots one at a time, waiting for each to come back online and sending a desktop notification when it does. |
|
||||
| `skyg remote reboot-pending` | Reboot every remote host with a pending reboot (`booted-system` != next-boot generation, so `boot` deploys count). Lists them, asks for confirmation, then reboots one at a time. Each host must prove it rebooted (boot id changed) and come back on its next-boot generation, otherwise it is reported as failed. Sends a desktop notification per host. |
|
||||
| `skyg hm switch` | Apply Home Manager config |
|
||||
| `skyg flake update [input]` | Update flake inputs |
|
||||
| `skyg flake check` | Run `nix flake check --no-build` |
|
||||
|
||||
+31
-12
@@ -212,9 +212,10 @@ export def "skyg remote status-all" [] {
|
||||
# Print each host's status as one block, as soon as it finishes.
|
||||
let booted = (if ($info.booted | is-empty) { "-" } else { $info.booted | path basename })
|
||||
let current = (if ($info.current | is-empty) { "-" } else { $info.current | path basename })
|
||||
let next_boot = (if ($info.next_boot | is-empty) { "-" } else { $info.next_boot | path basename })
|
||||
let reboot_line = if $info.reboot_pending { $"(ansi yellow)REBOOT PENDING(ansi reset)\n" } else { "" }
|
||||
let build_line = if $info.needs_build { $"(ansi red)NEEDS NEW BUILD(ansi reset)" } else { $"(ansi green)UP TO DATE(ansi reset)" }
|
||||
print $"($host) | Booted: ($booted) | Current: ($current)"
|
||||
print $"($host) | Booted: ($booted) | Current: ($current) | Next boot: ($next_boot)"
|
||||
|
||||
$info
|
||||
})
|
||||
@@ -222,7 +223,7 @@ export def "skyg remote status-all" [] {
|
||||
return ($results | select host status reboot_pending needs_build)
|
||||
}
|
||||
|
||||
# Reboot every remote host that is pending a reboot (booted-system != current-system).
|
||||
# Reboot every remote host that is pending a reboot (booted-system != next-boot generation).
|
||||
# Detects pending hosts, lists them, asks for confirmation, then reboots each one at a
|
||||
# time, waiting for it to come back online and sending a desktop notification when it does.
|
||||
export def "skyg remote reboot-pending" [] {
|
||||
@@ -253,15 +254,14 @@ export def "skyg remote reboot-pending" [] {
|
||||
}
|
||||
|
||||
let info = (try {
|
||||
let booted = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system' 2>/dev/null" | str trim
|
||||
let current = bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/current-system' 2>/dev/null" | str trim
|
||||
{booted: $booted, current: $current, pending: ($booted != $current)}
|
||||
let s = (get-boot-state $target)
|
||||
{booted: $s.booted, next_boot: $s.next_boot, pending: ($s.booted != "" and $s.booted != $s.next_boot)}
|
||||
} catch {
|
||||
{booted: "", current: "", pending: false}
|
||||
{booted: "", next_boot: "", pending: false}
|
||||
})
|
||||
|
||||
if $info.pending {
|
||||
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) current=($info.current | path basename)"
|
||||
print $"(ansi blue)REBOOT PENDING(ansi reset): ($host) booted=($info.booted | path basename) next=($info.next_boot | path basename)"
|
||||
$pending = ($pending | append $host)
|
||||
} else {
|
||||
print $"(ansi green)UP TO DATE(ansi reset): ($host)"
|
||||
@@ -320,12 +320,31 @@ export def "skyg remote reboot-pending" [] {
|
||||
let boot_id_after = (try {
|
||||
bash -c $"ssh -o BatchMode=yes ($target) 'cat /proc/sys/kernel/random/boot_id' 2>/dev/null" | str trim
|
||||
} catch { "" })
|
||||
let confirmed = ($boot_id_before != "" and $boot_id_after != "" and $boot_id_before != $boot_id_after)
|
||||
if $confirmed {
|
||||
print $"(ansi green)($host) is back online (reboot confirmed)(ansi reset)"
|
||||
# Proof of reboot: boot id changed. If we can't read it, fall back to "it went offline".
|
||||
let did_reboot = if $boot_id_before != "" and $boot_id_after != "" {
|
||||
$boot_id_before != $boot_id_after
|
||||
} else {
|
||||
print $"(ansi green)($host) is back online(ansi reset)"
|
||||
$went_down
|
||||
}
|
||||
if not $did_reboot {
|
||||
print $"(ansi red)($host) answered but did NOT reboot (same boot id, never went offline)(ansi reset)"
|
||||
notify $"❌ ($host) did not reboot" "Reboot command had no effect"
|
||||
$failed = ($failed | append $host)
|
||||
continue
|
||||
}
|
||||
|
||||
# Check the host really came up on the generation we wanted.
|
||||
let after = (try { get-boot-state $target } catch { null })
|
||||
if $after != null and $after.booted != $after.next_boot {
|
||||
print $"(ansi red)($host) rebooted but is NOT on its latest generation(ansi reset)"
|
||||
print $" booted=($after.booted | path basename)"
|
||||
print $" wanted=($after.next_boot | path basename)"
|
||||
notify $"⚠️ ($host) wrong generation" "Rebooted but not on the latest generation"
|
||||
$failed = ($failed | append $host)
|
||||
continue
|
||||
}
|
||||
|
||||
print $"(ansi green)($host) rebooted and is back online(ansi reset)"
|
||||
notify $"✅ ($host) rebooted" "Back online and reachable"
|
||||
$rebooted = ($rebooted | append $host)
|
||||
} else {
|
||||
@@ -342,7 +361,7 @@ export def "skyg remote reboot-pending" [] {
|
||||
Total hosts considered: ($all_hosts | length)
|
||||
Pending reboot: ($pending | length) - ($pending | str join ', ')
|
||||
Rebooted: ($rebooted | length) - ($rebooted | str join ', ')
|
||||
Failed to return: ($failed | length) - ($failed | str join ', ')
|
||||
Failed: ($failed | length) - ($failed | str join ', ')
|
||||
Offline: ($offline | length) - ($offline | str join ', ')
|
||||
Up to date: ($up_to_date | length) - ($up_to_date | str join ', ')
|
||||
"
|
||||
|
||||
+26
-10
@@ -1,6 +1,19 @@
|
||||
use types.nu *
|
||||
use profile.nu *
|
||||
|
||||
# Read the 3 system paths on a remote host:
|
||||
# booted = what it booted from
|
||||
# current = what is running now (changed by switch/test)
|
||||
# next_boot = what the next reboot will use (changed by switch/boot)
|
||||
export def get-boot-state [target: string] {
|
||||
let lines = (bash -c $"ssh -o BatchMode=yes ($target) 'readlink /run/booted-system; readlink /run/current-system; readlink -f /nix/var/nix/profiles/system' 2>/dev/null" | lines)
|
||||
let booted = ($lines | get 0? | default "")
|
||||
let current = ($lines | get 1? | default "")
|
||||
let profile = ($lines | get 2? | default "")
|
||||
let next_boot = if $profile == "" { $current } else { $profile }
|
||||
{booted: $booted, current: $current, next_boot: $next_boot}
|
||||
}
|
||||
|
||||
export def get-remote-boot-info [ host: string@remote_targets ] {
|
||||
let target = $"root@($host).lab.internal"
|
||||
let is_online = (try {
|
||||
@@ -8,13 +21,13 @@ export def get-remote-boot-info [ host: string@remote_targets ] {
|
||||
true
|
||||
} catch { false })
|
||||
if (not $is_online) {
|
||||
return {host: $host, status: "offline", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||
return {host: $host, status: "offline", booted: "", current: "", next_boot: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||
}
|
||||
return (try {
|
||||
# Fetch remote system paths and the local toplevel concurrently.
|
||||
let fetched = (["remote" "local"] | par-each {|what|
|
||||
if $what == "remote" {
|
||||
bash -c $"ssh ($target) 'readlink /run/booted-system; readlink /run/current-system' 2>/dev/null" | lines
|
||||
get-boot-state $target
|
||||
} else {
|
||||
try {
|
||||
bash -c $"nix eval --raw '.#nixosConfigurations.hl-($host).config.system.build.toplevel.outPath' 2>/dev/null || true" | str trim
|
||||
@@ -22,22 +35,25 @@ export def get-remote-boot-info [ host: string@remote_targets ] {
|
||||
}
|
||||
})
|
||||
|
||||
let booted = ($fetched.0 | get 0? | default "")
|
||||
let current = ($fetched.0 | get 1? | default "")
|
||||
let state = $fetched.0
|
||||
let booted = $state.booted
|
||||
let current = $state.current
|
||||
let next_boot = $state.next_boot
|
||||
let local_toplevel = $fetched.1
|
||||
|
||||
let reboot_pending = ($booted != $current)
|
||||
let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $current }
|
||||
# A reboot only matters if it would land on a different system than we booted.
|
||||
let reboot_pending = ($booted != "" and $booted != $next_boot)
|
||||
let needs_build = if $local_toplevel == "" { "unknown" } else { $local_toplevel != $next_boot }
|
||||
|
||||
{host: $host, status: "online", booted: $booted, current: $current, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel}
|
||||
{host: $host, status: "online", booted: $booted, current: $current, next_boot: $next_boot, reboot_pending: $reboot_pending, needs_build: $needs_build, local_toplevel: $local_toplevel}
|
||||
} catch {
|
||||
{host: $host, status: "error", booted: "", current: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||
{host: $host, status: "error", booted: "", current: "", next_boot: "", reboot_pending: false, needs_build: false, local_toplevel: ""}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
# Poll SSH until the host reaches the desired reachability, or the timeout elapses.
|
||||
def wait-for-host [target: string, want_online: bool, timeout: int] {
|
||||
export def wait-for-host [target: string, want_online: bool, timeout: int] {
|
||||
let deadline = (date now) + ($timeout * 1sec)
|
||||
while (date now) < $deadline {
|
||||
let online = (try {
|
||||
@@ -53,7 +69,7 @@ def wait-for-host [target: string, want_online: bool, timeout: int] {
|
||||
}
|
||||
|
||||
# Send a desktop notification if notify-send is available; never fail the caller.
|
||||
def notify [title: string, body: string] {
|
||||
export def notify [title: string, body: string] {
|
||||
try {
|
||||
^notify-send $title $body
|
||||
} catch { }
|
||||
|
||||
Generated
+21
@@ -3116,6 +3116,7 @@
|
||||
"nixpkgs": "nixpkgs_14",
|
||||
"nixpkgs-unstable": "nixpkgs-unstable",
|
||||
"noctalia": "noctalia",
|
||||
"skyg-linux-tracker": "skyg-linux-tracker",
|
||||
"skygqts": "skygqts",
|
||||
"stylix": "stylix",
|
||||
"systems": "systems_6",
|
||||
@@ -3182,6 +3183,26 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"skyg-linux-tracker": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1792243194,
|
||||
"narHash": "sha256-GYgEoGZOoqJjebzMZ+SFr/i47QmQdp5/uUbDPjzzakU=",
|
||||
"ref": "refs/heads/main",
|
||||
"rev": "81e99f86673a478be4b57c47f75db77a743ee7a1",
|
||||
"revCount": 66,
|
||||
"type": "git",
|
||||
"url": "https://gitea.skyg.ca/ari/skyg-linux-tracker.git"
|
||||
},
|
||||
"original": {
|
||||
"type": "git",
|
||||
"url": "https://gitea.skyg.ca/ari/skyg-linux-tracker.git"
|
||||
}
|
||||
},
|
||||
"skygqts": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
|
||||
@@ -4,6 +4,11 @@
|
||||
skygqts.url = "git+ssh://gitea@gitea.app.internal:22/ari/skygqts.git";
|
||||
skygqts.inputs.nixpkgs.follows = "nixpkgs";
|
||||
hermenix.url = "git+ssh://gitea@gitea.app.internal:22/ari/hermenix.git";
|
||||
# Read-only host monitor (systemd, journal, Docker) over REST + MCP.
|
||||
skyg-linux-tracker = {
|
||||
url = "git+https://gitea.skyg.ca/ari/skyg-linux-tracker.git";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
# Headless Hyprland session streamed over a Unix socket.
|
||||
hhstream = {
|
||||
url = "git+ssh://gitea@gitea.app.internal:22/Hermenix/hyprland-headless-stream-plugin.git";
|
||||
@@ -104,6 +109,7 @@
|
||||
{ self
|
||||
, skygqts
|
||||
, hermenix
|
||||
, skyg-linux-tracker
|
||||
, hhstream
|
||||
, nixpkgs-unstable
|
||||
, flox
|
||||
@@ -174,6 +180,7 @@
|
||||
buildbot-gitea-webhook-secret = ./secrets/buildbot-gitea-webhook-secret.age;
|
||||
buildbot-gitea-oauth-secret = ./secrets/buildbot-gitea-oauth-secret.age;
|
||||
immich-shared = ./secrets/immich-shared.age;
|
||||
grafana-secret-key = ./secrets/grafana-secret-key.age;
|
||||
};
|
||||
inherit
|
||||
hyprlauncher
|
||||
@@ -188,6 +195,7 @@
|
||||
hermes-agent
|
||||
skygqts
|
||||
hermenix
|
||||
skyg-linux-tracker
|
||||
hhstream
|
||||
flox
|
||||
determinate
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ hosts/
|
||||
| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node |
|
||||
| `hl-bigbox1/` | hl-bigbox1 | AI/compute server (NVIDIA + AMD); Immich machine-learning (CUDA) |
|
||||
| `hl-bigbox2/` | hl-bigbox2 | NFS storage server; Immich server/db (docker, `immich.home.sosnovsky.ca`) |
|
||||
| `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI) |
|
||||
| `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI), monitoring (Grafana, Prometheus, Loki, OTel collector) |
|
||||
| `hl-minipc2/` | hl-minipc2 | Container/media server |
|
||||
| `hl-minipc3/` | hl-minipc3 | K3s worker node; central Caddy reverse-proxy (`*.home.sosnovsky.ca`, plus `gitea.skyg.ca` HTTP/SSH); buildbot-nix CI master + worker (2 cores) |
|
||||
| `hl-pi1/` | hl-pi1 | Raspberry Pi CM4 (aarch64) — headless server, SSH only |
|
||||
|
||||
@@ -137,7 +137,7 @@ in
|
||||
|
||||
vscode
|
||||
unstablePkgs.zed-editor-fhs
|
||||
devenv
|
||||
unstablePkgs.devenv
|
||||
just
|
||||
rpi-imager
|
||||
rpiboot
|
||||
|
||||
@@ -4,5 +4,6 @@
|
||||
./immich-ml.nix
|
||||
./portainer.nix
|
||||
./timers.nix
|
||||
./linux-tracker.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "bigbox1.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,40 @@
|
||||
{ ... }:
|
||||
{ config, pkgs, ... }:
|
||||
{
|
||||
services.fwupd.enable = true;
|
||||
|
||||
# Interim: the box loses power under load. Use less power and heat, log more.
|
||||
# Remove this block once the new board is in and stable.
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
# No turbo. Lost on reboot unless set here.
|
||||
"w /sys/devices/system/cpu/cpufreq/boost - - - - 0"
|
||||
# Max clock 2.5 GHz. Steps on this CPU: 3.5, 3.0, 2.5, 2.0, 1.4.
|
||||
"w /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq - - - - 2500000"
|
||||
];
|
||||
|
||||
# Keep logs on disk and flush often, so the last seconds before a crash survive.
|
||||
services.journald.extraConfig = ''
|
||||
Storage=persistent
|
||||
SystemMaxUse=2G
|
||||
SyncIntervalSec=5s
|
||||
'';
|
||||
|
||||
# Reboot 10 s after a kernel panic instead of hanging.
|
||||
boot.kernel.sysctl."kernel.panic" = 10;
|
||||
|
||||
# Watch disk health and log warnings. No mail set up.
|
||||
services.smartd = {
|
||||
enable = true;
|
||||
autodetect = true;
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
pkgs.smartmontools
|
||||
pkgs.lm_sensors
|
||||
pkgs.dmidecode
|
||||
pkgs.stress-ng
|
||||
pkgs.memtester
|
||||
pkgs.ethtool
|
||||
config.boot.kernelPackages.cpupower
|
||||
];
|
||||
}
|
||||
@@ -4,5 +4,6 @@
|
||||
./nix-serve.nix
|
||||
./nfs-server.nix
|
||||
./immich.nix
|
||||
./linux-tracker.nix
|
||||
];
|
||||
}
|
||||
@@ -34,7 +34,7 @@ in
|
||||
environmentFiles = [ config.age.secrets.immich-shared.path ];
|
||||
environment = envVars;
|
||||
deploy.resources.limits = {
|
||||
cpus = "4";
|
||||
cpus = "1";
|
||||
memory = "8G";
|
||||
};
|
||||
networks = {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "bigbox2.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -16,6 +16,13 @@
|
||||
lmstudio
|
||||
signal-cli
|
||||
blueman
|
||||
unstablePkgs.devenv
|
||||
|
||||
unstablePkgs.grok-build
|
||||
unstablePkgs.pi-coding-agent
|
||||
unstablePkgs.goose-cli
|
||||
unstablePkgs.claude-agent-acp
|
||||
unstablePkgs.claude-code
|
||||
])
|
||||
++ [ myPkgs.ds4-rocm ];
|
||||
}
|
||||
@@ -7,5 +7,6 @@
|
||||
./wyoming.nix
|
||||
./steam.nix
|
||||
./buildbot.nix
|
||||
./linux-tracker.nix
|
||||
];
|
||||
}
|
||||
@@ -39,6 +39,8 @@ in
|
||||
{ address = "10.0.101.3"; port = 8080; }
|
||||
{ address = "10.0.101.3"; port = 9102; }
|
||||
{ address = "10.0.101.8"; port = 3000; }
|
||||
# skyg-linux-tracker (REST + MCP) on every lab host.
|
||||
{ address = "10.0.0.0/16"; port = 1984; }
|
||||
];
|
||||
publish = [
|
||||
{ hostPort = ports.hermes; guestPort = 8642; }
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "fwdesk.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -31,4 +31,18 @@
|
||||
enable = true;
|
||||
arguments = "-o pipewire";
|
||||
};
|
||||
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
allowedHosts = [ "fws1.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -12,5 +12,10 @@
|
||||
./iu.nix
|
||||
./audiobookshelf.nix
|
||||
./buzz.nix
|
||||
./linux-tracker.nix
|
||||
./otel-collector.nix
|
||||
./prometheus.nix
|
||||
./loki.nix
|
||||
./grafana.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{ config, skyg-secrets, ... }:
|
||||
let
|
||||
ports = {
|
||||
grafana = 3001;
|
||||
prometheus = 9090;
|
||||
loki = 3100;
|
||||
};
|
||||
in
|
||||
{
|
||||
age.secrets.grafana-secret-key = {
|
||||
file = skyg-secrets.grafana-secret-key;
|
||||
owner = "grafana";
|
||||
group = "grafana";
|
||||
};
|
||||
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
|
||||
settings = {
|
||||
server = {
|
||||
http_addr = "0.0.0.0";
|
||||
http_port = ports.grafana;
|
||||
};
|
||||
security.secret_key = "$__(file){${config.age.secrets.grafana-secret-key.path}}";
|
||||
};
|
||||
|
||||
provision = {
|
||||
enable = true;
|
||||
datasources.settings.datasources = [
|
||||
{
|
||||
name = "Prometheus";
|
||||
type = "prometheus";
|
||||
access = "proxy";
|
||||
url = "http://127.0.0.1:${toString ports.prometheus}";
|
||||
isDefault = true;
|
||||
}
|
||||
{
|
||||
name = "Loki";
|
||||
type = "loki";
|
||||
access = "proxy";
|
||||
url = "http://127.0.0.1:${toString ports.loki}";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "minipc1.lab.internal" ];
|
||||
# OTLP out to the collector on this box (see ./otel-collector.nix).
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://127.0.0.1:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
{ ... }:
|
||||
let
|
||||
ports = {
|
||||
loki = 3100;
|
||||
grpc = 9096;
|
||||
};
|
||||
dataDir = "/var/lib/loki";
|
||||
in
|
||||
{
|
||||
services.loki = {
|
||||
enable = true;
|
||||
|
||||
# Single-binary Loki on the filesystem: no object store, no ring.
|
||||
configuration = {
|
||||
auth_enabled = false;
|
||||
|
||||
server = {
|
||||
http_listen_port = ports.loki;
|
||||
grpc_listen_port = ports.grpc;
|
||||
};
|
||||
|
||||
common = {
|
||||
instance_addr = "127.0.0.1";
|
||||
path_prefix = dataDir;
|
||||
replication_factor = 1;
|
||||
ring.kvstore.store = "inmemory";
|
||||
storage.filesystem = {
|
||||
chunks_directory = "${dataDir}/chunks";
|
||||
rules_directory = "${dataDir}/rules";
|
||||
};
|
||||
};
|
||||
|
||||
schema_config.configs = [
|
||||
{
|
||||
from = "2026-10-01";
|
||||
store = "tsdb";
|
||||
object_store = "filesystem";
|
||||
schema = "v13";
|
||||
index = {
|
||||
prefix = "index_";
|
||||
period = "24h";
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
# The compactor is what enforces the 24h window below.
|
||||
compactor = {
|
||||
working_directory = "${dataDir}/compactor";
|
||||
retention_enabled = true;
|
||||
delete_request_store = "filesystem";
|
||||
};
|
||||
|
||||
limits_config = {
|
||||
retention_period = "24h";
|
||||
reject_old_samples = true;
|
||||
reject_old_samples_max_age = "168h";
|
||||
# Required for OTLP ingestion; on by default since 3.0.
|
||||
allow_structured_metadata = true;
|
||||
};
|
||||
|
||||
ruler = {
|
||||
storage.type = "local";
|
||||
storage.local.directory = "${dataDir}/rules";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# Loki data lives on tnas1. Prometheus and Grafana stay on local disk, since
|
||||
# their databases do not work well on NFS.
|
||||
# The export must exist on the NAS and be writable by uid 985 / gid 981.
|
||||
# The mount is ordered after network-online so tnas1's name resolves at boot.
|
||||
fileSystems.${dataDir} = {
|
||||
device = "tnas1.lab.internal:/mnt/SmallG/monitoring/loki";
|
||||
fsType = "nfs";
|
||||
options = [
|
||||
"nfsvers=4.0"
|
||||
"hard"
|
||||
"noatime"
|
||||
"nofail"
|
||||
"_netdev"
|
||||
"x-systemd.automount"
|
||||
"x-systemd.requires=network-online.target"
|
||||
"x-systemd.after=network-online.target"
|
||||
];
|
||||
};
|
||||
systemd.services.loki.unitConfig.RequiresMountsFor = [ dataDir ];
|
||||
|
||||
# NFS ownership is numeric, so pin the ids instead of letting NixOS assign
|
||||
# them by name order (a new user sorting before "loki" would shift them).
|
||||
users.users.loki.uid = 985;
|
||||
users.groups.loki.gid = 981;
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ ports.loki ];
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
ports = {
|
||||
otlpGrpc = 4317;
|
||||
otlpHttp = 4318;
|
||||
# Loopback scrape target for the Prometheus below.
|
||||
metrics = 9464;
|
||||
};
|
||||
in
|
||||
{
|
||||
services.opentelemetry-collector = {
|
||||
enable = true;
|
||||
# The prometheus exporter is a contrib component, not core.
|
||||
package = pkgs.opentelemetry-collector-contrib;
|
||||
|
||||
settings = {
|
||||
receivers.otlp.protocols = {
|
||||
grpc.endpoint = "0.0.0.0:${toString ports.otlpGrpc}";
|
||||
http.endpoint = "0.0.0.0:${toString ports.otlpHttp}";
|
||||
};
|
||||
|
||||
processors.batch.timeout = "5s";
|
||||
|
||||
exporters = {
|
||||
# Pull endpoint. Resource attributes (host.name, service.name) only
|
||||
# become labels when conversion is enabled — off by default, which
|
||||
# would leave every host's series with an identical label set.
|
||||
prometheus = {
|
||||
endpoint = "127.0.0.1:${toString ports.metrics}";
|
||||
resource_to_telemetry_conversion.enabled = true;
|
||||
};
|
||||
# The contrib `loki` exporter was removed upstream; Loki ingests
|
||||
# OTLP/HTTP natively at /otlp instead.
|
||||
"otlphttp/loki".endpoint = "http://127.0.0.1:3100/otlp";
|
||||
nop = { };
|
||||
};
|
||||
|
||||
service.pipelines = {
|
||||
metrics = {
|
||||
receivers = [ "otlp" ];
|
||||
processors = [ "batch" ];
|
||||
exporters = [ "prometheus" ];
|
||||
};
|
||||
logs = {
|
||||
receivers = [ "otlp" ];
|
||||
processors = [ "batch" ];
|
||||
exporters = [ "otlphttp/loki" ];
|
||||
};
|
||||
# The tracker ships its own spans; nothing here stores them.
|
||||
traces = {
|
||||
receivers = [ "otlp" ];
|
||||
processors = [ "batch" ];
|
||||
exporters = [ "nop" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# Only the OTLP listeners face the lab; 9464 stays on loopback.
|
||||
networking.firewall.allowedTCPPorts = [ ports.otlpGrpc ports.otlpHttp ];
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{ ... }:
|
||||
let
|
||||
ports = { prometheus = 9090; };
|
||||
in
|
||||
{
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
port = ports.prometheus;
|
||||
listenAddress = "0.0.0.0";
|
||||
retentionTime = "48h";
|
||||
|
||||
scrapeConfigs = [
|
||||
{
|
||||
job_name = "lab-trackers";
|
||||
static_configs = [{ targets = [ "127.0.0.1:9464" ]; }];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ ports.prometheus ];
|
||||
}
|
||||
@@ -6,5 +6,6 @@
|
||||
./stack1.nix
|
||||
./stack2.nix
|
||||
./timers.nix
|
||||
./linux-tracker.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "minipc2.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
imports = [
|
||||
./caddy.nix
|
||||
./acme.nix
|
||||
./linux-tracker.nix
|
||||
# ./buildbot.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "minipc3.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,20 @@
|
||||
{ ... }:
|
||||
{
|
||||
skyg.nixos.common.networking.nfsServer.enable = true;
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "terra1.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
services.nfs.server = {
|
||||
lockdPort = 4001;
|
||||
mountdPort = 4002;
|
||||
|
||||
@@ -51,6 +51,7 @@ let
|
||||
specialArgs.hermes-agent.nixosModules.default
|
||||
specialArgs.tether.nixosModules.default
|
||||
specialArgs.hhstream.nixosModules.default
|
||||
specialArgs.skyg-linux-tracker.nixosModules.default
|
||||
{ programs.nix-index-database.comma.enable = true; }
|
||||
];
|
||||
makeHomeConfig = {}: import ./home { };
|
||||
|
||||
@@ -26,7 +26,9 @@ server/
|
||||
- **`timers.nix`** is a reusable helper: anything added to `skyg.server.timers.<name>`
|
||||
(with `script`, `OnCalendar`, `wantedBy`) becomes a systemd timer + oneshot service. Host
|
||||
files (e.g. `hosts/hl-minipc2.nix`'s scrypted-backups) use it for backups/auto-updates.
|
||||
- No Prometheus exporters: the node exporter module was removed (unauthenticated, no scraper).
|
||||
- No per-host Prometheus exporters: metrics are **pushed**. Every lab host runs
|
||||
`skyg-linux-tracker` and ships OTLP to the collector on bigbox2
|
||||
(`hosts/hl-bigbox2/services/otel-collector.nix`), which feeds Prometheus and Loki.
|
||||
- Naming: this tree mostly uses the legacy `skyg.server.*` prefix for cross-cutting modules
|
||||
(admin/timers/arrs/dns) and `skyg.nixos.server.*` for the K3s/K8s roles and the
|
||||
service modules under `services/`.
|
||||
|
||||
@@ -56,4 +56,7 @@ in
|
||||
|
||||
# immich
|
||||
"secrets/immich-shared.age".publicKeys = [ ari bigbox2 ];
|
||||
|
||||
# Grafana signing key (settings.security.secret_key) on hl-minipc1
|
||||
"secrets/grafana-secret-key.age".publicKeys = [ ari minipc1 ];
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
age-encryption.org/v1
|
||||
-> ssh-ed25519 CIR0mA MZHddRKKINlOjaZq4KeZv0ZbUlqHF72rSWaM8Gp9YEQ
|
||||
dYvdCXVxJl/jqfzVCvWFOzpnZVLr4zDbeyo384EHUqA
|
||||
-> ssh-ed25519 So2Y2A lGsCiYTRNOTqY+XPtA/4b3L6aoS277CPBpJAYNBR+Bs
|
||||
Q6rwNPEPENCz1f7RL1ugsz4UG3zh3bwIIIIFQzNrhDI
|
||||
--- PECFTeXNGLXQn7oRM3/RQD4RoPYl+Udyt30iWLWvQ/s
|
||||
Uèzy¡íÈC�œ7ŒŸJJè÷;ÓiÍdî÷‘ê†dGN((d‹‘YñX9£.‚#*š¦Ll·–N¼óêå/t:ÓrÅ·Æw9€®‘�hZ”eÎ0‹‘o²¢
|
||||
-> ssh-ed25519 CIR0mA xqQTxyhPrikYagGRGiZgquAAdZRRuJ6vY6QkhRrMsk4
|
||||
/hyH926q9x8OyK+sNFTmmDZA5X3cQJFA3+xiAP+7s7M
|
||||
-> ssh-ed25519 So2Y2A NFDlVnErG2qjx0cP5Ud4nmdxZMm0KSJLFFXtjUBCwhM
|
||||
SXbvegO8aC5My91ciXhu4Gq09FiRDmRIha/l0niyuL0
|
||||
--- thUe2VQJDpYQhtROqB1IbILqJ3C+XUoOq+ikZlv5FBQ
|
||||
ˆú¼åøó
ˆ;ÃPטr솀·8UYÁh‡‚ºi³¦MÿFÑ®b7E±ÖÃébs<é’Ô)e`�dш€lۡߎ®fñõ˜‚EpHjô}c!¨°ÿ�Ŭö
|
||||
@@ -1,7 +1,7 @@
|
||||
age-encryption.org/v1
|
||||
-> ssh-ed25519 CIR0mA R/k75aPb/6UY7hC2sNLDnJBN0SzS1gLxCKV5/6k14lo
|
||||
Ia2f/UVHOOyAOKIWQDQBGUA4tBQCC9I+jI8HKngdDgc
|
||||
-> ssh-ed25519 So2Y2A kMKHgmMXFv/FFpxG4afwDFu3EYY6dRHEDIMB+413VyQ
|
||||
ohCZhQSOc8nB6F+VOMqLJsmzkydg/+wDJTeGbPyiWz4
|
||||
--- YJueYHrRUK8D0Jz+V9vXPPNOIpT0GGcIyKJ4u72qluY
|
||||
Pp6~_Ñ} ЉêÝc,a}Õár£8í¥Y=dA¨ €ßŠñ|90%ay·�FÊ(HTª»±” ñ°gó§?m^a˜*Ÿ2�
|
||||
-> ssh-ed25519 CIR0mA qOPQimH/RGijxXLsUFYnoRBYaRlgkTkdakveVD/rxHY
|
||||
I54XAAaZGHuREobPbErwvpjBDCutqv5qE/ZFivvonLc
|
||||
-> ssh-ed25519 So2Y2A yEvFPWoKOHDebfR4fTVNooHahbqoNdkNEA6hR6bATGA
|
||||
HHwKok4lDVU65Ty5f7ZIpeUZvTlBXdqvOlg83PQz7EU
|
||||
--- abAtzYT8rbVVTB93m0IYUymq/RxV6X2sk6ZTxi4FIsw
|
||||
§‘ø¦Zã,&¼F‘+׎ÉfÁ¨u£v¤eÙ#šŽ`óˆ]3 }Dq™98€Û�™5Çd]–õµì;°MC,ïbZø€„ip_Q
|
||||
@@ -1,7 +1,7 @@
|
||||
age-encryption.org/v1
|
||||
-> ssh-ed25519 CIR0mA q9gHASCHMVma89r38UB+SQhOfvJh+fTn6cDde+VokyI
|
||||
NwJV0BDtHtR76S1gnobJ/sCf72X7Jgh/aX9zdppJQrg
|
||||
-> ssh-ed25519 So2Y2A bA0FjhApSVVz5KihvhT/dYrSH5amAUX2j7b9IaZ1YSQ
|
||||
1m1Tq0ib8kQbRXTCx+kiBEN/LG8mmfrlnVqCFZAYQtM
|
||||
--- ClysLz304nxJUBJH8gx5VPE3FKJTtgntVj2b4czSJC8
|
||||
³=@b)[|[HÜúcmd1‰ÏÝ2L˜8Ú_`ùèo³§0Ñ^"¡Ö<¬ÁІúO>af¥æiü~$¥ÿŸ…\q:±Pfåÿ¨3€#„c1‘ö—xByóxÈñÀò»EÏÇ
|
||||
-> ssh-ed25519 CIR0mA NcvzVtUfM/i7MmeLG7CV2Ngt8QplSC0nQ2m3Qo8/dWI
|
||||
B1kUDAkLfcq6BRrfd1EYAohLnuF0rpC+QM4Un/9nVjQ
|
||||
-> ssh-ed25519 So2Y2A 6pu6KIHQy0VqYAuxbWTTy4s1Mr1pNlugyNDsw3EXASU
|
||||
ebUCvyFTKsxKKuTWPGVnb8FVpmtGsyRcXONN/L2SGKI
|
||||
--- V80No7zOTrxlfBfSEorOEeLBe/2ve+lX3K/3ixY5QFg
|
||||
µåHÖ¾1S¼uf´,¶7ò>.%Jµ�ëïç�hó_D8³†Ô8%æÖ¥cø‚ùÓZ‚é³|ß�¨íY¸CK1}¢ëQ•Z
�šçÑ5Bå‘jç…çôµß¬àX
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,8 +1,7 @@
|
||||
age-encryption.org/v1
|
||||
-> ssh-ed25519 CIR0mA CxgrXCVHrJnqoGzZEz0oKGie2XEHLu809dcFhmIUGA4
|
||||
X25TGuWbIwRMkAE9EEedn9hb6bWCuMVmJi+61d5KIys
|
||||
-> ssh-ed25519 LuJNQA kpVx1opi4cjdpzic4Xg8phRyR/AHZnVpsfqj62FBvFc
|
||||
1siGAQsjaKNzx2XRrszSpjz75mzsOm23ya+QXSZFXtg
|
||||
--- SZyPEECt2N8gsCZqrBJY6he5OohAZBJxh/uQs8BKWBE
|
||||
çÒX®ÂåÀX„=ê�ÍØágÌÇ!©Y{ �‡Þ
“ºÒ�fÓÚÍàÀð§wìŒ
|
||||
Â?~gæK�O´dXÍiü©„.×™šEÄíá
|
||||
-> ssh-ed25519 CIR0mA u5gJNbu+l/8Fkd5yxYvtqKJ/ZDSxQqjcpP4g9yqzllg
|
||||
I6C5pleesVsD66KfLsXfXyi5FWsPpTQH/01T9l3eCUM
|
||||
-> ssh-ed25519 LuJNQA On/rlQtXl9Rr3OhJZ9zapT2GLcX3caGmeRUokUCwbD4
|
||||
v1zV8ri6IcLzTWGMne6DpjCJBVr/Lja4Ah1M5Bj0IDs
|
||||
--- GSIODQOv5D+vQgI95cQxOwiVYZYhOqaYzNTPHCHoSlA
|
||||
h
ZÚ«ţ›‰í¶qšźYoá˙±N˙µJ'€ßÚ€ˇbVŤö$ŰŹü±,|Ľb•8ö8^FKűK�/��±^€řˇ,_˵s
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,8 @@
|
||||
age-encryption.org/v1
|
||||
-> ssh-ed25519 CIR0mA Ef4yK/jaOy9aJoGACo8Hfl4/O3R+dOuBNNKmjf9mNnE
|
||||
DGL2wY1SjH8j1n8cjl7FrYI4bki5fpPdcToGBh1gess
|
||||
-> ssh-ed25519 LuJNQA 02n9TO/NsaebvWz1bvU/PMj151pZYVyGeDzZalJexwo
|
||||
DLpad+VkRn9TnMrfE6yGoP2TrfZrjyqrgsXGqOPvUmI
|
||||
--- l3k5gUk4hXjTmEVtvQhSz6vsreh3knXGcIgHAIiwJ7Y
|
||||
¶#±3ॎ�Ö»á¼%NgØt¬ór? Ç«<íÝ6 @˜“¾VšèwG4¯ qæP§
|
||||
?¶üBæò2'lÕbå©:U#5ë4çÒ¸¤~myyôÎb¯¿àýJÀ
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,7 +1,7 @@
|
||||
age-encryption.org/v1
|
||||
-> ssh-ed25519 CIR0mA wbZRuR6TE7oD+sTfCZU561XShPsv0CMxMEpLebi+1hw
|
||||
b1McWvJNoxYG6K5hjiuc0WLPViyCtgdyvmjwhxPfpZ0
|
||||
-> ssh-ed25519 LuJNQA aTL4kxlpMBqABK/CJ5KjAiBDpL3m5ETnlK8hDF61cgw
|
||||
gFIydWCNJH/iur3L7pj0R9cl+/i63jcHhVh0AlJwQvA
|
||||
--- q2VnKs8u6d4NbPJoTDd0dJLuYVW8Ne3NaanbGlCakbQ
|
||||
Q§ËÃï`·Ÿ�þgæëø¶ôPIqA•Ôjâø6bù*`÷"Ly¿Nt£Ôýð›ºgS÷ƒbz({Š�a®£¹(ƒR×Ò¤mT,ÀÔÓù~Œ×¦ÃŒÄÇ5ä‹£¸ï_t8ö!·èF¤*vï £÷&i<œ
|
||||
-> ssh-ed25519 CIR0mA tptdMYirvCYTTTQKPR8ZrMKud6BsQkabKFqQppTJ8DM
|
||||
ZPWGoj47fBimtTEkldCAY/UcYMM+uYp68MhSiW/K0u4
|
||||
-> ssh-ed25519 LuJNQA oXZz25+1kHT92d8ksBkIgZYmw+TsSlFU7GmQugsIWD8
|
||||
J2GqEFhaSs/JVWxH2q1vP39Hlxq5mGMlWPDMZAiRWus
|
||||
--- pdLoyc19kNV5o4tLoYjw4334Xi6Pr4Adno0TUkxiJoM
|
||||
ª%ñМ²‰(;“šªËÈ8â0QƒÈK¤±–¦*ˆß¥ôþ¯•½�'Gª*²z[P×Ô¹€’¾:Âþá�k™Ê³…^ÁáÚk:àôÈÖÏ�\ ôãÕÛME+ØUóvtÍ5A½ñ#`k¶Ç:ÿs¯(E�ì Šx>Õ
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Reference in new issue
Block a user