This commit is contained in:
ari committed 2026-10-19 14:49:47 -04:00
1 parent da93290953
commit 4f3a95b42c
50 files changed
+486 -37

No files matched your search

+1 -1
View File
@@ -35,7 +35,7 @@ hosts/
| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node |
| `hl-bigbox1/` | hl-bigbox1 | AI/compute server (NVIDIA + AMD); Immich machine-learning (CUDA) |
| `hl-bigbox2/` | hl-bigbox2 | NFS storage server; Immich server/db (docker, `immich.home.sosnovsky.ca`) |
| `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI) |
| `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI), monitoring (Grafana, Prometheus, Loki, OTel collector) |
| `hl-minipc2/` | hl-minipc2 | Container/media server |
| `hl-minipc3/` | hl-minipc3 | K3s worker node; central Caddy reverse-proxy (`*.home.sosnovsky.ca`, plus `gitea.skyg.ca` HTTP/SSH); buildbot-nix CI master + worker (2 cores) |
| `hl-pi1/` | hl-pi1 | Raspberry Pi CM4 (aarch64) — headless server, SSH only |
+1
View File
@@ -4,5 +4,6 @@
./immich-ml.nix
./portainer.nix
./timers.nix
./linux-tracker.nix
];
}
@@ -0,0 +1,18 @@
{ config, ... }:
{
services.skyg-linux-tracker = {
enable = true;
bind = "0.0.0.0";
openFirewall = true;
docker.enable = config.virtualisation.docker.enable;
# Reachable only by its lab FQDN; the short name does not resolve.
allowedHosts = [ "bigbox1.lab.internal" ];
# OTLP out to the collector on minipc1.
settings.otel = {
enabled = true;
endpoint = "http://minipc1.lab.internal:4317";
protocol = "grpc";
export_logs = true;
};
};
}
+37 -1
View File
@@ -1,4 +1,40 @@
{ ... }:
{ config, pkgs, ... }:
{
services.fwupd.enable = true;
# Interim: the box loses power under load. Use less power and heat, log more.
# Remove this block once the new board is in and stable.
systemd.tmpfiles.rules = [
# No turbo. Lost on reboot unless set here.
"w /sys/devices/system/cpu/cpufreq/boost - - - - 0"
# Max clock 2.5 GHz. Steps on this CPU: 3.5, 3.0, 2.5, 2.0, 1.4.
"w /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq - - - - 2500000"
];
# Keep logs on disk and flush often, so the last seconds before a crash survive.
services.journald.extraConfig = ''
Storage=persistent
SystemMaxUse=2G
SyncIntervalSec=5s
'';
# Reboot 10 s after a kernel panic instead of hanging.
boot.kernel.sysctl."kernel.panic" = 10;
# Watch disk health and log warnings. No mail set up.
services.smartd = {
enable = true;
autodetect = true;
};
environment.systemPackages = [
pkgs.smartmontools
pkgs.lm_sensors
pkgs.dmidecode
pkgs.stress-ng
pkgs.memtester
pkgs.ethtool
config.boot.kernelPackages.cpupower
];
}
+1
View File
@@ -4,5 +4,6 @@
./nix-serve.nix
./nfs-server.nix
./immich.nix
./linux-tracker.nix
];
}
+1 -1
View File
@@ -34,7 +34,7 @@ in
environmentFiles = [ config.age.secrets.immich-shared.path ];
environment = envVars;
deploy.resources.limits = {
cpus = "4";
cpus = "1";
memory = "8G";
};
networks = {
@@ -0,0 +1,18 @@
{ config, ... }:
{
services.skyg-linux-tracker = {
enable = true;
bind = "0.0.0.0";
openFirewall = true;
docker.enable = config.virtualisation.docker.enable;
# Reachable only by its lab FQDN; the short name does not resolve.
allowedHosts = [ "bigbox2.lab.internal" ];
# OTLP out to the collector on minipc1.
settings.otel = {
enabled = true;
endpoint = "http://minipc1.lab.internal:4317";
protocol = "grpc";
export_logs = true;
};
};
}
+1
View File
@@ -7,5 +7,6 @@
./wyoming.nix
./steam.nix
./buildbot.nix
./linux-tracker.nix
];
}
+2
View File
@@ -39,6 +39,8 @@ in
{ address = "10.0.101.3"; port = 8080; }
{ address = "10.0.101.3"; port = 9102; }
{ address = "10.0.101.8"; port = 3000; }
# skyg-linux-tracker (REST + MCP) on every lab host.
{ address = "10.0.0.0/16"; port = 1984; }
];
publish = [
{ hostPort = ports.hermes; guestPort = 8642; }
@@ -0,0 +1,18 @@
{ config, ... }:
{
services.skyg-linux-tracker = {
enable = true;
bind = "0.0.0.0";
openFirewall = true;
docker.enable = config.virtualisation.docker.enable;
# Reachable only by its lab FQDN; the short name does not resolve.
allowedHosts = [ "fwdesk.lab.internal" ];
# OTLP out to the collector on minipc1.
settings.otel = {
enabled = true;
endpoint = "http://minipc1.lab.internal:4317";
protocol = "grpc";
export_logs = true;
};
};
}
+14
View File
@@ -31,4 +31,18 @@
enable = true;
arguments = "-o pipewire";
};
services.skyg-linux-tracker = {
enable = true;
bind = "0.0.0.0";
openFirewall = true;
allowedHosts = [ "fws1.lab.internal" ];
# OTLP out to the collector on minipc1.
settings.otel = {
enabled = true;
endpoint = "http://minipc1.lab.internal:4317";
protocol = "grpc";
export_logs = true;
};
};
}
+5
View File
@@ -12,5 +12,10 @@
./iu.nix
./audiobookshelf.nix
./buzz.nix
./linux-tracker.nix
./otel-collector.nix
./prometheus.nix
./loki.nix
./grafana.nix
];
}
+47
View File
@@ -0,0 +1,47 @@
{ config, skyg-secrets, ... }:
let
ports = {
grafana = 3001;
prometheus = 9090;
loki = 3100;
};
in
{
age.secrets.grafana-secret-key = {
file = skyg-secrets.grafana-secret-key;
owner = "grafana";
group = "grafana";
};
services.grafana = {
enable = true;
openFirewall = true;
settings = {
server = {
http_addr = "0.0.0.0";
http_port = ports.grafana;
};
security.secret_key = "$__(file){${config.age.secrets.grafana-secret-key.path}}";
};
provision = {
enable = true;
datasources.settings.datasources = [
{
name = "Prometheus";
type = "prometheus";
access = "proxy";
url = "http://127.0.0.1:${toString ports.prometheus}";
isDefault = true;
}
{
name = "Loki";
type = "loki";
access = "proxy";
url = "http://127.0.0.1:${toString ports.loki}";
}
];
};
};
}
@@ -0,0 +1,18 @@
{ config, ... }:
{
services.skyg-linux-tracker = {
enable = true;
bind = "0.0.0.0";
openFirewall = true;
docker.enable = config.virtualisation.docker.enable;
# Reachable only by its lab FQDN; the short name does not resolve.
allowedHosts = [ "minipc1.lab.internal" ];
# OTLP out to the collector on this box (see ./otel-collector.nix).
settings.otel = {
enabled = true;
endpoint = "http://127.0.0.1:4317";
protocol = "grpc";
export_logs = true;
};
};
}
+94
View File
@@ -0,0 +1,94 @@
{ ... }:
let
ports = {
loki = 3100;
grpc = 9096;
};
dataDir = "/var/lib/loki";
in
{
services.loki = {
enable = true;
# Single-binary Loki on the filesystem: no object store, no ring.
configuration = {
auth_enabled = false;
server = {
http_listen_port = ports.loki;
grpc_listen_port = ports.grpc;
};
common = {
instance_addr = "127.0.0.1";
path_prefix = dataDir;
replication_factor = 1;
ring.kvstore.store = "inmemory";
storage.filesystem = {
chunks_directory = "${dataDir}/chunks";
rules_directory = "${dataDir}/rules";
};
};
schema_config.configs = [
{
from = "2026-10-01";
store = "tsdb";
object_store = "filesystem";
schema = "v13";
index = {
prefix = "index_";
period = "24h";
};
}
];
# The compactor is what enforces the 24h window below.
compactor = {
working_directory = "${dataDir}/compactor";
retention_enabled = true;
delete_request_store = "filesystem";
};
limits_config = {
retention_period = "24h";
reject_old_samples = true;
reject_old_samples_max_age = "168h";
# Required for OTLP ingestion; on by default since 3.0.
allow_structured_metadata = true;
};
ruler = {
storage.type = "local";
storage.local.directory = "${dataDir}/rules";
};
};
};
# Loki data lives on tnas1. Prometheus and Grafana stay on local disk, since
# their databases do not work well on NFS.
# The export must exist on the NAS and be writable by uid 985 / gid 981.
# The mount is ordered after network-online so tnas1's name resolves at boot.
fileSystems.${dataDir} = {
device = "tnas1.lab.internal:/mnt/SmallG/monitoring/loki";
fsType = "nfs";
options = [
"nfsvers=4.0"
"hard"
"noatime"
"nofail"
"_netdev"
"x-systemd.automount"
"x-systemd.requires=network-online.target"
"x-systemd.after=network-online.target"
];
};
systemd.services.loki.unitConfig.RequiresMountsFor = [ dataDir ];
# NFS ownership is numeric, so pin the ids instead of letting NixOS assign
# them by name order (a new user sorting before "loki" would shift them).
users.users.loki.uid = 985;
users.groups.loki.gid = 981;
networking.firewall.allowedTCPPorts = [ ports.loki ];
}
@@ -0,0 +1,61 @@
{ pkgs, ... }:
let
ports = {
otlpGrpc = 4317;
otlpHttp = 4318;
# Loopback scrape target for the Prometheus below.
metrics = 9464;
};
in
{
services.opentelemetry-collector = {
enable = true;
# The prometheus exporter is a contrib component, not core.
package = pkgs.opentelemetry-collector-contrib;
settings = {
receivers.otlp.protocols = {
grpc.endpoint = "0.0.0.0:${toString ports.otlpGrpc}";
http.endpoint = "0.0.0.0:${toString ports.otlpHttp}";
};
processors.batch.timeout = "5s";
exporters = {
# Pull endpoint. Resource attributes (host.name, service.name) only
# become labels when conversion is enabled — off by default, which
# would leave every host's series with an identical label set.
prometheus = {
endpoint = "127.0.0.1:${toString ports.metrics}";
resource_to_telemetry_conversion.enabled = true;
};
# The contrib `loki` exporter was removed upstream; Loki ingests
# OTLP/HTTP natively at /otlp instead.
"otlphttp/loki".endpoint = "http://127.0.0.1:3100/otlp";
nop = { };
};
service.pipelines = {
metrics = {
receivers = [ "otlp" ];
processors = [ "batch" ];
exporters = [ "prometheus" ];
};
logs = {
receivers = [ "otlp" ];
processors = [ "batch" ];
exporters = [ "otlphttp/loki" ];
};
# The tracker ships its own spans; nothing here stores them.
traces = {
receivers = [ "otlp" ];
processors = [ "batch" ];
exporters = [ "nop" ];
};
};
};
};
# Only the OTLP listeners face the lab; 9464 stays on loopback.
networking.firewall.allowedTCPPorts = [ ports.otlpGrpc ports.otlpHttp ];
}
+21
View File
@@ -0,0 +1,21 @@
{ ... }:
let
ports = { prometheus = 9090; };
in
{
services.prometheus = {
enable = true;
port = ports.prometheus;
listenAddress = "0.0.0.0";
retentionTime = "48h";
scrapeConfigs = [
{
job_name = "lab-trackers";
static_configs = [{ targets = [ "127.0.0.1:9464" ]; }];
}
];
};
networking.firewall.allowedTCPPorts = [ ports.prometheus ];
}
+1
View File
@@ -6,5 +6,6 @@
./stack1.nix
./stack2.nix
./timers.nix
./linux-tracker.nix
];
}
@@ -0,0 +1,18 @@
{ config, ... }:
{
services.skyg-linux-tracker = {
enable = true;
bind = "0.0.0.0";
openFirewall = true;
docker.enable = config.virtualisation.docker.enable;
# Reachable only by its lab FQDN; the short name does not resolve.
allowedHosts = [ "minipc2.lab.internal" ];
# OTLP out to the collector on minipc1.
settings.otel = {
enabled = true;
endpoint = "http://minipc1.lab.internal:4317";
protocol = "grpc";
export_logs = true;
};
};
}
+1
View File
@@ -2,6 +2,7 @@
imports = [
./caddy.nix
./acme.nix
./linux-tracker.nix
# ./buildbot.nix
];
}
@@ -0,0 +1,18 @@
{ config, ... }:
{
services.skyg-linux-tracker = {
enable = true;
bind = "0.0.0.0";
openFirewall = true;
docker.enable = config.virtualisation.docker.enable;
# Reachable only by its lab FQDN; the short name does not resolve.
allowedHosts = [ "minipc3.lab.internal" ];
# OTLP out to the collector on minipc1.
settings.otel = {
enabled = true;
endpoint = "http://minipc1.lab.internal:4317";
protocol = "grpc";
export_logs = true;
};
};
}
+14
View File
@@ -1,6 +1,20 @@
{ ... }:
{
skyg.nixos.common.networking.nfsServer.enable = true;
services.skyg-linux-tracker = {
enable = true;
bind = "0.0.0.0";
openFirewall = true;
# Reachable only by its lab FQDN; the short name does not resolve.
allowedHosts = [ "terra1.lab.internal" ];
# OTLP out to the collector on minipc1.
settings.otel = {
enabled = true;
endpoint = "http://minipc1.lab.internal:4317";
protocol = "grpc";
export_logs = true;
};
};
services.nfs.server = {
lockdPort = 4001;
mountdPort = 4002;