tracking
This commit is contained in:
1 parent
da93290953
commit
4f3a95b42c
50 files changed
+486
-37
No files matched your search
+1
-1
@@ -35,7 +35,7 @@ hosts/
|
||||
| `hl-fws1/` | hl-fws1 | Framework 11th Intel — K3s server node |
|
||||
| `hl-bigbox1/` | hl-bigbox1 | AI/compute server (NVIDIA + AMD); Immich machine-learning (CUDA) |
|
||||
| `hl-bigbox2/` | hl-bigbox2 | NFS storage server; Immich server/db (docker, `immich.home.sosnovsky.ca`) |
|
||||
| `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI) |
|
||||
| `hl-minipc1/` | hl-minipc1 | Core infra: Gitea (native or containerized), nix-serve, registry, Buzz (docker, `buzz-manage` CLI), monitoring (Grafana, Prometheus, Loki, OTel collector) |
|
||||
| `hl-minipc2/` | hl-minipc2 | Container/media server |
|
||||
| `hl-minipc3/` | hl-minipc3 | K3s worker node; central Caddy reverse-proxy (`*.home.sosnovsky.ca`, plus `gitea.skyg.ca` HTTP/SSH); buildbot-nix CI master + worker (2 cores) |
|
||||
| `hl-pi1/` | hl-pi1 | Raspberry Pi CM4 (aarch64) — headless server, SSH only |
|
||||
|
||||
@@ -4,5 +4,6 @@
|
||||
./immich-ml.nix
|
||||
./portainer.nix
|
||||
./timers.nix
|
||||
./linux-tracker.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "bigbox1.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,40 @@
|
||||
{ ... }:
|
||||
{ config, pkgs, ... }:
|
||||
{
|
||||
services.fwupd.enable = true;
|
||||
|
||||
# Interim: the box loses power under load. Use less power and heat, log more.
|
||||
# Remove this block once the new board is in and stable.
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
# No turbo. Lost on reboot unless set here.
|
||||
"w /sys/devices/system/cpu/cpufreq/boost - - - - 0"
|
||||
# Max clock 2.5 GHz. Steps on this CPU: 3.5, 3.0, 2.5, 2.0, 1.4.
|
||||
"w /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq - - - - 2500000"
|
||||
];
|
||||
|
||||
# Keep logs on disk and flush often, so the last seconds before a crash survive.
|
||||
services.journald.extraConfig = ''
|
||||
Storage=persistent
|
||||
SystemMaxUse=2G
|
||||
SyncIntervalSec=5s
|
||||
'';
|
||||
|
||||
# Reboot 10 s after a kernel panic instead of hanging.
|
||||
boot.kernel.sysctl."kernel.panic" = 10;
|
||||
|
||||
# Watch disk health and log warnings. No mail set up.
|
||||
services.smartd = {
|
||||
enable = true;
|
||||
autodetect = true;
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
pkgs.smartmontools
|
||||
pkgs.lm_sensors
|
||||
pkgs.dmidecode
|
||||
pkgs.stress-ng
|
||||
pkgs.memtester
|
||||
pkgs.ethtool
|
||||
config.boot.kernelPackages.cpupower
|
||||
];
|
||||
}
|
||||
@@ -4,5 +4,6 @@
|
||||
./nix-serve.nix
|
||||
./nfs-server.nix
|
||||
./immich.nix
|
||||
./linux-tracker.nix
|
||||
];
|
||||
}
|
||||
@@ -34,7 +34,7 @@ in
|
||||
environmentFiles = [ config.age.secrets.immich-shared.path ];
|
||||
environment = envVars;
|
||||
deploy.resources.limits = {
|
||||
cpus = "4";
|
||||
cpus = "1";
|
||||
memory = "8G";
|
||||
};
|
||||
networks = {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "bigbox2.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -7,5 +7,6 @@
|
||||
./wyoming.nix
|
||||
./steam.nix
|
||||
./buildbot.nix
|
||||
./linux-tracker.nix
|
||||
];
|
||||
}
|
||||
@@ -39,6 +39,8 @@ in
|
||||
{ address = "10.0.101.3"; port = 8080; }
|
||||
{ address = "10.0.101.3"; port = 9102; }
|
||||
{ address = "10.0.101.8"; port = 3000; }
|
||||
# skyg-linux-tracker (REST + MCP) on every lab host.
|
||||
{ address = "10.0.0.0/16"; port = 1984; }
|
||||
];
|
||||
publish = [
|
||||
{ hostPort = ports.hermes; guestPort = 8642; }
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "fwdesk.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -31,4 +31,18 @@
|
||||
enable = true;
|
||||
arguments = "-o pipewire";
|
||||
};
|
||||
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
allowedHosts = [ "fws1.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -12,5 +12,10 @@
|
||||
./iu.nix
|
||||
./audiobookshelf.nix
|
||||
./buzz.nix
|
||||
./linux-tracker.nix
|
||||
./otel-collector.nix
|
||||
./prometheus.nix
|
||||
./loki.nix
|
||||
./grafana.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{ config, skyg-secrets, ... }:
|
||||
let
|
||||
ports = {
|
||||
grafana = 3001;
|
||||
prometheus = 9090;
|
||||
loki = 3100;
|
||||
};
|
||||
in
|
||||
{
|
||||
age.secrets.grafana-secret-key = {
|
||||
file = skyg-secrets.grafana-secret-key;
|
||||
owner = "grafana";
|
||||
group = "grafana";
|
||||
};
|
||||
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
|
||||
settings = {
|
||||
server = {
|
||||
http_addr = "0.0.0.0";
|
||||
http_port = ports.grafana;
|
||||
};
|
||||
security.secret_key = "$__(file){${config.age.secrets.grafana-secret-key.path}}";
|
||||
};
|
||||
|
||||
provision = {
|
||||
enable = true;
|
||||
datasources.settings.datasources = [
|
||||
{
|
||||
name = "Prometheus";
|
||||
type = "prometheus";
|
||||
access = "proxy";
|
||||
url = "http://127.0.0.1:${toString ports.prometheus}";
|
||||
isDefault = true;
|
||||
}
|
||||
{
|
||||
name = "Loki";
|
||||
type = "loki";
|
||||
access = "proxy";
|
||||
url = "http://127.0.0.1:${toString ports.loki}";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "minipc1.lab.internal" ];
|
||||
# OTLP out to the collector on this box (see ./otel-collector.nix).
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://127.0.0.1:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
{ ... }:
|
||||
let
|
||||
ports = {
|
||||
loki = 3100;
|
||||
grpc = 9096;
|
||||
};
|
||||
dataDir = "/var/lib/loki";
|
||||
in
|
||||
{
|
||||
services.loki = {
|
||||
enable = true;
|
||||
|
||||
# Single-binary Loki on the filesystem: no object store, no ring.
|
||||
configuration = {
|
||||
auth_enabled = false;
|
||||
|
||||
server = {
|
||||
http_listen_port = ports.loki;
|
||||
grpc_listen_port = ports.grpc;
|
||||
};
|
||||
|
||||
common = {
|
||||
instance_addr = "127.0.0.1";
|
||||
path_prefix = dataDir;
|
||||
replication_factor = 1;
|
||||
ring.kvstore.store = "inmemory";
|
||||
storage.filesystem = {
|
||||
chunks_directory = "${dataDir}/chunks";
|
||||
rules_directory = "${dataDir}/rules";
|
||||
};
|
||||
};
|
||||
|
||||
schema_config.configs = [
|
||||
{
|
||||
from = "2026-10-01";
|
||||
store = "tsdb";
|
||||
object_store = "filesystem";
|
||||
schema = "v13";
|
||||
index = {
|
||||
prefix = "index_";
|
||||
period = "24h";
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
# The compactor is what enforces the 24h window below.
|
||||
compactor = {
|
||||
working_directory = "${dataDir}/compactor";
|
||||
retention_enabled = true;
|
||||
delete_request_store = "filesystem";
|
||||
};
|
||||
|
||||
limits_config = {
|
||||
retention_period = "24h";
|
||||
reject_old_samples = true;
|
||||
reject_old_samples_max_age = "168h";
|
||||
# Required for OTLP ingestion; on by default since 3.0.
|
||||
allow_structured_metadata = true;
|
||||
};
|
||||
|
||||
ruler = {
|
||||
storage.type = "local";
|
||||
storage.local.directory = "${dataDir}/rules";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# Loki data lives on tnas1. Prometheus and Grafana stay on local disk, since
|
||||
# their databases do not work well on NFS.
|
||||
# The export must exist on the NAS and be writable by uid 985 / gid 981.
|
||||
# The mount is ordered after network-online so tnas1's name resolves at boot.
|
||||
fileSystems.${dataDir} = {
|
||||
device = "tnas1.lab.internal:/mnt/SmallG/monitoring/loki";
|
||||
fsType = "nfs";
|
||||
options = [
|
||||
"nfsvers=4.0"
|
||||
"hard"
|
||||
"noatime"
|
||||
"nofail"
|
||||
"_netdev"
|
||||
"x-systemd.automount"
|
||||
"x-systemd.requires=network-online.target"
|
||||
"x-systemd.after=network-online.target"
|
||||
];
|
||||
};
|
||||
systemd.services.loki.unitConfig.RequiresMountsFor = [ dataDir ];
|
||||
|
||||
# NFS ownership is numeric, so pin the ids instead of letting NixOS assign
|
||||
# them by name order (a new user sorting before "loki" would shift them).
|
||||
users.users.loki.uid = 985;
|
||||
users.groups.loki.gid = 981;
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ ports.loki ];
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
ports = {
|
||||
otlpGrpc = 4317;
|
||||
otlpHttp = 4318;
|
||||
# Loopback scrape target for the Prometheus below.
|
||||
metrics = 9464;
|
||||
};
|
||||
in
|
||||
{
|
||||
services.opentelemetry-collector = {
|
||||
enable = true;
|
||||
# The prometheus exporter is a contrib component, not core.
|
||||
package = pkgs.opentelemetry-collector-contrib;
|
||||
|
||||
settings = {
|
||||
receivers.otlp.protocols = {
|
||||
grpc.endpoint = "0.0.0.0:${toString ports.otlpGrpc}";
|
||||
http.endpoint = "0.0.0.0:${toString ports.otlpHttp}";
|
||||
};
|
||||
|
||||
processors.batch.timeout = "5s";
|
||||
|
||||
exporters = {
|
||||
# Pull endpoint. Resource attributes (host.name, service.name) only
|
||||
# become labels when conversion is enabled — off by default, which
|
||||
# would leave every host's series with an identical label set.
|
||||
prometheus = {
|
||||
endpoint = "127.0.0.1:${toString ports.metrics}";
|
||||
resource_to_telemetry_conversion.enabled = true;
|
||||
};
|
||||
# The contrib `loki` exporter was removed upstream; Loki ingests
|
||||
# OTLP/HTTP natively at /otlp instead.
|
||||
"otlphttp/loki".endpoint = "http://127.0.0.1:3100/otlp";
|
||||
nop = { };
|
||||
};
|
||||
|
||||
service.pipelines = {
|
||||
metrics = {
|
||||
receivers = [ "otlp" ];
|
||||
processors = [ "batch" ];
|
||||
exporters = [ "prometheus" ];
|
||||
};
|
||||
logs = {
|
||||
receivers = [ "otlp" ];
|
||||
processors = [ "batch" ];
|
||||
exporters = [ "otlphttp/loki" ];
|
||||
};
|
||||
# The tracker ships its own spans; nothing here stores them.
|
||||
traces = {
|
||||
receivers = [ "otlp" ];
|
||||
processors = [ "batch" ];
|
||||
exporters = [ "nop" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# Only the OTLP listeners face the lab; 9464 stays on loopback.
|
||||
networking.firewall.allowedTCPPorts = [ ports.otlpGrpc ports.otlpHttp ];
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{ ... }:
|
||||
let
|
||||
ports = { prometheus = 9090; };
|
||||
in
|
||||
{
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
port = ports.prometheus;
|
||||
listenAddress = "0.0.0.0";
|
||||
retentionTime = "48h";
|
||||
|
||||
scrapeConfigs = [
|
||||
{
|
||||
job_name = "lab-trackers";
|
||||
static_configs = [{ targets = [ "127.0.0.1:9464" ]; }];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ ports.prometheus ];
|
||||
}
|
||||
@@ -6,5 +6,6 @@
|
||||
./stack1.nix
|
||||
./stack2.nix
|
||||
./timers.nix
|
||||
./linux-tracker.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "minipc2.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
imports = [
|
||||
./caddy.nix
|
||||
./acme.nix
|
||||
./linux-tracker.nix
|
||||
# ./buildbot.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{ config, ... }:
|
||||
{
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
docker.enable = config.virtualisation.docker.enable;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "minipc3.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,20 @@
|
||||
{ ... }:
|
||||
{
|
||||
skyg.nixos.common.networking.nfsServer.enable = true;
|
||||
services.skyg-linux-tracker = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
openFirewall = true;
|
||||
# Reachable only by its lab FQDN; the short name does not resolve.
|
||||
allowedHosts = [ "terra1.lab.internal" ];
|
||||
# OTLP out to the collector on minipc1.
|
||||
settings.otel = {
|
||||
enabled = true;
|
||||
endpoint = "http://minipc1.lab.internal:4317";
|
||||
protocol = "grpc";
|
||||
export_logs = true;
|
||||
};
|
||||
};
|
||||
services.nfs.server = {
|
||||
lockdPort = 4001;
|
||||
mountdPort = 4002;
|
||||
|
||||
Reference in new issue
Block a user