buz support

This commit is contained in:
ari committed 2026-09-20 20:13:39 -04:00
1 parent 5d727c26cd
commit beb83f21b3
13 files changed
+397 -42

No files matched your search

+14 -14
View File
@@ -20,20 +20,20 @@ hosts/
## Managed Hosts
| File | Host | Role |
|---|---|---|
| `fwbook.nix` | fwbook | Framework 13 AMD — personal laptop / daily driver |
| `hl-fwdesk.nix` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation |
| `hl-fws1.nix` | hl-fws1 | Framework 11th Intel — K3s server node |
| `hl-bigbox1.nix` | hl-bigbox1 | AI/compute server (NVIDIA + AMD) |
| `hl-bigbox2.nix` | hl-bigbox2 | NFS storage server |
| `hl-minipc1.nix` | hl-minipc1 | Core infra: Gitea, nix-serve, Docker registry |
| `hl-minipc2.nix` | hl-minipc2 | Container/media server |
| `hl-minipc3.nix` | hl-minipc3 | K3s worker node |
| `hl-pi1.nix` | hl-pi1 | Raspberry Pi CM4 (aarch64) — headless server, SSH only. Also buildable as a bootable SD image via the `hl-pi1-sd-image` nixosConfiguration (`skyg build-pi-sd-image`) |
| `hl-pi2.nix` | hl-pi2 | Raspberry Pi 4 (aarch64) — headless server, SSH only. Also buildable as a bootable SD image via the `hl-pi2-sd-image` nixosConfiguration |
| `hl-terra1.nix` | hl-terra1 | NFS storage + Jellyfin backup |
| `iso.nix` | iso | Bootable NixOS installer image |
| File | Host | Role |
| ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `fwbook.nix` | fwbook | Framework 13 AMD — personal laptop / daily driver |
| `hl-fwdesk.nix` | hl-fwdesk | Framework Desktop AI Max 300 — AI workstation |
| `hl-fws1.nix` | hl-fws1 | Framework 11th Intel — K3s server node |
| `hl-bigbox1.nix` | hl-bigbox1 | AI/compute server (NVIDIA + AMD) |
| `hl-bigbox2.nix` | hl-bigbox2 | NFS storage server |
| `hl-minipc1.nix` | hl-minipc1 | Core infra: Gitea, nix-serve, registry, Buzz (podman) |
| `hl-minipc2.nix` | hl-minipc2 | Container/media server |
| `hl-minipc3.nix` | hl-minipc3 | K3s worker node |
| `hl-pi1.nix` | hl-pi1 | Raspberry Pi CM4 (aarch64) — headless server, SSH only. Also buildable as a bootable SD image via the `hl-pi1-sd-image` nixosConfiguration (`skyg build-pi-sd-image`) |
| `hl-pi2.nix` | hl-pi2 | Raspberry Pi 4 (aarch64) — headless server, SSH only. Also buildable as a bootable SD image via the `hl-pi2-sd-image` nixosConfiguration |
| `hl-terra1.nix` | hl-terra1 | NFS storage + Jellyfin backup |
| `iso.nix` | iso | Bootable NixOS installer image |
## Rules
@@ -14,22 +14,14 @@
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-intel" ];
boot.extraModulePackages = [ ];
fileSystems."/" = {
device = "/dev/disk/by-uuid/142c88f4-b1bc-4ed9-92ee-2c52cdfa4274";
fsType = "ext4";
};
fileSystems."/boot" = {
device = "/dev/disk/by-uuid/01FE-EDCC";
fsType = "vfat";
};
# Data
# fileSystems."/mnt/Data" = {
# device = "/dev/disk/by-uuid/ee4a60a8-b0d1-4f5c-a554-1d1d84c89e34";
# fsType = "ext4";
# options = [ "defaults" ];
# };
fileSystems."/nix" = {
device = "/dev/disk/by-uuid/ee4a60a8-b0d1-4f5c-a554-1d1d84c89e34";
fsType = "ext4";
+149
View File
@@ -25,6 +25,21 @@ let
ports.ssh
ports.iu
];
# Buzz stack
buzz = {
domain = "buzz.lab.internal";
port = 3000;
ip = "10.0.101.3";
bucket = "buzz-media";
};
# NFS-backed volume on tnas1 (repo pattern: compose volume with nfs driver_opts).
nfsVolume = subpath: {
driver_opts = {
type = "nfs";
o = "addr=tnas1.lab.internal,rw,nfsvers=4.0,nolock,hard,noatime";
device = ":/mnt/SmallG/buzz/${subpath}";
};
};
in
{
skyg.user.enable = true;
@@ -34,7 +49,17 @@ in
openFirewall = true;
addressesSecretName = "dns-addresses.conf";
};
skyg.nixos.common.containers.runtime = "podman";
skyg.nixos.common.containers.openMetricsPort = true;
# One shared macvlan network, created once at boot.
skyg.nixos.common.containers.networks.lab = {
driver = "macvlan";
driverOpts.parent = "eno1";
subnet = "10.0.0.0/16";
gateway = "10.0.0.1";
ipRange = "10.0.101.16/28";
};
skyg.server.admin.enable = true;
skyg.server.exporters.enable = true;
skyg.nixos.server.k3s.enable = false;
@@ -199,6 +224,130 @@ in
};
};
# Buzz — relay + postgres + redis + garage (S3). Reachable at
# http://buzz.lab.internal:3000 on its own macvlan IP (10.0.101.3).
# Data lives on tnas1 via NFS-backed compose volumes (/mnt/SmallG/buzz).
age.secrets.buzz-env.file = ../secrets/buzz-env.age;
skyg.nixos.common.container-services.buzz = {
enable = true;
autoUpdate.enable = true;
networks = {
internal = { driver = "bridge"; };
# Shared macvlan network created by skyg.nixos.common.containers.networks.lab.
lan = config.skyg.nixos.common.containers.networks.lab.compose;
};
volumes = {
buzz-postgres = nfsVolume "postgres";
buzz-redis = nfsVolume "redis";
buzz-garage = nfsVolume "garage";
buzz-git = nfsVolume "git";
};
services = {
relay = {
image = "ghcr.io/block/buzz:main";
dependsOn = [ "postgres" "redis" "garage" ];
networks = {
internal = { };
lan = { ipv4_address = buzz.ip; };
};
environmentFiles = [ config.age.secrets.buzz-env.path ];
environment = {
BUZZ_BIND_ADDR = "0.0.0.0:${toString buzz.port}";
BUZZ_HEALTH_PORT = "8080";
BUZZ_METRICS_PORT = "9102";
RELAY_URL = "ws://${buzz.domain}:${toString buzz.port}";
BUZZ_MEDIA_BASE_URL = "http://${buzz.domain}:${toString buzz.port}/media";
BUZZ_MEDIA_SERVER_DOMAIN = "${buzz.domain}:${toString buzz.port}";
BUZZ_CORS_ORIGINS = "http://${buzz.domain}:${toString buzz.port}";
BUZZ_S3_ENDPOINT = "http://garage:3900";
BUZZ_S3_REGION = "garage";
BUZZ_S3_ADDRESSING_STYLE = "path";
BUZZ_S3_BUCKET = buzz.bucket;
BUZZ_GIT_REPO_PATH = "/data/git";
BUZZ_AUTO_MIGRATE = "true";
BUZZ_GIT_CONFORMANCE_PROBE = "true";
RUST_LOG = "buzz_relay=info,buzz_db=info,buzz_auth=info,buzz_pubsub=info,tower_http=info";
};
volumes = [ "buzz-git:/data/git" ];
healthcheck = {
test = [
"CMD-SHELL"
"bash -ec 'exec 3<>/dev/tcp/127.0.0.1/8080; printf \"GET /_readiness HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n\" >&3; grep -q \"200 OK\" <&3'"
];
interval = "10s";
timeout = "3s";
retries = 12;
start_period = "30s";
};
};
postgres = {
image = "postgres:17-alpine";
networks = [ "internal" ];
environmentFiles = [ config.age.secrets.buzz-env.path ];
environment = {
POSTGRES_DB = "buzz";
POSTGRES_USER = "buzz";
};
volumes = [ "buzz-postgres:/var/lib/postgresql/data" ];
healthcheck = {
test = [ "CMD-SHELL" "pg_isready -U buzz -d buzz" ];
interval = "5s";
timeout = "5s";
retries = 12;
start_period = "10s";
};
};
redis = {
image = "redis:7-alpine";
networks = [ "internal" ];
environmentFiles = [ config.age.secrets.buzz-env.path ];
volumes = [ "buzz-redis:/data" ];
files."/usr/local/bin/redis-entrypoint.sh" = ''
#!/bin/sh
set -eu
exec redis-server --appendonly yes --requirepass "$REDIS_PASSWORD"
'';
command = [ "sh" "/usr/local/bin/redis-entrypoint.sh" ];
};
garage = {
image = "dxflrs/garage:v2.4.1";
networks = [ "internal" ];
environmentFiles = [ config.age.secrets.buzz-env.path ];
environment.GARAGE_DEFAULT_BUCKET = buzz.bucket;
volumes = [ "buzz-garage:/var/lib/garage" ];
command = [ "/garage" "server" "--single-node" "--default-bucket" ];
files."/etc/garage.toml" = ''
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
replication_factor = 1
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "127.0.0.1:3901"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage.localhost"
'';
healthcheck = {
test = [ "CMD" "/garage" "status" ];
interval = "30s";
timeout = "10s";
retries = 5;
start_period = "20s";
};
};
};
};
# Certbot TLS service
skyg.server.dns.certbot = {
enable = false;
+2 -2
View File
@@ -17,7 +17,7 @@ common/
├── ssh-notify.nix # skyg.nixos.common.ssh-notify — forward notify-send from SSH sessions to a desktop client
├── qemu.nix # skyg.core.qemu — QEMU/quickemu + SPICE guest tooling
├── binary-cache.nix # skyg.nixos.common.cachePush — push finished builds to the bigbox2 cache
├── containers/ # Docker vs Podman runtime selection
├── containers/ # Docker vs Podman runtime selection + host-created container networks
├── hardware/ # GPU, fan control, audio, laptop power, Coral TPU udev
└── pritunl/ # Pritunl VPN client service
```
@@ -53,6 +53,6 @@ skyg.nixos.common.pritunl.* → pritunl/
## Conventions
- This tree is for behaviour that is reasonable on *any* Linux host. Anything role-specific
- This tree is for behaviour that is reasonable on _any_ Linux host. Anything role-specific
belongs in `server/` or `desktop/`.
- Hostname-independent defaults can be set directly; anything optional gets an `enable` gate.
+35 -2
View File
@@ -8,7 +8,8 @@ Container runtime configuration and compose-style service groups.
containers/
├── default.nix # Shared runtime options (runtime, enableOnBoot, registries, metrics)
├── docker.nix # Applied when runtime == "docker"
├── podman.nix # Applied when runtime == "podman" (dockerCompat + socket)
├── podman.nix # Applied when runtime == "podman" (dockerCompat + socket + registries)
├── networks.nix # Host-created container networks (skyg.nixos.common.containers.networks)
├── container-services/ # Compose-style service groups (modular implementation)
│ ├── default.nix
│ ├── options.nix
@@ -31,7 +32,8 @@ consistently (compose tooling, user group membership, OCI backend, metrics).
(default `minipc1.lab.internal:5001`), exposes metrics on `metricsPort` (9323), adds the
user to the `docker` group, and installs `docker-compose`.
- **Podman** enables `dockerCompat` + docker socket so `docker`/`oci-containers` keep working,
and installs `podman-compose` + `podman-tui`.
allows plain-HTTP pulls from `localDockerRegistries` via
`virtualisation.containers.registries.settings`, and installs `podman-compose` + `podman-tui`.
- `openMetricsPort` opens the metrics port in the firewall (Docker only).
### Option namespace
@@ -43,8 +45,39 @@ skyg.nixos.common.containers.enableOnBoot
skyg.nixos.common.containers.localDockerRegistries
skyg.nixos.common.containers.metricsPort
skyg.nixos.common.containers.openMetricsPort
skyg.nixos.common.containers.networks.<name>.* → networks.nix (host-created networks)
```
## Host-created networks (`skyg.nixos.common.containers.networks`)
Creates container networks once on the host (one systemd oneshot each, ordered before
`container-networks.target`), independent of any compose project. This is how a _single_
network — e.g. one macvlan network — is shared by several `container-services` groups
instead of being re-declared (and re-created) per stack.
```nix
skyg.nixos.common.containers.networks.lab = {
driver = "macvlan";
driverOpts.parent = "eno1";
subnet = "10.0.0.0/16";
gateway = "10.0.0.1";
ipRange = "10.0.101.16/28";
};
```
Each network exposes a computed `compose` attribute (an `external: true` entry) for use in a
group's `networks` option:
```nix
skyg.nixos.common.container-services.my-stack = {
networks.lan = config.skyg.nixos.common.containers.networks.lab.compose;
services.app.networks.lan = { ipv4_address = "10.0.101.3"; };
};
```
Fields: `enable`, `name` (default = attr name), `driver`, `driverOpts`, `subnet`, `gateway`,
`ipRange`, `internal`. Works with both docker and podman.
## Container service groups (`skyg.nixos.common.container-services`)
A compose-like abstraction that:
@@ -4,14 +4,14 @@ Custom compose-style container service groups for systemd.
## Files
| File | Purpose |
|------|---------|
| `default.nix` | Main module entry point; imports options and wires everything |
| `options.nix` | All `skyg.nixos.common.container-services.*` option declarations |
| `lib.nix` | Shared helpers (runtime dispatch, path sanitization, env file collection) |
| File | Purpose |
| ------------- | ----------------------------------------------------------------------------------- |
| `default.nix` | Main module entry point; imports options and wires everything |
| `options.nix` | All `skyg.nixos.common.container-services.*` option declarations |
| `lib.nix` | Shared helpers (runtime dispatch, path sanitization, env file collection) |
| `compose.nix` | YAML composition (mkComposeService, mkComposeAttrs, mkComposeFile, mkOverridesFile) |
| `files.nix` | File mounting logic (getAllFiles, mkFileVolumesForService, mkFilesService) |
| `systemd.nix` | Systemd unit builders (mkSystemdService, mkPathUnit, mkEnvReloadService) |
| `files.nix` | File mounting logic (getAllFiles, mkFileVolumesForService, mkFilesService) |
| `systemd.nix` | Systemd unit builders (mkSystemdService, mkPathUnit, mkEnvReloadService) |
## How it works
@@ -42,6 +42,8 @@ If adding a new per-service option:
## Architecture
- Each group gets **one** systemd unit (`container-services-<group>.service`) that manages the whole stack
- Group units are ordered after `container-networks.target`, so any network declared in
`skyg.nixos.common.containers.networks` exists before a stack starts
- Custom files are collected and written by a separate unit (`container-services-<group>-files.service`)
- Env file changes trigger a path unit that restarts the stack (`container-services-<group>-env-reload.*`)
- Compose files are rendered to the store and staged to `/var/lib/container-services/<group>/compose.yml`
@@ -14,9 +14,10 @@
{
description = "Container service group '${groupName}'";
wantedBy = [ "multi-user.target" ];
wants = [ "network-online.target" "remote-fs.target" ];
wants = [ "network-online.target" "remote-fs.target" "container-networks.target" ];
after = [
runtimeService
"container-networks.target"
"network-online.target"
"agenix.service"
"remote-fs.target"
@@ -4,6 +4,7 @@
imports = [
./docker.nix
./podman.nix
./networks.nix
./container-services
];
options = {
@@ -0,0 +1,128 @@
{ config, lib, pkgs, ... }:
let
cfg = config.skyg.nixos.common.containers;
isDocker = cfg.runtime == "docker";
runtimeBin = if isDocker then "${pkgs.docker}/bin/docker" else "${pkgs.podman}/bin/podman";
runtimeService = if isDocker then "docker.service" else "podman.service";
enabledNetworks = lib.filterAttrs (_: net: net.enable) cfg.networks;
createArgs = net:
lib.concatStringsSep " " (
[ "network" "create" "--driver" net.driver ]
++ lib.mapAttrsToList (k: v: "--opt ${k}=${v}") net.driverOpts
++ lib.optional (net.subnet != null) "--subnet ${net.subnet}"
++ lib.optional (net.gateway != null) "--gateway ${net.gateway}"
++ lib.optional (net.ipRange != null) "--ip-range ${net.ipRange}"
++ lib.optional net.internal "--internal"
++ [ net.name ]
);
in
{
options.skyg.nixos.common.containers.networks = lib.mkOption {
default = { };
description = ''
Container networks created on the host at boot, one systemd oneshot per
network, independent of any compose project. This gives a single network
that several container-services groups can share.
Reference one from a container-services group through its computed
`compose` attribute:
networks.lan = config.skyg.nixos.common.containers.networks.lab.compose;
Works with both docker and podman (see runtime).
'';
type = lib.types.attrsOf (lib.types.submodule ({ name, config, ... }: {
options = {
enable = lib.mkEnableOption "creating this container network" // { default = true; };
name = lib.mkOption {
type = lib.types.str;
default = name;
description = "Runtime name of the network.";
};
driver = lib.mkOption {
type = lib.types.str;
default = "bridge";
example = "macvlan";
description = "Network driver.";
};
driverOpts = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
example = { parent = "eno1"; };
description = "Driver options (--opt key=value), e.g. the macvlan parent interface.";
};
subnet = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "IPv4 subnet (--subnet).";
};
gateway = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "IPv4 gateway (--gateway).";
};
ipRange = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "IPv4 range for dynamic address allocation (--ip-range).";
};
internal = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Restrict the network to internal traffic (--internal).";
};
compose = lib.mkOption {
type = lib.types.attrsOf lib.types.anything;
description = ''
Computed compose `networks:` entry (external) for referencing this
network from a container-services group.
'';
};
};
config.compose = {
name = config.name;
external = true;
};
}));
};
config = {
# Always defined so container-services units can order themselves after it.
systemd.targets.container-networks = {
description = "Container networks";
wantedBy = [ "multi-user.target" ];
};
systemd.services = lib.mkIf cfg.enable (lib.mapAttrs'
(n: net: lib.nameValuePair "container-network-${n}" {
description = "Create container network '${net.name}'";
after = [ runtimeService "network-online.target" ];
requires = [ runtimeService ];
wantedBy = [ "container-networks.target" ];
before = [ "container-networks.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
# Idempotent: reuse the network if it already exists.
script = ''
${runtimeBin} network inspect ${net.name} >/dev/null 2>&1 \
|| ${runtimeBin} ${createArgs net}
'';
})
enabledNetworks);
};
}
@@ -13,6 +13,10 @@ in
dockerSocket.enable = true;
autoPrune.enable = true;
};
# Allow pulling from local registries that serve plain HTTP
containers.registries.settings = lib.optionalAttrs (cfg.localDockerRegistries != [ ]) {
registries.insecure.registries = cfg.localDockerRegistries;
};
};
environment.systemPackages = with pkgs; [ podman-compose podman-tui ];
};
+50 -8
View File
@@ -5,14 +5,14 @@ staged on disk, and managed by a systemd oneshot unit.
## Where things live
| What | Path |
| ---- | ---- |
| Staged compose file | `/var/lib/container-services/<group>/compose.yml` |
| What | Path |
| -------------------------------------------------------------------------------- | ----------------------------------------------------------- |
| Staged compose file | `/var/lib/container-services/<group>/compose.yml` |
| Staged overrides file (composeFile groups with networks/volumes/extraConfig set) | `/var/lib/container-services/<group>/compose.overrides.yml` |
| Nix store source | `/nix/store/…-compose.yml` (copied on each activation) |
| Systemd unit | `container-services-<group>.service` |
| Container logs | `docker compose -p <group> logs -f <service>` |
| Unit journal | `journalctl -u container-services-<group> -f` |
| Nix store source | `/nix/store/…-compose.yml` (copied on each activation) |
| Systemd unit | `container-services-<group>.service` |
| Container logs | `docker compose -p <group> logs -f <service>` |
| Unit journal | `journalctl -u container-services-<group> -f` |
## Quick reference
@@ -51,6 +51,7 @@ journalctl -u container-services-<group> -n 50
```
Common causes:
- **Docker not running** — `systemctl status docker`
- **Secret path missing** — agenix may not have decrypted yet; check `systemctl status agenix`
- **Port already in use** — another service holds the host port; `ss -tlnp | grep <port>`
@@ -123,6 +124,7 @@ services.app = {
```
Files are:
- Written to `/var/lib/container-services/<group>/files/` on the host
- Mounted read-only into the container at the specified paths
- Updated automatically when you rebuild
@@ -155,17 +157,19 @@ skyg.nixos.common.container-services.my-stack = {
```
When `composeFile` is set, the module:
- Copies that file to `/var/lib/container-services/<group>/compose.yml` at
startup, just like a rendered one. It must define `services` itself —
ignores this group's `services`/`files` blocks.
- If this group also sets `networks`/`volumes`/`extraConfig`, those are
rendered to `/var/lib/container-services/<group>/compose.overrides.yml` and
passed to compose **before** the composeFile (`-f compose.overrides.yml -f
compose.yml`), so compose's native multi-file merge applies —
compose.yml`), so compose's native multi-file merge applies —
composeFile's own definitions win on any overlapping keys.
- Requires exactly one of `composeFile` or `services` to be set (assertion).
Notes:
- `env_file:` entries in the external file resolve **relative to**
`/var/lib/container-services/<group>/`. Use absolute paths (e.g. another
agenix secret path) or inline `environment:` instead.
@@ -173,6 +177,44 @@ Notes:
---
## Shared Networks (created on the host)
Compose networks declared inside a group are scoped to that group's compose project, so two
groups would each get their own copy. To share **one** network across several stacks (e.g. a
single macvlan network), declare it once with `skyg.nixos.common.containers.networks` and
reference it as `external` via its computed `compose` attribute:
```nix
# once, e.g. in the host file
skyg.nixos.common.containers.networks.lab = {
driver = "macvlan";
driverOpts.parent = "eno1";
subnet = "10.0.0.0/16";
gateway = "10.0.0.1";
ipRange = "10.0.101.16/28";
};
skyg.nixos.common.container-services.my-stack = {
enable = true;
networks = {
internal = { driver = "bridge"; };
lan = config.skyg.nixos.common.containers.networks.lab.compose;
};
services.app = {
image = "example/app:latest";
networks = {
internal = { };
lan = { ipv4_address = "10.0.101.3"; };
};
};
};
```
The network is created by a host systemd unit (`container-network-<name>.service`) before any
group starts; groups only reference it (`external: true`) and never create it.
---
## Limitations
- **One unit per group.** No per-container systemd units. Use `docker compose logs` for
+3
View File
@@ -25,6 +25,9 @@ in
"secrets/iu-project.age".publicKeys = [ ari minipc1 ];
# Buzz stack env (minipc1)
"secrets/buzz-env.age".publicKeys = [ ari minipc1 ];
# stack compose definition — minipc2
"secrets/stack1.age".publicKeys = [ ari minipc2 ];
"secrets/stack2.age".publicKeys = [ ari minipc2 ];
Binary file not shown.