diff --git a/hosts/hl-fwdesk.nix b/hosts/hl-fwdesk.nix index db8279d..450b097 100644 --- a/hosts/hl-fwdesk.nix +++ b/hosts/hl-fwdesk.nix @@ -1,5 +1,5 @@ { pkgs -, lib +, config , ... }: let @@ -11,6 +11,8 @@ let wyoming = 10400; comfyui = 8188; libretranslate = 5000; + signal = 8080; + hermes = 8642; }; openPorts = [ ports.ollama @@ -165,6 +167,33 @@ in enable = true; host = "0.0.0.0"; port = ports.libretranslate; + threads = 10; + }; + systemd.services.libretranslate.environment.ARGOS_DEVICE_TYPE = "cpu"; + + # Hermes Agent gateway (Docker container) + age.secrets.hermes-env.file = ../secrets/hermes-env.age; + skyg.nixos.common.container-services.hermes-agent = { + timeoutStopSec = 210; + services.hermes = { + image = "nousresearch/hermes-agent"; + command = [ "gateway" "run" ]; + ports = [ "${toString ports.hermes}:8642" ]; + volumes = [ "/var/lib/hermes:/opt/data" ]; + environmentFiles = [ config.age.secrets.hermes-env.path ]; + environment = { + PUID = "1000"; # ari's UID + PGID = "100"; # GID of the 'users' group in NixOS + }; + extraConfig.shm_size = "1g"; + }; + }; + # Signal CLI + skyg.nixos.server.services.signal-cli = { + enable = false; + host = "0.0.0.0"; + port = ports.signal; + environmentFile = config.age.secrets.hermes-env.path; }; hardware.enableAllFirmware = true; diff --git a/modules/nixos/common/containers/ABOUTME.md b/modules/nixos/common/containers/ABOUTME.md index 0d034bf..6832611 100644 --- a/modules/nixos/common/containers/ABOUTME.md +++ b/modules/nixos/common/containers/ABOUTME.md @@ -1,19 +1,22 @@ # modules/nixos/common/containers/ -Container runtime configuration. Picks **one** of Docker or Podman based on -`skyg.nixos.common.containers.runtime` and configures it consistently (compose tooling, -user group membership, OCI backend, metrics). +Container runtime configuration and compose-style service groups. ## Files ``` containers/ -├── default.nix # Shared options: runtime, enableOnBoot, registries, metrics port -├── docker.nix # Applied when runtime == "docker" -└── podman.nix # Applied when runtime == "podman" (dockerCompat + socket) +├── default.nix # Shared runtime options (runtime, enableOnBoot, registries, metrics) +├── docker.nix # Applied when runtime == "docker" +├── podman.nix # Applied when runtime == "podman" (dockerCompat + socket) +├── container-services.nix # Compose-style groups → rendered compose.yml + systemd oneshot +└── user-guide.md # Full usage docs for container-services ``` -## Behaviour +## Runtime selection (`skyg.nixos.common.containers`) + +Picks **one** of Docker or Podman based on `runtime` and configures it +consistently (compose tooling, user group membership, OCI backend, metrics). - `runtime` defaults to `"docker"`. Each backend file gates on `cfg.runtime == ""`, so exactly one applies. @@ -24,7 +27,7 @@ containers/ and installs `podman-compose` + `podman-tui`. - `openMetricsPort` opens the metrics port in the firewall (Docker only). -## Option Namespace +### Option namespace ``` skyg.nixos.common.containers.runtime → "docker" | "podman" @@ -34,8 +37,35 @@ skyg.nixos.common.containers.metricsPort skyg.nixos.common.containers.openMetricsPort ``` +## Container service groups (`skyg.nixos.common.container-services`) + +A compose-like abstraction that: + +1. Converts a Nix attrset into a real `compose.yml` via `pkgs.formats.yaml`. +2. Stages it to `/var/lib/container-services//compose.yml` at activation. +3. Manages the whole stack with one systemd oneshot unit per group: + `container-services-.service`. + +```nix +skyg.nixos.common.container-services.my-stack = { + services.app = { + image = "example/app:latest"; + ports = [ "8080:8080" ]; + volumes = [ "/var/lib/app:/data" ]; + environmentFiles = [ config.age.secrets.app-env.path ]; + extraConfig.shm_size = "512m"; + }; +}; +``` + +See [user-guide.md](./user-guide.md) for the full option reference and day-2 ops. + ## Conventions -- Always select the runtime via the `runtime` option; do not enable both backends manually. +- Always select the runtime via `skyg.nixos.common.containers.runtime`; do not enable both + backends manually. - `virtualisation.oci-containers.backend` is set by whichever runtime is active, so server - service modules can declare containers backend-agnostically. + service modules can declare containers backend-agnostically via `virtualisation.oci-containers`. +- Prefer `container-services` for new stacks. Only reach for raw + `virtualisation.oci-containers.containers` for single-container services with no + inter-container networking needs. diff --git a/modules/nixos/common/containers/container-services.nix b/modules/nixos/common/containers/container-services.nix new file mode 100644 index 0000000..6a1d5e2 --- /dev/null +++ b/modules/nixos/common/containers/container-services.nix @@ -0,0 +1,268 @@ +{ config, pkgs, lib, ... }: + +let + containersCfg = config.skyg.nixos.common.containers; + cfg = config.skyg.nixos.common.container-services; + + isDocker = containersCfg.runtime == "docker"; + + composeBin = + if isDocker + then "${pkgs.docker-compose}/bin/docker-compose" + else "${pkgs.podman-compose}/bin/podman-compose"; + + runtimeService = + if isDocker then "docker.service" else "podman.service"; + + # ── Compose document builder ───────────────────────────────────────────── + + # Build one compose service entry from a service config. + # groupName is threaded in so the default network reference is correct. + mkComposeService = groupName: _svcName: svcCfg: + { + image = svcCfg.image; + restart = svcCfg.restart; + networks = + if svcCfg.networks == [ ] + then [ groupName ] + else svcCfg.networks; + } + // lib.optionalAttrs (svcCfg.command != [ ]) { + command = svcCfg.command; + } + // lib.optionalAttrs (svcCfg.ports != [ ]) { + ports = svcCfg.ports; + } + // lib.optionalAttrs (svcCfg.volumes != [ ]) { + volumes = svcCfg.volumes; + } + // lib.optionalAttrs (svcCfg.environment != { }) { + environment = svcCfg.environment; + } + // lib.optionalAttrs (svcCfg.environmentFiles != [ ]) { + env_file = map builtins.toString svcCfg.environmentFiles; + } + // lib.optionalAttrs (svcCfg.dependsOn != [ ]) { + depends_on = svcCfg.dependsOn; + } + // svcCfg.extraConfig; + + # Build the full compose document attrset for a group. + mkComposeAttrs = groupName: grpCfg: + let + networks = + if grpCfg.networks == { } + then { "${groupName}" = { driver = "bridge"; }; } + else grpCfg.networks; + in + { + services = lib.mapAttrs (mkComposeService groupName) grpCfg.services; + networks = networks; + } + // lib.optionalAttrs (grpCfg.volumes != [ ]) { + volumes = lib.genAttrs grpCfg.volumes (_: { }); + } + // grpCfg.extraConfig; + + # Render compose.yml to the Nix store. + mkComposeFile = groupName: grpCfg: + (pkgs.formats.yaml { }).generate "compose.yml" + (mkComposeAttrs groupName grpCfg); + + # ── Systemd service builder ────────────────────────────────────────────── + + mkSystemdService = groupName: grpCfg: + let + composeFile = mkComposeFile groupName grpCfg; + stateDir = grpCfg.stateDir; + in + { + description = "Container service group '${groupName}'"; + wantedBy = [ "multi-user.target" ]; + # agenix.service decrypts secrets before we try to read env_file paths + after = [ runtimeService "network-online.target" "agenix.service" ]; + requires = [ runtimeService ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + TimeoutStopSec = grpCfg.timeoutStopSec; + # Stage the store-rendered compose.yml into a stable, readable location + ExecStartPre = "${pkgs.coreutils}/bin/cp ${composeFile} ${stateDir}/compose.yml"; + ExecStart = "${composeBin} -p ${groupName} -f ${stateDir}/compose.yml up -d --remove-orphans"; + ExecStop = "${composeBin} -p ${groupName} -f ${stateDir}/compose.yml down"; + }; + }; + + enabledGroups = lib.filterAttrs (_: g: g.enable) cfg; + +in +{ + # ── Option declarations ────────────────────────────────────────────────── + + options.skyg.nixos.common.container-services = lib.mkOption { + description = '' + Compose-style container service groups. + + Each group is rendered to a compose.yml in the Nix store, staged into + its stateDir on activation, and managed by a single systemd oneshot unit: + + container-services-.service + + See modules/nixos/common/containers/user-guide.md for full documentation. + ''; + default = { }; + type = lib.types.attrsOf (lib.types.submodule ({ name, ... }: { + options = { + + enable = lib.mkEnableOption "this container service group" // { default = true; }; + + stateDir = lib.mkOption { + type = lib.types.str; + default = "/var/lib/container-services/${name}"; + description = "Directory where the staged compose.yml lives at runtime."; + }; + + timeoutStopSec = lib.mkOption { + type = lib.types.int; + default = 120; + description = "Seconds to wait for 'compose down' before forcibly killing containers."; + }; + + services = lib.mkOption { + description = "Container services in this group (maps to compose services: block)."; + default = { }; + type = lib.types.attrsOf (lib.types.submodule { + options = { + + image = lib.mkOption { + type = lib.types.str; + description = "Container image (registry/name:tag)."; + }; + + command = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Entrypoint command override (compose command:)."; + }; + + ports = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "8642:8642" ]; + description = "Port mappings in HOST:CONTAINER format."; + }; + + volumes = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "/var/lib/hermes:/opt/data" ]; + description = "Volume mounts in src:dst[:opts] format."; + }; + + environment = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + description = "Environment variables as key/value pairs."; + }; + + environmentFiles = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Paths to env files loaded by compose at runtime. + Accepts runtime paths such as agenix secret paths + (e.g. config.age.secrets.my-secret.path). + ''; + }; + + dependsOn = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Sibling service names that must start first (compose depends_on:)."; + }; + + restart = lib.mkOption { + type = lib.types.str; + default = "unless-stopped"; + description = "Compose restart policy."; + }; + + networks = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Networks to join. Empty list = attach to the group's + auto-created bridge network (named after the group). + ''; + }; + + extraConfig = lib.mkOption { + type = lib.types.attrsOf lib.types.anything; + default = { }; + description = '' + Free-form attrset merged directly into the compose service + block. Use this for keys not modelled above: + shm_size, cap_add, devices, healthcheck, deploy, etc. + ''; + }; + + }; + }); + }; + + volumes = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Named volumes declared at the top level of the compose document."; + }; + + networks = lib.mkOption { + type = lib.types.attrsOf lib.types.anything; + default = { }; + description = '' + Network definitions for the top-level compose networks: block. + Empty = auto-create a bridge network named after the group. + ''; + }; + + extraConfig = lib.mkOption { + type = lib.types.attrsOf lib.types.anything; + default = { }; + description = "Free-form attrset merged into the top-level compose document."; + }; + + }; + })); + }; + + # ── Configuration ──────────────────────────────────────────────────────── + + config = { + assertions = lib.optionals (enabledGroups != { }) [ + { + assertion = config.virtualisation.docker.enable or false + || config.virtualisation.podman.enable or false; + message = '' + skyg.nixos.common.container-services: at least one container runtime + must be enabled. Set skyg.nixos.common.containers.runtime (or enable + virtualisation.docker / virtualisation.podman directly). + ''; + } + ]; + + # Create each group's stateDir via tmpfiles so the cp in ExecStartPre succeeds. + systemd.tmpfiles.rules = + lib.mapAttrsToList + (_: grpCfg: "d ${grpCfg.stateDir} 0750 root root -") + enabledGroups; + + # One oneshot unit per group — manages the whole compose stack. + systemd.services = + lib.mapAttrs' + (groupName: grpCfg: + lib.nameValuePair + "container-services-${groupName}" + (mkSystemdService groupName grpCfg)) + enabledGroups; + }; +} diff --git a/modules/nixos/common/containers/default.nix b/modules/nixos/common/containers/default.nix index 047a7d9..aa17baf 100644 --- a/modules/nixos/common/containers/default.nix +++ b/modules/nixos/common/containers/default.nix @@ -4,6 +4,7 @@ imports = [ ./docker.nix ./podman.nix + ./container-services.nix ]; options = { skyg.nixos.common.containers = { diff --git a/modules/nixos/common/containers/user-guide.md b/modules/nixos/common/containers/user-guide.md new file mode 100644 index 0000000..c72ff2b --- /dev/null +++ b/modules/nixos/common/containers/user-guide.md @@ -0,0 +1,111 @@ +# Container Services — User Guide + +Declares Docker Compose stacks in Nix. Each group is rendered to a `compose.yml`, +staged on disk, and managed by a systemd oneshot unit. + +## Where things live + +| What | Path | +| ---- | ---- | +| Staged compose file | `/var/lib/container-services//compose.yml` | +| Nix store source | `/nix/store/…-compose.yml` (copied on each activation) | +| Systemd unit | `container-services-.service` | +| Container logs | `docker compose -p logs -f ` | +| Unit journal | `journalctl -u container-services- -f` | + +## Quick reference + +```bash +# See what's running +docker compose -p ps + +# Follow container logs +docker compose -p logs -f + +# Inspect the generated compose file +cat /var/lib/container-services//compose.yml + +# Restart the stack (e.g. after nixos-rebuild switch) +systemctl restart container-services- + +# Stop / start without a rebuild +systemctl stop container-services- +systemctl start container-services- + +# Shell into a running container +docker compose -p exec bash + +# Pull a new image and recreate +docker compose -p pull +systemctl restart container-services- +``` + +## Debugging + +### Unit fails to start + +```bash +systemctl status container-services- +journalctl -u container-services- -n 50 +``` + +Common causes: +- **Docker not running** — `systemctl status docker` +- **Secret path missing** — agenix may not have decrypted yet; check `systemctl status agenix` +- **Port already in use** — another service holds the host port; `ss -tlnp | grep ` +- **Image not pullable** — network issue or wrong image name; try `docker pull ` manually + +### Container starts but misbehaves + +```bash +# Last 100 lines from the container +docker compose -p logs --tail=100 + +# Inspect the compose file Nix generated +cat /var/lib/container-services//compose.yml + +# Verify env file was read correctly (check what the container sees) +docker compose -p exec env | grep +``` + +### Permission errors on the data volume + +The container's init runs as root and chowns `/opt/data` to `PUID:PGID`. If it still +fails, check ownership on the host: + +```bash +ls -la /var/lib/ +# Should match PUID:PGID set in environment {} +``` + +If the directory was created by a previous run as root and never chowned, remove it and +let Docker recreate it (named volumes are unaffected; only bind mounts need this): + +```bash +sudo rm -rf /var/lib/ +systemctl restart container-services- +``` + +### compose.yml looks wrong + +The file at `/var/lib/container-services//compose.yml` is overwritten on every +`ExecStartPre`. If it looks wrong, check the Nix-rendered source in the store directly: + +```bash +# Find the store path +systemctl cat container-services- | grep ExecStartPre +# Then cat it +cat /nix/store/…-compose.yml +``` + +If the store file is wrong, the bug is in your Nix options — rebuild with +`nixos-rebuild build` and inspect the new store path before switching. + +## Limitations + +- **One unit per group.** No per-container systemd units. Use `docker compose logs` for + container output, `journalctl` only for the start/stop lifecycle. +- **`compose down` on stop.** Containers are removed when the unit stops. Data in + bind mounts and named volumes is preserved. +- **Compose project name == group name.** Keep group names DNS-label–safe (letters, + digits, hyphens). diff --git a/modules/nixos/server/services/signal-cli.nix b/modules/nixos/server/services/signal-cli.nix index 3045c46..175d927 100644 --- a/modules/nixos/server/services/signal-cli.nix +++ b/modules/nixos/server/services/signal-cli.nix @@ -4,6 +4,7 @@ with lib; let cfg = config.skyg.nixos.server.services.signal-cli; + skygUser = config.skyg.user; in { options = { @@ -71,10 +72,8 @@ in "skyg.nixos.server.services.signal-cli: set either `account` or `environmentFile` (providing SIGNAL_ACCOUNT)."; }]; - # Ensure the config/state directory exists (the account is linked into it - # manually). 0700 since it holds Signal account credentials. systemd.tmpfiles.rules = [ "d ${cfg.configDir} 0700 root root - -" ]; - + environment.systemPackages = [ pkgs.signal-cli ]; systemd.services.signal-cli-daemon = { description = "signal-cli daemon (HTTP) for the Hermes Signal adapter"; @@ -84,16 +83,19 @@ in serviceConfig = { Type = "simple"; + User = skygUser.name; + Environment = "JAVA_OPTS=--enable-native-access=ALL-UNNAMED"; EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; - ExecStart = - let - account = - if cfg.account != null then cfg.account else "\${SIGNAL_ACCOUNT}"; - in - "${cfg.package}/bin/signal-cli --config ${cfg.configDir} --account ${account} daemon --http ${cfg.host}:${toString cfg.port}"; + ExecStart = pkgs.writeShellScript "signal-cli-daemon-start" '' + set -eu + account=${if cfg.account != null then lib.escapeShellArg cfg.account else "\"\${SIGNAL_ACCOUNT:?SIGNAL_ACCOUNT is unset/empty in the env file}\""} + export JAVA_OPTS="--enable-native-access=ALL-UNNAMED" + exec ${cfg.package}/bin/signal-cli --config ${cfg.configDir} --account "$account" daemon --http ${cfg.host}:${toString cfg.port} + ''; Restart = "on-failure"; RestartSec = 5; + StartLimitBurst = 2; }; }; diff --git a/secrets/hermes-env.age b/secrets/hermes-env.age index 604185e..d56a0ae 100644 Binary files a/secrets/hermes-env.age and b/secrets/hermes-env.age differ