container framework + initial work on signal-cli + hermes

This commit is contained in:
ari committed 2026-06-28 15:30:15 -04:00
1 parent 50ef9c7238
commit cf2f221a94
7 files changed
+461 -20

No files matched your search

+30 -1
View File
@@ -1,5 +1,5 @@
{ pkgs
, lib
, config
, ...
}:
let
@@ -11,6 +11,8 @@ let
wyoming = 10400;
comfyui = 8188;
libretranslate = 5000;
signal = 8080;
hermes = 8642;
};
openPorts = [
ports.ollama
@@ -165,6 +167,33 @@ in
enable = true;
host = "0.0.0.0";
port = ports.libretranslate;
threads = 10;
};
systemd.services.libretranslate.environment.ARGOS_DEVICE_TYPE = "cpu";
# Hermes Agent gateway (Docker container)
age.secrets.hermes-env.file = ../secrets/hermes-env.age;
skyg.nixos.common.container-services.hermes-agent = {
timeoutStopSec = 210;
services.hermes = {
image = "nousresearch/hermes-agent";
command = [ "gateway" "run" ];
ports = [ "${toString ports.hermes}:8642" ];
volumes = [ "/var/lib/hermes:/opt/data" ];
environmentFiles = [ config.age.secrets.hermes-env.path ];
environment = {
PUID = "1000"; # ari's UID
PGID = "100"; # GID of the 'users' group in NixOS
};
extraConfig.shm_size = "1g";
};
};
# Signal CLI
skyg.nixos.server.services.signal-cli = {
enable = false;
host = "0.0.0.0";
port = ports.signal;
environmentFile = config.age.secrets.hermes-env.path;
};
hardware.enableAllFirmware = true;
+40 -10
View File
@@ -1,19 +1,22 @@
# modules/nixos/common/containers/
Container runtime configuration. Picks **one** of Docker or Podman based on
`skyg.nixos.common.containers.runtime` and configures it consistently (compose tooling,
user group membership, OCI backend, metrics).
Container runtime configuration and compose-style service groups.
## Files
```
containers/
├── default.nix # Shared options: runtime, enableOnBoot, registries, metrics port
├── docker.nix # Applied when runtime == "docker"
└── podman.nix # Applied when runtime == "podman" (dockerCompat + socket)
├── default.nix # Shared runtime options (runtime, enableOnBoot, registries, metrics)
├── docker.nix # Applied when runtime == "docker"
├── podman.nix # Applied when runtime == "podman" (dockerCompat + socket)
├── container-services.nix # Compose-style groups → rendered compose.yml + systemd oneshot
└── user-guide.md # Full usage docs for container-services
```
## Behaviour
## Runtime selection (`skyg.nixos.common.containers`)
Picks **one** of Docker or Podman based on `runtime` and configures it
consistently (compose tooling, user group membership, OCI backend, metrics).
- `runtime` defaults to `"docker"`. Each backend file gates on `cfg.runtime == "<name>"`,
so exactly one applies.
@@ -24,7 +27,7 @@ containers/
and installs `podman-compose` + `podman-tui`.
- `openMetricsPort` opens the metrics port in the firewall (Docker only).
## Option Namespace
### Option namespace
```
skyg.nixos.common.containers.runtime → "docker" | "podman"
@@ -34,8 +37,35 @@ skyg.nixos.common.containers.metricsPort
skyg.nixos.common.containers.openMetricsPort
```
## Container service groups (`skyg.nixos.common.container-services`)
A compose-like abstraction that:
1. Converts a Nix attrset into a real `compose.yml` via `pkgs.formats.yaml`.
2. Stages it to `/var/lib/container-services/<group>/compose.yml` at activation.
3. Manages the whole stack with one systemd oneshot unit per group:
`container-services-<group>.service`.
```nix
skyg.nixos.common.container-services.my-stack = {
services.app = {
image = "example/app:latest";
ports = [ "8080:8080" ];
volumes = [ "/var/lib/app:/data" ];
environmentFiles = [ config.age.secrets.app-env.path ];
extraConfig.shm_size = "512m";
};
};
```
See [user-guide.md](./user-guide.md) for the full option reference and day-2 ops.
## Conventions
- Always select the runtime via the `runtime` option; do not enable both backends manually.
- Always select the runtime via `skyg.nixos.common.containers.runtime`; do not enable both
backends manually.
- `virtualisation.oci-containers.backend` is set by whichever runtime is active, so server
service modules can declare containers backend-agnostically.
service modules can declare containers backend-agnostically via `virtualisation.oci-containers`.
- Prefer `container-services` for new stacks. Only reach for raw
`virtualisation.oci-containers.containers` for single-container services with no
inter-container networking needs.
@@ -0,0 +1,268 @@
{ config, pkgs, lib, ... }:
let
containersCfg = config.skyg.nixos.common.containers;
cfg = config.skyg.nixos.common.container-services;
isDocker = containersCfg.runtime == "docker";
composeBin =
if isDocker
then "${pkgs.docker-compose}/bin/docker-compose"
else "${pkgs.podman-compose}/bin/podman-compose";
runtimeService =
if isDocker then "docker.service" else "podman.service";
# ── Compose document builder ─────────────────────────────────────────────
# Build one compose service entry from a service config.
# groupName is threaded in so the default network reference is correct.
mkComposeService = groupName: _svcName: svcCfg:
{
image = svcCfg.image;
restart = svcCfg.restart;
networks =
if svcCfg.networks == [ ]
then [ groupName ]
else svcCfg.networks;
}
// lib.optionalAttrs (svcCfg.command != [ ]) {
command = svcCfg.command;
}
// lib.optionalAttrs (svcCfg.ports != [ ]) {
ports = svcCfg.ports;
}
// lib.optionalAttrs (svcCfg.volumes != [ ]) {
volumes = svcCfg.volumes;
}
// lib.optionalAttrs (svcCfg.environment != { }) {
environment = svcCfg.environment;
}
// lib.optionalAttrs (svcCfg.environmentFiles != [ ]) {
env_file = map builtins.toString svcCfg.environmentFiles;
}
// lib.optionalAttrs (svcCfg.dependsOn != [ ]) {
depends_on = svcCfg.dependsOn;
}
// svcCfg.extraConfig;
# Build the full compose document attrset for a group.
mkComposeAttrs = groupName: grpCfg:
let
networks =
if grpCfg.networks == { }
then { "${groupName}" = { driver = "bridge"; }; }
else grpCfg.networks;
in
{
services = lib.mapAttrs (mkComposeService groupName) grpCfg.services;
networks = networks;
}
// lib.optionalAttrs (grpCfg.volumes != [ ]) {
volumes = lib.genAttrs grpCfg.volumes (_: { });
}
// grpCfg.extraConfig;
# Render compose.yml to the Nix store.
mkComposeFile = groupName: grpCfg:
(pkgs.formats.yaml { }).generate "compose.yml"
(mkComposeAttrs groupName grpCfg);
# ── Systemd service builder ──────────────────────────────────────────────
mkSystemdService = groupName: grpCfg:
let
composeFile = mkComposeFile groupName grpCfg;
stateDir = grpCfg.stateDir;
in
{
description = "Container service group '${groupName}'";
wantedBy = [ "multi-user.target" ];
# agenix.service decrypts secrets before we try to read env_file paths
after = [ runtimeService "network-online.target" "agenix.service" ];
requires = [ runtimeService ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
TimeoutStopSec = grpCfg.timeoutStopSec;
# Stage the store-rendered compose.yml into a stable, readable location
ExecStartPre = "${pkgs.coreutils}/bin/cp ${composeFile} ${stateDir}/compose.yml";
ExecStart = "${composeBin} -p ${groupName} -f ${stateDir}/compose.yml up -d --remove-orphans";
ExecStop = "${composeBin} -p ${groupName} -f ${stateDir}/compose.yml down";
};
};
enabledGroups = lib.filterAttrs (_: g: g.enable) cfg;
in
{
# ── Option declarations ──────────────────────────────────────────────────
options.skyg.nixos.common.container-services = lib.mkOption {
description = ''
Compose-style container service groups.
Each group is rendered to a compose.yml in the Nix store, staged into
its stateDir on activation, and managed by a single systemd oneshot unit:
container-services-<group>.service
See modules/nixos/common/containers/user-guide.md for full documentation.
'';
default = { };
type = lib.types.attrsOf (lib.types.submodule ({ name, ... }: {
options = {
enable = lib.mkEnableOption "this container service group" // { default = true; };
stateDir = lib.mkOption {
type = lib.types.str;
default = "/var/lib/container-services/${name}";
description = "Directory where the staged compose.yml lives at runtime.";
};
timeoutStopSec = lib.mkOption {
type = lib.types.int;
default = 120;
description = "Seconds to wait for 'compose down' before forcibly killing containers.";
};
services = lib.mkOption {
description = "Container services in this group (maps to compose services: block).";
default = { };
type = lib.types.attrsOf (lib.types.submodule {
options = {
image = lib.mkOption {
type = lib.types.str;
description = "Container image (registry/name:tag).";
};
command = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Entrypoint command override (compose command:).";
};
ports = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "8642:8642" ];
description = "Port mappings in HOST:CONTAINER format.";
};
volumes = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "/var/lib/hermes:/opt/data" ];
description = "Volume mounts in src:dst[:opts] format.";
};
environment = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
description = "Environment variables as key/value pairs.";
};
environmentFiles = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = ''
Paths to env files loaded by compose at runtime.
Accepts runtime paths such as agenix secret paths
(e.g. config.age.secrets.my-secret.path).
'';
};
dependsOn = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Sibling service names that must start first (compose depends_on:).";
};
restart = lib.mkOption {
type = lib.types.str;
default = "unless-stopped";
description = "Compose restart policy.";
};
networks = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = ''
Networks to join. Empty list = attach to the group's
auto-created bridge network (named after the group).
'';
};
extraConfig = lib.mkOption {
type = lib.types.attrsOf lib.types.anything;
default = { };
description = ''
Free-form attrset merged directly into the compose service
block. Use this for keys not modelled above:
shm_size, cap_add, devices, healthcheck, deploy, etc.
'';
};
};
});
};
volumes = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Named volumes declared at the top level of the compose document.";
};
networks = lib.mkOption {
type = lib.types.attrsOf lib.types.anything;
default = { };
description = ''
Network definitions for the top-level compose networks: block.
Empty = auto-create a bridge network named after the group.
'';
};
extraConfig = lib.mkOption {
type = lib.types.attrsOf lib.types.anything;
default = { };
description = "Free-form attrset merged into the top-level compose document.";
};
};
}));
};
# ── Configuration ────────────────────────────────────────────────────────
config = {
assertions = lib.optionals (enabledGroups != { }) [
{
assertion = config.virtualisation.docker.enable or false
|| config.virtualisation.podman.enable or false;
message = ''
skyg.nixos.common.container-services: at least one container runtime
must be enabled. Set skyg.nixos.common.containers.runtime (or enable
virtualisation.docker / virtualisation.podman directly).
'';
}
];
# Create each group's stateDir via tmpfiles so the cp in ExecStartPre succeeds.
systemd.tmpfiles.rules =
lib.mapAttrsToList
(_: grpCfg: "d ${grpCfg.stateDir} 0750 root root -")
enabledGroups;
# One oneshot unit per group — manages the whole compose stack.
systemd.services =
lib.mapAttrs'
(groupName: grpCfg:
lib.nameValuePair
"container-services-${groupName}"
(mkSystemdService groupName grpCfg))
enabledGroups;
};
}
@@ -4,6 +4,7 @@
imports = [
./docker.nix
./podman.nix
./container-services.nix
];
options = {
skyg.nixos.common.containers = {
@@ -0,0 +1,111 @@
# Container Services — User Guide
Declares Docker Compose stacks in Nix. Each group is rendered to a `compose.yml`,
staged on disk, and managed by a systemd oneshot unit.
## Where things live
| What | Path |
| ---- | ---- |
| Staged compose file | `/var/lib/container-services/<group>/compose.yml` |
| Nix store source | `/nix/store/…-compose.yml` (copied on each activation) |
| Systemd unit | `container-services-<group>.service` |
| Container logs | `docker compose -p <group> logs -f <service>` |
| Unit journal | `journalctl -u container-services-<group> -f` |
## Quick reference
```bash
# See what's running
docker compose -p <group> ps
# Follow container logs
docker compose -p <group> logs -f <service>
# Inspect the generated compose file
cat /var/lib/container-services/<group>/compose.yml
# Restart the stack (e.g. after nixos-rebuild switch)
systemctl restart container-services-<group>
# Stop / start without a rebuild
systemctl stop container-services-<group>
systemctl start container-services-<group>
# Shell into a running container
docker compose -p <group> exec <service> bash
# Pull a new image and recreate
docker compose -p <group> pull
systemctl restart container-services-<group>
```
## Debugging
### Unit fails to start
```bash
systemctl status container-services-<group>
journalctl -u container-services-<group> -n 50
```
Common causes:
- **Docker not running** — `systemctl status docker`
- **Secret path missing** — agenix may not have decrypted yet; check `systemctl status agenix`
- **Port already in use** — another service holds the host port; `ss -tlnp | grep <port>`
- **Image not pullable** — network issue or wrong image name; try `docker pull <image>` manually
### Container starts but misbehaves
```bash
# Last 100 lines from the container
docker compose -p <group> logs --tail=100 <service>
# Inspect the compose file Nix generated
cat /var/lib/container-services/<group>/compose.yml
# Verify env file was read correctly (check what the container sees)
docker compose -p <group> exec <service> env | grep <KEY>
```
### Permission errors on the data volume
The container's init runs as root and chowns `/opt/data` to `PUID:PGID`. If it still
fails, check ownership on the host:
```bash
ls -la /var/lib/<service-data-dir>
# Should match PUID:PGID set in environment {}
```
If the directory was created by a previous run as root and never chowned, remove it and
let Docker recreate it (named volumes are unaffected; only bind mounts need this):
```bash
sudo rm -rf /var/lib/<service-data-dir>
systemctl restart container-services-<group>
```
### compose.yml looks wrong
The file at `/var/lib/container-services/<group>/compose.yml` is overwritten on every
`ExecStartPre`. If it looks wrong, check the Nix-rendered source in the store directly:
```bash
# Find the store path
systemctl cat container-services-<group> | grep ExecStartPre
# Then cat it
cat /nix/store/…-compose.yml
```
If the store file is wrong, the bug is in your Nix options — rebuild with
`nixos-rebuild build` and inspect the new store path before switching.
## Limitations
- **One unit per group.** No per-container systemd units. Use `docker compose logs` for
container output, `journalctl` only for the start/stop lifecycle.
- **`compose down` on stop.** Containers are removed when the unit stops. Data in
bind mounts and named volumes is preserved.
- **Compose project name == group name.** Keep group names DNS-label–safe (letters,
digits, hyphens).
+11 -9
View File
@@ -4,6 +4,7 @@ with lib;
let
cfg = config.skyg.nixos.server.services.signal-cli;
skygUser = config.skyg.user;
in
{
options = {
@@ -71,10 +72,8 @@ in
"skyg.nixos.server.services.signal-cli: set either `account` or `environmentFile` (providing SIGNAL_ACCOUNT).";
}];
# Ensure the config/state directory exists (the account is linked into it
# manually). 0700 since it holds Signal account credentials.
systemd.tmpfiles.rules = [ "d ${cfg.configDir} 0700 root root - -" ];
environment.systemPackages = [ pkgs.signal-cli ];
systemd.services.signal-cli-daemon = {
description = "signal-cli daemon (HTTP) for the Hermes Signal adapter";
@@ -84,16 +83,19 @@ in
serviceConfig = {
Type = "simple";
User = skygUser.name;
Environment = "JAVA_OPTS=--enable-native-access=ALL-UNNAMED";
EnvironmentFile =
mkIf (cfg.environmentFile != null) cfg.environmentFile;
ExecStart =
let
account =
if cfg.account != null then cfg.account else "\${SIGNAL_ACCOUNT}";
in
"${cfg.package}/bin/signal-cli --config ${cfg.configDir} --account ${account} daemon --http ${cfg.host}:${toString cfg.port}";
ExecStart = pkgs.writeShellScript "signal-cli-daemon-start" ''
set -eu
account=${if cfg.account != null then lib.escapeShellArg cfg.account else "\"\${SIGNAL_ACCOUNT:?SIGNAL_ACCOUNT is unset/empty in the env file}\""}
export JAVA_OPTS="--enable-native-access=ALL-UNNAMED"
exec ${cfg.package}/bin/signal-cli --config ${cfg.configDir} --account "$account" daemon --http ${cfg.host}:${toString cfg.port}
'';
Restart = "on-failure";
RestartSec = 5;
StartLimitBurst = 2;
};
};
Binary file not shown.