container framework + initial work on signal-cli + hermes
This commit is contained in:
1 parent
50ef9c7238
commit
cf2f221a94
7 files changed
+461
-20
No files matched your search
+30
-1
@@ -1,5 +1,5 @@
|
||||
{ pkgs
|
||||
, lib
|
||||
, config
|
||||
, ...
|
||||
}:
|
||||
let
|
||||
@@ -11,6 +11,8 @@ let
|
||||
wyoming = 10400;
|
||||
comfyui = 8188;
|
||||
libretranslate = 5000;
|
||||
signal = 8080;
|
||||
hermes = 8642;
|
||||
};
|
||||
openPorts = [
|
||||
ports.ollama
|
||||
@@ -165,6 +167,33 @@ in
|
||||
enable = true;
|
||||
host = "0.0.0.0";
|
||||
port = ports.libretranslate;
|
||||
threads = 10;
|
||||
};
|
||||
systemd.services.libretranslate.environment.ARGOS_DEVICE_TYPE = "cpu";
|
||||
|
||||
# Hermes Agent gateway (Docker container)
|
||||
age.secrets.hermes-env.file = ../secrets/hermes-env.age;
|
||||
skyg.nixos.common.container-services.hermes-agent = {
|
||||
timeoutStopSec = 210;
|
||||
services.hermes = {
|
||||
image = "nousresearch/hermes-agent";
|
||||
command = [ "gateway" "run" ];
|
||||
ports = [ "${toString ports.hermes}:8642" ];
|
||||
volumes = [ "/var/lib/hermes:/opt/data" ];
|
||||
environmentFiles = [ config.age.secrets.hermes-env.path ];
|
||||
environment = {
|
||||
PUID = "1000"; # ari's UID
|
||||
PGID = "100"; # GID of the 'users' group in NixOS
|
||||
};
|
||||
extraConfig.shm_size = "1g";
|
||||
};
|
||||
};
|
||||
# Signal CLI
|
||||
skyg.nixos.server.services.signal-cli = {
|
||||
enable = false;
|
||||
host = "0.0.0.0";
|
||||
port = ports.signal;
|
||||
environmentFile = config.age.secrets.hermes-env.path;
|
||||
};
|
||||
|
||||
hardware.enableAllFirmware = true;
|
||||
|
||||
@@ -1,19 +1,22 @@
|
||||
# modules/nixos/common/containers/
|
||||
|
||||
Container runtime configuration. Picks **one** of Docker or Podman based on
|
||||
`skyg.nixos.common.containers.runtime` and configures it consistently (compose tooling,
|
||||
user group membership, OCI backend, metrics).
|
||||
Container runtime configuration and compose-style service groups.
|
||||
|
||||
## Files
|
||||
|
||||
```
|
||||
containers/
|
||||
├── default.nix # Shared options: runtime, enableOnBoot, registries, metrics port
|
||||
├── docker.nix # Applied when runtime == "docker"
|
||||
└── podman.nix # Applied when runtime == "podman" (dockerCompat + socket)
|
||||
├── default.nix # Shared runtime options (runtime, enableOnBoot, registries, metrics)
|
||||
├── docker.nix # Applied when runtime == "docker"
|
||||
├── podman.nix # Applied when runtime == "podman" (dockerCompat + socket)
|
||||
├── container-services.nix # Compose-style groups → rendered compose.yml + systemd oneshot
|
||||
└── user-guide.md # Full usage docs for container-services
|
||||
```
|
||||
|
||||
## Behaviour
|
||||
## Runtime selection (`skyg.nixos.common.containers`)
|
||||
|
||||
Picks **one** of Docker or Podman based on `runtime` and configures it
|
||||
consistently (compose tooling, user group membership, OCI backend, metrics).
|
||||
|
||||
- `runtime` defaults to `"docker"`. Each backend file gates on `cfg.runtime == "<name>"`,
|
||||
so exactly one applies.
|
||||
@@ -24,7 +27,7 @@ containers/
|
||||
and installs `podman-compose` + `podman-tui`.
|
||||
- `openMetricsPort` opens the metrics port in the firewall (Docker only).
|
||||
|
||||
## Option Namespace
|
||||
### Option namespace
|
||||
|
||||
```
|
||||
skyg.nixos.common.containers.runtime → "docker" | "podman"
|
||||
@@ -34,8 +37,35 @@ skyg.nixos.common.containers.metricsPort
|
||||
skyg.nixos.common.containers.openMetricsPort
|
||||
```
|
||||
|
||||
## Container service groups (`skyg.nixos.common.container-services`)
|
||||
|
||||
A compose-like abstraction that:
|
||||
|
||||
1. Converts a Nix attrset into a real `compose.yml` via `pkgs.formats.yaml`.
|
||||
2. Stages it to `/var/lib/container-services/<group>/compose.yml` at activation.
|
||||
3. Manages the whole stack with one systemd oneshot unit per group:
|
||||
`container-services-<group>.service`.
|
||||
|
||||
```nix
|
||||
skyg.nixos.common.container-services.my-stack = {
|
||||
services.app = {
|
||||
image = "example/app:latest";
|
||||
ports = [ "8080:8080" ];
|
||||
volumes = [ "/var/lib/app:/data" ];
|
||||
environmentFiles = [ config.age.secrets.app-env.path ];
|
||||
extraConfig.shm_size = "512m";
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
See [user-guide.md](./user-guide.md) for the full option reference and day-2 ops.
|
||||
|
||||
## Conventions
|
||||
|
||||
- Always select the runtime via the `runtime` option; do not enable both backends manually.
|
||||
- Always select the runtime via `skyg.nixos.common.containers.runtime`; do not enable both
|
||||
backends manually.
|
||||
- `virtualisation.oci-containers.backend` is set by whichever runtime is active, so server
|
||||
service modules can declare containers backend-agnostically.
|
||||
service modules can declare containers backend-agnostically via `virtualisation.oci-containers`.
|
||||
- Prefer `container-services` for new stacks. Only reach for raw
|
||||
`virtualisation.oci-containers.containers` for single-container services with no
|
||||
inter-container networking needs.
|
||||
@@ -0,0 +1,268 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
let
|
||||
containersCfg = config.skyg.nixos.common.containers;
|
||||
cfg = config.skyg.nixos.common.container-services;
|
||||
|
||||
isDocker = containersCfg.runtime == "docker";
|
||||
|
||||
composeBin =
|
||||
if isDocker
|
||||
then "${pkgs.docker-compose}/bin/docker-compose"
|
||||
else "${pkgs.podman-compose}/bin/podman-compose";
|
||||
|
||||
runtimeService =
|
||||
if isDocker then "docker.service" else "podman.service";
|
||||
|
||||
# ── Compose document builder ─────────────────────────────────────────────
|
||||
|
||||
# Build one compose service entry from a service config.
|
||||
# groupName is threaded in so the default network reference is correct.
|
||||
mkComposeService = groupName: _svcName: svcCfg:
|
||||
{
|
||||
image = svcCfg.image;
|
||||
restart = svcCfg.restart;
|
||||
networks =
|
||||
if svcCfg.networks == [ ]
|
||||
then [ groupName ]
|
||||
else svcCfg.networks;
|
||||
}
|
||||
// lib.optionalAttrs (svcCfg.command != [ ]) {
|
||||
command = svcCfg.command;
|
||||
}
|
||||
// lib.optionalAttrs (svcCfg.ports != [ ]) {
|
||||
ports = svcCfg.ports;
|
||||
}
|
||||
// lib.optionalAttrs (svcCfg.volumes != [ ]) {
|
||||
volumes = svcCfg.volumes;
|
||||
}
|
||||
// lib.optionalAttrs (svcCfg.environment != { }) {
|
||||
environment = svcCfg.environment;
|
||||
}
|
||||
// lib.optionalAttrs (svcCfg.environmentFiles != [ ]) {
|
||||
env_file = map builtins.toString svcCfg.environmentFiles;
|
||||
}
|
||||
// lib.optionalAttrs (svcCfg.dependsOn != [ ]) {
|
||||
depends_on = svcCfg.dependsOn;
|
||||
}
|
||||
// svcCfg.extraConfig;
|
||||
|
||||
# Build the full compose document attrset for a group.
|
||||
mkComposeAttrs = groupName: grpCfg:
|
||||
let
|
||||
networks =
|
||||
if grpCfg.networks == { }
|
||||
then { "${groupName}" = { driver = "bridge"; }; }
|
||||
else grpCfg.networks;
|
||||
in
|
||||
{
|
||||
services = lib.mapAttrs (mkComposeService groupName) grpCfg.services;
|
||||
networks = networks;
|
||||
}
|
||||
// lib.optionalAttrs (grpCfg.volumes != [ ]) {
|
||||
volumes = lib.genAttrs grpCfg.volumes (_: { });
|
||||
}
|
||||
// grpCfg.extraConfig;
|
||||
|
||||
# Render compose.yml to the Nix store.
|
||||
mkComposeFile = groupName: grpCfg:
|
||||
(pkgs.formats.yaml { }).generate "compose.yml"
|
||||
(mkComposeAttrs groupName grpCfg);
|
||||
|
||||
# ── Systemd service builder ──────────────────────────────────────────────
|
||||
|
||||
mkSystemdService = groupName: grpCfg:
|
||||
let
|
||||
composeFile = mkComposeFile groupName grpCfg;
|
||||
stateDir = grpCfg.stateDir;
|
||||
in
|
||||
{
|
||||
description = "Container service group '${groupName}'";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
# agenix.service decrypts secrets before we try to read env_file paths
|
||||
after = [ runtimeService "network-online.target" "agenix.service" ];
|
||||
requires = [ runtimeService ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
TimeoutStopSec = grpCfg.timeoutStopSec;
|
||||
# Stage the store-rendered compose.yml into a stable, readable location
|
||||
ExecStartPre = "${pkgs.coreutils}/bin/cp ${composeFile} ${stateDir}/compose.yml";
|
||||
ExecStart = "${composeBin} -p ${groupName} -f ${stateDir}/compose.yml up -d --remove-orphans";
|
||||
ExecStop = "${composeBin} -p ${groupName} -f ${stateDir}/compose.yml down";
|
||||
};
|
||||
};
|
||||
|
||||
enabledGroups = lib.filterAttrs (_: g: g.enable) cfg;
|
||||
|
||||
in
|
||||
{
|
||||
# ── Option declarations ──────────────────────────────────────────────────
|
||||
|
||||
options.skyg.nixos.common.container-services = lib.mkOption {
|
||||
description = ''
|
||||
Compose-style container service groups.
|
||||
|
||||
Each group is rendered to a compose.yml in the Nix store, staged into
|
||||
its stateDir on activation, and managed by a single systemd oneshot unit:
|
||||
|
||||
container-services-<group>.service
|
||||
|
||||
See modules/nixos/common/containers/user-guide.md for full documentation.
|
||||
'';
|
||||
default = { };
|
||||
type = lib.types.attrsOf (lib.types.submodule ({ name, ... }: {
|
||||
options = {
|
||||
|
||||
enable = lib.mkEnableOption "this container service group" // { default = true; };
|
||||
|
||||
stateDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/var/lib/container-services/${name}";
|
||||
description = "Directory where the staged compose.yml lives at runtime.";
|
||||
};
|
||||
|
||||
timeoutStopSec = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 120;
|
||||
description = "Seconds to wait for 'compose down' before forcibly killing containers.";
|
||||
};
|
||||
|
||||
services = lib.mkOption {
|
||||
description = "Container services in this group (maps to compose services: block).";
|
||||
default = { };
|
||||
type = lib.types.attrsOf (lib.types.submodule {
|
||||
options = {
|
||||
|
||||
image = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Container image (registry/name:tag).";
|
||||
};
|
||||
|
||||
command = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Entrypoint command override (compose command:).";
|
||||
};
|
||||
|
||||
ports = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
example = [ "8642:8642" ];
|
||||
description = "Port mappings in HOST:CONTAINER format.";
|
||||
};
|
||||
|
||||
volumes = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
example = [ "/var/lib/hermes:/opt/data" ];
|
||||
description = "Volume mounts in src:dst[:opts] format.";
|
||||
};
|
||||
|
||||
environment = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.str;
|
||||
default = { };
|
||||
description = "Environment variables as key/value pairs.";
|
||||
};
|
||||
|
||||
environmentFiles = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = ''
|
||||
Paths to env files loaded by compose at runtime.
|
||||
Accepts runtime paths such as agenix secret paths
|
||||
(e.g. config.age.secrets.my-secret.path).
|
||||
'';
|
||||
};
|
||||
|
||||
dependsOn = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Sibling service names that must start first (compose depends_on:).";
|
||||
};
|
||||
|
||||
restart = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "unless-stopped";
|
||||
description = "Compose restart policy.";
|
||||
};
|
||||
|
||||
networks = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = ''
|
||||
Networks to join. Empty list = attach to the group's
|
||||
auto-created bridge network (named after the group).
|
||||
'';
|
||||
};
|
||||
|
||||
extraConfig = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.anything;
|
||||
default = { };
|
||||
description = ''
|
||||
Free-form attrset merged directly into the compose service
|
||||
block. Use this for keys not modelled above:
|
||||
shm_size, cap_add, devices, healthcheck, deploy, etc.
|
||||
'';
|
||||
};
|
||||
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
volumes = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Named volumes declared at the top level of the compose document.";
|
||||
};
|
||||
|
||||
networks = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.anything;
|
||||
default = { };
|
||||
description = ''
|
||||
Network definitions for the top-level compose networks: block.
|
||||
Empty = auto-create a bridge network named after the group.
|
||||
'';
|
||||
};
|
||||
|
||||
extraConfig = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.anything;
|
||||
default = { };
|
||||
description = "Free-form attrset merged into the top-level compose document.";
|
||||
};
|
||||
|
||||
};
|
||||
}));
|
||||
};
|
||||
|
||||
# ── Configuration ────────────────────────────────────────────────────────
|
||||
|
||||
config = {
|
||||
assertions = lib.optionals (enabledGroups != { }) [
|
||||
{
|
||||
assertion = config.virtualisation.docker.enable or false
|
||||
|| config.virtualisation.podman.enable or false;
|
||||
message = ''
|
||||
skyg.nixos.common.container-services: at least one container runtime
|
||||
must be enabled. Set skyg.nixos.common.containers.runtime (or enable
|
||||
virtualisation.docker / virtualisation.podman directly).
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# Create each group's stateDir via tmpfiles so the cp in ExecStartPre succeeds.
|
||||
systemd.tmpfiles.rules =
|
||||
lib.mapAttrsToList
|
||||
(_: grpCfg: "d ${grpCfg.stateDir} 0750 root root -")
|
||||
enabledGroups;
|
||||
|
||||
# One oneshot unit per group — manages the whole compose stack.
|
||||
systemd.services =
|
||||
lib.mapAttrs'
|
||||
(groupName: grpCfg:
|
||||
lib.nameValuePair
|
||||
"container-services-${groupName}"
|
||||
(mkSystemdService groupName grpCfg))
|
||||
enabledGroups;
|
||||
};
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
imports = [
|
||||
./docker.nix
|
||||
./podman.nix
|
||||
./container-services.nix
|
||||
];
|
||||
options = {
|
||||
skyg.nixos.common.containers = {
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# Container Services — User Guide
|
||||
|
||||
Declares Docker Compose stacks in Nix. Each group is rendered to a `compose.yml`,
|
||||
staged on disk, and managed by a systemd oneshot unit.
|
||||
|
||||
## Where things live
|
||||
|
||||
| What | Path |
|
||||
| ---- | ---- |
|
||||
| Staged compose file | `/var/lib/container-services/<group>/compose.yml` |
|
||||
| Nix store source | `/nix/store/…-compose.yml` (copied on each activation) |
|
||||
| Systemd unit | `container-services-<group>.service` |
|
||||
| Container logs | `docker compose -p <group> logs -f <service>` |
|
||||
| Unit journal | `journalctl -u container-services-<group> -f` |
|
||||
|
||||
## Quick reference
|
||||
|
||||
```bash
|
||||
# See what's running
|
||||
docker compose -p <group> ps
|
||||
|
||||
# Follow container logs
|
||||
docker compose -p <group> logs -f <service>
|
||||
|
||||
# Inspect the generated compose file
|
||||
cat /var/lib/container-services/<group>/compose.yml
|
||||
|
||||
# Restart the stack (e.g. after nixos-rebuild switch)
|
||||
systemctl restart container-services-<group>
|
||||
|
||||
# Stop / start without a rebuild
|
||||
systemctl stop container-services-<group>
|
||||
systemctl start container-services-<group>
|
||||
|
||||
# Shell into a running container
|
||||
docker compose -p <group> exec <service> bash
|
||||
|
||||
# Pull a new image and recreate
|
||||
docker compose -p <group> pull
|
||||
systemctl restart container-services-<group>
|
||||
```
|
||||
|
||||
## Debugging
|
||||
|
||||
### Unit fails to start
|
||||
|
||||
```bash
|
||||
systemctl status container-services-<group>
|
||||
journalctl -u container-services-<group> -n 50
|
||||
```
|
||||
|
||||
Common causes:
|
||||
- **Docker not running** — `systemctl status docker`
|
||||
- **Secret path missing** — agenix may not have decrypted yet; check `systemctl status agenix`
|
||||
- **Port already in use** — another service holds the host port; `ss -tlnp | grep <port>`
|
||||
- **Image not pullable** — network issue or wrong image name; try `docker pull <image>` manually
|
||||
|
||||
### Container starts but misbehaves
|
||||
|
||||
```bash
|
||||
# Last 100 lines from the container
|
||||
docker compose -p <group> logs --tail=100 <service>
|
||||
|
||||
# Inspect the compose file Nix generated
|
||||
cat /var/lib/container-services/<group>/compose.yml
|
||||
|
||||
# Verify env file was read correctly (check what the container sees)
|
||||
docker compose -p <group> exec <service> env | grep <KEY>
|
||||
```
|
||||
|
||||
### Permission errors on the data volume
|
||||
|
||||
The container's init runs as root and chowns `/opt/data` to `PUID:PGID`. If it still
|
||||
fails, check ownership on the host:
|
||||
|
||||
```bash
|
||||
ls -la /var/lib/<service-data-dir>
|
||||
# Should match PUID:PGID set in environment {}
|
||||
```
|
||||
|
||||
If the directory was created by a previous run as root and never chowned, remove it and
|
||||
let Docker recreate it (named volumes are unaffected; only bind mounts need this):
|
||||
|
||||
```bash
|
||||
sudo rm -rf /var/lib/<service-data-dir>
|
||||
systemctl restart container-services-<group>
|
||||
```
|
||||
|
||||
### compose.yml looks wrong
|
||||
|
||||
The file at `/var/lib/container-services/<group>/compose.yml` is overwritten on every
|
||||
`ExecStartPre`. If it looks wrong, check the Nix-rendered source in the store directly:
|
||||
|
||||
```bash
|
||||
# Find the store path
|
||||
systemctl cat container-services-<group> | grep ExecStartPre
|
||||
# Then cat it
|
||||
cat /nix/store/…-compose.yml
|
||||
```
|
||||
|
||||
If the store file is wrong, the bug is in your Nix options — rebuild with
|
||||
`nixos-rebuild build` and inspect the new store path before switching.
|
||||
|
||||
## Limitations
|
||||
|
||||
- **One unit per group.** No per-container systemd units. Use `docker compose logs` for
|
||||
container output, `journalctl` only for the start/stop lifecycle.
|
||||
- **`compose down` on stop.** Containers are removed when the unit stops. Data in
|
||||
bind mounts and named volumes is preserved.
|
||||
- **Compose project name == group name.** Keep group names DNS-label–safe (letters,
|
||||
digits, hyphens).
|
||||
@@ -4,6 +4,7 @@ with lib;
|
||||
|
||||
let
|
||||
cfg = config.skyg.nixos.server.services.signal-cli;
|
||||
skygUser = config.skyg.user;
|
||||
in
|
||||
{
|
||||
options = {
|
||||
@@ -71,10 +72,8 @@ in
|
||||
"skyg.nixos.server.services.signal-cli: set either `account` or `environmentFile` (providing SIGNAL_ACCOUNT).";
|
||||
}];
|
||||
|
||||
# Ensure the config/state directory exists (the account is linked into it
|
||||
# manually). 0700 since it holds Signal account credentials.
|
||||
systemd.tmpfiles.rules = [ "d ${cfg.configDir} 0700 root root - -" ];
|
||||
|
||||
environment.systemPackages = [ pkgs.signal-cli ];
|
||||
systemd.services.signal-cli-daemon = {
|
||||
description = "signal-cli daemon (HTTP) for the Hermes Signal adapter";
|
||||
|
||||
@@ -84,16 +83,19 @@ in
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
User = skygUser.name;
|
||||
Environment = "JAVA_OPTS=--enable-native-access=ALL-UNNAMED";
|
||||
EnvironmentFile =
|
||||
mkIf (cfg.environmentFile != null) cfg.environmentFile;
|
||||
ExecStart =
|
||||
let
|
||||
account =
|
||||
if cfg.account != null then cfg.account else "\${SIGNAL_ACCOUNT}";
|
||||
in
|
||||
"${cfg.package}/bin/signal-cli --config ${cfg.configDir} --account ${account} daemon --http ${cfg.host}:${toString cfg.port}";
|
||||
ExecStart = pkgs.writeShellScript "signal-cli-daemon-start" ''
|
||||
set -eu
|
||||
account=${if cfg.account != null then lib.escapeShellArg cfg.account else "\"\${SIGNAL_ACCOUNT:?SIGNAL_ACCOUNT is unset/empty in the env file}\""}
|
||||
export JAVA_OPTS="--enable-native-access=ALL-UNNAMED"
|
||||
exec ${cfg.package}/bin/signal-cli --config ${cfg.configDir} --account "$account" daemon --http ${cfg.host}:${toString cfg.port}
|
||||
'';
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
StartLimitBurst = 2;
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
Binary file not shown.
Reference in new issue
Block a user