326 lines
11 KiB
Nix
326 lines
11 KiB
Nix
{ config, lib, pkgs, ... }:
|
|
|
|
with lib;
|
|
|
|
let
|
|
cfg = config.skyg.nixos.server.services.colibri;
|
|
skygUser = config.skyg.user;
|
|
|
|
# The colibri package variant to use: colibri (cpu), colibri-rocm.
|
|
colibriPkg = cfg.package;
|
|
|
|
# Build the `coli serve` command line from options.
|
|
# optionalString returns "" when false; filter those out so we don't get
|
|
# spurious leading/trailing whitespace in the final command line.
|
|
serverArgs = with cfg;
|
|
concatStringsSep " " (
|
|
filter (x: x != "") [
|
|
"--model ${escapeShellArg model}"
|
|
"--host ${escapeShellArg host}"
|
|
"--port ${toString port}"
|
|
|
|
(optionalString (modelId != null) "--model-id ${escapeShellArg modelId}")
|
|
(optionalString (ram != null) "--ram ${toString ram}")
|
|
(optionalString (ctx != null) "--ctx ${toString ctx}")
|
|
(optionalString (cap != null) "--cap ${toString cap}")
|
|
"--ngen ${toString ngen}"
|
|
(optionalString (topp != null) "--topp ${toString topp}")
|
|
(optionalString (topk != null) "--topk ${toString topk}")
|
|
(optionalString (temp != null) "--temp ${toString temp}")
|
|
(optionalString (policy != null) "--policy ${policy}")
|
|
(optionalString (gpu != null) "--gpu ${escapeShellArg gpu}")
|
|
(optionalString (vram != null) "--vram ${toString vram}")
|
|
"--max-queue ${toString maxQueue}"
|
|
"--queue-timeout ${toString queueTimeout}"
|
|
"--kv-slots ${toString kvSlots}"
|
|
]
|
|
++ map (o: "--cors-origin ${escapeShellArg o}") corsOrigins
|
|
++ map (h: "--allowed-host ${escapeShellArg h}") allowedHosts
|
|
);
|
|
in
|
|
{
|
|
options = {
|
|
skyg.nixos.server.services.colibri = {
|
|
enable = mkEnableOption
|
|
"colibri-serve, the OpenAI-compatible HTTP API for colibrì (GLM-5.2/OLMoE local inference).";
|
|
|
|
package = mkPackageOption pkgs "colibri" { };
|
|
|
|
user = mkOption {
|
|
description = "The user to run coli serve as.";
|
|
type = types.str;
|
|
default = skygUser.name;
|
|
};
|
|
|
|
group = mkOption {
|
|
description = "The group to run coli serve as.";
|
|
type = types.str;
|
|
default = "users";
|
|
};
|
|
|
|
model = mkOption {
|
|
description = "Path to the (converted) model directory.";
|
|
type = types.str;
|
|
example = "/nvme/glm52_i4";
|
|
};
|
|
|
|
modelMirror = mkOption {
|
|
description = ''
|
|
Second copy of the model on another drive: expert reads are split
|
|
across both SSDs. `coli` only reads this via the COLI_MODEL_MIRROR
|
|
environment variable, so it's exported rather than passed as a flag.
|
|
'';
|
|
default = null;
|
|
example = "/nvme2/glm52_i4";
|
|
type = types.nullOr types.str;
|
|
};
|
|
|
|
diskWeights = mkOption {
|
|
description = "Primary,mirror disk bandwidth ratio (e.g. \"9,3\"); default is measured at startup. Exported as COLI_DISK_WEIGHTS.";
|
|
default = null;
|
|
example = "9,3";
|
|
type = types.nullOr types.str;
|
|
};
|
|
|
|
host = mkOption {
|
|
description = "Address the coli serve HTTP API binds to.";
|
|
default = "127.0.0.1";
|
|
example = "0.0.0.0";
|
|
type = types.str;
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "TCP port the coli serve HTTP API listens on.";
|
|
default = 8000;
|
|
type = types.port;
|
|
};
|
|
|
|
modelId = mkOption {
|
|
description = "Model name reported by the OpenAI-compatible API (default: derived from the model's architecture).";
|
|
default = null;
|
|
example = "glm-5.2-colibri";
|
|
type = types.nullOr types.str;
|
|
};
|
|
|
|
environmentFile = mkOption {
|
|
description = ''
|
|
Path to an EnvironmentFile (e.g. an agenix-decrypted secret) providing
|
|
COLI_API_KEY, the bearer token clients must send. Avoids putting the
|
|
key in the world-readable Nix store or on the process command line.
|
|
Leave null to run without authentication (fine for host-only binding
|
|
on 127.0.0.1).
|
|
'';
|
|
default = null;
|
|
example = "/run/agenix/colibri-api-key";
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
ram = mkOption {
|
|
description = "RAM budget in GB (0 or null = auto: the engine uses ~88% of available RAM).";
|
|
default = null;
|
|
example = 64;
|
|
type = types.nullOr types.int;
|
|
};
|
|
|
|
ctx = mkOption {
|
|
description = "Maximum context length (number of tokens; null = auto).";
|
|
default = null;
|
|
example = 32768;
|
|
type = types.nullOr types.int;
|
|
};
|
|
|
|
cap = mkOption {
|
|
description = "Cache slots per layer (null = auto).";
|
|
default = null;
|
|
example = 4;
|
|
type = types.nullOr types.int;
|
|
};
|
|
|
|
ngen = mkOption {
|
|
description = "Maximum response tokens (a safety net; stop tokens usually end generation first).";
|
|
default = 1024;
|
|
type = types.int;
|
|
};
|
|
|
|
topp = mkOption {
|
|
description = "Adaptive expert top-p (null = engine default).";
|
|
default = null;
|
|
example = 0.9;
|
|
type = types.nullOr types.float;
|
|
};
|
|
|
|
topk = mkOption {
|
|
description = "Fixed expert top-k (null = engine default).";
|
|
default = null;
|
|
type = types.nullOr types.int;
|
|
};
|
|
|
|
temp = mkOption {
|
|
description = "Sampling temperature (0 = greedy; null = engine default, nucleus ~0.95).";
|
|
default = null;
|
|
example = 0.7;
|
|
type = types.nullOr types.float;
|
|
};
|
|
|
|
policy = mkOption {
|
|
description = "Resource policy. Explicit topk/topp still override it.";
|
|
default = null;
|
|
type = types.nullOr (types.enum [ "quality" "balanced" "experimental-fast" ]);
|
|
};
|
|
|
|
gpu = mkOption {
|
|
description = "GPU device selection: \"auto\", \"none\", or a device list such as \"0,1\".";
|
|
default = null;
|
|
example = "auto";
|
|
type = types.nullOr types.str;
|
|
};
|
|
|
|
vram = mkOption {
|
|
description = "Total VRAM budget in GB (null = auto).";
|
|
default = null;
|
|
example = 24;
|
|
type = types.nullOr types.int;
|
|
};
|
|
|
|
maxQueue = mkOption {
|
|
description = "Maximum queued requests before the API rejects new ones.";
|
|
default = 8;
|
|
type = types.int;
|
|
};
|
|
|
|
queueTimeout = mkOption {
|
|
description = "Seconds a request may wait in queue before timing out.";
|
|
default = 300;
|
|
type = types.int;
|
|
};
|
|
|
|
kvSlots = mkOption {
|
|
description = "KV cache slots (non-GLM architectures currently support exactly 1).";
|
|
default = 1;
|
|
type = types.int;
|
|
};
|
|
|
|
corsOrigins = mkOption {
|
|
description = "Origins allowed by CORS (repeatable). Empty = CORS disabled.";
|
|
default = [ ];
|
|
example = [ "https://chat.example.com" ];
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
allowedHosts = mkOption {
|
|
description = "Additional Host headers accepted by the DNS-rebinding guard (repeatable).";
|
|
default = [ ];
|
|
example = [ "colibri.lan" ];
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
environment = mkOption {
|
|
description = ''
|
|
Extra environment variables for the coli serve process. Useful for
|
|
GPU backends — e.g. ROCm Strix Halo may need HSA_OVERRIDE_GFX_VERSION.
|
|
|
|
Example for ROCm on Strix Halo:
|
|
{ HSA_OVERRIDE_GFX_VERSION = "11.0.2"; }
|
|
'';
|
|
default = { };
|
|
example = { HSA_OVERRIDE_GFX_VERSION = "11.0.2"; };
|
|
type = types.attrsOf types.str;
|
|
};
|
|
|
|
openFirewall = mkOption {
|
|
description = "Open the coli serve port in the firewall.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
autoStart = mkOption {
|
|
description = ''
|
|
Whether to start colibri-serve automatically at boot.
|
|
Set to false to only start it on demand via `systemctl start colibri-serve`.
|
|
'';
|
|
default = true;
|
|
type = types.bool;
|
|
};
|
|
|
|
extraArgs = mkOption {
|
|
description = "Additional arguments to append to the coli serve command line.";
|
|
default = [ ];
|
|
example = [ "--auto-tier" ];
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
extraServiceConfig = mkOption {
|
|
description = "Extra systemd service unit config (e.g. SupplementaryGroups, TimeoutStopSec).";
|
|
default = { };
|
|
example = {
|
|
SupplementaryGroups = [ "render" "video" ];
|
|
TimeoutStopSec = 120;
|
|
};
|
|
type = types.attrsOf types.str;
|
|
};
|
|
};
|
|
};
|
|
|
|
config = mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = cfg.model != "";
|
|
message = "skyg.nixos.server.services.colibri: `model` must be set to a model directory path.";
|
|
}
|
|
];
|
|
|
|
# Ensure the colibri package is available in the system closure (for manual use).
|
|
environment.systemPackages = [ colibriPkg ];
|
|
|
|
systemd.services.colibri-serve = {
|
|
description = "coli serve — colibrì local inference HTTP API (GLM-5.2/OLMoE)";
|
|
documentation = [
|
|
"https://github.com/JustVugg/colibri"
|
|
"https://github.com/JustVugg/colibri#readme"
|
|
];
|
|
|
|
after = [ "network-online.target" ];
|
|
wants = [ "network-online.target" ];
|
|
wantedBy = lib.optionals cfg.autoStart [ "multi-user.target" ];
|
|
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
User = cfg.user;
|
|
Group = cfg.group;
|
|
|
|
# Construct the ExecStart as a shell script so we can export the extra
|
|
# env vars coli only reads (not flags), e.g. COLI_MODEL_MIRROR. Use
|
|
# writeShellScript for a clean script in the store.
|
|
EnvironmentFile =
|
|
mkIf (cfg.environmentFile != null) cfg.environmentFile;
|
|
|
|
ExecStart = pkgs.writeShellScript "colibri-serve-start" ''
|
|
set -eu
|
|
${optionalString (cfg.modelMirror != null) "export COLI_MODEL_MIRROR=${escapeShellArg cfg.modelMirror}"}
|
|
${optionalString (cfg.diskWeights != null) "export COLI_DISK_WEIGHTS=${escapeShellArg cfg.diskWeights}"}
|
|
${concatStringsSep "\n" (mapAttrsToList (name: value: "export ${name}=${escapeShellArg value}") cfg.environment)}
|
|
exec ${colibriPkg}/bin/coli serve ${serverArgs} ${escapeShellArgs cfg.extraArgs}
|
|
'';
|
|
|
|
# No custom ExecStop: `coli serve` runs in the foreground and systemd's
|
|
# default cgroup-wide kill on stop takes down its spawned engine
|
|
# subprocess with it (same reasoning as ds4-server).
|
|
Restart = "on-failure";
|
|
RestartSec = 10;
|
|
StartLimitBurst = 3;
|
|
|
|
# Security hardening
|
|
NoNewPrivileges = true;
|
|
ProtectHome = true;
|
|
ProtectSystem = "full";
|
|
PrivateTmp = true;
|
|
CapabilityBoundingSet = "";
|
|
SystemCallFilter = "@system-service @resources";
|
|
} // cfg.extraServiceConfig;
|
|
};
|
|
|
|
networking.firewall =
|
|
mkIf cfg.openFirewall { allowedTCPPorts = [ cfg.port ]; };
|
|
};
|
|
}
|