Merge branch 'main' of gitea.app.internal:ari/nixos-setup
This commit is contained in:
commit
6db82ed10a
11 files changed
+445
No files matched your search
@@ -22,6 +22,10 @@
|
||||
# colibrì (JustVugg/colibri) — GLM-5.2/OLMoE local inference, see pkgs/colibri.
|
||||
colibri = final.callPackage ../../pkgs/colibri { };
|
||||
colibri-rocm = final.callPackage ../../pkgs/colibri { backend = "rocm"; };
|
||||
# llama.cpp v0.8.0 with DFlash2 speculative decoding, see pkgs/llama-cpp.
|
||||
llama-cpp-halo = final.callPackage ../../pkgs/llama-cpp { };
|
||||
llama-cpp-halo-cpu = final.callPackage ../../pkgs/llama-cpp { backend = "cpu"; };
|
||||
llama-cpp-cuda = final.callPackage ../../pkgs/llama-cpp { backend = "cuda"; };
|
||||
# Touchscreen gesture bridge for niri, see pkgs/niri-touchscreen-gestures.
|
||||
niri-touchscreen-gestures = final.callPackage ../../pkgs/niri-touchscreen-gestures { };
|
||||
# Buzz Desktop AppImage wrapper, see pkgs/buzz-desktop.
|
||||
|
||||
@@ -16,6 +16,7 @@ services/
|
||||
├── nbot-agent.nix # skyg.nixos.server.services.nbot-agent — nbot's agent system + gh, for nbot's `system`
|
||||
├── signal-cli.nix # skyg.nixos.server.services.signal-cli — signal-cli HTTP daemon (Hermes Signal bridge)
|
||||
├── colibri.nix # skyg.nixos.server.services.colibri — coli serve HTTP API (GLM-5.2/OLMoE local inference)
|
||||
├── llama-cpp.nix # skyg.nixos.server.services.llama-cpp — llama-server (GGUF, DFlash2 speculative decoding)
|
||||
└── comfyui/ # skyg.nixos.server.services.comfyui — ComfyUI with ROCm/CUDA GPU support
|
||||
```
|
||||
|
||||
@@ -31,6 +32,11 @@ services/
|
||||
service. `package` picks the backend (`pkgs.colibri`
|
||||
cpu / `pkgs.colibri-rocm`); `environmentFile` supplies `COLI_API_KEY` via agenix instead of
|
||||
a CLI flag.
|
||||
- **`llama-cpp.nix`** runs `llama-server` as a native systemd service. `draftModel` plus
|
||||
`specType = "draft-dflash"` turns on DFlash2 speculative decoding; `cacheReuse` turns on
|
||||
prefix reuse for repeated prompts (agents). Model files are read from disk, never packaged —
|
||||
the unit refuses to start with a clear message when one is missing. Enabled on fwdesk
|
||||
(off by default: ~19 GB of weights, first build is the ROCm package).
|
||||
|
||||
## Option Namespace
|
||||
|
||||
@@ -41,6 +47,7 @@ skyg.nixos.server.services.jellyfin.enable
|
||||
skyg.nixos.server.services.nbot-agent.system
|
||||
skyg.nixos.server.services.signal-cli.{enable,host,port,configDir,account,environmentFile,openFirewall}
|
||||
skyg.nixos.server.services.colibri.{enable,package,model,host,port,environmentFile,openFirewall,...}
|
||||
skyg.nixos.server.services.llama-cpp.{enable,package,model,draftModel,specType,specDraftNMax,cacheReuse,ctx,host,port,openFirewall,...}
|
||||
skyg.nixos.server.services.comfyui.{enable,mode,port,openFirewall}
|
||||
```
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
./audiobookshelf.nix
|
||||
./ai-services.nix
|
||||
./colibri.nix
|
||||
./llama-cpp.nix
|
||||
./comfyui
|
||||
./jellyfin.nix
|
||||
./nbot-agent.nix
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.skyg.nixos.server.services.llama-cpp;
|
||||
skygUser = config.skyg.user;
|
||||
|
||||
# Target model plus optional drafter. The drafter is only useful for a
|
||||
# strategy that matches its architecture: DFlash2 sidecars want
|
||||
# `--spec-type draft-dflash`.
|
||||
modelFiles = [ cfg.model ] ++ optional (cfg.draftModel != null) cfg.draftModel;
|
||||
|
||||
serverArgs =
|
||||
[
|
||||
"--model"
|
||||
cfg.model
|
||||
"--host"
|
||||
cfg.host
|
||||
"--port"
|
||||
(toString cfg.port)
|
||||
"--ctx-size"
|
||||
(toString cfg.ctx)
|
||||
"--n-gpu-layers"
|
||||
(toString cfg.gpuLayers)
|
||||
"--parallel"
|
||||
(toString cfg.parallel)
|
||||
"--flash-attn"
|
||||
cfg.flashAttention
|
||||
"--cache-type-k"
|
||||
cfg.kvCacheType
|
||||
"--cache-type-v"
|
||||
cfg.kvCacheType
|
||||
# Qwen-style tool calls arrive as Jinja templates.
|
||||
"--jinja"
|
||||
]
|
||||
++ optionals (cfg.draftModel != null) [
|
||||
"--model-draft"
|
||||
cfg.draftModel
|
||||
]
|
||||
++ optionals (cfg.specType != null) [
|
||||
"--spec-type"
|
||||
cfg.specType
|
||||
]
|
||||
++ optionals (cfg.specDraftNMax != null) [
|
||||
"--spec-draft-n-max"
|
||||
(toString cfg.specDraftNMax)
|
||||
]
|
||||
# Reuse the longest matching prefix of the previous prompt instead of
|
||||
# reprocessing it. This is the "prefix caching" half of the speedup, and it
|
||||
# works with any model.
|
||||
++ optionals (cfg.cacheReuse != null) [
|
||||
"--cache-reuse"
|
||||
(toString cfg.cacheReuse)
|
||||
]
|
||||
++ cfg.extraArgs;
|
||||
|
||||
checkModels = pkgs.writeShellScript "llama-cpp-check-models" ''
|
||||
set -eu
|
||||
${concatMapStringsSep "\n" (
|
||||
f: ''
|
||||
if [ ! -f ${escapeShellArg f} ]; then
|
||||
echo "llama-cpp: missing model file: ${f}" >&2
|
||||
exit 1
|
||||
fi
|
||||
''
|
||||
) modelFiles}
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.skyg.nixos.server.services.llama-cpp = {
|
||||
enable = mkEnableOption "the llama.cpp HTTP API (llama-server)";
|
||||
|
||||
package = mkOption {
|
||||
type = types.package;
|
||||
default = pkgs.llama-cpp-halo;
|
||||
defaultText = literalExpression "pkgs.llama-cpp-halo";
|
||||
description = ''
|
||||
Server package. `pkgs.llama-cpp-halo` is the ROCm build for gfx1151;
|
||||
`pkgs.llama-cpp-halo-cpu` is the same tree without the GPU backend.
|
||||
'';
|
||||
};
|
||||
|
||||
model = mkOption {
|
||||
type = types.str;
|
||||
example = "/var/lib/llama-cpp/Qwen3.8-27B-Q4_K_M.gguf";
|
||||
description = "Path to the target model (GGUF).";
|
||||
};
|
||||
|
||||
draftModel = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
example = "/var/lib/llama-cpp/Qwen3.8-27B-DFlash2-Q8_0.gguf";
|
||||
description = ''
|
||||
Optional draft model (GGUF). A drafter must be trained for the exact
|
||||
target model — a mismatch loads and then drafts nothing useful.
|
||||
'';
|
||||
};
|
||||
|
||||
specType = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
example = "draft-dflash";
|
||||
description = ''
|
||||
Speculative decoding strategy passed to `--spec-type`. Use
|
||||
`draft-dflash` for a DFlash/DFlash2 drafter, `draft-mtp` for a model's
|
||||
own MTP heads, or `ngram-simple` for prompt-copying workloads (code
|
||||
edits, document rewriting) where no drafter is needed at all.
|
||||
'';
|
||||
};
|
||||
|
||||
specDraftNMax = mkOption {
|
||||
type = types.nullOr types.int;
|
||||
default = null;
|
||||
example = 7;
|
||||
description = ''
|
||||
Tokens drafted per verification step. Clamped by the drafter's trained
|
||||
block size; 7 is the value upstream uses for DFlash2.
|
||||
'';
|
||||
};
|
||||
|
||||
cacheReuse = mkOption {
|
||||
type = types.nullOr types.int;
|
||||
default = null;
|
||||
example = 256;
|
||||
description = ''
|
||||
`--cache-reuse`: minimum run of tokens to reuse from a matching prompt
|
||||
prefix. 256 is a reasonable default for coding agents.
|
||||
'';
|
||||
};
|
||||
|
||||
ctx = mkOption {
|
||||
type = types.int;
|
||||
default = 32768;
|
||||
description = "Context size in tokens.";
|
||||
};
|
||||
|
||||
kvCacheType = mkOption {
|
||||
type = types.enum [
|
||||
"f16"
|
||||
"bf16"
|
||||
"q8_0"
|
||||
"q4_0"
|
||||
];
|
||||
default = "q8_0";
|
||||
description = "KV cache type for both K and V (q8_0 halves KV memory).";
|
||||
};
|
||||
|
||||
flashAttention = mkOption {
|
||||
type = types.enum [
|
||||
"on"
|
||||
"off"
|
||||
"auto"
|
||||
];
|
||||
default = "auto";
|
||||
description = "`--flash-attn` value.";
|
||||
};
|
||||
|
||||
gpuLayers = mkOption {
|
||||
type = types.int;
|
||||
default = 99;
|
||||
description = ''
|
||||
Layers offloaded to the GPU. On Strix Halo the GPU shares system RAM,
|
||||
so all layers fit and 99 is the right value.
|
||||
'';
|
||||
};
|
||||
|
||||
parallel = mkOption {
|
||||
type = types.int;
|
||||
default = 1;
|
||||
description = "Parallel request slots.";
|
||||
};
|
||||
|
||||
host = mkOption {
|
||||
type = types.str;
|
||||
default = "127.0.0.1";
|
||||
example = "0.0.0.0";
|
||||
description = "Address llama-server binds to.";
|
||||
};
|
||||
|
||||
port = mkOption {
|
||||
type = types.port;
|
||||
default = 8081;
|
||||
description = "TCP port for the HTTP API.";
|
||||
};
|
||||
|
||||
user = mkOption {
|
||||
type = types.str;
|
||||
default = skygUser.name;
|
||||
description = "User to run the server as. Needs the render/video groups for GPU access.";
|
||||
};
|
||||
|
||||
group = mkOption {
|
||||
type = types.str;
|
||||
default = "users";
|
||||
description = "Group to run the server as.";
|
||||
};
|
||||
|
||||
environment = mkOption {
|
||||
type = types.attrsOf types.str;
|
||||
default = {
|
||||
# Same workaround the box's Ollama and ds4 units use.
|
||||
HSA_ENABLE_SDMA = "0";
|
||||
};
|
||||
description = "Environment variables for the service.";
|
||||
};
|
||||
|
||||
extraArgs = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ ];
|
||||
example = [
|
||||
"--metrics"
|
||||
"--no-mmap"
|
||||
];
|
||||
description = "Extra arguments appended to the llama-server command line.";
|
||||
};
|
||||
|
||||
openFirewall = mkEnableOption "the llama-server port in the firewall";
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.specType == null || cfg.draftModel != null || cfg.specType != "draft-dflash";
|
||||
message = "services.llama-cpp: specType = \"draft-dflash\" needs a draftModel.";
|
||||
}
|
||||
];
|
||||
|
||||
systemd.services.llama-cpp = {
|
||||
description = "llama.cpp server (local LLM inference)";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
environment = cfg.environment;
|
||||
serviceConfig = {
|
||||
User = cfg.user;
|
||||
Group = cfg.group;
|
||||
ExecStartPre = checkModels;
|
||||
ExecStart = "${lib.getExe' cfg.package "llama-server"} ${escapeShellArgs serverArgs}";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
# Model files live here; the unit may create it, weights are downloaded
|
||||
# by hand (they are tens of GB and not part of the system closure).
|
||||
StateDirectory = "llama-cpp";
|
||||
WorkingDirectory = "/var/lib/llama-cpp";
|
||||
LimitNOFILE = 65536;
|
||||
TimeoutStopSec = 120;
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.port ];
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user