Files
nixos-setup/modules/nixos/server/services/llama-cpp.nix
T
hermenix ce92096042 llama-cpp: llama-server with DFlash2 speculative decoding for gfx1151
v0.8.0 is the first upstream release with DFlash2 drafters; nixpkgs has 0.6.0 (unstable) and an older tree (stable), so the engine is built here. HIP build for Strix Halo. The NixOS module exposes llama-server behind skyg.nixos.server.services.llama-cpp; fwdesk wires it to Qwen3.8-27B + its DFlash2 drafter, disabled until the weights are on disk.
2026-11-11 16:01:32 +00:00

253 lines
6.9 KiB
Nix

{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.skyg.nixos.server.services.llama-cpp;
skygUser = config.skyg.user;
# Target model plus optional drafter. The drafter is only useful for a
# strategy that matches its architecture: DFlash2 sidecars want
# `--spec-type draft-dflash`.
modelFiles = [ cfg.model ] ++ optional (cfg.draftModel != null) cfg.draftModel;
serverArgs =
[
"--model"
cfg.model
"--host"
cfg.host
"--port"
(toString cfg.port)
"--ctx-size"
(toString cfg.ctx)
"--n-gpu-layers"
(toString cfg.gpuLayers)
"--parallel"
(toString cfg.parallel)
"--flash-attn"
cfg.flashAttention
"--cache-type-k"
cfg.kvCacheType
"--cache-type-v"
cfg.kvCacheType
# Qwen-style tool calls arrive as Jinja templates.
"--jinja"
]
++ optionals (cfg.draftModel != null) [
"--model-draft"
cfg.draftModel
]
++ optionals (cfg.specType != null) [
"--spec-type"
cfg.specType
]
++ optionals (cfg.specDraftNMax != null) [
"--spec-draft-n-max"
(toString cfg.specDraftNMax)
]
# Reuse the longest matching prefix of the previous prompt instead of
# reprocessing it. This is the "prefix caching" half of the speedup, and it
# works with any model.
++ optionals (cfg.cacheReuse != null) [
"--cache-reuse"
(toString cfg.cacheReuse)
]
++ cfg.extraArgs;
checkModels = pkgs.writeShellScript "llama-cpp-check-models" ''
set -eu
${concatMapStringsSep "\n" (
f: ''
if [ ! -f ${escapeShellArg f} ]; then
echo "llama-cpp: missing model file: ${f}" >&2
exit 1
fi
''
) modelFiles}
'';
in
{
options.skyg.nixos.server.services.llama-cpp = {
enable = mkEnableOption "the llama.cpp HTTP API (llama-server)";
package = mkOption {
type = types.package;
default = pkgs.llama-cpp-halo;
defaultText = literalExpression "pkgs.llama-cpp-halo";
description = ''
Server package. `pkgs.llama-cpp-halo` is the ROCm build for gfx1151;
`pkgs.llama-cpp-halo-cpu` is the same tree without the GPU backend.
'';
};
model = mkOption {
type = types.str;
example = "/var/lib/llama-cpp/Qwen3.8-27B-Q4_K_M.gguf";
description = "Path to the target model (GGUF).";
};
draftModel = mkOption {
type = types.nullOr types.str;
default = null;
example = "/var/lib/llama-cpp/Qwen3.8-27B-DFlash2-Q8_0.gguf";
description = ''
Optional draft model (GGUF). A drafter must be trained for the exact
target model — a mismatch loads and then drafts nothing useful.
'';
};
specType = mkOption {
type = types.nullOr types.str;
default = null;
example = "draft-dflash";
description = ''
Speculative decoding strategy passed to `--spec-type`. Use
`draft-dflash` for a DFlash/DFlash2 drafter, `draft-mtp` for a model's
own MTP heads, or `ngram-simple` for prompt-copying workloads (code
edits, document rewriting) where no drafter is needed at all.
'';
};
specDraftNMax = mkOption {
type = types.nullOr types.int;
default = null;
example = 7;
description = ''
Tokens drafted per verification step. Clamped by the drafter's trained
block size; 7 is the value upstream uses for DFlash2.
'';
};
cacheReuse = mkOption {
type = types.nullOr types.int;
default = null;
example = 256;
description = ''
`--cache-reuse`: minimum run of tokens to reuse from a matching prompt
prefix. 256 is a reasonable default for coding agents.
'';
};
ctx = mkOption {
type = types.int;
default = 32768;
description = "Context size in tokens.";
};
kvCacheType = mkOption {
type = types.enum [
"f16"
"bf16"
"q8_0"
"q4_0"
];
default = "q8_0";
description = "KV cache type for both K and V (q8_0 halves KV memory).";
};
flashAttention = mkOption {
type = types.enum [
"on"
"off"
"auto"
];
default = "auto";
description = "`--flash-attn` value.";
};
gpuLayers = mkOption {
type = types.int;
default = 99;
description = ''
Layers offloaded to the GPU. On Strix Halo the GPU shares system RAM,
so all layers fit and 99 is the right value.
'';
};
parallel = mkOption {
type = types.int;
default = 1;
description = "Parallel request slots.";
};
host = mkOption {
type = types.str;
default = "127.0.0.1";
example = "0.0.0.0";
description = "Address llama-server binds to.";
};
port = mkOption {
type = types.port;
default = 8081;
description = "TCP port for the HTTP API.";
};
user = mkOption {
type = types.str;
default = skygUser.name;
description = "User to run the server as. Needs the render/video groups for GPU access.";
};
group = mkOption {
type = types.str;
default = "users";
description = "Group to run the server as.";
};
environment = mkOption {
type = types.attrsOf types.str;
default = {
# Same workaround the box's Ollama and ds4 units use.
HSA_ENABLE_SDMA = "0";
};
description = "Environment variables for the service.";
};
extraArgs = mkOption {
type = types.listOf types.str;
default = [ ];
example = [
"--metrics"
"--no-mmap"
];
description = "Extra arguments appended to the llama-server command line.";
};
openFirewall = mkEnableOption "the llama-server port in the firewall";
};
config = mkIf cfg.enable {
assertions = [
{
assertion = cfg.specType == null || cfg.draftModel != null || cfg.specType != "draft-dflash";
message = "services.llama-cpp: specType = \"draft-dflash\" needs a draftModel.";
}
];
systemd.services.llama-cpp = {
description = "llama.cpp server (local LLM inference)";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
environment = cfg.environment;
serviceConfig = {
User = cfg.user;
Group = cfg.group;
ExecStartPre = checkModels;
ExecStart = "${lib.getExe' cfg.package "llama-server"} ${escapeShellArgs serverArgs}";
Restart = "on-failure";
RestartSec = 5;
# Model files live here; the unit may create it, weights are downloaded
# by hand (they are tens of GB and not part of the system closure).
StateDirectory = "llama-cpp";
WorkingDirectory = "/var/lib/llama-cpp";
LimitNOFILE = 65536;
TimeoutStopSec = 120;
};
};
networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.port ];
};
}