Files
nixos-setup/hosts/hl-fwdesk/services/llama-cpp.nix
T
hermenix ce92096042 llama-cpp: llama-server with DFlash2 speculative decoding for gfx1151
v0.8.0 is the first upstream release with DFlash2 drafters; nixpkgs has 0.6.0 (unstable) and an older tree (stable), so the engine is built here. HIP build for Strix Halo. The NixOS module exposes llama-server behind skyg.nixos.server.services.llama-cpp; fwdesk wires it to Qwen3.8-27B + its DFlash2 drafter, disabled until the weights are on disk.
2026-11-11 16:01:32 +00:00

33 lines
1.0 KiB
Nix

{ ... }:
{
# llama.cpp with DFlash2 speculative decoding.
#
# Off by default: it needs ~19 GB of weights on disk and a first build of the
# ROCm package. Fetch the files, then flip enable:
#
# sudo install -d -o ari -g users /var/lib/llama-cpp
# cd /var/lib/llama-cpp
# curl -L -O https://huggingface.co/ggml-org/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf
# curl -L -O https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q8_0.gguf
#
# Measured expectation for this box: ~25-40 tok/s, not the 380 tok/s in the
# post that prompted this — that figure is a 3090 copying a document.
skyg.nixos.server.services.llama-cpp = {
enable = false;
model = "/var/lib/llama-cpp/Qwen3.8-27B-Q4_K_M.gguf";
draftModel = "/var/lib/llama-cpp/Qwen3.8-27B-DFlash2-Q8_0.gguf";
specType = "draft-dflash";
specDraftNMax = 7;
ctx = 32768;
kvCacheType = "q8_0";
cacheReuse = 256;
host = "0.0.0.0";
port = 8081;
openFirewall = true;
};
}