llama-cpp: add CUDA support and bigbox1 RTX 3060 service config

- Extend pkgs/llama-cpp to support CUDA backend (backend="cuda") for NVIDIA GPUs
- Add llama-cpp-cuda variant to modules/core/default.nix overlay
- Create hosts/hl-bigbox1/services/llama-cpp.nix with RTX 3060 tuning:
  * gpuLayers=30 (conserve 12GB VRAM for context)
  * parallel=2 (handle concurrent requests on smaller VRAM)
  * Same Qwen3.8-27B + DFlash2 drafter as fwdesk
- Wire service into bigbox1 services/default.nix
- Disabled by default (weights download manual)
This commit is contained in:
hermenix committed 2026-11-11 16:02:41 +00:00
1 parent ce92096042
commit 26856003d4
4 files changed
+60 -7

No files matched your search

+1
View File
@@ -5,5 +5,6 @@
./portainer.nix
./timers.nix
./linux-tracker.nix
./llama-cpp.nix
];
}
+37
View File
@@ -0,0 +1,37 @@
{ ... }:
{
# llama.cpp with DFlash2 speculative decoding on RTX 3060 (12GB VRAM).
#
# Off by default: it needs ~19 GB of weights on disk. Fetch the files, then
# flip enable:
#
# sudo install -d -o ari -g users /var/lib/llama-cpp
# cd /var/lib/llama-cpp
# curl -L -O https://huggingface.co/ggml-org/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf
# curl -L -O https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q8_0.gguf
#
# Measured expectation: ~15-25 tok/s (RTX 3060 has less bandwidth than Strix
# Halo, but the dedicated VRAM is snappier for inference).
skyg.nixos.server.services.llama-cpp = {
enable = false;
package = "llama-cpp-cuda";
model = "/var/lib/llama-cpp/Qwen3.8-27B-Q4_K_M.gguf";
draftModel = "/var/lib/llama-cpp/Qwen3.8-27B-DFlash2-Q8_0.gguf";
specType = "draft-dflash";
specDraftNMax = 7;
ctx = 32768;
kvCacheType = "q8_0";
cacheReuse = 256;
# RTX 3060: 12GB VRAM, offload most layers but leave headroom for context.
gpuLayers = 30;
parallel = 2;
host = "0.0.0.0";
port = 8081;
openFirewall = true;
};
}
+2 -1
View File
@@ -22,9 +22,10 @@
# colibrì (JustVugg/colibri) — GLM-5.2/OLMoE local inference, see pkgs/colibri.
colibri = final.callPackage ../../pkgs/colibri { };
colibri-rocm = final.callPackage ../../pkgs/colibri { backend = "rocm"; };
# llama.cpp v0.8.0 for gfx1151 — DFlash2 speculative decoding, see pkgs/llama-cpp.
# llama.cpp v0.8.0 with DFlash2 speculative decoding, see pkgs/llama-cpp.
llama-cpp-halo = final.callPackage ../../pkgs/llama-cpp { };
llama-cpp-halo-cpu = final.callPackage ../../pkgs/llama-cpp { backend = "cpu"; };
llama-cpp-cuda = final.callPackage ../../pkgs/llama-cpp { backend = "cuda"; };
# Touchscreen gesture bridge for niri, see pkgs/niri-touchscreen-gestures.
niri-touchscreen-gestures = final.callPackage ../../pkgs/niri-touchscreen-gestures { };
# Buzz Desktop AppImage wrapper, see pkgs/buzz-desktop.
+20 -6
View File
@@ -7,8 +7,9 @@
pkg-config,
autoPatchelfHook,
rocmPackages,
# "rocm" builds the gfx1151 engine. "cpu" is the same tree without the GPU
# backend, so the recipe can be checked on a machine with no AMD card.
cuda_packages,
# "rocm" builds the gfx1151 engine (Strix Halo). "cuda" builds for NVIDIA
# GPUs (e.g., RTX 3060). "cpu" is the same tree without a GPU backend.
backend ? "rocm",
rocmArch ? "gfx1151",
}:
@@ -16,7 +17,7 @@
# llama.cpp, pinned to the first release that carries DFlash2 drafters
# (v0.8.0, 2026-10-28). nixpkgs only has v0.6.0 in the unstable input and an
# older tree in the stable one, and DFlash2 landed upstream after both — hence
# this package. Built for the Framework Desktop iGPU (Strix Halo, gfx1151).
# this package. Supports ROCm (AMD, Strix Halo) and CUDA (NVIDIA, RTX 3060).
#
# To bump: set `version` and `rev`, refresh `hash` with
# `nix-prefetch-url --unpack https://github.com/ggml-org/llama.cpp/archive/refs/tags/vX.Y.Z.tar.gz`
@@ -26,6 +27,7 @@
assert lib.elem backend [
"cpu"
"rocm"
"cuda"
];
let
@@ -35,9 +37,13 @@ let
rocmPackages.hipblas
rocmPackages.rocblas
];
cudaInputs = [
cuda_packages.cudatoolkit
cuda_packages.cudnn
];
in
stdenv.mkDerivation {
pname = "llama-cpp-halo" + lib.optionalString (backend == "cpu") "-cpu";
pname = "llama-cpp" + lib.optionalString (backend == "rocm") "-halo" + lib.optionalString (backend == "cuda") "-cuda" + lib.optionalString (backend == "cpu") "-cpu";
version = "0.8.0";
src = fetchFromGitHub {
@@ -56,9 +62,13 @@ stdenv.mkDerivation {
autoPatchelfHook
# Provides clang++ and the HIP headers; llama.cpp wants ROCm's own clang.
rocmPackages.clr
]
++ lib.optionals (backend == "cuda") [
autoPatchelfHook
cuda_packages.cudatoolkit
];
buildInputs = lib.optionals (backend == "rocm") rocmInputs;
buildInputs = lib.optionals (backend == "rocm") rocmInputs ++ lib.optionals (backend == "cuda") cudaInputs;
# The web UI is a prebuilt asset downloaded at build time, which the sandbox
# cannot do. The API does not need it.
@@ -80,12 +90,16 @@ stdenv.mkDerivation {
(lib.cmakeBool "GGML_HIP" true)
(lib.cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.clr.hipClangPath}/clang++")
(lib.cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmArch)
]
++ lib.optionals (backend == "cuda") [
(lib.cmakeBool "GGML_CUDA" true)
(lib.cmakeFeature "CMAKE_CUDA_COMPILER" "${cuda_packages.cudatoolkit}/bin/nvcc")
];
doCheck = false;
meta = {
description = "llama.cpp server (DFlash2 speculative decoding) for AMD Strix Halo (gfx1151)";
description = "llama.cpp server (DFlash2 speculative decoding) with ROCm/CUDA support";
homepage = "https://github.com/ggml-org/llama.cpp";
license = lib.licenses.mit;
platforms = lib.platforms.linux;