From 26856003d4b4317e22f92d99e3b011487f4d9e26 Mon Sep 17 00:00:00 2001 From: hermenix Date: Wed, 11 Nov 2026 16:02:14 +0000 Subject: [PATCH] llama-cpp: add CUDA support and bigbox1 RTX 3060 service config - Extend pkgs/llama-cpp to support CUDA backend (backend="cuda") for NVIDIA GPUs - Add llama-cpp-cuda variant to modules/core/default.nix overlay - Create hosts/hl-bigbox1/services/llama-cpp.nix with RTX 3060 tuning: * gpuLayers=30 (conserve 12GB VRAM for context) * parallel=2 (handle concurrent requests on smaller VRAM) * Same Qwen3.8-27B + DFlash2 drafter as fwdesk - Wire service into bigbox1 services/default.nix - Disabled by default (weights download manual) --- hosts/hl-bigbox1/services/default.nix | 1 + hosts/hl-bigbox1/services/llama-cpp.nix | 37 +++++++++++++++++++++++++ modules/core/default.nix | 3 +- pkgs/llama-cpp/default.nix | 26 +++++++++++++---- 4 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 hosts/hl-bigbox1/services/llama-cpp.nix diff --git a/hosts/hl-bigbox1/services/default.nix b/hosts/hl-bigbox1/services/default.nix index 290a05d..338d766 100644 --- a/hosts/hl-bigbox1/services/default.nix +++ b/hosts/hl-bigbox1/services/default.nix @@ -5,5 +5,6 @@ ./portainer.nix ./timers.nix ./linux-tracker.nix + ./llama-cpp.nix ]; } diff --git a/hosts/hl-bigbox1/services/llama-cpp.nix b/hosts/hl-bigbox1/services/llama-cpp.nix new file mode 100644 index 0000000..47bbf31 --- /dev/null +++ b/hosts/hl-bigbox1/services/llama-cpp.nix @@ -0,0 +1,37 @@ +{ ... }: + +{ + # llama.cpp with DFlash2 speculative decoding on RTX 3060 (12GB VRAM). + # + # Off by default: it needs ~19 GB of weights on disk. Fetch the files, then + # flip enable: + # + # sudo install -d -o ari -g users /var/lib/llama-cpp + # cd /var/lib/llama-cpp + # curl -L -O https://huggingface.co/ggml-org/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf + # curl -L -O https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q8_0.gguf + # + # Measured expectation: ~15-25 tok/s (RTX 3060 has less bandwidth than Strix + # Halo, but the dedicated VRAM is snappier for inference). + skyg.nixos.server.services.llama-cpp = { + enable = false; + + package = "llama-cpp-cuda"; + model = "/var/lib/llama-cpp/Qwen3.8-27B-Q4_K_M.gguf"; + draftModel = "/var/lib/llama-cpp/Qwen3.8-27B-DFlash2-Q8_0.gguf"; + specType = "draft-dflash"; + specDraftNMax = 7; + + ctx = 32768; + kvCacheType = "q8_0"; + cacheReuse = 256; + + # RTX 3060: 12GB VRAM, offload most layers but leave headroom for context. + gpuLayers = 30; + parallel = 2; + + host = "0.0.0.0"; + port = 8081; + openFirewall = true; + }; +} diff --git a/modules/core/default.nix b/modules/core/default.nix index c56cc86..246a6fe 100644 --- a/modules/core/default.nix +++ b/modules/core/default.nix @@ -22,9 +22,10 @@ # colibrì (JustVugg/colibri) — GLM-5.2/OLMoE local inference, see pkgs/colibri. colibri = final.callPackage ../../pkgs/colibri { }; colibri-rocm = final.callPackage ../../pkgs/colibri { backend = "rocm"; }; - # llama.cpp v0.8.0 for gfx1151 — DFlash2 speculative decoding, see pkgs/llama-cpp. + # llama.cpp v0.8.0 with DFlash2 speculative decoding, see pkgs/llama-cpp. llama-cpp-halo = final.callPackage ../../pkgs/llama-cpp { }; llama-cpp-halo-cpu = final.callPackage ../../pkgs/llama-cpp { backend = "cpu"; }; + llama-cpp-cuda = final.callPackage ../../pkgs/llama-cpp { backend = "cuda"; }; # Touchscreen gesture bridge for niri, see pkgs/niri-touchscreen-gestures. niri-touchscreen-gestures = final.callPackage ../../pkgs/niri-touchscreen-gestures { }; # Buzz Desktop AppImage wrapper, see pkgs/buzz-desktop. diff --git a/pkgs/llama-cpp/default.nix b/pkgs/llama-cpp/default.nix index 78b5cfe..243bd3b 100644 --- a/pkgs/llama-cpp/default.nix +++ b/pkgs/llama-cpp/default.nix @@ -7,8 +7,9 @@ pkg-config, autoPatchelfHook, rocmPackages, - # "rocm" builds the gfx1151 engine. "cpu" is the same tree without the GPU - # backend, so the recipe can be checked on a machine with no AMD card. + cuda_packages, + # "rocm" builds the gfx1151 engine (Strix Halo). "cuda" builds for NVIDIA + # GPUs (e.g., RTX 3060). "cpu" is the same tree without a GPU backend. backend ? "rocm", rocmArch ? "gfx1151", }: @@ -16,7 +17,7 @@ # llama.cpp, pinned to the first release that carries DFlash2 drafters # (v0.8.0, 2026-10-28). nixpkgs only has v0.6.0 in the unstable input and an # older tree in the stable one, and DFlash2 landed upstream after both — hence -# this package. Built for the Framework Desktop iGPU (Strix Halo, gfx1151). +# this package. Supports ROCm (AMD, Strix Halo) and CUDA (NVIDIA, RTX 3060). # # To bump: set `version` and `rev`, refresh `hash` with # `nix-prefetch-url --unpack https://github.com/ggml-org/llama.cpp/archive/refs/tags/vX.Y.Z.tar.gz` @@ -26,6 +27,7 @@ assert lib.elem backend [ "cpu" "rocm" + "cuda" ]; let @@ -35,9 +37,13 @@ let rocmPackages.hipblas rocmPackages.rocblas ]; + cudaInputs = [ + cuda_packages.cudatoolkit + cuda_packages.cudnn + ]; in stdenv.mkDerivation { - pname = "llama-cpp-halo" + lib.optionalString (backend == "cpu") "-cpu"; + pname = "llama-cpp" + lib.optionalString (backend == "rocm") "-halo" + lib.optionalString (backend == "cuda") "-cuda" + lib.optionalString (backend == "cpu") "-cpu"; version = "0.8.0"; src = fetchFromGitHub { @@ -56,9 +62,13 @@ stdenv.mkDerivation { autoPatchelfHook # Provides clang++ and the HIP headers; llama.cpp wants ROCm's own clang. rocmPackages.clr + ] + ++ lib.optionals (backend == "cuda") [ + autoPatchelfHook + cuda_packages.cudatoolkit ]; - buildInputs = lib.optionals (backend == "rocm") rocmInputs; + buildInputs = lib.optionals (backend == "rocm") rocmInputs ++ lib.optionals (backend == "cuda") cudaInputs; # The web UI is a prebuilt asset downloaded at build time, which the sandbox # cannot do. The API does not need it. @@ -80,12 +90,16 @@ stdenv.mkDerivation { (lib.cmakeBool "GGML_HIP" true) (lib.cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.clr.hipClangPath}/clang++") (lib.cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmArch) + ] + ++ lib.optionals (backend == "cuda") [ + (lib.cmakeBool "GGML_CUDA" true) + (lib.cmakeFeature "CMAKE_CUDA_COMPILER" "${cuda_packages.cudatoolkit}/bin/nvcc") ]; doCheck = false; meta = { - description = "llama.cpp server (DFlash2 speculative decoding) for AMD Strix Halo (gfx1151)"; + description = "llama.cpp server (DFlash2 speculative decoding) with ROCm/CUDA support"; homepage = "https://github.com/ggml-org/llama.cpp"; license = lib.licenses.mit; platforms = lib.platforms.linux;