diff --git a/flake.nix b/flake.nix index 304609a..c05aa9d 100644 --- a/flake.nix +++ b/flake.nix @@ -339,6 +339,7 @@ buzz-desktop = lib.pkgs.${system}.buzz-desktop; delta-editor = lib.pkgs.${system}.delta-editor; colibri = lib.pkgs.${system}.colibri; + llama-cpp-halo = lib.pkgs.${system}.llama-cpp-halo; herdr = herdr.packages.${system}.default; } ); diff --git a/hosts/hl-fwdesk/services/default.nix b/hosts/hl-fwdesk/services/default.nix index c444d7d..e738a37 100644 --- a/hosts/hl-fwdesk/services/default.nix +++ b/hosts/hl-fwdesk/services/default.nix @@ -1,6 +1,7 @@ { imports = [ ./ds4.nix + ./llama-cpp.nix ./ollama.nix ./libretranslate.nix ./hermenix.nix diff --git a/hosts/hl-fwdesk/services/llama-cpp.nix b/hosts/hl-fwdesk/services/llama-cpp.nix new file mode 100644 index 0000000..e2ee2cc --- /dev/null +++ b/hosts/hl-fwdesk/services/llama-cpp.nix @@ -0,0 +1,32 @@ +{ ... }: + +{ + # llama.cpp with DFlash2 speculative decoding. + # + # Off by default: it needs ~19 GB of weights on disk and a first build of the + # ROCm package. Fetch the files, then flip enable: + # + # sudo install -d -o ari -g users /var/lib/llama-cpp + # cd /var/lib/llama-cpp + # curl -L -O https://huggingface.co/ggml-org/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf + # curl -L -O https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q8_0.gguf + # + # Measured expectation for this box: ~25-40 tok/s, not the 380 tok/s in the + # post that prompted this — that figure is a 3090 copying a document. + skyg.nixos.server.services.llama-cpp = { + enable = false; + + model = "/var/lib/llama-cpp/Qwen3.8-27B-Q4_K_M.gguf"; + draftModel = "/var/lib/llama-cpp/Qwen3.8-27B-DFlash2-Q8_0.gguf"; + specType = "draft-dflash"; + specDraftNMax = 7; + + ctx = 32768; + kvCacheType = "q8_0"; + cacheReuse = 256; + + host = "0.0.0.0"; + port = 8081; + openFirewall = true; + }; +} diff --git a/modules/core/default.nix b/modules/core/default.nix index 2e3269f..c56cc86 100644 --- a/modules/core/default.nix +++ b/modules/core/default.nix @@ -22,6 +22,9 @@ # colibrì (JustVugg/colibri) — GLM-5.2/OLMoE local inference, see pkgs/colibri. colibri = final.callPackage ../../pkgs/colibri { }; colibri-rocm = final.callPackage ../../pkgs/colibri { backend = "rocm"; }; + # llama.cpp v0.8.0 for gfx1151 — DFlash2 speculative decoding, see pkgs/llama-cpp. + llama-cpp-halo = final.callPackage ../../pkgs/llama-cpp { }; + llama-cpp-halo-cpu = final.callPackage ../../pkgs/llama-cpp { backend = "cpu"; }; # Touchscreen gesture bridge for niri, see pkgs/niri-touchscreen-gestures. niri-touchscreen-gestures = final.callPackage ../../pkgs/niri-touchscreen-gestures { }; # Buzz Desktop AppImage wrapper, see pkgs/buzz-desktop. diff --git a/modules/nixos/server/services/ABOUTME.md b/modules/nixos/server/services/ABOUTME.md index 500417e..ee6a7be 100644 --- a/modules/nixos/server/services/ABOUTME.md +++ b/modules/nixos/server/services/ABOUTME.md @@ -16,6 +16,7 @@ services/ ├── nbot-agent.nix # skyg.nixos.server.services.nbot-agent — nbot's agent system + gh, for nbot's `system` ├── signal-cli.nix # skyg.nixos.server.services.signal-cli — signal-cli HTTP daemon (Hermes Signal bridge) ├── colibri.nix # skyg.nixos.server.services.colibri — coli serve HTTP API (GLM-5.2/OLMoE local inference) +├── llama-cpp.nix # skyg.nixos.server.services.llama-cpp — llama-server (GGUF, DFlash2 speculative decoding) └── comfyui/ # skyg.nixos.server.services.comfyui — ComfyUI with ROCm/CUDA GPU support ``` @@ -31,6 +32,11 @@ services/ service. `package` picks the backend (`pkgs.colibri` cpu / `pkgs.colibri-rocm`); `environmentFile` supplies `COLI_API_KEY` via agenix instead of a CLI flag. +- **`llama-cpp.nix`** runs `llama-server` as a native systemd service. `draftModel` plus + `specType = "draft-dflash"` turns on DFlash2 speculative decoding; `cacheReuse` turns on + prefix reuse for repeated prompts (agents). Model files are read from disk, never packaged — + the unit refuses to start with a clear message when one is missing. Enabled on fwdesk + (off by default: ~19 GB of weights, first build is the ROCm package). ## Option Namespace @@ -41,6 +47,7 @@ skyg.nixos.server.services.jellyfin.enable skyg.nixos.server.services.nbot-agent.system skyg.nixos.server.services.signal-cli.{enable,host,port,configDir,account,environmentFile,openFirewall} skyg.nixos.server.services.colibri.{enable,package,model,host,port,environmentFile,openFirewall,...} +skyg.nixos.server.services.llama-cpp.{enable,package,model,draftModel,specType,specDraftNMax,cacheReuse,ctx,host,port,openFirewall,...} skyg.nixos.server.services.comfyui.{enable,mode,port,openFirewall} ``` diff --git a/modules/nixos/server/services/default.nix b/modules/nixos/server/services/default.nix index 9374053..4a513d7 100644 --- a/modules/nixos/server/services/default.nix +++ b/modules/nixos/server/services/default.nix @@ -4,6 +4,7 @@ ./audiobookshelf.nix ./ai-services.nix ./colibri.nix + ./llama-cpp.nix ./comfyui ./jellyfin.nix ./nbot-agent.nix diff --git a/modules/nixos/server/services/llama-cpp.nix b/modules/nixos/server/services/llama-cpp.nix new file mode 100644 index 0000000..3f9101e --- /dev/null +++ b/modules/nixos/server/services/llama-cpp.nix @@ -0,0 +1,252 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.skyg.nixos.server.services.llama-cpp; + skygUser = config.skyg.user; + + # Target model plus optional drafter. The drafter is only useful for a + # strategy that matches its architecture: DFlash2 sidecars want + # `--spec-type draft-dflash`. + modelFiles = [ cfg.model ] ++ optional (cfg.draftModel != null) cfg.draftModel; + + serverArgs = + [ + "--model" + cfg.model + "--host" + cfg.host + "--port" + (toString cfg.port) + "--ctx-size" + (toString cfg.ctx) + "--n-gpu-layers" + (toString cfg.gpuLayers) + "--parallel" + (toString cfg.parallel) + "--flash-attn" + cfg.flashAttention + "--cache-type-k" + cfg.kvCacheType + "--cache-type-v" + cfg.kvCacheType + # Qwen-style tool calls arrive as Jinja templates. + "--jinja" + ] + ++ optionals (cfg.draftModel != null) [ + "--model-draft" + cfg.draftModel + ] + ++ optionals (cfg.specType != null) [ + "--spec-type" + cfg.specType + ] + ++ optionals (cfg.specDraftNMax != null) [ + "--spec-draft-n-max" + (toString cfg.specDraftNMax) + ] + # Reuse the longest matching prefix of the previous prompt instead of + # reprocessing it. This is the "prefix caching" half of the speedup, and it + # works with any model. + ++ optionals (cfg.cacheReuse != null) [ + "--cache-reuse" + (toString cfg.cacheReuse) + ] + ++ cfg.extraArgs; + + checkModels = pkgs.writeShellScript "llama-cpp-check-models" '' + set -eu + ${concatMapStringsSep "\n" ( + f: '' + if [ ! -f ${escapeShellArg f} ]; then + echo "llama-cpp: missing model file: ${f}" >&2 + exit 1 + fi + '' + ) modelFiles} + ''; +in +{ + options.skyg.nixos.server.services.llama-cpp = { + enable = mkEnableOption "the llama.cpp HTTP API (llama-server)"; + + package = mkOption { + type = types.package; + default = pkgs.llama-cpp-halo; + defaultText = literalExpression "pkgs.llama-cpp-halo"; + description = '' + Server package. `pkgs.llama-cpp-halo` is the ROCm build for gfx1151; + `pkgs.llama-cpp-halo-cpu` is the same tree without the GPU backend. + ''; + }; + + model = mkOption { + type = types.str; + example = "/var/lib/llama-cpp/Qwen3.8-27B-Q4_K_M.gguf"; + description = "Path to the target model (GGUF)."; + }; + + draftModel = mkOption { + type = types.nullOr types.str; + default = null; + example = "/var/lib/llama-cpp/Qwen3.8-27B-DFlash2-Q8_0.gguf"; + description = '' + Optional draft model (GGUF). A drafter must be trained for the exact + target model — a mismatch loads and then drafts nothing useful. + ''; + }; + + specType = mkOption { + type = types.nullOr types.str; + default = null; + example = "draft-dflash"; + description = '' + Speculative decoding strategy passed to `--spec-type`. Use + `draft-dflash` for a DFlash/DFlash2 drafter, `draft-mtp` for a model's + own MTP heads, or `ngram-simple` for prompt-copying workloads (code + edits, document rewriting) where no drafter is needed at all. + ''; + }; + + specDraftNMax = mkOption { + type = types.nullOr types.int; + default = null; + example = 7; + description = '' + Tokens drafted per verification step. Clamped by the drafter's trained + block size; 7 is the value upstream uses for DFlash2. + ''; + }; + + cacheReuse = mkOption { + type = types.nullOr types.int; + default = null; + example = 256; + description = '' + `--cache-reuse`: minimum run of tokens to reuse from a matching prompt + prefix. 256 is a reasonable default for coding agents. + ''; + }; + + ctx = mkOption { + type = types.int; + default = 32768; + description = "Context size in tokens."; + }; + + kvCacheType = mkOption { + type = types.enum [ + "f16" + "bf16" + "q8_0" + "q4_0" + ]; + default = "q8_0"; + description = "KV cache type for both K and V (q8_0 halves KV memory)."; + }; + + flashAttention = mkOption { + type = types.enum [ + "on" + "off" + "auto" + ]; + default = "auto"; + description = "`--flash-attn` value."; + }; + + gpuLayers = mkOption { + type = types.int; + default = 99; + description = '' + Layers offloaded to the GPU. On Strix Halo the GPU shares system RAM, + so all layers fit and 99 is the right value. + ''; + }; + + parallel = mkOption { + type = types.int; + default = 1; + description = "Parallel request slots."; + }; + + host = mkOption { + type = types.str; + default = "127.0.0.1"; + example = "0.0.0.0"; + description = "Address llama-server binds to."; + }; + + port = mkOption { + type = types.port; + default = 8081; + description = "TCP port for the HTTP API."; + }; + + user = mkOption { + type = types.str; + default = skygUser.name; + description = "User to run the server as. Needs the render/video groups for GPU access."; + }; + + group = mkOption { + type = types.str; + default = "users"; + description = "Group to run the server as."; + }; + + environment = mkOption { + type = types.attrsOf types.str; + default = { + # Same workaround the box's Ollama and ds4 units use. + HSA_ENABLE_SDMA = "0"; + }; + description = "Environment variables for the service."; + }; + + extraArgs = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ + "--metrics" + "--no-mmap" + ]; + description = "Extra arguments appended to the llama-server command line."; + }; + + openFirewall = mkEnableOption "the llama-server port in the firewall"; + }; + + config = mkIf cfg.enable { + assertions = [ + { + assertion = cfg.specType == null || cfg.draftModel != null || cfg.specType != "draft-dflash"; + message = "services.llama-cpp: specType = \"draft-dflash\" needs a draftModel."; + } + ]; + + systemd.services.llama-cpp = { + description = "llama.cpp server (local LLM inference)"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + environment = cfg.environment; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + ExecStartPre = checkModels; + ExecStart = "${lib.getExe' cfg.package "llama-server"} ${escapeShellArgs serverArgs}"; + Restart = "on-failure"; + RestartSec = 5; + # Model files live here; the unit may create it, weights are downloaded + # by hand (they are tens of GB and not part of the system closure). + StateDirectory = "llama-cpp"; + WorkingDirectory = "/var/lib/llama-cpp"; + LimitNOFILE = 65536; + TimeoutStopSec = 120; + }; + }; + + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.port ]; + }; +} diff --git a/pkgs/ABOUTME.md b/pkgs/ABOUTME.md index 756d135..f371cb1 100644 --- a/pkgs/ABOUTME.md +++ b/pkgs/ABOUTME.md @@ -12,6 +12,7 @@ Packages here are exposed via an overlay defined in `modules/core/default.nix`. | `buzz-desktop/` | Block Buzz desktop (Type-2 AppImage from `block/buzz`). Wrapped with `appimageTools.wrapType2` plus extra FHS libs (`elfutils`, `zstd`, WebKit/tray, GStreamer good/bad) so it runs on NixOS. Exposed as `pkgs.buzz-desktop` (`buzz-desktop` binary + desktop entry / icons). | | `buzz-cli/` | Buzz CLI (`buzz`, `crates/buzz-cli` of `block/buzz`), built with `rustPlatform.buildRustPackage` from the same tag as `buzz-desktop`. Hermes's Buzz platform shells out to it. Bump with `update.sh` (sets the source hash, resets `cargoHash` to `lib.fakeHash`; build once and paste the `got:` hash). Robots call it with `pkgs.callPackage` (their nixpkgs has no host overlay). | | `delta-editor/` | Delta Editor (Zed Industries' AI-native editor, `delta.dev`) prebuilt tarball. No stable URL exists — the release API returns a short-lived signed R2 link — so `src` is a fixed-output derivation that resolves the API at build time. Interpreter/RPATH patched (`--force-rpath`, `dontPatchELF`), dlopen'd graphics libs (`libglvnd`, `vulkan-loader`, `wayland`) added via wrapper, `XKB_CONFIG_ROOT` set for the bundled libxkbcommon. Exposed as `pkgs.delta-editor` (`delta-editor` binary + desktop entry / icons); named `-editor` to avoid clashing with the `delta` git diff pager. | +| `llama-cpp/` | llama.cpp pinned to v0.8.0, the first release with DFlash2 drafters (`--spec-type draft-dflash`). nixpkgs only carries v0.6.0 in the unstable input and an older tree in the stable one, so the server is built here. HIP build for Strix Halo (`CMAKE_HIP_ARCHITECTURES=gfx1151`, ROCm clang++, `autoPatchelfHook`). Backend-parameterized: `pkgs.llama-cpp-halo` (rocm, default) and `pkgs.llama-cpp-halo-cpu` (same tree, no GPU backend). Web UI is off — it downloads prebuilt assets at build time, which the sandbox cannot do. Model weights are not packaged. | ## Adding a New Package diff --git a/pkgs/llama-cpp/default.nix b/pkgs/llama-cpp/default.nix new file mode 100644 index 0000000..78b5cfe --- /dev/null +++ b/pkgs/llama-cpp/default.nix @@ -0,0 +1,94 @@ +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + ninja, + pkg-config, + autoPatchelfHook, + rocmPackages, + # "rocm" builds the gfx1151 engine. "cpu" is the same tree without the GPU + # backend, so the recipe can be checked on a machine with no AMD card. + backend ? "rocm", + rocmArch ? "gfx1151", +}: + +# llama.cpp, pinned to the first release that carries DFlash2 drafters +# (v0.8.0, 2026-10-28). nixpkgs only has v0.6.0 in the unstable input and an +# older tree in the stable one, and DFlash2 landed upstream after both — hence +# this package. Built for the Framework Desktop iGPU (Strix Halo, gfx1151). +# +# To bump: set `version` and `rev`, refresh `hash` with +# `nix-prefetch-url --unpack https://github.com/ggml-org/llama.cpp/archive/refs/tags/vX.Y.Z.tar.gz` +# then `nix hash convert --to sri `. Check the release notes for a new +# `--spec-type` before assuming a drafter still loads. + +assert lib.elem backend [ + "cpu" + "rocm" +]; + +let + rocmInputs = [ + rocmPackages.clr + rocmPackages.rocm-runtime + rocmPackages.hipblas + rocmPackages.rocblas + ]; +in +stdenv.mkDerivation { + pname = "llama-cpp-halo" + lib.optionalString (backend == "cpu") "-cpu"; + version = "0.8.0"; + + src = fetchFromGitHub { + owner = "ggml-org"; + repo = "llama.cpp"; + tag = "v0.8.0"; + hash = "sha256-WQAtD8Hos6fEtgPQId7nZiLGyP4asAMKJYEXZid1nqc="; + }; + + nativeBuildInputs = [ + cmake + ninja + pkg-config + ] + ++ lib.optionals (backend == "rocm") [ + autoPatchelfHook + # Provides clang++ and the HIP headers; llama.cpp wants ROCm's own clang. + rocmPackages.clr + ]; + + buildInputs = lib.optionals (backend == "rocm") rocmInputs; + + # The web UI is a prebuilt asset downloaded at build time, which the sandbox + # cannot do. The API does not need it. + cmakeFlags = [ + (lib.cmakeBool "GGML_NATIVE" false) + (lib.cmakeBool "BUILD_SHARED_LIBS" false) + (lib.cmakeBool "LLAMA_BUILD_SERVER" true) + (lib.cmakeBool "LLAMA_BUILD_TESTS" false) + (lib.cmakeBool "LLAMA_BUILD_EXAMPLES" false) + (lib.cmakeBool "LLAMA_BUILD_APP" false) + (lib.cmakeBool "LLAMA_BUILD_UI" false) + (lib.cmakeBool "LLAMA_USE_PREBUILT_UI" false) + (lib.cmakeBool "LLAMA_OPENSSL" false) + # Upstream reads these from git, which the release tarball does not carry. + (lib.cmakeFeature "LLAMA_BUILD_NUMBER" "31309") + (lib.cmakeFeature "LLAMA_BUILD_COMMIT" "8642541b2") + ] + ++ lib.optionals (backend == "rocm") [ + (lib.cmakeBool "GGML_HIP" true) + (lib.cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.clr.hipClangPath}/clang++") + (lib.cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmArch) + ]; + + doCheck = false; + + meta = { + description = "llama.cpp server (DFlash2 speculative decoding) for AMD Strix Halo (gfx1151)"; + homepage = "https://github.com/ggml-org/llama.cpp"; + license = lib.licenses.mit; + platforms = lib.platforms.linux; + mainProgram = "llama-server"; + }; +}