From 4cc2123effd5fe601be768a817dd160952e3142c Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Wed, 5 Aug 2026 16:42:57 -0400 Subject: [PATCH] buzz + update ds4 --- hosts/fwbook.nix | 1 + hosts/hl-fwdesk.nix | 1 + modules/core/default.nix | 2 + modules/lib.nix | 1 + modules/nixos/server/services/colibri.nix | 316 ++++++++++++++++++++++ modules/nixos/server/services/default.nix | 1 + modules/nixos/server/services/ds4.nix | 11 +- pkgs/ABOUTME.md | 1 + pkgs/buzz-desktop/default.nix | 128 +++++++++ pkgs/colibri/default.nix | 115 ++++++++ pkgs/colibri/flake.nix | 194 ------------- pkgs/ds4/default.nix | 6 +- pkgs/ds4/user-guide.md | 46 +++- 13 files changed, 615 insertions(+), 208 deletions(-) create mode 100644 modules/nixos/server/services/colibri.nix create mode 100644 pkgs/buzz-desktop/default.nix create mode 100644 pkgs/colibri/default.nix delete mode 100644 pkgs/colibri/flake.nix diff --git a/hosts/fwbook.nix b/hosts/fwbook.nix index abeee71..891b6ff 100644 --- a/hosts/fwbook.nix +++ b/hosts/fwbook.nix @@ -246,6 +246,7 @@ in hermes-agent.packages.${pkgs.stdenv.hostPlatform.system}.default # claude-desktop.packages.x86_64-linux.default claude-code + buzz-desktop ]); diff --git a/hosts/hl-fwdesk.nix b/hosts/hl-fwdesk.nix index 572d0f4..1931364 100644 --- a/hosts/hl-fwdesk.nix +++ b/hosts/hl-fwdesk.nix @@ -76,6 +76,7 @@ in HSA_ENABLE_SDMA = "0"; }; openFirewall = true; + autoStart = false; }; }; networkDrives = { diff --git a/modules/core/default.nix b/modules/core/default.nix index 9f0c800..fccf735 100644 --- a/modules/core/default.nix +++ b/modules/core/default.nix @@ -21,6 +21,8 @@ ds4-cuda = final.callPackage ../../pkgs/ds4 { backend = "cuda"; }; # Touchscreen gesture bridge for niri, see pkgs/niri-touchscreen-gestures. niri-touchscreen-gestures = final.callPackage ../../pkgs/niri-touchscreen-gestures { }; + # Buzz Desktop AppImage wrapper, see pkgs/buzz-desktop. + buzz-desktop = final.callPackage ../../pkgs/buzz-desktop { }; }) (final: prev: { pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ diff --git a/modules/lib.nix b/modules/lib.nix index 65abaf5..fc9f839 100644 --- a/modules/lib.nix +++ b/modules/lib.nix @@ -30,6 +30,7 @@ let # need their toolchains and are wired in modules/core/default.nix. ds4 = final.callPackage ../pkgs/ds4 { }; niri-touchscreen-gestures = final.callPackage ../pkgs/niri-touchscreen-gestures { }; + buzz-desktop = final.callPackage ../pkgs/buzz-desktop { }; }) ]; } diff --git a/modules/nixos/server/services/colibri.nix b/modules/nixos/server/services/colibri.nix new file mode 100644 index 0000000..612faa0 --- /dev/null +++ b/modules/nixos/server/services/colibri.nix @@ -0,0 +1,316 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.skyg.nixos.server.services.colibri; + skygUser = config.skyg.user; + + # The colibri package variant to use: colibri (cpu), colibri-rocm. + colibriPkg = cfg.package; + + # Build the `coli serve` command line from options. + # optionalString returns "" when false; filter those out so we don't get + # spurious leading/trailing whitespace in the final command line. + serverArgs = with cfg; + concatStringsSep " " ( + filter (x: x != "") [ + "--model ${escapeShellArg model}" + "--host ${escapeShellArg host}" + "--port ${toString port}" + + (optionalString (modelId != null) "--model-id ${escapeShellArg modelId}") + (optionalString (ram != null) "--ram ${toString ram}") + (optionalString (ctx != null) "--ctx ${toString ctx}") + (optionalString (cap != null) "--cap ${toString cap}") + "--ngen ${toString ngen}" + (optionalString (topp != null) "--topp ${toString topp}") + (optionalString (topk != null) "--topk ${toString topk}") + (optionalString (temp != null) "--temp ${toString temp}") + (optionalString (policy != null) "--policy ${policy}") + (optionalString (gpu != null) "--gpu ${escapeShellArg gpu}") + (optionalString (vram != null) "--vram ${toString vram}") + "--max-queue ${toString maxQueue}" + "--queue-timeout ${toString queueTimeout}" + "--kv-slots ${toString kvSlots}" + ] + ++ map (o: "--cors-origin ${escapeShellArg o}") corsOrigins + ++ map (h: "--allowed-host ${escapeShellArg h}") allowedHosts + ); +in +{ + options = { + skyg.nixos.server.services.colibri = { + enable = mkEnableOption + "colibri-serve, the OpenAI-compatible HTTP API for colibrì (GLM-5.2/OLMoE local inference)."; + + package = mkPackageOption pkgs "colibri" { + default = pkgs.colibri; + }; + + user = mkOption { + description = "The user to run coli serve as."; + type = types.str; + default = skygUser.name; + }; + + group = mkOption { + description = "The group to run coli serve as."; + type = types.str; + default = "users"; + }; + + model = mkOption { + description = "Path to the (converted) model directory."; + type = types.str; + example = "/nvme/glm52_i4"; + }; + + modelMirror = mkOption { + description = '' + Second copy of the model on another drive: expert reads are split + across both SSDs. `coli` only reads this via the COLI_MODEL_MIRROR + environment variable, so it's exported rather than passed as a flag. + ''; + default = null; + example = "/nvme2/glm52_i4"; + type = types.nullOr types.str; + }; + + diskWeights = mkOption { + description = "Primary,mirror disk bandwidth ratio (e.g. \"9,3\"); default is measured at startup. Exported as COLI_DISK_WEIGHTS."; + default = null; + example = "9,3"; + type = types.nullOr types.str; + }; + + host = mkOption { + description = "Address the coli serve HTTP API binds to."; + default = "127.0.0.1"; + example = "0.0.0.0"; + type = types.str; + }; + + port = mkOption { + description = "TCP port the coli serve HTTP API listens on."; + default = 8000; + type = types.port; + }; + + modelId = mkOption { + description = "Model name reported by the OpenAI-compatible API (default: derived from the model's architecture)."; + default = null; + example = "glm-5.2-colibri"; + type = types.nullOr types.str; + }; + + apiKeyFile = mkOption { + description = '' + Path to a file containing the bearer token clients must send. Read at + service start and exported as COLI_API_KEY (never passed as a CLI + argument, so it doesn't show up in `ps`). Leave null to run without + authentication (fine for host-only binding on 127.0.0.1). + ''; + default = null; + example = "/run/agenix/colibri-api-key"; + type = types.nullOr types.path; + }; + + ram = mkOption { + description = "RAM budget in GB (0 or null = auto: the engine uses ~88% of available RAM)."; + default = null; + example = 64; + type = types.nullOr types.int; + }; + + ctx = mkOption { + description = "Maximum context length (number of tokens; null = auto)."; + default = null; + example = 32768; + type = types.nullOr types.int; + }; + + cap = mkOption { + description = "Cache slots per layer (null = auto)."; + default = null; + example = 4; + type = types.nullOr types.int; + }; + + ngen = mkOption { + description = "Maximum response tokens (a safety net; stop tokens usually end generation first)."; + default = 1024; + type = types.int; + }; + + topp = mkOption { + description = "Adaptive expert top-p (null = engine default)."; + default = null; + example = 0.9; + type = types.nullOr types.float; + }; + + topk = mkOption { + description = "Fixed expert top-k (null = engine default)."; + default = null; + type = types.nullOr types.int; + }; + + temp = mkOption { + description = "Sampling temperature (0 = greedy; null = engine default, nucleus ~0.95)."; + default = null; + example = 0.7; + type = types.nullOr types.float; + }; + + policy = mkOption { + description = "Resource policy. Explicit topk/topp still override it."; + default = null; + type = types.nullOr (types.enum [ "quality" "balanced" "experimental-fast" ]); + }; + + gpu = mkOption { + description = "GPU device selection: \"auto\", \"none\", or a device list such as \"0,1\"."; + default = null; + example = "auto"; + type = types.nullOr types.str; + }; + + vram = mkOption { + description = "Total VRAM budget in GB (null = auto)."; + default = null; + example = 24; + type = types.nullOr types.int; + }; + + maxQueue = mkOption { + description = "Maximum queued requests before the API rejects new ones."; + default = 8; + type = types.int; + }; + + queueTimeout = mkOption { + description = "Seconds a request may wait in queue before timing out."; + default = 300; + type = types.int; + }; + + kvSlots = mkOption { + description = "KV cache slots (non-GLM architectures currently support exactly 1)."; + default = 1; + type = types.int; + }; + + corsOrigins = mkOption { + description = "Origins allowed by CORS (repeatable). Empty = CORS disabled."; + default = [ ]; + example = [ "https://chat.example.com" ]; + type = types.listOf types.str; + }; + + allowedHosts = mkOption { + description = "Additional Host headers accepted by the DNS-rebinding guard (repeatable)."; + default = [ ]; + example = [ "colibri.lan" ]; + type = types.listOf types.str; + }; + + environment = mkOption { + description = '' + Extra environment variables for the coli serve process. Useful for + GPU backends — e.g. ROCm Strix Halo may need HSA_OVERRIDE_GFX_VERSION. + + Example for ROCm on Strix Halo: + { HSA_OVERRIDE_GFX_VERSION = "11.0.2"; } + ''; + default = { }; + example = { HSA_OVERRIDE_GFX_VERSION = "11.0.2"; }; + type = types.attrsOf types.str; + }; + + openFirewall = mkOption { + description = "Open the coli serve port in the firewall."; + default = false; + type = types.bool; + }; + + extraArgs = mkOption { + description = "Additional arguments to append to the coli serve command line."; + default = [ ]; + example = [ "--auto-tier" ]; + type = types.listOf types.str; + }; + + extraServiceConfig = mkOption { + description = "Extra systemd service unit config (e.g. SupplementaryGroups, TimeoutStopSec)."; + default = { }; + example = { + SupplementaryGroups = [ "render" "video" ]; + TimeoutStopSec = 120; + }; + type = types.attrsOf types.str; + }; + }; + }; + + config = mkIf cfg.enable { + assertions = [ + { + assertion = cfg.model != ""; + message = "skyg.nixos.server.services.colibri: `model` must be set to a model directory path."; + } + ]; + + # Ensure the colibri package is available in the system closure (for manual use). + environment.systemPackages = [ colibriPkg ]; + + systemd.services.colibri-serve = { + description = "coli serve — colibrì local inference HTTP API (GLM-5.2/OLMoE)"; + documentation = [ + "https://github.com/JustVugg/colibri" + "https://github.com/JustVugg/colibri#readme" + ]; + + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "simple"; + User = cfg.user; + Group = cfg.group; + + # Construct the ExecStart as a shell script so we can export env vars + # that would otherwise be lost with a raw binary + Environment= (e.g. + # COLI_MODEL_MIRROR, COLI_API_KEY read from a file). Use + # writeShellScript for a clean script in the store. + ExecStart = pkgs.writeShellScript "colibri-serve-start" '' + set -eu + ${optionalString (cfg.modelMirror != null) "export COLI_MODEL_MIRROR=${escapeShellArg cfg.modelMirror}"} + ${optionalString (cfg.diskWeights != null) "export COLI_DISK_WEIGHTS=${escapeShellArg cfg.diskWeights}"} + ${optionalString (cfg.apiKeyFile != null) ''export COLI_API_KEY="$(cat ${escapeShellArg cfg.apiKeyFile})"''} + ${concatStringsSep "\n" (mapAttrsToList (name: value: "export ${name}=${escapeShellArg value}") cfg.environment)} + exec ${colibriPkg}/bin/coli serve ${serverArgs} ${escapeShellArgs cfg.extraArgs} + ''; + + # No custom ExecStop: `coli serve` runs in the foreground and systemd's + # default cgroup-wide kill on stop takes down its spawned engine + # subprocess with it (same reasoning as ds4-server). + Restart = "on-failure"; + RestartSec = 10; + StartLimitBurst = 3; + + # Security hardening + NoNewPrivileges = true; + ProtectHome = true; + ProtectSystem = "full"; + PrivateTmp = true; + CapabilityBoundingSet = ""; + SystemCallFilter = "@system-service @resources"; + } // cfg.extraServiceConfig; + }; + + networking.firewall = + mkIf cfg.openFirewall { allowedTCPPorts = [ cfg.port ]; }; + }; +} diff --git a/modules/nixos/server/services/default.nix b/modules/nixos/server/services/default.nix index dd864f7..a80aa28 100644 --- a/modules/nixos/server/services/default.nix +++ b/modules/nixos/server/services/default.nix @@ -3,6 +3,7 @@ imports = [ ./audiobookshelf.nix ./ai-services.nix + ./colibri.nix ./comfyui ./ds4.nix ./jellyfin.nix diff --git a/modules/nixos/server/services/ds4.nix b/modules/nixos/server/services/ds4.nix index 40074d5..a4ed888 100644 --- a/modules/nixos/server/services/ds4.nix +++ b/modules/nixos/server/services/ds4.nix @@ -143,6 +143,15 @@ in type = types.bool; }; + autoStart = mkOption { + description = '' + Whether to start ds4-server automatically at boot. + Set to false to only start it on demand via `systemctl start ds4-server`. + ''; + default = true; + type = types.bool; + }; + extraArgs = mkOption { description = "Additional arguments to append to the ds4-server command line."; default = [ ]; @@ -182,7 +191,7 @@ in after = [ "network-online.target" ]; wants = [ "network-online.target" ]; - wantedBy = [ "multi-user.target" ]; + wantedBy = lib.optionals cfg.autoStart [ "multi-user.target" ]; serviceConfig = { Type = "simple"; diff --git a/pkgs/ABOUTME.md b/pkgs/ABOUTME.md index b8d62a1..4f3a8f9 100644 --- a/pkgs/ABOUTME.md +++ b/pkgs/ABOUTME.md @@ -10,6 +10,7 @@ Packages here are exposed via an overlay defined in `modules/core/default.nix`. | `grok-cli/` | xAI Grok CLI — prebuilt static binary, fetched per-platform. Exposed as `pkgs.grok-cli`. | | `ds4/` | DwarfStar (`antirez/ds4`) DeepSeek V4 local inference engine, built from source. Backend-parameterized (`backend = "cpu" \| "rocm" \| "cuda"`); tracks `main` and builds per-host. Exposed as `pkgs.ds4` (cpu, default), `pkgs.ds4-rocm` (Strix Halo / gfx1151), `pkgs.ds4-cuda`. Installs the `ds4`, `ds4-server`, `ds4-bench`, `ds4-eval`, `ds4-agent` binaries plus a `ds4-download-model` helper for fetching GGUF weights. Model weights are a runtime concern and not packaged. | | `niri-touchscreen-gestures/` | Python daemon that reads multi-touch events from a touchscreen via `evdev`, detects 2/3/4-finger swipes (up/down/left/right), and dispatches configurable `niri msg action` commands. Configured via TOML; exposed as `pkgs.niri-touchscreen-gestures`. | +| `buzz-desktop/` | Block Buzz desktop (Type-2 AppImage from `block/buzz`). Wrapped with `appimageTools.wrapType2` plus extra FHS libs (`elfutils`, `zstd`, WebKit/tray, GStreamer good/bad) so it runs on NixOS. Exposed as `pkgs.buzz-desktop` (`buzz-desktop` binary + desktop entry / icons). | ## Adding a New Package diff --git a/pkgs/buzz-desktop/default.nix b/pkgs/buzz-desktop/default.nix new file mode 100644 index 0000000..09c07d3 --- /dev/null +++ b/pkgs/buzz-desktop/default.nix @@ -0,0 +1,128 @@ +{ lib +, fetchurl +, appimageTools +, runCommand +}: + +# Buzz Desktop — self-hostable workspace for humans + AI agents (block/buzz). +# +# Upstream ships a Type-2 AppImage. On NixOS we need: +# 1. FHS libs missing from appimageTools.defaultFhsEnvArgs (elfutils, zstd, …) +# 2. WebKit DMA-BUF disabled — Buzz only auto-enables this when it detects a +# real AppImage (`APPIMAGE` env). wrapAppImage runs the extracted AppDir. +# 3. Explicit GST_PLUGIN_SYSTEM_PATH_1_0 for WebKit media elements. +# 4. Drop the upstream AppRun hook that hardcodes `GDK_BACKEND=x11` (breaks +# Wayland clipboard under niri). +# 5. Drop AppImage-bundled OpenSSL 3.0 — AppRun prepends $APPDIR/usr/lib to +# LD_LIBRARY_PATH, so that old libssl shadows nixpkgs OpenSSL 3.2+ and +# breaks GStreamer plugins / curl (`OPENSSL_3.2.0' not found`). +# +# To bump: set `version`, then: +# nix hash file --type sha256 --sri <(curl -fsSL \ +# https://github.com/block/buzz/releases/download/desktop-v${version}/Buzz_${version}_amd64.AppImage) + +let + pname = "buzz-desktop"; + version = "0.5.5"; + + src = fetchurl { + url = "https://github.com/block/buzz/releases/download/desktop-v${version}/Buzz_${version}_amd64.AppImage"; + hash = "sha256-zFHK2mN9YZcSHpXwgyisGcu/7t0+mSIotVLPQ4k+K90="; + }; + + extracted = appimageTools.extractType2 { + inherit pname version src; + }; + + appdir = runCommand "${pname}-appdir" { } '' + cp -a ${extracted}/. "$out" + chmod -R u+w "$out" + + # Prefer Wayland (clipboard) over hard-coded X11 from linuxdeploy. + hook="$out/apprun-hooks/linuxdeploy-plugin-gtk.sh" + if [ -f "$hook" ]; then + sed -i \ + -e 's/^export GDK_BACKEND=x11/# export GDK_BACKEND=x11 # nixos: allow Wayland for clipboard/' \ + "$hook" + fi + + # AppRun.wrapped forces LD_LIBRARY_PATH=$APPDIR/usr/lib first. The bundled + # libssl/libcrypto are OpenSSL 3.0.x and lack OPENSSL_3.2.0, which nixpkgs + # curl / gstreamer plugins need. Remove them so the FHS OpenSSL is used + # (libssl.so.3 soname is ABI-compatible for this use). + rm -f \ + "$out/usr/lib"/libssl.so \ + "$out/usr/lib"/libssl.so.* \ + "$out/usr/lib"/libcrypto.so \ + "$out/usr/lib"/libcrypto.so.* + ''; +in +appimageTools.wrapAppImage { + inherit pname version; + src = appdir; + + extraPkgs = pkgs: with pkgs; [ + elfutils # libelf.so.1 + zstd # libzstd.so.1 + openssl # replaces removed AppImage OpenSSL + curl + webkitgtk_4_1 + libsoup_3 + libayatana-appindicator + # WebKit media: appsink/appsrc (base), autoaudiosink (good) + gst_all_1.gstreamer + gst_all_1.gst-plugins-base + gst_all_1.gst-plugins-good + gst_all_1.gst-plugins-bad + gst_all_1.gst-plugins-ugly + gst_all_1.gst-libav + # Clipboard / desktop integration + wl-clipboard + xdg-utils + ]; + + profile = '' + # WebKit blank-window workaround (Buzz sets this itself only for real AppImages). + export WEBKIT_DISABLE_DMABUF_RENDERER=1 + export WEBKIT_DISABLE_COMPOSITING_MODE=1 + + # Do not force X11 — Wayland clipboard with niri. + unset GDK_BACKEND + export GTK_USE_PORTAL=1 + + # GStreamer plugins from the FHS root (after AppDir OpenSSL removal they + # can load nixpkgs plugins that need OpenSSL 3.2+). + export GST_PLUGIN_SYSTEM_PATH_1_0="/usr/lib/gstreamer-1.0''${GST_PLUGIN_SYSTEM_PATH_1_0:+:$GST_PLUGIN_SYSTEM_PATH_1_0}" + export GST_PLUGIN_PATH="/usr/lib/gstreamer-1.0''${GST_PLUGIN_PATH:+:$GST_PLUGIN_PATH}" + ''; + + extraInstallCommands = '' + install -Dm444 ${appdir}/usr/share/applications/Buzz.desktop \ + $out/share/applications/buzz-desktop.desktop + substituteInPlace $out/share/applications/buzz-desktop.desktop \ + --replace-fail 'Exec=buzz-desktop' 'Exec=${pname}' \ + --replace-fail 'Icon=buzz-desktop' 'Icon=${pname}' + + for size in 32x32 128x128 '256x256@2'; do + if [ -f ${appdir}/usr/share/icons/hicolor/$size/apps/buzz-desktop.png ]; then + install -Dm644 ${appdir}/usr/share/icons/hicolor/$size/apps/buzz-desktop.png \ + $out/share/icons/hicolor/$size/apps/${pname}.png + fi + done + if [ ! -e $out/share/icons/hicolor/128x128/apps/${pname}.png ] \ + && [ -f ${appdir}/buzz-desktop.png ]; then + install -Dm644 ${appdir}/buzz-desktop.png \ + $out/share/icons/hicolor/128x128/apps/${pname}.png + fi + ''; + + meta = { + description = "Buzz desktop — self-hostable workspace for humans and AI agents"; + homepage = "https://github.com/block/buzz"; + changelog = "https://github.com/block/buzz/releases/tag/desktop-v${version}"; + license = lib.licenses.asl20; + sourceProvenance = [ lib.sourceTypes.binaryNativeCode ]; + platforms = [ "x86_64-linux" ]; + mainProgram = "buzz-desktop"; + }; +} diff --git a/pkgs/colibri/default.nix b/pkgs/colibri/default.nix new file mode 100644 index 0000000..1247384 --- /dev/null +++ b/pkgs/colibri/default.nix @@ -0,0 +1,115 @@ +{ lib +, stdenv +, fetchFromGitHub +, makeWrapper +, python3 +, rocmPackages +, backend ? "cpu" # one of "cpu" | "rocm" +, rocmArch ? "gfx1151" # AMD Strix Halo (Framework Desktop iGPU) +}: + +# colibrì (JustVugg/colibri) — run GLM-5.2 (744B MoE) and other frontier MoE +# models locally from a pure-C engine with experts streamed from disk. +# Upstream ships its own `make install` target (installs `coli` to bin/ and +# the engine binaries + support .py modules to libexec/colibri/), and `coli` +# already knows how to find libexec/colibri/ relative to its own install +# path — so we lean on that instead of re-deriving the file layout by hand. +# +# This derivation is backend-parameterized like pkgs/ds4: pick the GPU +# backend with `backend` ("cpu" | "rocm"). The overlay in +# modules/core/default.nix wires up `colibri` (cpu) and `colibri-rocm`. +# +# To bump: set `rev` to the new tag's commit and refresh `hash` (start from +# lib.fakeHash and copy the SRI hash from the build error). + +assert lib.elem backend [ "cpu" "rocm" ]; + +let + # Only `coli convert` / `coli bench` need these; chat/serve just exec the + # compiled engine binary. + pythonEnv = python3.withPackages ( + ps: with ps; [ + torch + safetensors + huggingface-hub + numpy + tokenizers + datasets + ] + ); + + hipMakeFlags = lib.optionals (backend == "rocm") [ + "HIP=1" + "HIP_ARCH=${rocmArch}" + "ROCM_HOME=${rocmPackages.clr}" + "HIPCC=${rocmPackages.clr}/bin/hipcc" + ]; +in +stdenv.mkDerivation { + pname = "colibri" + lib.optionalString (backend == "rocm") "-rocm"; + version = "1.5.0"; + + src = fetchFromGitHub { + owner = "JustVugg"; + repo = "colibri"; + rev = "8f512fc8c2f48ffa18cd624cd4a5bcaae4a4abfc"; # tag v1.5.0 + hash = lib.fakeHash; # replace with the real hash on first build attempt + }; + + nativeBuildInputs = [ makeWrapper ]; + buildInputs = lib.optionals (backend == "rocm") [ + rocmPackages.clr + rocmPackages.rocm-runtime + rocmPackages.hipblas + ]; + + # python3 is needed by checkPhase: `make test-c` shells out to + # `python3 tools/run_tests.py`. + nativeCheckInputs = [ python3 ]; + + # Portable AVX2 binary by default; ARCH=native for single-host builds. + ARCH = if stdenv.hostPlatform.isx86_64 then "x86-64-v3" else "native"; + + makeFlags = [ "-C" "c" ] ++ hipMakeFlags; + buildFlags = [ "colibri" "olmoe" ]; + + # Use upstream's own install target instead of hand-copying files — it + # installs `coli` to bin/ and the engine + support modules to + # libexec/colibri/, which `coli` already knows how to find on its own. + installPhase = '' + runHook preInstall + make -C c install PREFIX=$out ARCH="$ARCH" ${lib.escapeShellArgs hipMakeFlags} + runHook postInstall + ''; + + # coli ships `#!/usr/bin/env python3`; point env at pythonEnv so the + # converter/bench subcommands resolve torch etc. No COLI_ENGINE/PYTHONPATH + # override needed — coli auto-detects bin/ + libexec/colibri/ relative to + # its own installed path. + postFixup = '' + wrapProgram $out/bin/coli --prefix PATH : ${lib.makeBinPath [ pythonEnv ]} + ''; + + # rocm skips: test-c would relink the engine without HIP=1, clobbering the + # GPU build, and needs real hardware anyway. + doCheck = backend == "cpu"; + checkPhase = '' + runHook preCheck + cd c + make test-c + cd .. + runHook postCheck + ''; + + meta = { + description = "Run GLM-5.2 (744B MoE) and other frontier MoE models on hardware you already own"; + homepage = "https://github.com/JustVugg/colibri"; + license = lib.licenses.asl20; + sourceProvenance = [ lib.sourceTypes.fromSource ]; + platforms = + if backend == "rocm" + then lib.platforms.linux + else lib.platforms.linux ++ lib.platforms.darwin; + mainProgram = "coli"; + }; +} diff --git a/pkgs/colibri/flake.nix b/pkgs/colibri/flake.nix deleted file mode 100644 index fde6295..0000000 --- a/pkgs/colibri/flake.nix +++ /dev/null @@ -1,194 +0,0 @@ -{ - description = "colibrì — run GLM-5.2 (744B MoE) on a consumer machine with ~25 GB RAM"; - - # Reproducibility: these inputs track a branch, so torch/numpy/etc. float across - # rebuilds. For deterministic builds run `nix flake lock` once and COMMIT the - # generated flake.lock (it pins each input to a commit SHA). (#D2) - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; - flake-utils.url = "github:numtide/flake-utils"; - }; - - outputs = - { self - , nixpkgs - , flake-utils - , - }: - flake-utils.lib.eachDefaultSystem ( - system: - let - pkgs = import nixpkgs { inherit system; }; - - # Python with the packages needed by the offline converter tools - pythonEnv = pkgs.python3.withPackages ( - ps: - with ps; [ - torch - safetensors - huggingface-hub - numpy - tokenizers - datasets - ] - ); - - colibri = pkgs.stdenv.mkDerivation { - pname = "colibri"; - version = "1.0"; - src = ./.; - - nativeBuildInputs = with pkgs; [ makeWrapper ]; - - buildInputs = with pkgs; [ - gcc - gmp - ]; - - # python3 is needed by checkPhase: `make test-c` shells out to - # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). - nativeCheckInputs = with pkgs; [ python3 ]; - - # Use x86-64-v3 (AVX2) for a portable binary; override with ARCH=native for local builds - ARCH = - if pkgs.stdenv.hostPlatform.isx86_64 - then "x86-64-v3" - else "native"; - - buildPhase = '' - runHook preBuild - make -C c glm ARCH="$ARCH" - runHook postBuild - ''; - - installPhase = '' - runHook preInstall - - # Self-contained layout under $out/lib/colibri that mirrors the - # source tree `coli` runs in (see the path-resolution logic at the - # top of c/coli): the engine, the coli CLI script, the support - # modules it imports (openai_server.py, resource_plan.py, - # doctor.py), and tools/ all sit next to each other. - mkdir -p $out/lib/colibri/tools $out/bin - cp c/glm $out/lib/colibri/glm - cp c/coli $out/lib/colibri/coli - chmod +x $out/lib/colibri/coli - cp c/openai_server.py c/resource_plan.py c/doctor.py c/version.py \ - $out/lib/colibri/ - cp -r c/tools/* $out/lib/colibri/tools/ - - # $out/bin holds the user-facing entry points. - ln -s ../lib/colibri/glm $out/bin/glm - - # Wrap coli: point it at the bundled engine (COLI_ENGINE) so it is - # found by default, and at the module dir (PYTHONPATH) so - # `import openai_server` / `resource_plan` / `doctor` resolve. - makeWrapper ${pythonEnv}/bin/python $out/bin/coli \ - --add-flags "$out/lib/colibri/coli" \ - --set-default COLI_ENGINE "$out/lib/colibri/glm" \ - --set PYTHONPATH "$out/lib/colibri:${pythonEnv}/${pkgs.python3.sitePackages}" - runHook postInstall - ''; - - checkPhase = '' - runHook preCheck - cd c - make test-c - cd .. - runHook postCheck - ''; - - doCheck = true; - - meta = with pkgs.lib; { - description = "Run GLM-5.2 (744B MoE) on a consumer machine with ~25 GB RAM"; - homepage = "https://github.com/JustVugg/colibri"; - license = licenses.asl20; - platforms = with platforms; linux ++ darwin; - mainProgram = "coli"; - }; - }; - - # ROCm/HIP variant: same package as `colibri`, plus the optional GPU - # expert tier built for AMD via the engine's HIP=1 path (see c/Makefile). - # Tuned for the Framework Desktop's Radeon 8060S/8050S iGPU == gfx1151. - # Linux-only (HIP=1 errors out on non-Linux in the Makefile). - # - # nix build .#colibri-rocm - # COLI_MODEL=/nvme/glm52_i4 CUDA_EXPERT_GB=auto nix run .#colibri-rocm -- chat - # - # If a gfx1151 kernel misbehaves on your ROCm version, run with - # HSA_OVERRIDE_GFX_VERSION=11.0.0 - colibri-rocm = colibri.overrideAttrs (old: { - pname = "colibri-rocm"; - - buildInputs = - old.buildInputs - ++ (with pkgs.rocmPackages; [ clr rocm-runtime hipblas ]); - - buildPhase = '' - runHook preBuild - make -C c glm \ - ARCH="$ARCH" \ - HIP=1 \ - HIP_ARCH="gfx1151" \ - ROCM_HOME="${pkgs.rocmPackages.clr}" \ - HIPCC="${pkgs.rocmPackages.clr}/bin/hipcc" - runHook postBuild - ''; - - # test-c would relink the engine WITHOUT HIP=1 (clobbering the GPU - # build), and the GPU path needs real hardware — so skip the suite. - doCheck = false; - - meta = old.meta // { platforms = pkgs.lib.platforms.linux; }; - }); - in - { - packages = - { - default = colibri; - inherit colibri; - } - # Only expose the ROCm package on Linux (HIP build is Linux-only). - // pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { - inherit colibri-rocm; - }; - - apps = { - default = { - type = "app"; - program = pkgs.lib.getExe colibri; - }; - glm = { - type = "app"; - program = "${colibri}/share/colibri/glm"; - }; - }; - - formatter = pkgs.alejandra; - - devShells.default = pkgs.mkShell { - inputsFrom = [ colibri ]; - - packages = with pkgs; [ - pythonEnv - gcc - gnumake - clang-tools # clangd / clang-tidy for IDE support - pkg-config - ]; - - shellHook = '' - echo "🐦 colibrì dev shell" - echo " gcc: $(gcc --version | head -1)" - echo " python: $(python3 --version)" - echo "" - echo "Build the engine: make -C c glm" - echo "Run the converter: python c/coli convert --model /path/to/glm52_i4" - echo "Chat: COLI_MODEL=/path/to/glm52_i4 ./c/glm ..." - ''; - }; - } - ); -} diff --git a/pkgs/ds4/default.nix b/pkgs/ds4/default.nix index c7e4c11..8e736ed 100644 --- a/pkgs/ds4/default.nix +++ b/pkgs/ds4/default.nix @@ -111,13 +111,13 @@ let in stdenv.mkDerivation (finalAttrs: { pname = "ds4" + lib.optionalString (backend != "cpu") "-${backend}"; - version = "0-unstable-2026-06-17"; + version = "0-unstable-2026-08-05"; src = fetchFromGitHub { owner = "antirez"; repo = "ds4"; - rev = "80ebbc396aee40eedc1d829222f3362d10fa4c6c"; - hash = "sha256-Ieuc72GHZs20ModQfnvI5Me31n4Pj+WFYtsuqaKJceo="; + rev = "b0309611041655f4e45671cfd9c9886aff161406"; + hash = "sha256-yBPQqX8oI9fElGiXfz72iWwIS8ZQpK4Bl9TWLZSa6JU="; }; enableParallelBuilding = true; diff --git a/pkgs/ds4/user-guide.md b/pkgs/ds4/user-guide.md index c7d2a1d..36e4a39 100644 --- a/pkgs/ds4/user-guide.md +++ b/pkgs/ds4/user-guide.md @@ -24,23 +24,42 @@ OpenAI-compatible client at. ## 1. Get the weights -Weights are **not** packaged. They live at the project's Hugging Face repo -[`antirez/deepseek-v4-gguf`](https://huggingface.co/antirez/deepseek-v4-gguf) -(public, MIT, no token needed) and only these custom GGUFs work — not arbitrary -DeepSeek GGUFs. +Weights are **not** packaged. They live at the project's Hugging Face repos +([`antirez/deepseek-v4-gguf`](https://huggingface.co/antirez/deepseek-v4-gguf) +for DeepSeek V4, [`antirez/GLM-5.2-GGUF`](https://huggingface.co/antirez/GLM-5.2-GGUF) +and [`unsloth/GLM-5.2-GGUF`](https://huggingface.co/unsloth/GLM-5.2-GGUF) for +GLM 5.2) — public, MIT, no token needed — and only these custom GGUFs work, not +arbitrary DeepSeek/GLM GGUFs. `ds4-download-model` writes into `./gguf` and links `./ds4flash.gguf` in the current directory (override the root with `DS4_HOME`, or the gguf dir with `DS4_GGUF_DIR`): ```sh -ds4-download-model q2-imatrix # ~81 GB, the pick for 96/128 GB machines -ds4-download-model --help # all targets (q4-imatrix, pro-*, mtp, ...) +# DeepSeek V4 Flash (0731 checkpoints) +ds4-download-model ds4f-q2 # ~81 GB, the pick for 96/128 GB machines +ds4-download-model ds4f-q2-q4 # ~98 GB, last 6 expert layers at q4 +ds4-download-model ds4f-q4 # ~153 GB, for >=256 GB machines +ds4-download-model ds4f-mxfp4 # ~156 GB, native MXFP4 (Metal/CUDA) +ds4-download-model ds4f-dspark # ~6 GB, optional DSpark speculative draft + +# DeepSeek V4 PRO (512 GB / distributed) +ds4-download-model pro-q2-imatrix # single-file PRO q2 +ds4-download-model pro-q4-split # both halves of PRO Q4 split + +# GLM 5.2 +ds4-download-model glm-unsloth-q4 # Unsloth UD-Q4_K_XL, 11 shards +ds4-download-model glm-antirez-iq2xxs # routed IQ2_XXS, reduced-memory +ds4-download-model glm-antirez-q2 # routed Q2_K, ~262 GB +ds4-download-model glm-antirez-q4 # routed Q4_K, ~434 GB + +ds4-download-model --help # full target list ``` -Pick by machine memory: `q2-imatrix` (128 GB), `q4-imatrix` (≥256 GB), -`pro-*` (512 GB / distributed). The small/Flash quants download via `curl`; the -huge **PRO** files need the `hf` CLI (not bundled — `nix shell nixpkgs#huggingface-hub`). +Pick by machine memory: `ds4f-q2` (96/128 GB), `ds4f-q4` (>=256 GB), +`pro-*` (512 GB / distributed), `glm-*` (per-target sizes above). The small +Flash quants download via `curl`; the MXFP4, PRO, and GLM files need the `hf` +CLI (not bundled — `nix shell nixpkgs#huggingface-hub`). ## 2. Run it @@ -49,7 +68,14 @@ huge **PRO** files need the `hf` CLI (not bundled — `nix shell nixpkgs#hugging ds4 # explicit model path -ds4 -m /path/to/gguf/DeepSeek-V4-Flash-IQ2XXS-...-imatrix.gguf +ds4 -m /path/to/gguf/DeepSeek-V4-Flash-IQ2XXS-...-imatrix-0731.gguf + +# DSpark speculative decoding (greedy only; needs the ds4f-dspark support GGUF) +ds4 -m ./ds4flash.gguf \ + --dspark --mtp ./gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf --temp 0 + +# GLM 5.2 (experimental greedy MTP via --glm-mtp) +ds4 -m gguf/GLM-5.2-UD-IQ2_XXS_RoutedIQ2XXS_blk78Q2K.gguf --glm-mtp-timing --temp 0 # HTTP server on the LAN ds4-server -m ./ds4flash.gguf \