ds4 glm support
This commit is contained in:
1 parent
4cc2123eff
commit
d090ebaad8
6 files changed
+40
-16
No files matched your search
@@ -188,6 +188,11 @@
|
||||
# GPU variants (ds4-rocm/ds4-cuda) are overlay-only — see modules/core.
|
||||
ds4 = lib.pkgs.${system}.ds4;
|
||||
claude-desktop = pkgs.callPackage ./pkgs/claude-desktop { };
|
||||
# Buzz Desktop AppImage (block/buzz) with NixOS FHS extras.
|
||||
buzz-desktop = lib.pkgs.${system}.buzz-desktop;
|
||||
# CPU variant of colibrì (JustVugg/colibri); the ROCm variant
|
||||
# (colibri-rocm) is overlay-only — see modules/core.
|
||||
colibri = lib.pkgs.${system}.colibri;
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ core/
|
||||
|
||||
## What lives here
|
||||
|
||||
- **Overlays** (`default.nix`): exposes `pkgs.grok-cli` (from `pkgs/grok-cli`) and patches
|
||||
- **Overlays** (`default.nix`): exposes `pkgs.grok-cli` (from `pkgs/grok-cli`), `pkgs.buzz-desktop` (from `pkgs/buzz-desktop`), and patches
|
||||
`python3Packages.pipx` to skip its flaky install check on nixpkgs 26.05.
|
||||
- **Distributed builds** (`default.nix`): registers `bigbox1.lab.internal` as a remote
|
||||
`x86_64-linux` build machine and enables `builders-use-substitutes`.
|
||||
|
||||
@@ -17,6 +17,7 @@ services/
|
||||
├── dockge.nix # skyg.nixos.server.services.dockge — compose stack manager (NFS volumes)
|
||||
├── signal-cli.nix # skyg.nixos.server.services.signal-cli — signal-cli HTTP daemon (Hermes Signal bridge)
|
||||
├── openclaw.nix # skyg.nixos.server.services.openclaw — AI gateway (NOT imported by default)
|
||||
├── colibri.nix # skyg.nixos.server.services.colibri — coli serve HTTP API (GLM-5.2/OLMoE local inference)
|
||||
└── comfyui/ # skyg.nixos.server.services.comfyui — ComfyUI with ROCm/CUDA GPU support
|
||||
```
|
||||
|
||||
@@ -29,6 +30,10 @@ services/
|
||||
- **`openclaw.nix`** is a native systemd service (with agenix-token support and security
|
||||
hardening). Note: it is **not** listed in `default.nix`'s imports, so it must be imported
|
||||
explicitly if used.
|
||||
- **`colibri.nix`** runs `coli serve` (colibrì's OpenAI-compatible API) as a native systemd
|
||||
service, mirroring `services/ds4.nix`'s shape. `package` picks the backend (`pkgs.colibri`
|
||||
cpu / `pkgs.colibri-rocm`); `environmentFile` supplies `COLI_API_KEY` via agenix instead of
|
||||
a CLI flag.
|
||||
|
||||
## Option Namespace
|
||||
|
||||
@@ -40,6 +45,7 @@ skyg.nixos.server.services.scrypted.enable
|
||||
skyg.nixos.server.services.dockge.{enable,openFirewall,port,volumes}
|
||||
skyg.nixos.server.services.signal-cli.{enable,host,port,configDir,account,environmentFile,openFirewall}
|
||||
skyg.nixos.server.services.openclaw.{enable,port,gatewayToken,...}
|
||||
skyg.nixos.server.services.colibri.{enable,package,model,host,port,environmentFile,openFirewall,...}
|
||||
skyg.nixos.server.services.comfyui.{enable,mode,port,openFirewall}
|
||||
```
|
||||
|
||||
|
||||
@@ -44,9 +44,7 @@ in
|
||||
enable = mkEnableOption
|
||||
"colibri-serve, the OpenAI-compatible HTTP API for colibrì (GLM-5.2/OLMoE local inference).";
|
||||
|
||||
package = mkPackageOption pkgs "colibri" {
|
||||
default = pkgs.colibri;
|
||||
};
|
||||
package = mkPackageOption pkgs "colibri" { };
|
||||
|
||||
user = mkOption {
|
||||
description = "The user to run coli serve as.";
|
||||
@@ -104,12 +102,13 @@ in
|
||||
type = types.nullOr types.str;
|
||||
};
|
||||
|
||||
apiKeyFile = mkOption {
|
||||
environmentFile = mkOption {
|
||||
description = ''
|
||||
Path to a file containing the bearer token clients must send. Read at
|
||||
service start and exported as COLI_API_KEY (never passed as a CLI
|
||||
argument, so it doesn't show up in `ps`). Leave null to run without
|
||||
authentication (fine for host-only binding on 127.0.0.1).
|
||||
Path to an EnvironmentFile (e.g. an agenix-decrypted secret) providing
|
||||
COLI_API_KEY, the bearer token clients must send. Avoids putting the
|
||||
key in the world-readable Nix store or on the process command line.
|
||||
Leave null to run without authentication (fine for host-only binding
|
||||
on 127.0.0.1).
|
||||
'';
|
||||
default = null;
|
||||
example = "/run/agenix/colibri-api-key";
|
||||
@@ -280,15 +279,16 @@ in
|
||||
User = cfg.user;
|
||||
Group = cfg.group;
|
||||
|
||||
# Construct the ExecStart as a shell script so we can export env vars
|
||||
# that would otherwise be lost with a raw binary + Environment= (e.g.
|
||||
# COLI_MODEL_MIRROR, COLI_API_KEY read from a file). Use
|
||||
# Construct the ExecStart as a shell script so we can export the extra
|
||||
# env vars coli only reads (not flags), e.g. COLI_MODEL_MIRROR. Use
|
||||
# writeShellScript for a clean script in the store.
|
||||
EnvironmentFile =
|
||||
mkIf (cfg.environmentFile != null) cfg.environmentFile;
|
||||
|
||||
ExecStart = pkgs.writeShellScript "colibri-serve-start" ''
|
||||
set -eu
|
||||
${optionalString (cfg.modelMirror != null) "export COLI_MODEL_MIRROR=${escapeShellArg cfg.modelMirror}"}
|
||||
${optionalString (cfg.diskWeights != null) "export COLI_DISK_WEIGHTS=${escapeShellArg cfg.diskWeights}"}
|
||||
${optionalString (cfg.apiKeyFile != null) ''export COLI_API_KEY="$(cat ${escapeShellArg cfg.apiKeyFile})"''}
|
||||
${concatStringsSep "\n" (mapAttrsToList (name: value: "export ${name}=${escapeShellArg value}") cfg.environment)}
|
||||
exec ${colibriPkg}/bin/coli serve ${serverArgs} ${escapeShellArgs cfg.extraArgs}
|
||||
'';
|
||||
|
||||
+13
-1
@@ -4,6 +4,7 @@
|
||||
, makeWrapper
|
||||
, curl
|
||||
, cacert
|
||||
, python3
|
||||
, rocmPackages
|
||||
, cudaPackages
|
||||
# backend selects the build target / toolchain.
|
||||
@@ -36,6 +37,17 @@
|
||||
assert lib.elem backend [ "cpu" "rocm" "cuda" ];
|
||||
|
||||
let
|
||||
# `hf` CLI used by ds4-download-model for the large/sharded GGUF files
|
||||
# (MXFP4, PRO, GLM). huggingface-hub provides the `hf` binary; hf-xet is the
|
||||
# optional Xet backend it uses to speed up big transfers. Default python3 is
|
||||
# 3.14, so this resolves to python314Packages.{huggingface-hub,hf-xet}.
|
||||
pythonEnv = python3.withPackages (
|
||||
ps: with ps; [
|
||||
huggingface-hub
|
||||
hf-xet
|
||||
]
|
||||
);
|
||||
|
||||
# ROCm libraries the gfx kernels and link flags (-lhipblas -lhipblaslt) need.
|
||||
rocmInputs = [
|
||||
rocmPackages.clr # provides hipcc + HIP runtime
|
||||
@@ -163,7 +175,7 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
# environment doesn't already set one.
|
||||
postFixup = ''
|
||||
wrapProgram "$out/bin/ds4-download-model" \
|
||||
--prefix PATH : ${lib.makeBinPath [ curl ]} \
|
||||
--prefix PATH : ${lib.makeBinPath [ curl pythonEnv ]} \
|
||||
--set-default SSL_CERT_FILE ${cacert}/etc/ssl/certs/ca-bundle.crt
|
||||
'';
|
||||
|
||||
|
||||
@@ -58,8 +58,9 @@ ds4-download-model --help # full target list
|
||||
|
||||
Pick by machine memory: `ds4f-q2` (96/128 GB), `ds4f-q4` (>=256 GB),
|
||||
`pro-*` (512 GB / distributed), `glm-*` (per-target sizes above). The small
|
||||
Flash quants download via `curl`; the MXFP4, PRO, and GLM files need the `hf`
|
||||
CLI (not bundled — `nix shell nixpkgs#huggingface-hub`).
|
||||
Flash quants download via `curl`; the MXFP4, PRO, and GLM files use the `hf`
|
||||
CLI (bundled in the package via `huggingface-hub` + `hf-xet`), which
|
||||
`ds4-download-model` puts on its PATH automatically.
|
||||
|
||||
## 2. Run it
|
||||
|
||||
|
||||
Reference in new issue
Block a user