From dd4965f030b7b0ab3efe0072e539707803cdb19b Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Sun, 1 Nov 2026 14:26:16 -0500 Subject: [PATCH] bazel --- flake.lock | 231 +++++++- flake.nix | 5 + hosts/hl-fwdesk/services/slackerbot.nix | 18 +- hosts/hl-fws1/services/slackerbot.nix | 18 +- modules/nixos/server/services/ABOUTME.md | 10 + modules/nixos/server/services/bazel-build.nix | 516 ++++++++++++++++++ modules/nixos/server/services/default.nix | 1 + pkgs/colibri/default.nix | 4 +- 8 files changed, 784 insertions(+), 19 deletions(-) create mode 100644 modules/nixos/server/services/bazel-build.nix diff --git a/flake.lock b/flake.lock index 8b0103a..7881972 100644 --- a/flake.lock +++ b/flake.lock @@ -279,6 +279,21 @@ "type": "github" } }, + "crane_2": { + "locked": { + "lastModified": 1780029330, + "narHash": "sha256-fWFKEKB5CP+NO8/3uOaEZIVbd03mvDJ//VsnKkXty08=", + "owner": "ipetkov", + "repo": "crane", + "rev": "6823b493a0bc2142082075576efa6633b537197e", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "crate2nix": { "flake": false, "locked": { @@ -614,6 +629,22 @@ "type": "github" } }, + "flake-compat_10": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, "flake-compat_2": { "flake": false, "locked": { @@ -829,6 +860,24 @@ } }, "flake-parts_5": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1769996383, + "narHash": "sha256-AnYjnFWgS49RlqX7LrC4uA+sCCDBj0Ry/WOJ5XWAsa0=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "57928607ea566b5db3ad13af0e57e921e6b12381", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_6": { "inputs": { "nixpkgs-lib": [ "stylix", @@ -849,6 +898,21 @@ "type": "github" } }, + "flake-root": { + "locked": { + "lastModified": 1723604017, + "narHash": "sha256-rBtQ8gg+Dn4Sx/s+pvjdq3CB2wQNzx9XGFq/JVGCB6k=", + "owner": "srid", + "repo": "flake-root", + "rev": "b759a56851e10cb13f6b8e5698af7b59c44be26e", + "type": "github" + }, + "original": { + "owner": "srid", + "repo": "flake-root", + "type": "github" + } + }, "flox": { "inputs": { "crane": "crane", @@ -1019,6 +1083,29 @@ "type": "github" } }, + "git-hooks_4": { + "inputs": { + "flake-compat": "flake-compat_9", + "gitignore": "gitignore_2", + "nixpkgs": [ + "nativelink", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1772024342, + "narHash": "sha256-+eXlIc4/7dE6EcPs9a2DaSY3fTA9AE526hGqkNID3Wg=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "6e34e97ed9788b17796ee43ccdbaf871a5c2b476", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "gitignore": { "inputs": { "nixpkgs": [ @@ -1041,6 +1128,28 @@ "type": "github" } }, + "gitignore_2": { + "inputs": { + "nixpkgs": [ + "nativelink", + "git-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, "gnome-shell": { "flake": false, "locked": { @@ -1579,7 +1688,7 @@ "hyprutils": "hyprutils_3", "hyprwayland-scanner": "hyprwayland-scanner_3", "hyprwire": "hyprwire_3", - "nixpkgs": "nixpkgs_11", + "nixpkgs": "nixpkgs_12", "pre-commit-hooks": "pre-commit-hooks_3", "systems": "systems_3", "xdph": "xdph_2" @@ -2197,6 +2306,31 @@ "type": "github" } }, + "nativelink": { + "inputs": { + "crane": "crane_2", + "flake-parts": "flake-parts_5", + "flake-root": "flake-root", + "git-hooks": "git-hooks_4", + "nix2container": "nix2container", + "nixpkgs": "nixpkgs_11", + "rust-overlay": "rust-overlay_3" + }, + "locked": { + "lastModified": 1793212176, + "narHash": "sha256-k9Vzp5nOq2D/JnYfnG1vlIZrca8dt7F6qz6SwirXLec=", + "owner": "TraceMachina", + "repo": "nativelink", + "rev": "bd336137d794a6a4e7ac32e561ab55e2d7e70ff3", + "type": "github" + }, + "original": { + "owner": "TraceMachina", + "ref": "v1.8.3", + "repo": "nativelink", + "type": "github" + } + }, "nix": { "inputs": { "bdwgc": "bdwgc", @@ -2313,6 +2447,28 @@ "type": "github" } }, + "nix2container": { + "inputs": { + "nixpkgs": [ + "nativelink", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1775487831, + "narHash": "sha256-2lguQpLPQaxpQCJjXhmEEAfabwsAhkP29Z7fgLzHARA=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e", + "type": "github" + } + }, "nix_2": { "inputs": { "flake-parts": "flake-parts_2", @@ -2531,7 +2687,7 @@ }, "nixos-hardware_2": { "inputs": { - "nixpkgs": "nixpkgs_12" + "nixpkgs": "nixpkgs_13" }, "locked": { "lastModified": 1792491056, @@ -2596,6 +2752,21 @@ "type": "github" } }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1769909678, + "narHash": "sha256-cBEymOf4/o3FD5AZnzC3J9hLbiZ+QDT/KDuyHXVJOpM=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "72716169fe93074c333e8d0173151350670b824c", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -2675,6 +2846,22 @@ } }, "nixpkgs_11": { + "locked": { + "lastModified": 1787498568, + "narHash": "sha256-9i/VTdusq/+NM/tz+J1Re+ojkMB8MBf0QshnYfzHz30=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "56c02bc00adcf003215cc4bd996d6efaf4cff188", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_12": { "locked": { "lastModified": 1788404924, "narHash": "sha256-lhEhY8X5EgkQ/eg6IFz4cc8jRuSYSvnxx1al7d1dvZ0=", @@ -2690,7 +2877,7 @@ "type": "github" } }, - "nixpkgs_12": { + "nixpkgs_13": { "locked": { "lastModified": 1789546076, "narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=", @@ -2703,7 +2890,7 @@ "url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz" } }, - "nixpkgs_13": { + "nixpkgs_14": { "locked": { "lastModified": 1792503283, "narHash": "sha256-vhe2jyvRCgufWPqDuDq1S3t3ypE+ka46IFutV02xKso=", @@ -2984,7 +3171,7 @@ }, "pre-commit-hooks_3": { "inputs": { - "flake-compat": "flake-compat_9", + "flake-compat": "flake-compat_10", "nixpkgs": [ "nixarchy", "hyprland", @@ -3102,12 +3289,13 @@ "hyprland": "hyprland", "hyprlauncher": "hyprlauncher", "my-nixpkgs": "my-nixpkgs", + "nativelink": "nativelink", "nix-darwin": "nix-darwin", "nix-flatpak": "nix-flatpak", "nix-index-database": "nix-index-database", "nixarchy": "nixarchy", "nixos-hardware": "nixos-hardware_2", - "nixpkgs": "nixpkgs_13", + "nixpkgs": "nixpkgs_14", "nixpkgs-unstable": "nixpkgs-unstable", "noctalia": "noctalia", "robotnix": "robotnix", @@ -3178,6 +3366,27 @@ "type": "github" } }, + "rust-overlay_3": { + "inputs": { + "nixpkgs": [ + "nativelink", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1787713867, + "narHash": "sha256-ZRYTEvDb8QZLCHRoMyIZle9V3Bvw905m0teeo1RvI7M=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "494680f9ffa2b4bf9f2df0f442c7096e44adfa00", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, "skyg-linux-tracker": { "inputs": { "nixpkgs": [ @@ -3225,11 +3434,11 @@ ] }, "locked": { - "lastModified": 1793556114, - "narHash": "sha256-TxDFlMNXyd9GXHa1FUCfZjauf5W//m73aMlhArE4vAQ=", + "lastModified": 1793561072, + "narHash": "sha256-5LTBU0P1Qbsu6SbO7iy73xt+djzpWn+qXtdJhdjdWqA=", "ref": "refs/heads/main", - "rev": "0ac61b4eceb017ccf94f86c8cde30d1fb6db3d3e", - "revCount": 2, + "rev": "3ba96528bbb56b4b4b3bde5b112cd839510ce8c3", + "revCount": 11, "type": "git", "url": "https://gitea.skyg.ca/ari/slacker-bot.git" }, @@ -3283,7 +3492,7 @@ "base16-helix": "base16-helix", "base16-vim": "base16-vim", "firefox-gnome-theme": "firefox-gnome-theme", - "flake-parts": "flake-parts_5", + "flake-parts": "flake-parts_6", "gnome-shell": "gnome-shell", "nixpkgs": [ "nixpkgs" diff --git a/flake.nix b/flake.nix index cd35f3a..cc41a32 100644 --- a/flake.nix +++ b/flake.nix @@ -25,6 +25,9 @@ url = "git+https://gitea.skyg.ca/ari/slacker-bot.git"; inputs.nixpkgs.follows = "nixpkgs-unstable"; }; + # Bazel remote build server for slackerbot containers. Pinned: its config + # format changes between releases (see bazel-build.nix). + nativelink.url = "github:TraceMachina/nativelink/v1.8.3"; # Headless Hyprland session streamed over a Unix socket. hhstream = { url = "git+ssh://gitea@gitea.app.internal:22/Hermenix/hyprland-headless-stream-plugin.git"; @@ -126,6 +129,7 @@ , robotnix , skyg-linux-tracker , slackerbot + , nativelink , hhstream , nixpkgs-unstable , flox @@ -225,6 +229,7 @@ robotnix skyg-linux-tracker slackerbot + nativelink hhstream flox determinate diff --git a/hosts/hl-fwdesk/services/slackerbot.nix b/hosts/hl-fwdesk/services/slackerbot.nix index 1a0c4c1..fe6613e 100644 --- a/hosts/hl-fwdesk/services/slackerbot.nix +++ b/hosts/hl-fwdesk/services/slackerbot.nix @@ -1,11 +1,16 @@ -{ config, lib, pkgs, slackerbot, skyg-secrets, ... }: +{ config, lib, pkgs, skyg-secrets, ... }: let ports = { cluster = 7946; }; homes = "/data/slackerbot"; secrets = [ "slackerbot-slack" "slackerbot-goose" "slackerbot-cluster" ]; + bazel = config.skyg.nixos.server.services.bazel-build; settings = { - # The system each container boots; updated on every rebuild. - system = "${slackerbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}"; + # The system each container boots (slackerbot's agent + bazel/rbuild); + # updated on every rebuild. + system = "${bazel.agentSystem}"; + # /handoff and /outputs for remote Bazel builds on this host. + rw_binds = bazel.binds.rw; + ro_binds = bazel.binds.ro; machines_dir = "/var/lib/machines"; state_dir = "/var/lib/slackerbot"; agent_cwd = "/root"; @@ -56,6 +61,13 @@ in clusterPort = ports.cluster; configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings; }; + # Bazel build server for the containers (16 cores / 32 threads). + skyg.nixos.server.services.bazel-build = { + enable = true; + cpuCount = 32; + cacheGB = 200; + }; + # The module takes one env file; these add to it. systemd.services.slackerbot.serviceConfig.EnvironmentFile = map (name: config.age.secrets.${name}.path) secrets; diff --git a/hosts/hl-fws1/services/slackerbot.nix b/hosts/hl-fws1/services/slackerbot.nix index c5af7de..8414706 100644 --- a/hosts/hl-fws1/services/slackerbot.nix +++ b/hosts/hl-fws1/services/slackerbot.nix @@ -1,9 +1,14 @@ -{ config, pkgs, slackerbot, skyg-secrets, ... }: +{ config, pkgs, skyg-secrets, ... }: let homes = "/var/lib/slackerbot/homes"; + bazel = config.skyg.nixos.server.services.bazel-build; settings = { - # The system each container boots; updated on every rebuild. - system = "${slackerbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}"; + # The system each container boots (slackerbot's agent + bazel/rbuild); + # updated on every rebuild. + system = "${bazel.agentSystem}"; + # /handoff and /outputs for remote Bazel builds on this host. + rw_binds = bazel.binds.rw; + ro_binds = bazel.binds.ro; machines_dir = "/var/lib/machines"; state_dir = "/var/lib/slackerbot"; agent_cwd = "/root"; @@ -43,6 +48,13 @@ in environmentFile = config.age.secrets.slackerbot-cluster.path; }; + # Bazel build server for the containers (4 cores / 8 threads). + skyg.nixos.server.services.bazel-build = { + enable = true; + cpuCount = 8; + cacheGB = 50; + }; + # Container links belong to systemd-networkd. networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; networking.dhcpcd.denyInterfaces = [ "ve-*" ]; diff --git a/modules/nixos/server/services/ABOUTME.md b/modules/nixos/server/services/ABOUTME.md index 7e12f6c..d97cfc1 100644 --- a/modules/nixos/server/services/ABOUTME.md +++ b/modules/nixos/server/services/ABOUTME.md @@ -11,6 +11,7 @@ independent `skyg.nixos.server.services..enable` flag. Most run as OCI con services/ ├── default.nix # imports the services below ├── ai-services.nix # skyg.nixos.server.services.ai — Wyoming (whisper/piper/wake) +├── bazel-build.nix # skyg.nixos.server.services.bazel-build — NativeLink Bazel build server for slackerbot containers ├── audiobookshelf.nix # skyg.nixos.server.services.audiobookshelf — audiobook/podcast server ├── jellyfin.nix # skyg.nixos.server.services.jellyfin — media server (uid/gid 7777) ├── signal-cli.nix # skyg.nixos.server.services.signal-cli — signal-cli HTTP daemon (Hermes Signal bridge) @@ -22,6 +23,14 @@ services/ - **`ai-services.nix`** openwakeword as containers and enables NixOS-native Wyoming faster-whisper + piper. Opens 11434/10200/10300/10400. +- **`bazel-build.nix`** runs NativeLink (from the `nativelink` flake input, pinned + to v1.8.3 because its config format changes) plus `build-materializer`, which + hardlinks finished outputs from the cache into `/var/lib/build-outputs/`. + Containers run `rbuild //target` (finds the host as the default gateway) and read + results at `/outputs`. It also exposes `agentSystem` (slackerbot's agent + bazel, + bazelrc, rbuild) and `binds` (slackerbot `ro_binds`/`rw_binds`). Used on hl-fws1 + and hl-fwdesk. Slackerbot must create `ro_binds` source folders, or containers fail + to start before their first build. - **`colibri.nix`** runs `coli serve` (colibrì's OpenAI-compatible API) as a native systemd service. `package` picks the backend (`pkgs.colibri` cpu / `pkgs.colibri-rocm`); `environmentFile` supplies `COLI_API_KEY` via agenix instead of @@ -32,6 +41,7 @@ services/ ``` skyg.nixos.server.services.ai.enable skyg.nixos.server.services.audiobookshelf.{enable,package,configDir,dataDir} +skyg.nixos.server.services.bazel-build.{enable,cpuCount,cacheGB,port,agentSystem,binds} skyg.nixos.server.services.jellyfin.enable skyg.nixos.server.services.signal-cli.{enable,host,port,configDir,account,environmentFile,openFirewall} skyg.nixos.server.services.colibri.{enable,package,model,host,port,environmentFile,openFirewall,...} diff --git a/modules/nixos/server/services/bazel-build.nix b/modules/nixos/server/services/bazel-build.nix new file mode 100644 index 0000000..0a5ac25 --- /dev/null +++ b/modules/nixos/server/services/bazel-build.nix @@ -0,0 +1,516 @@ +# Bazel build server for slackerbot containers on the same host. +# One NativeLink (cache + scheduler + worker) runs builds on the host CPUs. +# Final outputs are not sent over the network: a host service hardlinks them +# from NativeLink's cache into a folder each container sees at /outputs. +# +# container: rbuild //app ──grpc──▶ NativeLink (host) ──▶ host CPUs +# │ writes build events to /handoff/bep.json +# ▼ +# host: build-materializer follows bep.json and hardlinks each finished +# output into /var/lib/build-outputs// +# ▼ +# container: files appear at /outputs (read-only) +# +# slackerbot mounts the folders through `binds`. Containers run as root with +# no private user IDs, so no uid mapping is needed. +{ config, lib, pkgs, nativelink, slackerbot, ... }: + +with lib; + +let + cfg = config.skyg.nixos.server.services.bazel-build; + system = pkgs.stdenv.hostPlatform.system; + + stateDir = "/var/lib/nativelink"; + handoffRoot = "/var/lib/build-handoff"; + # Must be on the same disk as stateDir, or hardlinks fail (falls back to copies). + outputsRoot = "/var/lib/build-outputs"; + workerPort = 50061; # host only + + # Tools build actions run with. Same store paths on the host and in containers. + toolchain = pkgs.buildEnv { + name = "bazel-action-toolchain"; + paths = with pkgs; [ gcc binutils coreutils bash gnused gnugrep gawk findutils diffutils python3 ]; + }; + + # Matches NativeLink v1.8.3 (basic_cas example). Field names change between + # releases; check them when bumping the flake input. + instances = f: map (instance_name: { inherit instance_name; } // f) [ "" "main" ]; + nativelinkConfig = pkgs.writeText "nativelink.json" (builtins.toJSON { + stores = [ + { + # If any output of a cached result was evicted, report a miss so the + # action is rebuilt instead of returning a result with missing files. + name = "AC_MAIN_STORE"; + completeness_checking = { + backend.filesystem = { + content_path = "${stateDir}/ac/content"; + temp_path = "${stateDir}/ac/tmp"; + eviction_policy.max_bytes = 10 * 1000 * 1000 * 1000; + }; + cas_store.ref_store.name = "WORKER_FAST_SLOW_STORE"; + }; + } + { + name = "WORKER_FAST_SLOW_STORE"; + fast_slow = { + fast.filesystem = { + content_path = "${stateDir}/cas/content"; # materializer links from here + temp_path = "${stateDir}/cas/tmp"; + eviction_policy.max_bytes = cfg.cacheGB * 1000 * 1000 * 1000; + }; + slow.noop = { }; + }; + } + ]; + + schedulers = [{ + name = "MAIN_SCHEDULER"; + simple.supported_platform_properties = { + cpu_count = "minimum"; + OSFamily = "priority"; + "container-image" = "priority"; + }; + }]; + + workers = [{ + local = { + worker_api_endpoint.uri = "grpc://127.0.0.1:${toString workerPort}"; + cas_fast_slow_store = "WORKER_FAST_SLOW_STORE"; + upload_action_result.ac_store = "AC_MAIN_STORE"; + work_directory = "${stateDir}/work"; + platform_properties = { + cpu_count.values = [ (toString cfg.cpuCount) ]; + OSFamily.values = [ "" ]; + "container-image".values = [ "" ]; + }; + }; + }]; + + servers = [ + { + name = "public"; + # Only container links (ve-*) are trusted; the LAN port stays closed. + listener.http.socket_address = "0.0.0.0:${toString cfg.port}"; + services = { + cas = instances { cas_store = "WORKER_FAST_SLOW_STORE"; }; + ac = instances { ac_store = "AC_MAIN_STORE"; }; + execution = instances { cas_store = "WORKER_FAST_SLOW_STORE"; scheduler = "MAIN_SCHEDULER"; }; + capabilities = instances { remote_execution.scheduler = "MAIN_SCHEDULER"; }; + bytestream = instances { cas_store = "WORKER_FAST_SLOW_STORE"; }; + }; + } + { + name = "private_workers_servers"; + listener.http.socket_address = "127.0.0.1:${toString workerPort}"; + services = { + worker_api.scheduler = "MAIN_SCHEDULER"; + admin = { }; + health = { }; + }; + } + ]; + + global.max_open_files = 65536; + }); + + # One long-running service for all containers. When a container drops a + # "started" marker in its handoff folder, a thread follows its build event + # file. As each target finishes, it opens each output in the cache and + # hardlinks it into that container's outputs folder (opening first closes + # the eviction race). Outputs already evicted are reported back so the + # container can rebuild them. + # + # Handoff folders are written by the containers, so they are untrusted: no + # symlinks are followed and every destination is checked to stay inside + # that container's outputs folder. + # + # NativeLink v1.8 names cache files d2/--. The + # generation is a counter we can't guess, so the folder is indexed and + # re-read when a file isn't found. + materializer = pkgs.writeScript "build-materializer" '' + #!${pkgs.python3}/bin/python3 + import ctypes, errno, fcntl, json, os, re, shutil, sys, threading, time + + HANDOFF_ROOT = "${handoffRoot}" + OUT_ROOT = "${outputsRoot}" + DIGESTS = "${stateDir}/cas/content/d2" + NAME_OK = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") + BLOB = re.compile(r"/blobs/([0-9a-f]{64})/(\d+)") + FICLONE = 0x40049409 + LIBC = ctypes.CDLL(None, use_errno=True) + AT_FDCWD, AT_EMPTY_PATH = -100, 0x1000 + + def log(*a): + print(*a, file=sys.stderr, flush=True) + + class CasIndex: + # Maps "-" to the newest file name in the cache. + def __init__(self): + self.names, self.at, self.lock = {}, 0.0, threading.Lock() + + def rescan(self): + # Read the folder at most twice a second, however many threads ask. + wait = self.at + 0.5 - time.monotonic() + if wait > 0: + time.sleep(wait) + names = {} + try: + with os.scandir(DIGESTS) as it: + for e in it: + key, _, gen = e.name.rpartition("-") + if key and gen.isdigit() and int(gen) >= names.get(key, (-1, ""))[0]: + names[key] = (int(gen), e.name) + except FileNotFoundError: + pass + self.names, self.at = names, time.monotonic() + + def lookup(self, h, size, refresh): + key = f"{h}-{size}" + with self.lock: + if refresh or key not in self.names: + self.rescan() + hit = self.names.get(key) + return os.path.join(DIGESTS, hit[1]) if hit else None + + CAS = CasIndex() + + def link_fd(fd, dest): + # Hardlink the exact inode we hold open (root only). Fails with + # ENOENT if the file was evicted after we opened it. + if LIBC.linkat(fd, b"", AT_FDCWD, os.fsencode(dest), AT_EMPTY_PATH) != 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err), dest) + + def copy_fd(fd, dest): + # Reflink on btrfs/XFS (no extra disk), plain copy otherwise. + with open(dest, "wb") as out: + try: + fcntl.ioctl(out.fileno(), FICLONE, fd) + except OSError: + with os.fdopen(os.dup(fd), "rb") as src: + src.seek(0) + shutil.copyfileobj(src, out, 1 << 20) + + def place(h, size, dest): + os.makedirs(os.path.dirname(dest), exist_ok=True) + tmp = dest + ".materializing" + if os.path.lexists(tmp): + os.unlink(tmp) + if size == 0: + open(tmp, "wb").close() + os.replace(tmp, dest) + return True + for refresh in (False, True): + path = CAS.lookup(h, size, refresh) + if path is None: + continue + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + except FileNotFoundError: + continue # index was stale; re-read it once + try: + head = os.pread(fd, 4, 0) + if head.startswith(b"\x7fELF") or head.startswith(b"#!"): + # Executables get their own copy so chmod never touches the cache. + copy_fd(fd, tmp) + os.chmod(tmp, 0o755) + else: + try: + link_fd(fd, tmp) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.EXDEV): + raise + # Evicted after we opened it (or another disk): + # the open fd still has the data, so copy it. + copy_fd(fd, tmp) + os.replace(tmp, dest) + # Renaming a hardlink onto the same file is a no-op that + # leaves tmp behind (output unchanged since last build). + if os.path.lexists(tmp): + os.unlink(tmp) + return True + finally: + os.close(fd) + return False + + class Job: + def __init__(self, name): + self.name = name + self.handoff = os.path.join(HANDOFF_ROOT, name) + self.out = os.path.join(OUT_ROOT, name) + os.makedirs(self.out, exist_ok=True) + self.out = os.path.realpath(self.out) + self.started = os.path.join(self.handoff, "started") + self.sets = {} + self.missing = [] + + def safe_dest(self, parts): + dest = os.path.normpath(os.path.join(self.out, *parts)) + if not dest.startswith(self.out + "/"): + raise ValueError("path escapes outputs folder: " + dest) + return dest + + def files_in(self, set_id, seen): + if set_id in seen or set_id not in self.sets: + return [] + seen.add(set_id) + s = self.sets[set_id] + found = list(s.get("files", [])) + for sub in s.get("fileSets", []): + found += self.files_in(sub["id"], seen) + return found + + @staticmethod + def digest_of(f): + if f.get("digest"): + return f["digest"], int(f.get("length", 0)) + m = BLOB.search(f.get("uri", "")) + return (m.group(1), int(m.group(2))) if m else (None, None) + + def handle(self, ev): + if "namedSetOfFiles" in ev: + self.sets[ev["id"]["namedSet"]["id"]] = ev["namedSetOfFiles"] + elif "completed" in ev and ev["completed"].get("success"): + label = ev["id"]["targetCompleted"]["label"] + ok, seen = True, set() + for group in ev["completed"].get("outputGroup", []): + for fs in group.get("fileSets", []): + for f in self.files_in(fs["id"], seen): + h, size = self.digest_of(f) + if h is None: + continue + try: + dest = self.safe_dest(f.get("pathPrefix", []) + [f["name"]]) + except ValueError as e: + log(self.name, e) + continue + if not place(h, size, dest): + ok = False + if not ok: + self.missing.append(label) + + def open_bep(self): + path = os.path.join(self.handoff, "bep.json") + while os.path.lexists(self.started): + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + return os.fdopen(fd, "r") + except FileNotFoundError: + time.sleep(0.1) + return None + + def follow(self): + fh = self.open_bep() + if fh is None: + return + buf, finishing = "", False + with fh: + while True: + chunk = fh.readline() + if not chunk: + if finishing: + return # build over and file fully read + if not os.path.lexists(self.started): + finishing = True # one more pass for the last lines + else: + time.sleep(0.05) + continue + buf += chunk + if not buf.endswith("\n"): + continue + line, buf = buf, "" + if line.strip(): + self.handle(json.loads(line)) + + def write_result(self): + # O_EXCL + O_NOFOLLOW: a container can't aim this at a host file. + tmp = os.path.join(self.handoff, ".result.tmp") + if os.path.lexists(tmp): + os.unlink(tmp) + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644) + with os.fdopen(fd, "w") as fh: + json.dump({"missing": self.missing}, fh) + os.replace(tmp, os.path.join(self.handoff, "result.json")) + + def run(self): + try: + self.follow() + except Exception as e: + log(self.name, "build handoff failed:", e) + self.missing.append("//...") # tell rbuild to fall back to a download + while os.path.lexists(self.started): + time.sleep(0.1) + try: + self.write_result() + except Exception as e: + log(self.name, "could not write result:", e) + + active, lock = set(), threading.Lock() + + def worker(name): + try: + Job(name).run() + finally: + with lock: + active.discard(name) + + os.makedirs(HANDOFF_ROOT, exist_ok=True) + while True: + for name in os.listdir(HANDOFF_ROOT): + if not NAME_OK.match(name): + continue + d = os.path.join(HANDOFF_ROOT, name) + if os.path.islink(d) or not os.path.isdir(d): + continue + if not os.path.lexists(os.path.join(d, "started")): + continue + with lock: + if name in active: + continue + active.add(name) + threading.Thread(target=worker, args=(name,), daemon=True).start() + time.sleep(0.2) + ''; + + # Container side: `rbuild //target ...` instead of `bazel build`. + # The host is the container's default gateway (ve-* addresses are not fixed). + rbuild = pkgs.writeShellScriptBin "rbuild" '' + set -u + H=/handoff + gw=$(${pkgs.iproute2}/bin/ip -4 route show default | ${pkgs.gawk}/bin/awk '{print $3; exit}') + if [ -z "$gw" ]; then + echo "rbuild: no default route, can't find the build server" >&2 + exit 1 + fi + remote=--remote_executor=grpc://$gw:${toString cfg.port} + rm -f $H/bep.json $H/result.json + touch $H/started + ${pkgs.bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@" + rc=$? + rm -f $H/started + for _ in $(seq 600); do [ -f $H/result.json ] && break; sleep 0.1; done + if [ ! -f $H/result.json ]; then + echo "rbuild: host did not hand back outputs" >&2 + exit 1 + fi + mapfile -t missing < <(${pkgs.jq}/bin/jq -r '.missing[]' $H/result.json) + if [ ''${#missing[@]} -gt 0 ]; then + # Rare: outputs were evicted before the host could link them. Bazel + # finds them gone, rebuilds them, and downloads them over the network. + echo "rbuild: rebuilding ''${#missing[@]} evicted target(s)" >&2 + ${pkgs.bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$? + fi + echo "rbuild: outputs are in /outputs" >&2 + exit $rc + ''; + + bazelrc = '' + build --remote_instance_name=main + build --remote_download_minimal + build --jobs=${toString (cfg.cpuCount * 4)} + + # Rebuild instead of failing when the cache evicts something Bazel needs. + build --experimental_remote_cache_eviction_retries=5 + + build --action_env=PATH=${toolchain}/bin + build --host_action_env=PATH=${toolchain}/bin + build --repo_env=CC=${toolchain}/bin/gcc + ''; + + # Added to slackerbot's agent system. Uses host packages so bazel and the + # toolchain match the host exactly. + agentModule = { + environment.systemPackages = [ pkgs.bazel pkgs.git rbuild ]; + environment.etc."bazel.bazelrc".text = bazelrc; + }; +in +{ + options = { + skyg.nixos.server.services.bazel-build = { + enable = mkEnableOption "Bazel build server (NativeLink) for slackerbot containers"; + + cpuCount = mkOption { + description = "CPU threads the host worker offers to builds."; + type = types.ints.positive; + }; + + cacheGB = mkOption { + description = "Size of the build cache in GB."; + default = 100; + type = types.ints.positive; + }; + + port = mkOption { + description = "Port containers reach NativeLink on (container links only)."; + default = 50051; + type = types.port; + }; + + agentSystem = mkOption { + description = "slackerbot's agent system with bazel, bazelrc and rbuild added. Use for slackerbot's `system`."; + readOnly = true; + type = types.package; + default = (slackerbot.nixosConfigurations.agent.extendModules { + modules = [ agentModule ]; + }).config.system.build.toplevel; + # Keeps the manual from building it on every host. + defaultText = literalExpression "slackerbot agent + bazel, bazelrc, rbuild"; + }; + + binds = mkOption { + description = "slackerbot ro_binds / rw_binds that give each container /handoff and /outputs."; + readOnly = true; + type = types.attrsOf (types.listOf types.str); + default = { + rw = [ "${handoffRoot}/{machine}:/handoff" ]; + ro = [ "${outputsRoot}/{machine}:/outputs" ]; + }; + }; + }; + }; + + config = mkIf cfg.enable { + environment.systemPackages = [ pkgs.bazel ]; + + systemd.services.nativelink = { + description = "NativeLink remote cache and build service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ toolchain ]; + serviceConfig = { + ExecStart = "${nativelink.packages.${system}.nativelink}/bin/nativelink ${nativelinkConfig}"; + DynamicUser = true; + StateDirectory = "nativelink"; + # Cached files must be readable by the materializer and containers. + UMask = "0022"; + LimitNOFILE = 65536; + Restart = "on-failure"; + }; + }; + + systemd.services.build-materializer = { + description = "Hand Bazel outputs to slackerbot containers"; + wantedBy = [ "multi-user.target" ]; + after = [ "nativelink.service" ]; + serviceConfig = { + ExecStart = "${materializer}"; # root: needed to link cache files + UMask = "0022"; + Restart = "always"; + }; + }; + + # Outputs older than 7 days are removed. Hardlinked files keep their data + # alive after NativeLink evicts them, so this is what frees that space. + # `e` ages what's inside each container's folder, never the folder itself + # (it is a live bind mount source). + systemd.tmpfiles.rules = [ + "d ${handoffRoot} 0755 root root -" + "d ${outputsRoot} 0755 root root -" + "e ${outputsRoot}/* - - - 7d" + ]; + + # Keep the toolchain from garbage collection. + system.extraDependencies = [ toolchain ]; + }; +} diff --git a/modules/nixos/server/services/default.nix b/modules/nixos/server/services/default.nix index 4bc2978..eaa1339 100644 --- a/modules/nixos/server/services/default.nix +++ b/modules/nixos/server/services/default.nix @@ -3,6 +3,7 @@ imports = [ ./audiobookshelf.nix ./ai-services.nix + ./bazel-build.nix ./colibri.nix ./comfyui ./jellyfin.nix diff --git a/pkgs/colibri/default.nix b/pkgs/colibri/default.nix index a3898d9..b2b9c57 100644 --- a/pkgs/colibri/default.nix +++ b/pkgs/colibri/default.nix @@ -77,12 +77,12 @@ let in stdenv.mkDerivation { pname = "colibri" + lib.optionalString (backend == "rocm") "-rocm"; - version = "2.0.0"; + version = "2.1.0"; src = fetchFromGitHub { owner = "JustVugg"; repo = "colibri"; - rev = "a7f535a9b34ffa31a4eef18c8a0bb6775c973a6e"; # tag v1.5.0 + rev = "8579957378ca922ad4340a6392513485cbc50c2c"; # tag v1.5.0 hash = "sha256-4gxlmTiB3RfWdeqQfkr6nstBrQktRA0jUP7TY29KpM0="; };