From 594a1c19017f0caf3948df4a14a5b1a17390785e Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Sun, 1 Nov 2026 16:17:50 -0500 Subject: [PATCH] bazel related limit issues --- AGENTS.md | 2 +- flake.lock | 8 +- hosts/hl-fwdesk/services/slackerbot.nix | 2 +- hosts/hl-fws1/services/slackerbot.nix | 2 +- modules/nixos/server/services/ABOUTME.md | 15 ++- modules/nixos/server/services/bazel-build.nix | 99 +++++++++++++++++-- pkgs/colibri/default.nix | 2 +- 7 files changed, 114 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a4e2996..c5b9c17 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ Key non-negotiable rules from that file: changes — no matter how small the task. - **Never run builds, system switches, or remote installs** (`nixos-rebuild`, `nh os`, `skyg os`, `skyg remote install`, `nixos-anywhere`, `nix build`, - `nix flake check`, etc.) — zero exceptions, even if asked. + `nix flake check`, etc.). You may only run it after asking the user 3 times if the are sure about it! - Never edit `flake.lock`, `*.hardware-configuration.nix`, or anything in `secrets/`. - Never add `imports = [...]` to host files. diff --git a/flake.lock b/flake.lock index 7881972..0afd22e 100644 --- a/flake.lock +++ b/flake.lock @@ -3434,11 +3434,11 @@ ] }, "locked": { - "lastModified": 1793561072, - "narHash": "sha256-5LTBU0P1Qbsu6SbO7iy73xt+djzpWn+qXtdJhdjdWqA=", + "lastModified": 1793566366, + "narHash": "sha256-yytC8S47uJZNm7GraZQb84QH4YHA/tIOKIw6sis7sJA=", "ref": "refs/heads/main", - "rev": "3ba96528bbb56b4b4b3bde5b112cd839510ce8c3", - "revCount": 11, + "rev": "917a361370ab4da27ae10b232da34fcb40e04adf", + "revCount": 19, "type": "git", "url": "https://gitea.skyg.ca/ari/slacker-bot.git" }, diff --git a/hosts/hl-fwdesk/services/slackerbot.nix b/hosts/hl-fwdesk/services/slackerbot.nix index fe6613e..45fe777 100644 --- a/hosts/hl-fwdesk/services/slackerbot.nix +++ b/hosts/hl-fwdesk/services/slackerbot.nix @@ -28,7 +28,7 @@ let OPENROUTER_API_KEY = "$OPENROUTER_API_KEY"; }; - container = { memory_mb = 1024; cpus = 1.0; }; + container = { memory_mb = 2048; cpus = 1.0; }; max_container = { memory_mb = 4096; cpus = 2.0; }; # Leave room for ollama, ds4, comfyui, hermes and the robots. host_reserve = { memory_mb = 32768; cpus = 4.0; }; diff --git a/hosts/hl-fws1/services/slackerbot.nix b/hosts/hl-fws1/services/slackerbot.nix index 8414706..624099e 100644 --- a/hosts/hl-fws1/services/slackerbot.nix +++ b/hosts/hl-fws1/services/slackerbot.nix @@ -18,7 +18,7 @@ let allow_indefinite = true; boot_timeout = "2m"; - container = { memory_mb = 1024; cpus = 1.0; }; + container = { memory_mb = 2048; cpus = 1.0; }; max_container = { memory_mb = 4096; cpus = 2.0; }; host_reserve = { memory_mb = 8192; cpus = 2.0; }; storage = { min = "2G"; max = "5G"; limit = "20G"; }; diff --git a/modules/nixos/server/services/ABOUTME.md b/modules/nixos/server/services/ABOUTME.md index d97cfc1..2d848e9 100644 --- a/modules/nixos/server/services/ABOUTME.md +++ b/modules/nixos/server/services/ABOUTME.md @@ -27,10 +27,23 @@ services/ to v1.8.3 because its config format changes) plus `build-materializer`, which hardlinks finished outputs from the cache into `/var/lib/build-outputs/`. Containers run `rbuild //target` (finds the host as the default gateway) and read - results at `/outputs`. It also exposes `agentSystem` (slackerbot's agent + bazel, + results at `/outputs`. Plain `bazel` builds remotely too: at boot, + `bazel-remote-rc` writes the gateway into `/run/bazel-remote.bazelrc`, which the + container's `/etc/bazel.bazelrc` try-imports (missing = local build). Plain builds + download final outputs (`--remote_download_toplevel`). The Bazel server is capped at + 512M. Global goose hints (`/root/.config/goose/.goosehints` → `/etc/goose/goosehints`) + tell goose that a local build means something is wrong: stop and report it. It also exposes `agentSystem` (slackerbot's agent + bazel, go, bazelrc, rbuild) and `binds` (slackerbot `ro_binds`/`rw_binds`). Used on hl-fws1 and hl-fwdesk. Slackerbot must create `ro_binds` source folders, or containers fail to start before their first build. + `bazel` is a bazelisk wrapper on the host and in containers: a repo's + `.bazelversion` wins, otherwise 8.7.0 (`USE_BAZEL_FALLBACK_VERSION`). Bazel 8.7.0 + is fetched once per node by Nix (official release, sha256-checked) and the wrapper + enters it into each user's bazelisk cache as a link, so containers share that one + read-only copy and never download it. Other versions download into each container's + own `/root/.cache/bazelisk`. Upstream Bazel only runs on NixOS through + `programs.nix-ld`, enabled on the host and in containers. Bump `bazelVersion` and + `bazelSha256` together. - **`colibri.nix`** runs `coli serve` (colibrì's OpenAI-compatible API) as a native systemd service. `package` picks the backend (`pkgs.colibri` cpu / `pkgs.colibri-rocm`); `environmentFile` supplies `COLI_API_KEY` via agenix instead of diff --git a/modules/nixos/server/services/bazel-build.nix b/modules/nixos/server/services/bazel-build.nix index 0a5ac25..45d4cae 100644 --- a/modules/nixos/server/services/bazel-build.nix +++ b/modules/nixos/server/services/bazel-build.nix @@ -20,6 +20,34 @@ with lib; let cfg = config.skyg.nixos.server.services.bazel-build; system = pkgs.stdenv.hostPlatform.system; + # Official Bazel release, checked against its sha256. Containers see + # /nix/store read-only, so one copy per node serves all of them. Upstream + # builds only run on NixOS through nix-ld. To bump: change both values. + bazelVersion = "8.7.0"; + bazelSha256 = "d7606e679b78067c811096fb3d6cf135225b528835ca396e3a4dddf957859544"; + # Copied as-is: patching would break the archive appended to the binary. + bazelRelease = pkgs.runCommand "bazel-${bazelVersion}-release" { } '' + install -Dm755 ${pkgs.fetchurl { + url = "https://releases.bazel.build/${bazelVersion}/release/bazel-${bazelVersion}-linux-x86_64"; + sha256 = bazelSha256; + }} $out/bin/bazel + ''; + + # `bazel` runs bazelisk on the host and in containers: a repo's + # .bazelversion wins, otherwise bazelVersion. bazelVersion is pre-entered in + # bazelisk's cache (format of bazelisk 1.28) as a link to bazelRelease, so it + # is never downloaded. Other versions download into each user's own cache. + bazel = pkgs.writeShellScriptBin "bazel" '' + export USE_BAZEL_FALLBACK_VERSION=''${USE_BAZEL_FALLBACK_VERSION:-silent:${bazelVersion}} + dl=''${BAZELISK_HOME_LINUX:-''${BAZELISK_HOME:-''${XDG_CACHE_HOME:-$HOME/.cache}/bazelisk}}/downloads + meta=$dl/metadata/bazelbuild/bazel-${bazelVersion}-linux-x86_64 + if [ ! -e "$dl/sha256/${bazelSha256}/bin/bazel" ] || [ "$(cat "$meta" 2>/dev/null)" != ${bazelSha256} ]; then + mkdir -p "$dl/sha256" "''${meta%/*}" + ln -sfn ${bazelRelease} "$dl/sha256/${bazelSha256}" + printf %s ${bazelSha256} > "$meta" + fi + exec ${pkgs.bazelisk}/bin/bazelisk "$@" + ''; stateDir = "/var/lib/nativelink"; handoffRoot = "/var/lib/build-handoff"; @@ -386,7 +414,7 @@ let remote=--remote_executor=grpc://$gw:${toString cfg.port} rm -f $H/bep.json $H/result.json touch $H/started - ${pkgs.bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@" + ${bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@" rc=$? rm -f $H/started for _ in $(seq 600); do [ -f $H/result.json ] && break; sleep 0.1; done @@ -399,15 +427,22 @@ let # Rare: outputs were evicted before the host could link them. Bazel # finds them gone, rebuilds them, and downloads them over the network. echo "rbuild: rebuilding ''${#missing[@]} evicted target(s)" >&2 - ${pkgs.bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$? + ${bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$? fi echo "rbuild: outputs are in /outputs" >&2 exit $rc ''; + remoteRc = "/run/bazel-remote.bazelrc"; + bazelrc = '' + # Builds run on the host, so the Bazel server in a container stays small. + startup --host_jvm_args=-Xmx512m + build --remote_instance_name=main - build --remote_download_minimal + # Plain `bazel build` downloads final outputs; rbuild overrides this with + # minimal and gets them through /outputs instead. + build --remote_download_toplevel build --jobs=${toString (cfg.cpuCount * 4)} # Rebuild instead of failing when the cache evicts something Bazel needs. @@ -416,13 +451,62 @@ let build --action_env=PATH=${toolchain}/bin build --host_action_env=PATH=${toolchain}/bin build --repo_env=CC=${toolchain}/bin/gcc + + # Points plain `bazel` at the host (written at boot). Missing = local build. + try-import ${remoteRc} + ''; + + # Global goose instructions in each container. + goosehints = '' + # Bazel builds run on the host, never in this container + + This container is small (about 2 GB RAM, 1 CPU). `bazel build/test/run` is + set up to run every action on the host's build server, and `rbuild //target` + does the same and puts outputs in /outputs. + + A normal build ends with a line like + `INFO: 120 processes: 110 remote, 10 internal.` (or "remote cache hit"). + If instead you see `linux-sandbox`, `local` or `processwrapper-sandbox` + doing the work, or /run/bazel-remote.bazelrc is missing, or a build can't + reach the remote executor, something is wrong: + - stop the build (`bazel shutdown`), don't retry it locally, + - don't remove or override the --remote_* flags, + - tell the user what you saw, with the error lines. ''; # Added to slackerbot's agent system. Uses host packages so bazel and the # toolchain match the host exactly. agentModule = { - environment.systemPackages = [ pkgs.bazel pkgs.git rbuild ]; + environment.systemPackages = [ bazel pkgs.bazelisk pkgs.go pkgs.git rbuild ]; environment.etc."bazel.bazelrc".text = bazelrc; + environment.etc."goose/goosehints".text = goosehints; + programs.nix-ld.enable = true; + + systemd.tmpfiles.rules = [ + "d /root/.config/goose 0755 root root -" + "L+ /root/.config/goose/.goosehints - - - - /etc/goose/goosehints" + ]; + + # The host is the default gateway, whose address changes per container, + # so write it for bazel.bazelrc once the route is up. Doesn't hold up boot. + systemd.services.bazel-remote-rc = { + description = "Point Bazel at the host build server"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ pkgs.iproute2 pkgs.gawk pkgs.coreutils ]; + script = '' + for _ in $(seq 60); do + gw=$(ip -4 route show default | awk '{print $3; exit}') + if [ -n "$gw" ]; then + echo "build --remote_executor=grpc://$gw:${toString cfg.port}" > ${remoteRc} + exit 0 + fi + sleep 1 + done + echo "no default route after 60s; bazel will build locally" >&2 + exit 1 + ''; + }; }; in { @@ -448,14 +532,14 @@ in }; agentSystem = mkOption { - description = "slackerbot's agent system with bazel, bazelrc and rbuild added. Use for slackerbot's `system`."; + description = "slackerbot's agent system with bazel, go, bazelrc and rbuild added. Use for slackerbot's `system`."; readOnly = true; type = types.package; default = (slackerbot.nixosConfigurations.agent.extendModules { modules = [ agentModule ]; }).config.system.build.toplevel; # Keeps the manual from building it on every host. - defaultText = literalExpression "slackerbot agent + bazel, bazelrc, rbuild"; + defaultText = literalExpression "slackerbot agent + bazel, go, bazelrc, rbuild"; }; binds = mkOption { @@ -471,7 +555,8 @@ in }; config = mkIf cfg.enable { - environment.systemPackages = [ pkgs.bazel ]; + environment.systemPackages = [ bazel pkgs.bazelisk ]; + programs.nix-ld.enable = true; # for the Bazel bazelisk downloads systemd.services.nativelink = { description = "NativeLink remote cache and build service"; diff --git a/pkgs/colibri/default.nix b/pkgs/colibri/default.nix index b2b9c57..d9d1b30 100644 --- a/pkgs/colibri/default.nix +++ b/pkgs/colibri/default.nix @@ -83,7 +83,7 @@ stdenv.mkDerivation { owner = "JustVugg"; repo = "colibri"; rev = "8579957378ca922ad4340a6392513485cbc50c2c"; # tag v1.5.0 - hash = "sha256-4gxlmTiB3RfWdeqQfkr6nstBrQktRA0jUP7TY29KpM0="; + hash = "sha256-wuSLM50nAzcFWDC7fJb3dfAvuq1sc69quT1y0ANq9Pk="; }; nativeBuildInputs = [ makeWrapper ];