From 2a2b1d84219d511ef47bff41a4c436e6bd566ddc Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Sun, 1 Nov 2026 23:14:35 -0500 Subject: [PATCH] bazel-build: remote-only builds for slackerbot containers - Containers get an enforcing bazel wrapper: remote spawn strategy, no local fallback, and refused override flags. Gateway rc is written at boot. - NativeLink: ExecPaths for the DynamicUser state dir (was noexec). - Bazel server exits after 5 idle minutes; shared per-host repo cache (/repo-cache, idmap); goose hints say a local build means trouble. - Removed the /handoff -> /outputs materializer; outputs download into bazel-bin again. - slackerbot containers: 4G/1 CPU, up to 8G/2 CPUs. - fws1: disable networkd wait-online. - slackerbot 4db99ea (send_file symlinked home, trimmed tool lines). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0119iKoN3fSvRGUAYHupPW4U --- flake.lock | 8 +- hosts/hl-fwdesk/services/slackerbot.nix | 6 +- hosts/hl-fws1/services/slackerbot.nix | 9 +- modules/nixos/server/services/ABOUTME.md | 44 +- modules/nixos/server/services/bazel-build.nix | 413 ++++-------------- 5 files changed, 124 insertions(+), 356 deletions(-) diff --git a/flake.lock b/flake.lock index 0afd22e..e3b5dfb 100644 --- a/flake.lock +++ b/flake.lock @@ -3434,11 +3434,11 @@ ] }, "locked": { - "lastModified": 1793566366, - "narHash": "sha256-yytC8S47uJZNm7GraZQb84QH4YHA/tIOKIw6sis7sJA=", + "lastModified": 1793590815, + "narHash": "sha256-Z2HRkYW0547L309qm4e0w++xVKIJFpKbqBUDLEoWG4c=", "ref": "refs/heads/main", - "rev": "917a361370ab4da27ae10b232da34fcb40e04adf", - "revCount": 19, + "rev": "4db99eaa2a0e1529194983962a07109fdb716bcd", + "revCount": 22, "type": "git", "url": "https://gitea.skyg.ca/ari/slacker-bot.git" }, diff --git a/hosts/hl-fwdesk/services/slackerbot.nix b/hosts/hl-fwdesk/services/slackerbot.nix index 45fe777..d7c7b67 100644 --- a/hosts/hl-fwdesk/services/slackerbot.nix +++ b/hosts/hl-fwdesk/services/slackerbot.nix @@ -8,7 +8,7 @@ let # The system each container boots (slackerbot's agent + bazel/rbuild); # updated on every rebuild. system = "${bazel.agentSystem}"; - # /handoff and /outputs for remote Bazel builds on this host. + # Shared Bazel download cache (/repo-cache) for the containers. rw_binds = bazel.binds.rw; ro_binds = bazel.binds.ro; machines_dir = "/var/lib/machines"; @@ -28,8 +28,8 @@ let OPENROUTER_API_KEY = "$OPENROUTER_API_KEY"; }; - container = { memory_mb = 2048; cpus = 1.0; }; - max_container = { memory_mb = 4096; cpus = 2.0; }; + container = { memory_mb = 4096; cpus = 1.0; }; + max_container = { memory_mb = 8192; cpus = 2.0; }; # Leave room for ollama, ds4, comfyui, hermes and the robots. host_reserve = { memory_mb = 32768; cpus = 4.0; }; storage = { min = "2G"; max = "5G"; limit = "20G"; }; diff --git a/hosts/hl-fws1/services/slackerbot.nix b/hosts/hl-fws1/services/slackerbot.nix index 624099e..ff46e6c 100644 --- a/hosts/hl-fws1/services/slackerbot.nix +++ b/hosts/hl-fws1/services/slackerbot.nix @@ -6,7 +6,7 @@ let # The system each container boots (slackerbot's agent + bazel/rbuild); # updated on every rebuild. system = "${bazel.agentSystem}"; - # /handoff and /outputs for remote Bazel builds on this host. + # Shared Bazel download cache (/repo-cache) for the containers. rw_binds = bazel.binds.rw; ro_binds = bazel.binds.ro; machines_dir = "/var/lib/machines"; @@ -18,8 +18,8 @@ let allow_indefinite = true; boot_timeout = "2m"; - container = { memory_mb = 2048; cpus = 1.0; }; - max_container = { memory_mb = 4096; cpus = 2.0; }; + container = { memory_mb = 4096; cpus = 1.0; }; + max_container = { memory_mb = 8192; cpus = 2.0; }; host_reserve = { memory_mb = 8192; cpus = 2.0; }; storage = { min = "2G"; max = "5G"; limit = "20G"; }; drives = [{ path = homes; reserve = "100G"; }]; @@ -58,6 +58,9 @@ in # Container links belong to systemd-networkd. networking.networkmanager.unmanaged = [ "interface-name:ve-*" ]; networking.dhcpcd.denyInterfaces = [ "ve-*" ]; + # networkd only has the container links here (NetworkManager owns wlan), + # so wait-online would always time out. + systemd.network.wait-online.enable = false; systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ]; } diff --git a/modules/nixos/server/services/ABOUTME.md b/modules/nixos/server/services/ABOUTME.md index 2d848e9..9ae4f4b 100644 --- a/modules/nixos/server/services/ABOUTME.md +++ b/modules/nixos/server/services/ABOUTME.md @@ -24,18 +24,38 @@ services/ - **`ai-services.nix`** openwakeword as containers and enables NixOS-native Wyoming faster-whisper + piper. Opens 11434/10200/10300/10400. - **`bazel-build.nix`** runs NativeLink (from the `nativelink` flake input, pinned - to v1.8.3 because its config format changes) plus `build-materializer`, which - hardlinks finished outputs from the cache into `/var/lib/build-outputs/`. - Containers run `rbuild //target` (finds the host as the default gateway) and read - results at `/outputs`. Plain `bazel` builds remotely too: at boot, - `bazel-remote-rc` writes the gateway into `/run/bazel-remote.bazelrc`, which the - container's `/etc/bazel.bazelrc` try-imports (missing = local build). Plain builds - download final outputs (`--remote_download_toplevel`). The Bazel server is capped at - 512M. Global goose hints (`/root/.config/goose/.goosehints` → `/etc/goose/goosehints`) - tell goose that a local build means something is wrong: stop and report it. It also exposes `agentSystem` (slackerbot's agent + bazel, go, - bazelrc, rbuild) and `binds` (slackerbot `ro_binds`/`rw_binds`). Used on hl-fws1 - and hl-fwdesk. Slackerbot must create `ro_binds` source folders, or containers fail - to start before their first build. + to v1.8.3 because its config format changes) as a remote cache and executor for + slackerbot containers. Used on hl-fws1 and hl-fwdesk. Goal: as little as possible + runs in a container. Measured on slacker-bot (1 CPU container, 2026-11-01): local + 458 s, remote with no cache 65 s, remote with warm cache 17 s. + - NativeLink uses `DynamicUser`, whose state folder systemd mounts `noexec`; + `ExecPaths` re-allows it, or every remote action fails with "Permission denied". + - Containers get their own `bazel` (`agentBazel`), with no plain bazel/bazelisk on + PATH. At boot `bazel-remote-rc` writes the gateway into `/run/bazel-remote.bazelrc`, + which `/etc/bazel.bazelrc` try-imports. Only remote actions are allowed + (`--spawn_strategy=remote`, no local fallback), so a missing file or unreachable + host fails the build instead of building in the container. The wrapper refuses + command-line flags that change this (`--spawn_strategy`, `--strategy*`, + `--remote_executor`, `--bazelrc`, ...). Flags in a repo's own `.bazelrc` are not + checked. + - Final outputs are downloaded into `bazel-bin` (`--remote_download_toplevel`). + `rbuild` is just `bazel build`. A `/handoff` → `/outputs` path that hardlinked + outputs from the cache was tried and removed: outputs are a few MB over a local + link, so it saved nothing (download ~20 s vs handoff ~344 s). Most of that was a + lock bug: Bazel's server inherits open file handles, so a lock held on an fd in + the wrapper stays held until the server exits. Close such fds for Bazel (`9>&-`). + - Still in the container: goose, the Bazel server (512M heap, exits after 5 idle + minutes) and repository rules, which Bazel can't run remotely. + - Downloads share one cache per host (`/var/lib/bazel-repo-cache` → `/repo-cache`, + sha256-checked, 30-day cleanup). Fetched repo folders stay per container + (`--repo_contents_cache`), since their contents are not checked. + - Containers use private user IDs, so writable binds (`/repo-cache`) use + `idmap`; without it the folder shows as `nobody` and the container can't write. + - Global goose hints (`/root/.config/goose/.goosehints` → `/etc/goose/goosehints`): + a local build means something is wrong (stop and report); build Go through Bazel. + - Exposes `agentSystem` (slackerbot's agent + bazel, go, bazelrc, rbuild) and + `binds` (slackerbot `ro_binds`/`rw_binds`). Slackerbot creates missing bind + source folders, or containers fail to start. `bazel` is a bazelisk wrapper on the host and in containers: a repo's `.bazelversion` wins, otherwise 8.7.0 (`USE_BAZEL_FALLBACK_VERSION`). Bazel 8.7.0 is fetched once per node by Nix (official release, sha256-checked) and the wrapper diff --git a/modules/nixos/server/services/bazel-build.nix b/modules/nixos/server/services/bazel-build.nix index 45d4cae..c85dcea 100644 --- a/modules/nixos/server/services/bazel-build.nix +++ b/modules/nixos/server/services/bazel-build.nix @@ -1,18 +1,14 @@ # Bazel build server for slackerbot containers on the same host. # One NativeLink (cache + scheduler + worker) runs builds on the host CPUs. -# Final outputs are not sent over the network: a host service hardlinks them -# from NativeLink's cache into a folder each container sees at /outputs. # -# container: rbuild //app ──grpc──▶ NativeLink (host) ──▶ host CPUs -# │ writes build events to /handoff/bep.json -# ▼ -# host: build-materializer follows bep.json and hardlinks each finished -# output into /var/lib/build-outputs// -# ▼ -# container: files appear at /outputs (read-only) +# container: bazel build //app ──grpc──▶ NativeLink (host) ──▶ host CPUs +# ◀── final outputs downloaded (small, same machine) # -# slackerbot mounts the folders through `binds`. Containers run as root with -# no private user IDs, so no uid mapping is needed. +# A /outputs handoff (hardlinking outputs from the cache) was tried and +# removed: it saved nothing here, since outputs are a few MB over a local link. +# +# slackerbot mounts the shared download cache through `binds`. Containers use +# private user IDs, so writable binds need `idmap` (container root = host root). { config, lib, pkgs, nativelink, slackerbot, ... }: with lib; @@ -50,9 +46,8 @@ let ''; stateDir = "/var/lib/nativelink"; - handoffRoot = "/var/lib/build-handoff"; - # Must be on the same disk as stateDir, or hardlinks fail (falls back to copies). - outputsRoot = "/var/lib/build-outputs"; + # Bazel download cache shared by all containers (files are sha256-checked). + repoCacheRoot = "/var/lib/bazel-repo-cache"; workerPort = 50061; # host only # Tools build actions run with. Same store paths on the host and in containers. @@ -83,7 +78,7 @@ let name = "WORKER_FAST_SLOW_STORE"; fast_slow = { fast.filesystem = { - content_path = "${stateDir}/cas/content"; # materializer links from here + content_path = "${stateDir}/cas/content"; temp_path = "${stateDir}/cas/tmp"; eviction_policy.max_bytes = cfg.cacheGB * 1000 * 1000 * 1000; }; @@ -142,306 +137,62 @@ let global.max_open_files = 65536; }); - # One long-running service for all containers. When a container drops a - # "started" marker in its handoff folder, a thread follows its build event - # file. As each target finishes, it opens each output in the cache and - # hardlinks it into that container's outputs folder (opening first closes - # the eviction race). Outputs already evicted are reported back so the - # container can rebuild them. - # - # Handoff folders are written by the containers, so they are untrusted: no - # symlinks are followed and every destination is checked to stay inside - # that container's outputs folder. - # - # NativeLink v1.8 names cache files d2/--. The - # generation is a counter we can't guess, so the folder is indexed and - # re-read when a file isn't found. - materializer = pkgs.writeScript "build-materializer" '' - #!${pkgs.python3}/bin/python3 - import ctypes, errno, fcntl, json, os, re, shutil, sys, threading, time - - HANDOFF_ROOT = "${handoffRoot}" - OUT_ROOT = "${outputsRoot}" - DIGESTS = "${stateDir}/cas/content/d2" - NAME_OK = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") - BLOB = re.compile(r"/blobs/([0-9a-f]{64})/(\d+)") - FICLONE = 0x40049409 - LIBC = ctypes.CDLL(None, use_errno=True) - AT_FDCWD, AT_EMPTY_PATH = -100, 0x1000 - - def log(*a): - print(*a, file=sys.stderr, flush=True) - - class CasIndex: - # Maps "-" to the newest file name in the cache. - def __init__(self): - self.names, self.at, self.lock = {}, 0.0, threading.Lock() - - def rescan(self): - # Read the folder at most twice a second, however many threads ask. - wait = self.at + 0.5 - time.monotonic() - if wait > 0: - time.sleep(wait) - names = {} - try: - with os.scandir(DIGESTS) as it: - for e in it: - key, _, gen = e.name.rpartition("-") - if key and gen.isdigit() and int(gen) >= names.get(key, (-1, ""))[0]: - names[key] = (int(gen), e.name) - except FileNotFoundError: - pass - self.names, self.at = names, time.monotonic() - - def lookup(self, h, size, refresh): - key = f"{h}-{size}" - with self.lock: - if refresh or key not in self.names: - self.rescan() - hit = self.names.get(key) - return os.path.join(DIGESTS, hit[1]) if hit else None - - CAS = CasIndex() - - def link_fd(fd, dest): - # Hardlink the exact inode we hold open (root only). Fails with - # ENOENT if the file was evicted after we opened it. - if LIBC.linkat(fd, b"", AT_FDCWD, os.fsencode(dest), AT_EMPTY_PATH) != 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err), dest) - - def copy_fd(fd, dest): - # Reflink on btrfs/XFS (no extra disk), plain copy otherwise. - with open(dest, "wb") as out: - try: - fcntl.ioctl(out.fileno(), FICLONE, fd) - except OSError: - with os.fdopen(os.dup(fd), "rb") as src: - src.seek(0) - shutil.copyfileobj(src, out, 1 << 20) - - def place(h, size, dest): - os.makedirs(os.path.dirname(dest), exist_ok=True) - tmp = dest + ".materializing" - if os.path.lexists(tmp): - os.unlink(tmp) - if size == 0: - open(tmp, "wb").close() - os.replace(tmp, dest) - return True - for refresh in (False, True): - path = CAS.lookup(h, size, refresh) - if path is None: - continue - try: - fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) - except FileNotFoundError: - continue # index was stale; re-read it once - try: - head = os.pread(fd, 4, 0) - if head.startswith(b"\x7fELF") or head.startswith(b"#!"): - # Executables get their own copy so chmod never touches the cache. - copy_fd(fd, tmp) - os.chmod(tmp, 0o755) - else: - try: - link_fd(fd, tmp) - except OSError as e: - if e.errno not in (errno.ENOENT, errno.EXDEV): - raise - # Evicted after we opened it (or another disk): - # the open fd still has the data, so copy it. - copy_fd(fd, tmp) - os.replace(tmp, dest) - # Renaming a hardlink onto the same file is a no-op that - # leaves tmp behind (output unchanged since last build). - if os.path.lexists(tmp): - os.unlink(tmp) - return True - finally: - os.close(fd) - return False - - class Job: - def __init__(self, name): - self.name = name - self.handoff = os.path.join(HANDOFF_ROOT, name) - self.out = os.path.join(OUT_ROOT, name) - os.makedirs(self.out, exist_ok=True) - self.out = os.path.realpath(self.out) - self.started = os.path.join(self.handoff, "started") - self.sets = {} - self.missing = [] - - def safe_dest(self, parts): - dest = os.path.normpath(os.path.join(self.out, *parts)) - if not dest.startswith(self.out + "/"): - raise ValueError("path escapes outputs folder: " + dest) - return dest - - def files_in(self, set_id, seen): - if set_id in seen or set_id not in self.sets: - return [] - seen.add(set_id) - s = self.sets[set_id] - found = list(s.get("files", [])) - for sub in s.get("fileSets", []): - found += self.files_in(sub["id"], seen) - return found - - @staticmethod - def digest_of(f): - if f.get("digest"): - return f["digest"], int(f.get("length", 0)) - m = BLOB.search(f.get("uri", "")) - return (m.group(1), int(m.group(2))) if m else (None, None) - - def handle(self, ev): - if "namedSetOfFiles" in ev: - self.sets[ev["id"]["namedSet"]["id"]] = ev["namedSetOfFiles"] - elif "completed" in ev and ev["completed"].get("success"): - label = ev["id"]["targetCompleted"]["label"] - ok, seen = True, set() - for group in ev["completed"].get("outputGroup", []): - for fs in group.get("fileSets", []): - for f in self.files_in(fs["id"], seen): - h, size = self.digest_of(f) - if h is None: - continue - try: - dest = self.safe_dest(f.get("pathPrefix", []) + [f["name"]]) - except ValueError as e: - log(self.name, e) - continue - if not place(h, size, dest): - ok = False - if not ok: - self.missing.append(label) - - def open_bep(self): - path = os.path.join(self.handoff, "bep.json") - while os.path.lexists(self.started): - try: - fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) - return os.fdopen(fd, "r") - except FileNotFoundError: - time.sleep(0.1) - return None - - def follow(self): - fh = self.open_bep() - if fh is None: - return - buf, finishing = "", False - with fh: - while True: - chunk = fh.readline() - if not chunk: - if finishing: - return # build over and file fully read - if not os.path.lexists(self.started): - finishing = True # one more pass for the last lines - else: - time.sleep(0.05) - continue - buf += chunk - if not buf.endswith("\n"): - continue - line, buf = buf, "" - if line.strip(): - self.handle(json.loads(line)) - - def write_result(self): - # O_EXCL + O_NOFOLLOW: a container can't aim this at a host file. - tmp = os.path.join(self.handoff, ".result.tmp") - if os.path.lexists(tmp): - os.unlink(tmp) - fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644) - with os.fdopen(fd, "w") as fh: - json.dump({"missing": self.missing}, fh) - os.replace(tmp, os.path.join(self.handoff, "result.json")) - - def run(self): - try: - self.follow() - except Exception as e: - log(self.name, "build handoff failed:", e) - self.missing.append("//...") # tell rbuild to fall back to a download - while os.path.lexists(self.started): - time.sleep(0.1) - try: - self.write_result() - except Exception as e: - log(self.name, "could not write result:", e) - - active, lock = set(), threading.Lock() - - def worker(name): - try: - Job(name).run() - finally: - with lock: - active.discard(name) - - os.makedirs(HANDOFF_ROOT, exist_ok=True) - while True: - for name in os.listdir(HANDOFF_ROOT): - if not NAME_OK.match(name): - continue - d = os.path.join(HANDOFF_ROOT, name) - if os.path.islink(d) or not os.path.isdir(d): - continue - if not os.path.lexists(os.path.join(d, "started")): - continue - with lock: - if name in active: - continue - active.add(name) - threading.Thread(target=worker, args=(name,), daemon=True).start() - time.sleep(0.2) - ''; - - # Container side: `rbuild //target ...` instead of `bazel build`. - # The host is the container's default gateway (ve-* addresses are not fixed). - rbuild = pkgs.writeShellScriptBin "rbuild" '' + # Container side `bazel`. Refuses flags that would move work into the + # container and forces remote actions. The host address comes from + # /run/bazel-remote.bazelrc (see bazelrc). + agentBazel = pkgs.writeShellScriptBin "bazel" '' set -u - H=/handoff - gw=$(${pkgs.iproute2}/bin/ip -4 route show default | ${pkgs.gawk}/bin/awk '{print $3; exit}') - if [ -z "$gw" ]; then - echo "rbuild: no default route, can't find the build server" >&2 - exit 1 - fi - remote=--remote_executor=grpc://$gw:${toString cfg.port} - rm -f $H/bep.json $H/result.json - touch $H/started - ${bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@" - rc=$? - rm -f $H/started - for _ in $(seq 600); do [ -f $H/result.json ] && break; sleep 0.1; done - if [ ! -f $H/result.json ]; then - echo "rbuild: host did not hand back outputs" >&2 - exit 1 - fi - mapfile -t missing < <(${pkgs.jq}/bin/jq -r '.missing[]' $H/result.json) - if [ ''${#missing[@]} -gt 0 ]; then - # Rare: outputs were evicted before the host could link them. Bazel - # finds them gone, rebuilds them, and downloads them over the network. - echo "rbuild: rebuilding ''${#missing[@]} evicted target(s)" >&2 - ${bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$? - fi - echo "rbuild: outputs are in /outputs" >&2 - exit $rc + real=${bazel}/bin/bazel + enforce=(--spawn_strategy=remote --noremote_local_fallback) + + # Find the command (first word that isn't a startup option) and check flags. + # Arguments after `--` belong to the program being run, so they're skipped. + cmd="" at=0 i=0 + for a in "$@"; do + [ "$a" = "--" ] && break + case "$a" in + --spawn_strategy*|--strategy*|--test_strategy*|--remote_executor*|\ + --remote_local_fallback*|--bazelrc*|--ignore_all_rc_files|--nosystem_rc) + echo "bazel: $a isn't allowed here. Builds must run on the host's build server;" \ + "tell the user instead of working around it." >&2 + exit 2 ;; + esac + if [ -z "$cmd" ] && [ "''${a#-}" = "$a" ]; then cmd=$a at=$i; fi + i=$((i + 1)) + done + [ -z "$cmd" ] && exec "$real" "$@" + pre=("''${@:1:at}") + post=("''${@:at+2}") + + case "$cmd" in + build|test|run|coverage) exec "$real" "''${pre[@]}" "$cmd" "''${enforce[@]}" "''${post[@]}" ;; + *) exec "$real" "$@" ;; + esac ''; + # Kept for old habits and docs: same as `bazel build`. + rbuild = pkgs.writeShellScriptBin "rbuild" ''exec ${agentBazel}/bin/bazel build "$@"''; + remoteRc = "/run/bazel-remote.bazelrc"; bazelrc = '' - # Builds run on the host, so the Bazel server in a container stays small. + # Builds run on the host, so the Bazel server in a container stays small, + # and exits after 5 idle minutes instead of 3 hours. startup --host_jvm_args=-Xmx512m + startup --host_jvm_args=-XX:+UseSerialGC + startup --max_idle_secs=300 + + # Every action runs on the host; one that can't fails instead of running here. + build --spawn_strategy=remote + build --noremote_local_fallback + + # Downloads are shared by all containers on this host. Fetched repo folders + # are not (unchecked, so one container could tamper with another's). + common --repository_cache=/repo-cache + common --repo_contents_cache=/root/.cache/bazel/repo-contents build --remote_instance_name=main - # Plain `bazel build` downloads final outputs; rbuild overrides this with - # minimal and gets them through /outputs instead. + # Final outputs are downloaded into bazel-bin (a few MB, same machine). build --remote_download_toplevel build --jobs=${toString (cfg.cpuCount * 4)} @@ -452,7 +203,7 @@ let build --host_action_env=PATH=${toolchain}/bin build --repo_env=CC=${toolchain}/bin/gcc - # Points plain `bazel` at the host (written at boot). Missing = local build. + # Points plain `bazel` at the host (written at boot). Missing = builds fail. try-import ${remoteRc} ''; @@ -460,9 +211,12 @@ let goosehints = '' # Bazel builds run on the host, never in this container - This container is small (about 2 GB RAM, 1 CPU). `bazel build/test/run` is - set up to run every action on the host's build server, and `rbuild //target` - does the same and puts outputs in /outputs. + This container is small (a few GB RAM, 1 CPU). `bazel build/test/run` runs + every action on the host's build server. + + Outputs are downloaded into bazel-bin as usual. + Flags that change where actions run (--spawn_strategy, --strategy, + --remote_executor, --bazelrc, ...) are refused on purpose. A normal build ends with a line like `INFO: 120 processes: 110 remote, 10 internal.` (or "remote cache hit"). @@ -470,14 +224,19 @@ let doing the work, or /run/bazel-remote.bazelrc is missing, or a build can't reach the remote executor, something is wrong: - stop the build (`bazel shutdown`), don't retry it locally, - - don't remove or override the --remote_* flags, + - don't remove or override the --remote_* or --spawn_strategy flags, - tell the user what you saw, with the error lines. + + Build and test Go code through Bazel (`bazel build`, `bazel test`), not + `go build` / `go test`: those run here, in this small container. `go mod` + and other quick `go` commands are fine. ''; # Added to slackerbot's agent system. Uses host packages so bazel and the # toolchain match the host exactly. agentModule = { - environment.systemPackages = [ bazel pkgs.bazelisk pkgs.go pkgs.git rbuild ]; + # No plain bazel/bazelisk on PATH, so the checks in agentBazel can't be skipped by accident. + environment.systemPackages = [ agentBazel pkgs.go pkgs.git rbuild ]; environment.etc."bazel.bazelrc".text = bazelrc; environment.etc."goose/goosehints".text = goosehints; programs.nix-ld.enable = true; @@ -543,12 +302,13 @@ in }; binds = mkOption { - description = "slackerbot ro_binds / rw_binds that give each container /handoff and /outputs."; + description = "slackerbot ro_binds / rw_binds that give each container the shared /repo-cache."; readOnly = true; type = types.attrsOf (types.listOf types.str); default = { - rw = [ "${handoffRoot}/{machine}:/handoff" ]; - ro = [ "${outputsRoot}/{machine}:/outputs" ]; + # idmap: every container writes as the same host user, so all can share it. + rw = [ "${repoCacheRoot}:/repo-cache:idmap" ]; + ro = [ ]; }; }; }; @@ -567,32 +327,17 @@ in ExecStart = "${nativelink.packages.${system}.nativelink}/bin/nativelink ${nativelinkConfig}"; DynamicUser = true; StateDirectory = "nativelink"; - # Cached files must be readable by the materializer and containers. + # DynamicUser's state dir is mounted noexec; build actions run from work/. + ExecPaths = [ "/var/lib/private/nativelink" ]; UMask = "0022"; LimitNOFILE = 65536; Restart = "on-failure"; }; }; - systemd.services.build-materializer = { - description = "Hand Bazel outputs to slackerbot containers"; - wantedBy = [ "multi-user.target" ]; - after = [ "nativelink.service" ]; - serviceConfig = { - ExecStart = "${materializer}"; # root: needed to link cache files - UMask = "0022"; - Restart = "always"; - }; - }; - - # Outputs older than 7 days are removed. Hardlinked files keep their data - # alive after NativeLink evicts them, so this is what frees that space. - # `e` ages what's inside each container's folder, never the folder itself - # (it is a live bind mount source). systemd.tmpfiles.rules = [ - "d ${handoffRoot} 0755 root root -" - "d ${outputsRoot} 0755 root root -" - "e ${outputsRoot}/* - - - 7d" + # Downloads unused for 30 days are removed (Bazel never cleans this). + "d ${repoCacheRoot} 0755 root root 30d" ]; # Keep the toolchain from garbage collection.