bazel-build: remote-only builds for slackerbot containers
- Containers get an enforcing bazel wrapper: remote spawn strategy, no local fallback, and refused override flags. Gateway rc is written at boot. - NativeLink: ExecPaths for the DynamicUser state dir (was noexec). - Bazel server exits after 5 idle minutes; shared per-host repo cache (/repo-cache, idmap); goose hints say a local build means trouble. - Removed the /handoff -> /outputs materializer; outputs download into bazel-bin again. - slackerbot containers: 4G/1 CPU, up to 8G/2 CPUs. - fws1: disable networkd wait-online. - slackerbot 4db99ea (send_file symlinked home, trimmed tool lines). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0119iKoN3fSvRGUAYHupPW4U
This commit is contained in:
1 parent
594a1c1901
commit
2a2b1d8421
5 files changed
+124
-356
No files matched your search
Generated
+4
-4
@@ -3434,11 +3434,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1793566366,
|
"lastModified": 1793590815,
|
||||||
"narHash": "sha256-yytC8S47uJZNm7GraZQb84QH4YHA/tIOKIw6sis7sJA=",
|
"narHash": "sha256-Z2HRkYW0547L309qm4e0w++xVKIJFpKbqBUDLEoWG4c=",
|
||||||
"ref": "refs/heads/main",
|
"ref": "refs/heads/main",
|
||||||
"rev": "917a361370ab4da27ae10b232da34fcb40e04adf",
|
"rev": "4db99eaa2a0e1529194983962a07109fdb716bcd",
|
||||||
"revCount": 19,
|
"revCount": 22,
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://gitea.skyg.ca/ari/slacker-bot.git"
|
"url": "https://gitea.skyg.ca/ari/slacker-bot.git"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ let
|
|||||||
# The system each container boots (slackerbot's agent + bazel/rbuild);
|
# The system each container boots (slackerbot's agent + bazel/rbuild);
|
||||||
# updated on every rebuild.
|
# updated on every rebuild.
|
||||||
system = "${bazel.agentSystem}";
|
system = "${bazel.agentSystem}";
|
||||||
# /handoff and /outputs for remote Bazel builds on this host.
|
# Shared Bazel download cache (/repo-cache) for the containers.
|
||||||
rw_binds = bazel.binds.rw;
|
rw_binds = bazel.binds.rw;
|
||||||
ro_binds = bazel.binds.ro;
|
ro_binds = bazel.binds.ro;
|
||||||
machines_dir = "/var/lib/machines";
|
machines_dir = "/var/lib/machines";
|
||||||
@@ -28,8 +28,8 @@ let
|
|||||||
OPENROUTER_API_KEY = "$OPENROUTER_API_KEY";
|
OPENROUTER_API_KEY = "$OPENROUTER_API_KEY";
|
||||||
};
|
};
|
||||||
|
|
||||||
container = { memory_mb = 2048; cpus = 1.0; };
|
container = { memory_mb = 4096; cpus = 1.0; };
|
||||||
max_container = { memory_mb = 4096; cpus = 2.0; };
|
max_container = { memory_mb = 8192; cpus = 2.0; };
|
||||||
# Leave room for ollama, ds4, comfyui, hermes and the robots.
|
# Leave room for ollama, ds4, comfyui, hermes and the robots.
|
||||||
host_reserve = { memory_mb = 32768; cpus = 4.0; };
|
host_reserve = { memory_mb = 32768; cpus = 4.0; };
|
||||||
storage = { min = "2G"; max = "5G"; limit = "20G"; };
|
storage = { min = "2G"; max = "5G"; limit = "20G"; };
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ let
|
|||||||
# The system each container boots (slackerbot's agent + bazel/rbuild);
|
# The system each container boots (slackerbot's agent + bazel/rbuild);
|
||||||
# updated on every rebuild.
|
# updated on every rebuild.
|
||||||
system = "${bazel.agentSystem}";
|
system = "${bazel.agentSystem}";
|
||||||
# /handoff and /outputs for remote Bazel builds on this host.
|
# Shared Bazel download cache (/repo-cache) for the containers.
|
||||||
rw_binds = bazel.binds.rw;
|
rw_binds = bazel.binds.rw;
|
||||||
ro_binds = bazel.binds.ro;
|
ro_binds = bazel.binds.ro;
|
||||||
machines_dir = "/var/lib/machines";
|
machines_dir = "/var/lib/machines";
|
||||||
@@ -18,8 +18,8 @@ let
|
|||||||
allow_indefinite = true;
|
allow_indefinite = true;
|
||||||
boot_timeout = "2m";
|
boot_timeout = "2m";
|
||||||
|
|
||||||
container = { memory_mb = 2048; cpus = 1.0; };
|
container = { memory_mb = 4096; cpus = 1.0; };
|
||||||
max_container = { memory_mb = 4096; cpus = 2.0; };
|
max_container = { memory_mb = 8192; cpus = 2.0; };
|
||||||
host_reserve = { memory_mb = 8192; cpus = 2.0; };
|
host_reserve = { memory_mb = 8192; cpus = 2.0; };
|
||||||
storage = { min = "2G"; max = "5G"; limit = "20G"; };
|
storage = { min = "2G"; max = "5G"; limit = "20G"; };
|
||||||
drives = [{ path = homes; reserve = "100G"; }];
|
drives = [{ path = homes; reserve = "100G"; }];
|
||||||
@@ -58,6 +58,9 @@ in
|
|||||||
# Container links belong to systemd-networkd.
|
# Container links belong to systemd-networkd.
|
||||||
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
|
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
|
||||||
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
|
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
|
||||||
|
# networkd only has the container links here (NetworkManager owns wlan),
|
||||||
|
# so wait-online would always time out.
|
||||||
|
systemd.network.wait-online.enable = false;
|
||||||
|
|
||||||
systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ];
|
systemd.tmpfiles.rules = [ "d ${homes} 0700 root root -" ];
|
||||||
}
|
}
|
||||||
@@ -24,18 +24,38 @@ services/
|
|||||||
- **`ai-services.nix`** openwakeword as containers and
|
- **`ai-services.nix`** openwakeword as containers and
|
||||||
enables NixOS-native Wyoming faster-whisper + piper. Opens 11434/10200/10300/10400.
|
enables NixOS-native Wyoming faster-whisper + piper. Opens 11434/10200/10300/10400.
|
||||||
- **`bazel-build.nix`** runs NativeLink (from the `nativelink` flake input, pinned
|
- **`bazel-build.nix`** runs NativeLink (from the `nativelink` flake input, pinned
|
||||||
to v1.8.3 because its config format changes) plus `build-materializer`, which
|
to v1.8.3 because its config format changes) as a remote cache and executor for
|
||||||
hardlinks finished outputs from the cache into `/var/lib/build-outputs/<container>`.
|
slackerbot containers. Used on hl-fws1 and hl-fwdesk. Goal: as little as possible
|
||||||
Containers run `rbuild //target` (finds the host as the default gateway) and read
|
runs in a container. Measured on slacker-bot (1 CPU container, 2026-11-01): local
|
||||||
results at `/outputs`. Plain `bazel` builds remotely too: at boot,
|
458 s, remote with no cache 65 s, remote with warm cache 17 s.
|
||||||
`bazel-remote-rc` writes the gateway into `/run/bazel-remote.bazelrc`, which the
|
- NativeLink uses `DynamicUser`, whose state folder systemd mounts `noexec`;
|
||||||
container's `/etc/bazel.bazelrc` try-imports (missing = local build). Plain builds
|
`ExecPaths` re-allows it, or every remote action fails with "Permission denied".
|
||||||
download final outputs (`--remote_download_toplevel`). The Bazel server is capped at
|
- Containers get their own `bazel` (`agentBazel`), with no plain bazel/bazelisk on
|
||||||
512M. Global goose hints (`/root/.config/goose/.goosehints` → `/etc/goose/goosehints`)
|
PATH. At boot `bazel-remote-rc` writes the gateway into `/run/bazel-remote.bazelrc`,
|
||||||
tell goose that a local build means something is wrong: stop and report it. It also exposes `agentSystem` (slackerbot's agent + bazel, go,
|
which `/etc/bazel.bazelrc` try-imports. Only remote actions are allowed
|
||||||
bazelrc, rbuild) and `binds` (slackerbot `ro_binds`/`rw_binds`). Used on hl-fws1
|
(`--spawn_strategy=remote`, no local fallback), so a missing file or unreachable
|
||||||
and hl-fwdesk. Slackerbot must create `ro_binds` source folders, or containers fail
|
host fails the build instead of building in the container. The wrapper refuses
|
||||||
to start before their first build.
|
command-line flags that change this (`--spawn_strategy`, `--strategy*`,
|
||||||
|
`--remote_executor`, `--bazelrc`, ...). Flags in a repo's own `.bazelrc` are not
|
||||||
|
checked.
|
||||||
|
- Final outputs are downloaded into `bazel-bin` (`--remote_download_toplevel`).
|
||||||
|
`rbuild` is just `bazel build`. A `/handoff` → `/outputs` path that hardlinked
|
||||||
|
outputs from the cache was tried and removed: outputs are a few MB over a local
|
||||||
|
link, so it saved nothing (download ~20 s vs handoff ~344 s). Most of that was a
|
||||||
|
lock bug: Bazel's server inherits open file handles, so a lock held on an fd in
|
||||||
|
the wrapper stays held until the server exits. Close such fds for Bazel (`9>&-`).
|
||||||
|
- Still in the container: goose, the Bazel server (512M heap, exits after 5 idle
|
||||||
|
minutes) and repository rules, which Bazel can't run remotely.
|
||||||
|
- Downloads share one cache per host (`/var/lib/bazel-repo-cache` → `/repo-cache`,
|
||||||
|
sha256-checked, 30-day cleanup). Fetched repo folders stay per container
|
||||||
|
(`--repo_contents_cache`), since their contents are not checked.
|
||||||
|
- Containers use private user IDs, so writable binds (`/repo-cache`) use
|
||||||
|
`idmap`; without it the folder shows as `nobody` and the container can't write.
|
||||||
|
- Global goose hints (`/root/.config/goose/.goosehints` → `/etc/goose/goosehints`):
|
||||||
|
a local build means something is wrong (stop and report); build Go through Bazel.
|
||||||
|
- Exposes `agentSystem` (slackerbot's agent + bazel, go, bazelrc, rbuild) and
|
||||||
|
`binds` (slackerbot `ro_binds`/`rw_binds`). Slackerbot creates missing bind
|
||||||
|
source folders, or containers fail to start.
|
||||||
`bazel` is a bazelisk wrapper on the host and in containers: a repo's
|
`bazel` is a bazelisk wrapper on the host and in containers: a repo's
|
||||||
`.bazelversion` wins, otherwise 8.7.0 (`USE_BAZEL_FALLBACK_VERSION`). Bazel 8.7.0
|
`.bazelversion` wins, otherwise 8.7.0 (`USE_BAZEL_FALLBACK_VERSION`). Bazel 8.7.0
|
||||||
is fetched once per node by Nix (official release, sha256-checked) and the wrapper
|
is fetched once per node by Nix (official release, sha256-checked) and the wrapper
|
||||||
|
|||||||
@@ -1,18 +1,14 @@
|
|||||||
# Bazel build server for slackerbot containers on the same host.
|
# Bazel build server for slackerbot containers on the same host.
|
||||||
# One NativeLink (cache + scheduler + worker) runs builds on the host CPUs.
|
# One NativeLink (cache + scheduler + worker) runs builds on the host CPUs.
|
||||||
# Final outputs are not sent over the network: a host service hardlinks them
|
|
||||||
# from NativeLink's cache into a folder each container sees at /outputs.
|
|
||||||
#
|
#
|
||||||
# container: rbuild //app ──grpc──▶ NativeLink (host) ──▶ host CPUs
|
# container: bazel build //app ──grpc──▶ NativeLink (host) ──▶ host CPUs
|
||||||
# │ writes build events to /handoff/bep.json
|
# ◀── final outputs downloaded (small, same machine)
|
||||||
# ▼
|
|
||||||
# host: build-materializer follows bep.json and hardlinks each finished
|
|
||||||
# output into /var/lib/build-outputs/<container>/
|
|
||||||
# ▼
|
|
||||||
# container: files appear at /outputs (read-only)
|
|
||||||
#
|
#
|
||||||
# slackerbot mounts the folders through `binds`. Containers run as root with
|
# A /outputs handoff (hardlinking outputs from the cache) was tried and
|
||||||
# no private user IDs, so no uid mapping is needed.
|
# removed: it saved nothing here, since outputs are a few MB over a local link.
|
||||||
|
#
|
||||||
|
# slackerbot mounts the shared download cache through `binds`. Containers use
|
||||||
|
# private user IDs, so writable binds need `idmap` (container root = host root).
|
||||||
{ config, lib, pkgs, nativelink, slackerbot, ... }:
|
{ config, lib, pkgs, nativelink, slackerbot, ... }:
|
||||||
|
|
||||||
with lib;
|
with lib;
|
||||||
@@ -50,9 +46,8 @@ let
|
|||||||
'';
|
'';
|
||||||
|
|
||||||
stateDir = "/var/lib/nativelink";
|
stateDir = "/var/lib/nativelink";
|
||||||
handoffRoot = "/var/lib/build-handoff";
|
# Bazel download cache shared by all containers (files are sha256-checked).
|
||||||
# Must be on the same disk as stateDir, or hardlinks fail (falls back to copies).
|
repoCacheRoot = "/var/lib/bazel-repo-cache";
|
||||||
outputsRoot = "/var/lib/build-outputs";
|
|
||||||
workerPort = 50061; # host only
|
workerPort = 50061; # host only
|
||||||
|
|
||||||
# Tools build actions run with. Same store paths on the host and in containers.
|
# Tools build actions run with. Same store paths on the host and in containers.
|
||||||
@@ -83,7 +78,7 @@ let
|
|||||||
name = "WORKER_FAST_SLOW_STORE";
|
name = "WORKER_FAST_SLOW_STORE";
|
||||||
fast_slow = {
|
fast_slow = {
|
||||||
fast.filesystem = {
|
fast.filesystem = {
|
||||||
content_path = "${stateDir}/cas/content"; # materializer links from here
|
content_path = "${stateDir}/cas/content";
|
||||||
temp_path = "${stateDir}/cas/tmp";
|
temp_path = "${stateDir}/cas/tmp";
|
||||||
eviction_policy.max_bytes = cfg.cacheGB * 1000 * 1000 * 1000;
|
eviction_policy.max_bytes = cfg.cacheGB * 1000 * 1000 * 1000;
|
||||||
};
|
};
|
||||||
@@ -142,306 +137,62 @@ let
|
|||||||
global.max_open_files = 65536;
|
global.max_open_files = 65536;
|
||||||
});
|
});
|
||||||
|
|
||||||
# One long-running service for all containers. When a container drops a
|
# Container side `bazel`. Refuses flags that would move work into the
|
||||||
# "started" marker in its handoff folder, a thread follows its build event
|
# container and forces remote actions. The host address comes from
|
||||||
# file. As each target finishes, it opens each output in the cache and
|
# /run/bazel-remote.bazelrc (see bazelrc).
|
||||||
# hardlinks it into that container's outputs folder (opening first closes
|
agentBazel = pkgs.writeShellScriptBin "bazel" ''
|
||||||
# the eviction race). Outputs already evicted are reported back so the
|
|
||||||
# container can rebuild them.
|
|
||||||
#
|
|
||||||
# Handoff folders are written by the containers, so they are untrusted: no
|
|
||||||
# symlinks are followed and every destination is checked to stay inside
|
|
||||||
# that container's outputs folder.
|
|
||||||
#
|
|
||||||
# NativeLink v1.8 names cache files d2/<sha256>-<size>-<generation>. The
|
|
||||||
# generation is a counter we can't guess, so the folder is indexed and
|
|
||||||
# re-read when a file isn't found.
|
|
||||||
materializer = pkgs.writeScript "build-materializer" ''
|
|
||||||
#!${pkgs.python3}/bin/python3
|
|
||||||
import ctypes, errno, fcntl, json, os, re, shutil, sys, threading, time
|
|
||||||
|
|
||||||
HANDOFF_ROOT = "${handoffRoot}"
|
|
||||||
OUT_ROOT = "${outputsRoot}"
|
|
||||||
DIGESTS = "${stateDir}/cas/content/d2"
|
|
||||||
NAME_OK = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
|
||||||
BLOB = re.compile(r"/blobs/([0-9a-f]{64})/(\d+)")
|
|
||||||
FICLONE = 0x40049409
|
|
||||||
LIBC = ctypes.CDLL(None, use_errno=True)
|
|
||||||
AT_FDCWD, AT_EMPTY_PATH = -100, 0x1000
|
|
||||||
|
|
||||||
def log(*a):
|
|
||||||
print(*a, file=sys.stderr, flush=True)
|
|
||||||
|
|
||||||
class CasIndex:
|
|
||||||
# Maps "<sha256>-<size>" to the newest file name in the cache.
|
|
||||||
def __init__(self):
|
|
||||||
self.names, self.at, self.lock = {}, 0.0, threading.Lock()
|
|
||||||
|
|
||||||
def rescan(self):
|
|
||||||
# Read the folder at most twice a second, however many threads ask.
|
|
||||||
wait = self.at + 0.5 - time.monotonic()
|
|
||||||
if wait > 0:
|
|
||||||
time.sleep(wait)
|
|
||||||
names = {}
|
|
||||||
try:
|
|
||||||
with os.scandir(DIGESTS) as it:
|
|
||||||
for e in it:
|
|
||||||
key, _, gen = e.name.rpartition("-")
|
|
||||||
if key and gen.isdigit() and int(gen) >= names.get(key, (-1, ""))[0]:
|
|
||||||
names[key] = (int(gen), e.name)
|
|
||||||
except FileNotFoundError:
|
|
||||||
pass
|
|
||||||
self.names, self.at = names, time.monotonic()
|
|
||||||
|
|
||||||
def lookup(self, h, size, refresh):
|
|
||||||
key = f"{h}-{size}"
|
|
||||||
with self.lock:
|
|
||||||
if refresh or key not in self.names:
|
|
||||||
self.rescan()
|
|
||||||
hit = self.names.get(key)
|
|
||||||
return os.path.join(DIGESTS, hit[1]) if hit else None
|
|
||||||
|
|
||||||
CAS = CasIndex()
|
|
||||||
|
|
||||||
def link_fd(fd, dest):
|
|
||||||
# Hardlink the exact inode we hold open (root only). Fails with
|
|
||||||
# ENOENT if the file was evicted after we opened it.
|
|
||||||
if LIBC.linkat(fd, b"", AT_FDCWD, os.fsencode(dest), AT_EMPTY_PATH) != 0:
|
|
||||||
err = ctypes.get_errno()
|
|
||||||
raise OSError(err, os.strerror(err), dest)
|
|
||||||
|
|
||||||
def copy_fd(fd, dest):
|
|
||||||
# Reflink on btrfs/XFS (no extra disk), plain copy otherwise.
|
|
||||||
with open(dest, "wb") as out:
|
|
||||||
try:
|
|
||||||
fcntl.ioctl(out.fileno(), FICLONE, fd)
|
|
||||||
except OSError:
|
|
||||||
with os.fdopen(os.dup(fd), "rb") as src:
|
|
||||||
src.seek(0)
|
|
||||||
shutil.copyfileobj(src, out, 1 << 20)
|
|
||||||
|
|
||||||
def place(h, size, dest):
|
|
||||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
||||||
tmp = dest + ".materializing"
|
|
||||||
if os.path.lexists(tmp):
|
|
||||||
os.unlink(tmp)
|
|
||||||
if size == 0:
|
|
||||||
open(tmp, "wb").close()
|
|
||||||
os.replace(tmp, dest)
|
|
||||||
return True
|
|
||||||
for refresh in (False, True):
|
|
||||||
path = CAS.lookup(h, size, refresh)
|
|
||||||
if path is None:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
|
||||||
except FileNotFoundError:
|
|
||||||
continue # index was stale; re-read it once
|
|
||||||
try:
|
|
||||||
head = os.pread(fd, 4, 0)
|
|
||||||
if head.startswith(b"\x7fELF") or head.startswith(b"#!"):
|
|
||||||
# Executables get their own copy so chmod never touches the cache.
|
|
||||||
copy_fd(fd, tmp)
|
|
||||||
os.chmod(tmp, 0o755)
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
link_fd(fd, tmp)
|
|
||||||
except OSError as e:
|
|
||||||
if e.errno not in (errno.ENOENT, errno.EXDEV):
|
|
||||||
raise
|
|
||||||
# Evicted after we opened it (or another disk):
|
|
||||||
# the open fd still has the data, so copy it.
|
|
||||||
copy_fd(fd, tmp)
|
|
||||||
os.replace(tmp, dest)
|
|
||||||
# Renaming a hardlink onto the same file is a no-op that
|
|
||||||
# leaves tmp behind (output unchanged since last build).
|
|
||||||
if os.path.lexists(tmp):
|
|
||||||
os.unlink(tmp)
|
|
||||||
return True
|
|
||||||
finally:
|
|
||||||
os.close(fd)
|
|
||||||
return False
|
|
||||||
|
|
||||||
class Job:
|
|
||||||
def __init__(self, name):
|
|
||||||
self.name = name
|
|
||||||
self.handoff = os.path.join(HANDOFF_ROOT, name)
|
|
||||||
self.out = os.path.join(OUT_ROOT, name)
|
|
||||||
os.makedirs(self.out, exist_ok=True)
|
|
||||||
self.out = os.path.realpath(self.out)
|
|
||||||
self.started = os.path.join(self.handoff, "started")
|
|
||||||
self.sets = {}
|
|
||||||
self.missing = []
|
|
||||||
|
|
||||||
def safe_dest(self, parts):
|
|
||||||
dest = os.path.normpath(os.path.join(self.out, *parts))
|
|
||||||
if not dest.startswith(self.out + "/"):
|
|
||||||
raise ValueError("path escapes outputs folder: " + dest)
|
|
||||||
return dest
|
|
||||||
|
|
||||||
def files_in(self, set_id, seen):
|
|
||||||
if set_id in seen or set_id not in self.sets:
|
|
||||||
return []
|
|
||||||
seen.add(set_id)
|
|
||||||
s = self.sets[set_id]
|
|
||||||
found = list(s.get("files", []))
|
|
||||||
for sub in s.get("fileSets", []):
|
|
||||||
found += self.files_in(sub["id"], seen)
|
|
||||||
return found
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def digest_of(f):
|
|
||||||
if f.get("digest"):
|
|
||||||
return f["digest"], int(f.get("length", 0))
|
|
||||||
m = BLOB.search(f.get("uri", ""))
|
|
||||||
return (m.group(1), int(m.group(2))) if m else (None, None)
|
|
||||||
|
|
||||||
def handle(self, ev):
|
|
||||||
if "namedSetOfFiles" in ev:
|
|
||||||
self.sets[ev["id"]["namedSet"]["id"]] = ev["namedSetOfFiles"]
|
|
||||||
elif "completed" in ev and ev["completed"].get("success"):
|
|
||||||
label = ev["id"]["targetCompleted"]["label"]
|
|
||||||
ok, seen = True, set()
|
|
||||||
for group in ev["completed"].get("outputGroup", []):
|
|
||||||
for fs in group.get("fileSets", []):
|
|
||||||
for f in self.files_in(fs["id"], seen):
|
|
||||||
h, size = self.digest_of(f)
|
|
||||||
if h is None:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
dest = self.safe_dest(f.get("pathPrefix", []) + [f["name"]])
|
|
||||||
except ValueError as e:
|
|
||||||
log(self.name, e)
|
|
||||||
continue
|
|
||||||
if not place(h, size, dest):
|
|
||||||
ok = False
|
|
||||||
if not ok:
|
|
||||||
self.missing.append(label)
|
|
||||||
|
|
||||||
def open_bep(self):
|
|
||||||
path = os.path.join(self.handoff, "bep.json")
|
|
||||||
while os.path.lexists(self.started):
|
|
||||||
try:
|
|
||||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
|
||||||
return os.fdopen(fd, "r")
|
|
||||||
except FileNotFoundError:
|
|
||||||
time.sleep(0.1)
|
|
||||||
return None
|
|
||||||
|
|
||||||
def follow(self):
|
|
||||||
fh = self.open_bep()
|
|
||||||
if fh is None:
|
|
||||||
return
|
|
||||||
buf, finishing = "", False
|
|
||||||
with fh:
|
|
||||||
while True:
|
|
||||||
chunk = fh.readline()
|
|
||||||
if not chunk:
|
|
||||||
if finishing:
|
|
||||||
return # build over and file fully read
|
|
||||||
if not os.path.lexists(self.started):
|
|
||||||
finishing = True # one more pass for the last lines
|
|
||||||
else:
|
|
||||||
time.sleep(0.05)
|
|
||||||
continue
|
|
||||||
buf += chunk
|
|
||||||
if not buf.endswith("\n"):
|
|
||||||
continue
|
|
||||||
line, buf = buf, ""
|
|
||||||
if line.strip():
|
|
||||||
self.handle(json.loads(line))
|
|
||||||
|
|
||||||
def write_result(self):
|
|
||||||
# O_EXCL + O_NOFOLLOW: a container can't aim this at a host file.
|
|
||||||
tmp = os.path.join(self.handoff, ".result.tmp")
|
|
||||||
if os.path.lexists(tmp):
|
|
||||||
os.unlink(tmp)
|
|
||||||
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644)
|
|
||||||
with os.fdopen(fd, "w") as fh:
|
|
||||||
json.dump({"missing": self.missing}, fh)
|
|
||||||
os.replace(tmp, os.path.join(self.handoff, "result.json"))
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
try:
|
|
||||||
self.follow()
|
|
||||||
except Exception as e:
|
|
||||||
log(self.name, "build handoff failed:", e)
|
|
||||||
self.missing.append("//...") # tell rbuild to fall back to a download
|
|
||||||
while os.path.lexists(self.started):
|
|
||||||
time.sleep(0.1)
|
|
||||||
try:
|
|
||||||
self.write_result()
|
|
||||||
except Exception as e:
|
|
||||||
log(self.name, "could not write result:", e)
|
|
||||||
|
|
||||||
active, lock = set(), threading.Lock()
|
|
||||||
|
|
||||||
def worker(name):
|
|
||||||
try:
|
|
||||||
Job(name).run()
|
|
||||||
finally:
|
|
||||||
with lock:
|
|
||||||
active.discard(name)
|
|
||||||
|
|
||||||
os.makedirs(HANDOFF_ROOT, exist_ok=True)
|
|
||||||
while True:
|
|
||||||
for name in os.listdir(HANDOFF_ROOT):
|
|
||||||
if not NAME_OK.match(name):
|
|
||||||
continue
|
|
||||||
d = os.path.join(HANDOFF_ROOT, name)
|
|
||||||
if os.path.islink(d) or not os.path.isdir(d):
|
|
||||||
continue
|
|
||||||
if not os.path.lexists(os.path.join(d, "started")):
|
|
||||||
continue
|
|
||||||
with lock:
|
|
||||||
if name in active:
|
|
||||||
continue
|
|
||||||
active.add(name)
|
|
||||||
threading.Thread(target=worker, args=(name,), daemon=True).start()
|
|
||||||
time.sleep(0.2)
|
|
||||||
'';
|
|
||||||
|
|
||||||
# Container side: `rbuild //target ...` instead of `bazel build`.
|
|
||||||
# The host is the container's default gateway (ve-* addresses are not fixed).
|
|
||||||
rbuild = pkgs.writeShellScriptBin "rbuild" ''
|
|
||||||
set -u
|
set -u
|
||||||
H=/handoff
|
real=${bazel}/bin/bazel
|
||||||
gw=$(${pkgs.iproute2}/bin/ip -4 route show default | ${pkgs.gawk}/bin/awk '{print $3; exit}')
|
enforce=(--spawn_strategy=remote --noremote_local_fallback)
|
||||||
if [ -z "$gw" ]; then
|
|
||||||
echo "rbuild: no default route, can't find the build server" >&2
|
# Find the command (first word that isn't a startup option) and check flags.
|
||||||
exit 1
|
# Arguments after `--` belong to the program being run, so they're skipped.
|
||||||
fi
|
cmd="" at=0 i=0
|
||||||
remote=--remote_executor=grpc://$gw:${toString cfg.port}
|
for a in "$@"; do
|
||||||
rm -f $H/bep.json $H/result.json
|
[ "$a" = "--" ] && break
|
||||||
touch $H/started
|
case "$a" in
|
||||||
${bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@"
|
--spawn_strategy*|--strategy*|--test_strategy*|--remote_executor*|\
|
||||||
rc=$?
|
--remote_local_fallback*|--bazelrc*|--ignore_all_rc_files|--nosystem_rc)
|
||||||
rm -f $H/started
|
echo "bazel: $a isn't allowed here. Builds must run on the host's build server;" \
|
||||||
for _ in $(seq 600); do [ -f $H/result.json ] && break; sleep 0.1; done
|
"tell the user instead of working around it." >&2
|
||||||
if [ ! -f $H/result.json ]; then
|
exit 2 ;;
|
||||||
echo "rbuild: host did not hand back outputs" >&2
|
esac
|
||||||
exit 1
|
if [ -z "$cmd" ] && [ "''${a#-}" = "$a" ]; then cmd=$a at=$i; fi
|
||||||
fi
|
i=$((i + 1))
|
||||||
mapfile -t missing < <(${pkgs.jq}/bin/jq -r '.missing[]' $H/result.json)
|
done
|
||||||
if [ ''${#missing[@]} -gt 0 ]; then
|
[ -z "$cmd" ] && exec "$real" "$@"
|
||||||
# Rare: outputs were evicted before the host could link them. Bazel
|
pre=("''${@:1:at}")
|
||||||
# finds them gone, rebuilds them, and downloads them over the network.
|
post=("''${@:at+2}")
|
||||||
echo "rbuild: rebuilding ''${#missing[@]} evicted target(s)" >&2
|
|
||||||
${bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$?
|
case "$cmd" in
|
||||||
fi
|
build|test|run|coverage) exec "$real" "''${pre[@]}" "$cmd" "''${enforce[@]}" "''${post[@]}" ;;
|
||||||
echo "rbuild: outputs are in /outputs" >&2
|
*) exec "$real" "$@" ;;
|
||||||
exit $rc
|
esac
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
# Kept for old habits and docs: same as `bazel build`.
|
||||||
|
rbuild = pkgs.writeShellScriptBin "rbuild" ''exec ${agentBazel}/bin/bazel build "$@"'';
|
||||||
|
|
||||||
remoteRc = "/run/bazel-remote.bazelrc";
|
remoteRc = "/run/bazel-remote.bazelrc";
|
||||||
|
|
||||||
bazelrc = ''
|
bazelrc = ''
|
||||||
# Builds run on the host, so the Bazel server in a container stays small.
|
# Builds run on the host, so the Bazel server in a container stays small,
|
||||||
|
# and exits after 5 idle minutes instead of 3 hours.
|
||||||
startup --host_jvm_args=-Xmx512m
|
startup --host_jvm_args=-Xmx512m
|
||||||
|
startup --host_jvm_args=-XX:+UseSerialGC
|
||||||
|
startup --max_idle_secs=300
|
||||||
|
|
||||||
|
# Every action runs on the host; one that can't fails instead of running here.
|
||||||
|
build --spawn_strategy=remote
|
||||||
|
build --noremote_local_fallback
|
||||||
|
|
||||||
|
# Downloads are shared by all containers on this host. Fetched repo folders
|
||||||
|
# are not (unchecked, so one container could tamper with another's).
|
||||||
|
common --repository_cache=/repo-cache
|
||||||
|
common --repo_contents_cache=/root/.cache/bazel/repo-contents
|
||||||
|
|
||||||
build --remote_instance_name=main
|
build --remote_instance_name=main
|
||||||
# Plain `bazel build` downloads final outputs; rbuild overrides this with
|
# Final outputs are downloaded into bazel-bin (a few MB, same machine).
|
||||||
# minimal and gets them through /outputs instead.
|
|
||||||
build --remote_download_toplevel
|
build --remote_download_toplevel
|
||||||
build --jobs=${toString (cfg.cpuCount * 4)}
|
build --jobs=${toString (cfg.cpuCount * 4)}
|
||||||
|
|
||||||
@@ -452,7 +203,7 @@ let
|
|||||||
build --host_action_env=PATH=${toolchain}/bin
|
build --host_action_env=PATH=${toolchain}/bin
|
||||||
build --repo_env=CC=${toolchain}/bin/gcc
|
build --repo_env=CC=${toolchain}/bin/gcc
|
||||||
|
|
||||||
# Points plain `bazel` at the host (written at boot). Missing = local build.
|
# Points plain `bazel` at the host (written at boot). Missing = builds fail.
|
||||||
try-import ${remoteRc}
|
try-import ${remoteRc}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
@@ -460,9 +211,12 @@ let
|
|||||||
goosehints = ''
|
goosehints = ''
|
||||||
# Bazel builds run on the host, never in this container
|
# Bazel builds run on the host, never in this container
|
||||||
|
|
||||||
This container is small (about 2 GB RAM, 1 CPU). `bazel build/test/run` is
|
This container is small (a few GB RAM, 1 CPU). `bazel build/test/run` runs
|
||||||
set up to run every action on the host's build server, and `rbuild //target`
|
every action on the host's build server.
|
||||||
does the same and puts outputs in /outputs.
|
|
||||||
|
Outputs are downloaded into bazel-bin as usual.
|
||||||
|
Flags that change where actions run (--spawn_strategy, --strategy,
|
||||||
|
--remote_executor, --bazelrc, ...) are refused on purpose.
|
||||||
|
|
||||||
A normal build ends with a line like
|
A normal build ends with a line like
|
||||||
`INFO: 120 processes: 110 remote, 10 internal.` (or "remote cache hit").
|
`INFO: 120 processes: 110 remote, 10 internal.` (or "remote cache hit").
|
||||||
@@ -470,14 +224,19 @@ let
|
|||||||
doing the work, or /run/bazel-remote.bazelrc is missing, or a build can't
|
doing the work, or /run/bazel-remote.bazelrc is missing, or a build can't
|
||||||
reach the remote executor, something is wrong:
|
reach the remote executor, something is wrong:
|
||||||
- stop the build (`bazel shutdown`), don't retry it locally,
|
- stop the build (`bazel shutdown`), don't retry it locally,
|
||||||
- don't remove or override the --remote_* flags,
|
- don't remove or override the --remote_* or --spawn_strategy flags,
|
||||||
- tell the user what you saw, with the error lines.
|
- tell the user what you saw, with the error lines.
|
||||||
|
|
||||||
|
Build and test Go code through Bazel (`bazel build`, `bazel test`), not
|
||||||
|
`go build` / `go test`: those run here, in this small container. `go mod`
|
||||||
|
and other quick `go` commands are fine.
|
||||||
'';
|
'';
|
||||||
|
|
||||||
# Added to slackerbot's agent system. Uses host packages so bazel and the
|
# Added to slackerbot's agent system. Uses host packages so bazel and the
|
||||||
# toolchain match the host exactly.
|
# toolchain match the host exactly.
|
||||||
agentModule = {
|
agentModule = {
|
||||||
environment.systemPackages = [ bazel pkgs.bazelisk pkgs.go pkgs.git rbuild ];
|
# No plain bazel/bazelisk on PATH, so the checks in agentBazel can't be skipped by accident.
|
||||||
|
environment.systemPackages = [ agentBazel pkgs.go pkgs.git rbuild ];
|
||||||
environment.etc."bazel.bazelrc".text = bazelrc;
|
environment.etc."bazel.bazelrc".text = bazelrc;
|
||||||
environment.etc."goose/goosehints".text = goosehints;
|
environment.etc."goose/goosehints".text = goosehints;
|
||||||
programs.nix-ld.enable = true;
|
programs.nix-ld.enable = true;
|
||||||
@@ -543,12 +302,13 @@ in
|
|||||||
};
|
};
|
||||||
|
|
||||||
binds = mkOption {
|
binds = mkOption {
|
||||||
description = "slackerbot ro_binds / rw_binds that give each container /handoff and /outputs.";
|
description = "slackerbot ro_binds / rw_binds that give each container the shared /repo-cache.";
|
||||||
readOnly = true;
|
readOnly = true;
|
||||||
type = types.attrsOf (types.listOf types.str);
|
type = types.attrsOf (types.listOf types.str);
|
||||||
default = {
|
default = {
|
||||||
rw = [ "${handoffRoot}/{machine}:/handoff" ];
|
# idmap: every container writes as the same host user, so all can share it.
|
||||||
ro = [ "${outputsRoot}/{machine}:/outputs" ];
|
rw = [ "${repoCacheRoot}:/repo-cache:idmap" ];
|
||||||
|
ro = [ ];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
@@ -567,32 +327,17 @@ in
|
|||||||
ExecStart = "${nativelink.packages.${system}.nativelink}/bin/nativelink ${nativelinkConfig}";
|
ExecStart = "${nativelink.packages.${system}.nativelink}/bin/nativelink ${nativelinkConfig}";
|
||||||
DynamicUser = true;
|
DynamicUser = true;
|
||||||
StateDirectory = "nativelink";
|
StateDirectory = "nativelink";
|
||||||
# Cached files must be readable by the materializer and containers.
|
# DynamicUser's state dir is mounted noexec; build actions run from work/.
|
||||||
|
ExecPaths = [ "/var/lib/private/nativelink" ];
|
||||||
UMask = "0022";
|
UMask = "0022";
|
||||||
LimitNOFILE = 65536;
|
LimitNOFILE = 65536;
|
||||||
Restart = "on-failure";
|
Restart = "on-failure";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
systemd.services.build-materializer = {
|
|
||||||
description = "Hand Bazel outputs to slackerbot containers";
|
|
||||||
wantedBy = [ "multi-user.target" ];
|
|
||||||
after = [ "nativelink.service" ];
|
|
||||||
serviceConfig = {
|
|
||||||
ExecStart = "${materializer}"; # root: needed to link cache files
|
|
||||||
UMask = "0022";
|
|
||||||
Restart = "always";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
# Outputs older than 7 days are removed. Hardlinked files keep their data
|
|
||||||
# alive after NativeLink evicts them, so this is what frees that space.
|
|
||||||
# `e` ages what's inside each container's folder, never the folder itself
|
|
||||||
# (it is a live bind mount source).
|
|
||||||
systemd.tmpfiles.rules = [
|
systemd.tmpfiles.rules = [
|
||||||
"d ${handoffRoot} 0755 root root -"
|
# Downloads unused for 30 days are removed (Bazel never cleans this).
|
||||||
"d ${outputsRoot} 0755 root root -"
|
"d ${repoCacheRoot} 0755 root root 30d"
|
||||||
"e ${outputsRoot}/* - - - 7d"
|
|
||||||
];
|
];
|
||||||
|
|
||||||
# Keep the toolchain from garbage collection.
|
# Keep the toolchain from garbage collection.
|
||||||
|
|||||||
Reference in new issue
Block a user