602 lines
23 KiB
Nix
602 lines
23 KiB
Nix
# Bazel build server for slackerbot containers on the same host.
|
|
# One NativeLink (cache + scheduler + worker) runs builds on the host CPUs.
|
|
# Final outputs are not sent over the network: a host service hardlinks them
|
|
# from NativeLink's cache into a folder each container sees at /outputs.
|
|
#
|
|
# container: rbuild //app ──grpc──▶ NativeLink (host) ──▶ host CPUs
|
|
# │ writes build events to /handoff/bep.json
|
|
# ▼
|
|
# host: build-materializer follows bep.json and hardlinks each finished
|
|
# output into /var/lib/build-outputs/<container>/
|
|
# ▼
|
|
# container: files appear at /outputs (read-only)
|
|
#
|
|
# slackerbot mounts the folders through `binds`. Containers run as root with
|
|
# no private user IDs, so no uid mapping is needed.
|
|
{ config, lib, pkgs, nativelink, slackerbot, ... }:
|
|
|
|
with lib;
|
|
|
|
let
|
|
cfg = config.skyg.nixos.server.services.bazel-build;
|
|
system = pkgs.stdenv.hostPlatform.system;
|
|
# Official Bazel release, checked against its sha256. Containers see
|
|
# /nix/store read-only, so one copy per node serves all of them. Upstream
|
|
# builds only run on NixOS through nix-ld. To bump: change both values.
|
|
bazelVersion = "8.7.0";
|
|
bazelSha256 = "d7606e679b78067c811096fb3d6cf135225b528835ca396e3a4dddf957859544";
|
|
# Copied as-is: patching would break the archive appended to the binary.
|
|
bazelRelease = pkgs.runCommand "bazel-${bazelVersion}-release" { } ''
|
|
install -Dm755 ${pkgs.fetchurl {
|
|
url = "https://releases.bazel.build/${bazelVersion}/release/bazel-${bazelVersion}-linux-x86_64";
|
|
sha256 = bazelSha256;
|
|
}} $out/bin/bazel
|
|
'';
|
|
|
|
# `bazel` runs bazelisk on the host and in containers: a repo's
|
|
# .bazelversion wins, otherwise bazelVersion. bazelVersion is pre-entered in
|
|
# bazelisk's cache (format of bazelisk 1.28) as a link to bazelRelease, so it
|
|
# is never downloaded. Other versions download into each user's own cache.
|
|
bazel = pkgs.writeShellScriptBin "bazel" ''
|
|
export USE_BAZEL_FALLBACK_VERSION=''${USE_BAZEL_FALLBACK_VERSION:-silent:${bazelVersion}}
|
|
dl=''${BAZELISK_HOME_LINUX:-''${BAZELISK_HOME:-''${XDG_CACHE_HOME:-$HOME/.cache}/bazelisk}}/downloads
|
|
meta=$dl/metadata/bazelbuild/bazel-${bazelVersion}-linux-x86_64
|
|
if [ ! -e "$dl/sha256/${bazelSha256}/bin/bazel" ] || [ "$(cat "$meta" 2>/dev/null)" != ${bazelSha256} ]; then
|
|
mkdir -p "$dl/sha256" "''${meta%/*}"
|
|
ln -sfn ${bazelRelease} "$dl/sha256/${bazelSha256}"
|
|
printf %s ${bazelSha256} > "$meta"
|
|
fi
|
|
exec ${pkgs.bazelisk}/bin/bazelisk "$@"
|
|
'';
|
|
|
|
stateDir = "/var/lib/nativelink";
|
|
handoffRoot = "/var/lib/build-handoff";
|
|
# Must be on the same disk as stateDir, or hardlinks fail (falls back to copies).
|
|
outputsRoot = "/var/lib/build-outputs";
|
|
workerPort = 50061; # host only
|
|
|
|
# Tools build actions run with. Same store paths on the host and in containers.
|
|
toolchain = pkgs.buildEnv {
|
|
name = "bazel-action-toolchain";
|
|
paths = with pkgs; [ gcc binutils coreutils bash gnused gnugrep gawk findutils diffutils python3 ];
|
|
};
|
|
|
|
# Matches NativeLink v1.8.3 (basic_cas example). Field names change between
|
|
# releases; check them when bumping the flake input.
|
|
instances = f: map (instance_name: { inherit instance_name; } // f) [ "" "main" ];
|
|
nativelinkConfig = pkgs.writeText "nativelink.json" (builtins.toJSON {
|
|
stores = [
|
|
{
|
|
# If any output of a cached result was evicted, report a miss so the
|
|
# action is rebuilt instead of returning a result with missing files.
|
|
name = "AC_MAIN_STORE";
|
|
completeness_checking = {
|
|
backend.filesystem = {
|
|
content_path = "${stateDir}/ac/content";
|
|
temp_path = "${stateDir}/ac/tmp";
|
|
eviction_policy.max_bytes = 10 * 1000 * 1000 * 1000;
|
|
};
|
|
cas_store.ref_store.name = "WORKER_FAST_SLOW_STORE";
|
|
};
|
|
}
|
|
{
|
|
name = "WORKER_FAST_SLOW_STORE";
|
|
fast_slow = {
|
|
fast.filesystem = {
|
|
content_path = "${stateDir}/cas/content"; # materializer links from here
|
|
temp_path = "${stateDir}/cas/tmp";
|
|
eviction_policy.max_bytes = cfg.cacheGB * 1000 * 1000 * 1000;
|
|
};
|
|
slow.noop = { };
|
|
};
|
|
}
|
|
];
|
|
|
|
schedulers = [{
|
|
name = "MAIN_SCHEDULER";
|
|
simple.supported_platform_properties = {
|
|
cpu_count = "minimum";
|
|
OSFamily = "priority";
|
|
"container-image" = "priority";
|
|
};
|
|
}];
|
|
|
|
workers = [{
|
|
local = {
|
|
worker_api_endpoint.uri = "grpc://127.0.0.1:${toString workerPort}";
|
|
cas_fast_slow_store = "WORKER_FAST_SLOW_STORE";
|
|
upload_action_result.ac_store = "AC_MAIN_STORE";
|
|
work_directory = "${stateDir}/work";
|
|
platform_properties = {
|
|
cpu_count.values = [ (toString cfg.cpuCount) ];
|
|
OSFamily.values = [ "" ];
|
|
"container-image".values = [ "" ];
|
|
};
|
|
};
|
|
}];
|
|
|
|
servers = [
|
|
{
|
|
name = "public";
|
|
# Only container links (ve-*) are trusted; the LAN port stays closed.
|
|
listener.http.socket_address = "0.0.0.0:${toString cfg.port}";
|
|
services = {
|
|
cas = instances { cas_store = "WORKER_FAST_SLOW_STORE"; };
|
|
ac = instances { ac_store = "AC_MAIN_STORE"; };
|
|
execution = instances { cas_store = "WORKER_FAST_SLOW_STORE"; scheduler = "MAIN_SCHEDULER"; };
|
|
capabilities = instances { remote_execution.scheduler = "MAIN_SCHEDULER"; };
|
|
bytestream = instances { cas_store = "WORKER_FAST_SLOW_STORE"; };
|
|
};
|
|
}
|
|
{
|
|
name = "private_workers_servers";
|
|
listener.http.socket_address = "127.0.0.1:${toString workerPort}";
|
|
services = {
|
|
worker_api.scheduler = "MAIN_SCHEDULER";
|
|
admin = { };
|
|
health = { };
|
|
};
|
|
}
|
|
];
|
|
|
|
global.max_open_files = 65536;
|
|
});
|
|
|
|
# One long-running service for all containers. When a container drops a
|
|
# "started" marker in its handoff folder, a thread follows its build event
|
|
# file. As each target finishes, it opens each output in the cache and
|
|
# hardlinks it into that container's outputs folder (opening first closes
|
|
# the eviction race). Outputs already evicted are reported back so the
|
|
# container can rebuild them.
|
|
#
|
|
# Handoff folders are written by the containers, so they are untrusted: no
|
|
# symlinks are followed and every destination is checked to stay inside
|
|
# that container's outputs folder.
|
|
#
|
|
# NativeLink v1.8 names cache files d2/<sha256>-<size>-<generation>. The
|
|
# generation is a counter we can't guess, so the folder is indexed and
|
|
# re-read when a file isn't found.
|
|
materializer = pkgs.writeScript "build-materializer" ''
|
|
#!${pkgs.python3}/bin/python3
|
|
import ctypes, errno, fcntl, json, os, re, shutil, sys, threading, time
|
|
|
|
HANDOFF_ROOT = "${handoffRoot}"
|
|
OUT_ROOT = "${outputsRoot}"
|
|
DIGESTS = "${stateDir}/cas/content/d2"
|
|
NAME_OK = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
|
BLOB = re.compile(r"/blobs/([0-9a-f]{64})/(\d+)")
|
|
FICLONE = 0x40049409
|
|
LIBC = ctypes.CDLL(None, use_errno=True)
|
|
AT_FDCWD, AT_EMPTY_PATH = -100, 0x1000
|
|
|
|
def log(*a):
|
|
print(*a, file=sys.stderr, flush=True)
|
|
|
|
class CasIndex:
|
|
# Maps "<sha256>-<size>" to the newest file name in the cache.
|
|
def __init__(self):
|
|
self.names, self.at, self.lock = {}, 0.0, threading.Lock()
|
|
|
|
def rescan(self):
|
|
# Read the folder at most twice a second, however many threads ask.
|
|
wait = self.at + 0.5 - time.monotonic()
|
|
if wait > 0:
|
|
time.sleep(wait)
|
|
names = {}
|
|
try:
|
|
with os.scandir(DIGESTS) as it:
|
|
for e in it:
|
|
key, _, gen = e.name.rpartition("-")
|
|
if key and gen.isdigit() and int(gen) >= names.get(key, (-1, ""))[0]:
|
|
names[key] = (int(gen), e.name)
|
|
except FileNotFoundError:
|
|
pass
|
|
self.names, self.at = names, time.monotonic()
|
|
|
|
def lookup(self, h, size, refresh):
|
|
key = f"{h}-{size}"
|
|
with self.lock:
|
|
if refresh or key not in self.names:
|
|
self.rescan()
|
|
hit = self.names.get(key)
|
|
return os.path.join(DIGESTS, hit[1]) if hit else None
|
|
|
|
CAS = CasIndex()
|
|
|
|
def link_fd(fd, dest):
|
|
# Hardlink the exact inode we hold open (root only). Fails with
|
|
# ENOENT if the file was evicted after we opened it.
|
|
if LIBC.linkat(fd, b"", AT_FDCWD, os.fsencode(dest), AT_EMPTY_PATH) != 0:
|
|
err = ctypes.get_errno()
|
|
raise OSError(err, os.strerror(err), dest)
|
|
|
|
def copy_fd(fd, dest):
|
|
# Reflink on btrfs/XFS (no extra disk), plain copy otherwise.
|
|
with open(dest, "wb") as out:
|
|
try:
|
|
fcntl.ioctl(out.fileno(), FICLONE, fd)
|
|
except OSError:
|
|
with os.fdopen(os.dup(fd), "rb") as src:
|
|
src.seek(0)
|
|
shutil.copyfileobj(src, out, 1 << 20)
|
|
|
|
def place(h, size, dest):
|
|
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
tmp = dest + ".materializing"
|
|
if os.path.lexists(tmp):
|
|
os.unlink(tmp)
|
|
if size == 0:
|
|
open(tmp, "wb").close()
|
|
os.replace(tmp, dest)
|
|
return True
|
|
for refresh in (False, True):
|
|
path = CAS.lookup(h, size, refresh)
|
|
if path is None:
|
|
continue
|
|
try:
|
|
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
|
except FileNotFoundError:
|
|
continue # index was stale; re-read it once
|
|
try:
|
|
head = os.pread(fd, 4, 0)
|
|
if head.startswith(b"\x7fELF") or head.startswith(b"#!"):
|
|
# Executables get their own copy so chmod never touches the cache.
|
|
copy_fd(fd, tmp)
|
|
os.chmod(tmp, 0o755)
|
|
else:
|
|
try:
|
|
link_fd(fd, tmp)
|
|
except OSError as e:
|
|
if e.errno not in (errno.ENOENT, errno.EXDEV):
|
|
raise
|
|
# Evicted after we opened it (or another disk):
|
|
# the open fd still has the data, so copy it.
|
|
copy_fd(fd, tmp)
|
|
os.replace(tmp, dest)
|
|
# Renaming a hardlink onto the same file is a no-op that
|
|
# leaves tmp behind (output unchanged since last build).
|
|
if os.path.lexists(tmp):
|
|
os.unlink(tmp)
|
|
return True
|
|
finally:
|
|
os.close(fd)
|
|
return False
|
|
|
|
class Job:
|
|
def __init__(self, name):
|
|
self.name = name
|
|
self.handoff = os.path.join(HANDOFF_ROOT, name)
|
|
self.out = os.path.join(OUT_ROOT, name)
|
|
os.makedirs(self.out, exist_ok=True)
|
|
self.out = os.path.realpath(self.out)
|
|
self.started = os.path.join(self.handoff, "started")
|
|
self.sets = {}
|
|
self.missing = []
|
|
|
|
def safe_dest(self, parts):
|
|
dest = os.path.normpath(os.path.join(self.out, *parts))
|
|
if not dest.startswith(self.out + "/"):
|
|
raise ValueError("path escapes outputs folder: " + dest)
|
|
return dest
|
|
|
|
def files_in(self, set_id, seen):
|
|
if set_id in seen or set_id not in self.sets:
|
|
return []
|
|
seen.add(set_id)
|
|
s = self.sets[set_id]
|
|
found = list(s.get("files", []))
|
|
for sub in s.get("fileSets", []):
|
|
found += self.files_in(sub["id"], seen)
|
|
return found
|
|
|
|
@staticmethod
|
|
def digest_of(f):
|
|
if f.get("digest"):
|
|
return f["digest"], int(f.get("length", 0))
|
|
m = BLOB.search(f.get("uri", ""))
|
|
return (m.group(1), int(m.group(2))) if m else (None, None)
|
|
|
|
def handle(self, ev):
|
|
if "namedSetOfFiles" in ev:
|
|
self.sets[ev["id"]["namedSet"]["id"]] = ev["namedSetOfFiles"]
|
|
elif "completed" in ev and ev["completed"].get("success"):
|
|
label = ev["id"]["targetCompleted"]["label"]
|
|
ok, seen = True, set()
|
|
for group in ev["completed"].get("outputGroup", []):
|
|
for fs in group.get("fileSets", []):
|
|
for f in self.files_in(fs["id"], seen):
|
|
h, size = self.digest_of(f)
|
|
if h is None:
|
|
continue
|
|
try:
|
|
dest = self.safe_dest(f.get("pathPrefix", []) + [f["name"]])
|
|
except ValueError as e:
|
|
log(self.name, e)
|
|
continue
|
|
if not place(h, size, dest):
|
|
ok = False
|
|
if not ok:
|
|
self.missing.append(label)
|
|
|
|
def open_bep(self):
|
|
path = os.path.join(self.handoff, "bep.json")
|
|
while os.path.lexists(self.started):
|
|
try:
|
|
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
|
return os.fdopen(fd, "r")
|
|
except FileNotFoundError:
|
|
time.sleep(0.1)
|
|
return None
|
|
|
|
def follow(self):
|
|
fh = self.open_bep()
|
|
if fh is None:
|
|
return
|
|
buf, finishing = "", False
|
|
with fh:
|
|
while True:
|
|
chunk = fh.readline()
|
|
if not chunk:
|
|
if finishing:
|
|
return # build over and file fully read
|
|
if not os.path.lexists(self.started):
|
|
finishing = True # one more pass for the last lines
|
|
else:
|
|
time.sleep(0.05)
|
|
continue
|
|
buf += chunk
|
|
if not buf.endswith("\n"):
|
|
continue
|
|
line, buf = buf, ""
|
|
if line.strip():
|
|
self.handle(json.loads(line))
|
|
|
|
def write_result(self):
|
|
# O_EXCL + O_NOFOLLOW: a container can't aim this at a host file.
|
|
tmp = os.path.join(self.handoff, ".result.tmp")
|
|
if os.path.lexists(tmp):
|
|
os.unlink(tmp)
|
|
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644)
|
|
with os.fdopen(fd, "w") as fh:
|
|
json.dump({"missing": self.missing}, fh)
|
|
os.replace(tmp, os.path.join(self.handoff, "result.json"))
|
|
|
|
def run(self):
|
|
try:
|
|
self.follow()
|
|
except Exception as e:
|
|
log(self.name, "build handoff failed:", e)
|
|
self.missing.append("//...") # tell rbuild to fall back to a download
|
|
while os.path.lexists(self.started):
|
|
time.sleep(0.1)
|
|
try:
|
|
self.write_result()
|
|
except Exception as e:
|
|
log(self.name, "could not write result:", e)
|
|
|
|
active, lock = set(), threading.Lock()
|
|
|
|
def worker(name):
|
|
try:
|
|
Job(name).run()
|
|
finally:
|
|
with lock:
|
|
active.discard(name)
|
|
|
|
os.makedirs(HANDOFF_ROOT, exist_ok=True)
|
|
while True:
|
|
for name in os.listdir(HANDOFF_ROOT):
|
|
if not NAME_OK.match(name):
|
|
continue
|
|
d = os.path.join(HANDOFF_ROOT, name)
|
|
if os.path.islink(d) or not os.path.isdir(d):
|
|
continue
|
|
if not os.path.lexists(os.path.join(d, "started")):
|
|
continue
|
|
with lock:
|
|
if name in active:
|
|
continue
|
|
active.add(name)
|
|
threading.Thread(target=worker, args=(name,), daemon=True).start()
|
|
time.sleep(0.2)
|
|
'';
|
|
|
|
# Container side: `rbuild //target ...` instead of `bazel build`.
|
|
# The host is the container's default gateway (ve-* addresses are not fixed).
|
|
rbuild = pkgs.writeShellScriptBin "rbuild" ''
|
|
set -u
|
|
H=/handoff
|
|
gw=$(${pkgs.iproute2}/bin/ip -4 route show default | ${pkgs.gawk}/bin/awk '{print $3; exit}')
|
|
if [ -z "$gw" ]; then
|
|
echo "rbuild: no default route, can't find the build server" >&2
|
|
exit 1
|
|
fi
|
|
remote=--remote_executor=grpc://$gw:${toString cfg.port}
|
|
rm -f $H/bep.json $H/result.json
|
|
touch $H/started
|
|
${bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@"
|
|
rc=$?
|
|
rm -f $H/started
|
|
for _ in $(seq 600); do [ -f $H/result.json ] && break; sleep 0.1; done
|
|
if [ ! -f $H/result.json ]; then
|
|
echo "rbuild: host did not hand back outputs" >&2
|
|
exit 1
|
|
fi
|
|
mapfile -t missing < <(${pkgs.jq}/bin/jq -r '.missing[]' $H/result.json)
|
|
if [ ''${#missing[@]} -gt 0 ]; then
|
|
# Rare: outputs were evicted before the host could link them. Bazel
|
|
# finds them gone, rebuilds them, and downloads them over the network.
|
|
echo "rbuild: rebuilding ''${#missing[@]} evicted target(s)" >&2
|
|
${bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$?
|
|
fi
|
|
echo "rbuild: outputs are in /outputs" >&2
|
|
exit $rc
|
|
'';
|
|
|
|
remoteRc = "/run/bazel-remote.bazelrc";
|
|
|
|
bazelrc = ''
|
|
# Builds run on the host, so the Bazel server in a container stays small.
|
|
startup --host_jvm_args=-Xmx512m
|
|
|
|
build --remote_instance_name=main
|
|
# Plain `bazel build` downloads final outputs; rbuild overrides this with
|
|
# minimal and gets them through /outputs instead.
|
|
build --remote_download_toplevel
|
|
build --jobs=${toString (cfg.cpuCount * 4)}
|
|
|
|
# Rebuild instead of failing when the cache evicts something Bazel needs.
|
|
build --experimental_remote_cache_eviction_retries=5
|
|
|
|
build --action_env=PATH=${toolchain}/bin
|
|
build --host_action_env=PATH=${toolchain}/bin
|
|
build --repo_env=CC=${toolchain}/bin/gcc
|
|
|
|
# Points plain `bazel` at the host (written at boot). Missing = local build.
|
|
try-import ${remoteRc}
|
|
'';
|
|
|
|
# Global goose instructions in each container.
|
|
goosehints = ''
|
|
# Bazel builds run on the host, never in this container
|
|
|
|
This container is small (about 2 GB RAM, 1 CPU). `bazel build/test/run` is
|
|
set up to run every action on the host's build server, and `rbuild //target`
|
|
does the same and puts outputs in /outputs.
|
|
|
|
A normal build ends with a line like
|
|
`INFO: 120 processes: 110 remote, 10 internal.` (or "remote cache hit").
|
|
If instead you see `linux-sandbox`, `local` or `processwrapper-sandbox`
|
|
doing the work, or /run/bazel-remote.bazelrc is missing, or a build can't
|
|
reach the remote executor, something is wrong:
|
|
- stop the build (`bazel shutdown`), don't retry it locally,
|
|
- don't remove or override the --remote_* flags,
|
|
- tell the user what you saw, with the error lines.
|
|
'';
|
|
|
|
# Added to slackerbot's agent system. Uses host packages so bazel and the
|
|
# toolchain match the host exactly.
|
|
agentModule = {
|
|
environment.systemPackages = [ bazel pkgs.bazelisk pkgs.go pkgs.git rbuild ];
|
|
environment.etc."bazel.bazelrc".text = bazelrc;
|
|
environment.etc."goose/goosehints".text = goosehints;
|
|
programs.nix-ld.enable = true;
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d /root/.config/goose 0755 root root -"
|
|
"L+ /root/.config/goose/.goosehints - - - - /etc/goose/goosehints"
|
|
];
|
|
|
|
# The host is the default gateway, whose address changes per container,
|
|
# so write it for bazel.bazelrc once the route is up. Doesn't hold up boot.
|
|
systemd.services.bazel-remote-rc = {
|
|
description = "Point Bazel at the host build server";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network.target" ];
|
|
path = [ pkgs.iproute2 pkgs.gawk pkgs.coreutils ];
|
|
script = ''
|
|
for _ in $(seq 60); do
|
|
gw=$(ip -4 route show default | awk '{print $3; exit}')
|
|
if [ -n "$gw" ]; then
|
|
echo "build --remote_executor=grpc://$gw:${toString cfg.port}" > ${remoteRc}
|
|
exit 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "no default route after 60s; bazel will build locally" >&2
|
|
exit 1
|
|
'';
|
|
};
|
|
};
|
|
in
|
|
{
|
|
options = {
|
|
skyg.nixos.server.services.bazel-build = {
|
|
enable = mkEnableOption "Bazel build server (NativeLink) for slackerbot containers";
|
|
|
|
cpuCount = mkOption {
|
|
description = "CPU threads the host worker offers to builds.";
|
|
type = types.ints.positive;
|
|
};
|
|
|
|
cacheGB = mkOption {
|
|
description = "Size of the build cache in GB.";
|
|
default = 100;
|
|
type = types.ints.positive;
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Port containers reach NativeLink on (container links only).";
|
|
default = 50051;
|
|
type = types.port;
|
|
};
|
|
|
|
agentSystem = mkOption {
|
|
description = "slackerbot's agent system with bazel, go, bazelrc and rbuild added. Use for slackerbot's `system`.";
|
|
readOnly = true;
|
|
type = types.package;
|
|
default = (slackerbot.nixosConfigurations.agent.extendModules {
|
|
modules = [ agentModule ];
|
|
}).config.system.build.toplevel;
|
|
# Keeps the manual from building it on every host.
|
|
defaultText = literalExpression "slackerbot agent + bazel, go, bazelrc, rbuild";
|
|
};
|
|
|
|
binds = mkOption {
|
|
description = "slackerbot ro_binds / rw_binds that give each container /handoff and /outputs.";
|
|
readOnly = true;
|
|
type = types.attrsOf (types.listOf types.str);
|
|
default = {
|
|
rw = [ "${handoffRoot}/{machine}:/handoff" ];
|
|
ro = [ "${outputsRoot}/{machine}:/outputs" ];
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
config = mkIf cfg.enable {
|
|
environment.systemPackages = [ bazel pkgs.bazelisk ];
|
|
programs.nix-ld.enable = true; # for the Bazel bazelisk downloads
|
|
|
|
systemd.services.nativelink = {
|
|
description = "NativeLink remote cache and build service";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network.target" ];
|
|
path = [ toolchain ];
|
|
serviceConfig = {
|
|
ExecStart = "${nativelink.packages.${system}.nativelink}/bin/nativelink ${nativelinkConfig}";
|
|
DynamicUser = true;
|
|
StateDirectory = "nativelink";
|
|
# Cached files must be readable by the materializer and containers.
|
|
UMask = "0022";
|
|
LimitNOFILE = 65536;
|
|
Restart = "on-failure";
|
|
};
|
|
};
|
|
|
|
systemd.services.build-materializer = {
|
|
description = "Hand Bazel outputs to slackerbot containers";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "nativelink.service" ];
|
|
serviceConfig = {
|
|
ExecStart = "${materializer}"; # root: needed to link cache files
|
|
UMask = "0022";
|
|
Restart = "always";
|
|
};
|
|
};
|
|
|
|
# Outputs older than 7 days are removed. Hardlinked files keep their data
|
|
# alive after NativeLink evicts them, so this is what frees that space.
|
|
# `e` ages what's inside each container's folder, never the folder itself
|
|
# (it is a live bind mount source).
|
|
systemd.tmpfiles.rules = [
|
|
"d ${handoffRoot} 0755 root root -"
|
|
"d ${outputsRoot} 0755 root root -"
|
|
"e ${outputsRoot}/* - - - 7d"
|
|
];
|
|
|
|
# Keep the toolchain from garbage collection.
|
|
system.extraDependencies = [ toolchain ];
|
|
};
|
|
}
|