bazel related limit issues
This commit is contained in:
1 parent
dd4965f030
commit
594a1c1901
7 files changed
+114
-16
No files matched your search
@@ -10,7 +10,7 @@ Key non-negotiable rules from that file:
|
||||
changes — no matter how small the task.
|
||||
- **Never run builds, system switches, or remote installs** (`nixos-rebuild`,
|
||||
`nh os`, `skyg os`, `skyg remote install`, `nixos-anywhere`, `nix build`,
|
||||
`nix flake check`, etc.) — zero exceptions, even if asked.
|
||||
`nix flake check`, etc.). You may only run it after asking the user 3 times if the are sure about it!
|
||||
- Never edit `flake.lock`, `*.hardware-configuration.nix`, or anything in
|
||||
`secrets/`.
|
||||
- Never add `imports = [...]` to host files.
|
||||
|
||||
Generated
+4
-4
@@ -3434,11 +3434,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1793561072,
|
||||
"narHash": "sha256-5LTBU0P1Qbsu6SbO7iy73xt+djzpWn+qXtdJhdjdWqA=",
|
||||
"lastModified": 1793566366,
|
||||
"narHash": "sha256-yytC8S47uJZNm7GraZQb84QH4YHA/tIOKIw6sis7sJA=",
|
||||
"ref": "refs/heads/main",
|
||||
"rev": "3ba96528bbb56b4b4b3bde5b112cd839510ce8c3",
|
||||
"revCount": 11,
|
||||
"rev": "917a361370ab4da27ae10b232da34fcb40e04adf",
|
||||
"revCount": 19,
|
||||
"type": "git",
|
||||
"url": "https://gitea.skyg.ca/ari/slacker-bot.git"
|
||||
},
|
||||
|
||||
@@ -28,7 +28,7 @@ let
|
||||
OPENROUTER_API_KEY = "$OPENROUTER_API_KEY";
|
||||
};
|
||||
|
||||
container = { memory_mb = 1024; cpus = 1.0; };
|
||||
container = { memory_mb = 2048; cpus = 1.0; };
|
||||
max_container = { memory_mb = 4096; cpus = 2.0; };
|
||||
# Leave room for ollama, ds4, comfyui, hermes and the robots.
|
||||
host_reserve = { memory_mb = 32768; cpus = 4.0; };
|
||||
|
||||
@@ -18,7 +18,7 @@ let
|
||||
allow_indefinite = true;
|
||||
boot_timeout = "2m";
|
||||
|
||||
container = { memory_mb = 1024; cpus = 1.0; };
|
||||
container = { memory_mb = 2048; cpus = 1.0; };
|
||||
max_container = { memory_mb = 4096; cpus = 2.0; };
|
||||
host_reserve = { memory_mb = 8192; cpus = 2.0; };
|
||||
storage = { min = "2G"; max = "5G"; limit = "20G"; };
|
||||
|
||||
@@ -27,10 +27,23 @@ services/
|
||||
to v1.8.3 because its config format changes) plus `build-materializer`, which
|
||||
hardlinks finished outputs from the cache into `/var/lib/build-outputs/<container>`.
|
||||
Containers run `rbuild //target` (finds the host as the default gateway) and read
|
||||
results at `/outputs`. It also exposes `agentSystem` (slackerbot's agent + bazel,
|
||||
results at `/outputs`. Plain `bazel` builds remotely too: at boot,
|
||||
`bazel-remote-rc` writes the gateway into `/run/bazel-remote.bazelrc`, which the
|
||||
container's `/etc/bazel.bazelrc` try-imports (missing = local build). Plain builds
|
||||
download final outputs (`--remote_download_toplevel`). The Bazel server is capped at
|
||||
512M. Global goose hints (`/root/.config/goose/.goosehints` → `/etc/goose/goosehints`)
|
||||
tell goose that a local build means something is wrong: stop and report it. It also exposes `agentSystem` (slackerbot's agent + bazel, go,
|
||||
bazelrc, rbuild) and `binds` (slackerbot `ro_binds`/`rw_binds`). Used on hl-fws1
|
||||
and hl-fwdesk. Slackerbot must create `ro_binds` source folders, or containers fail
|
||||
to start before their first build.
|
||||
`bazel` is a bazelisk wrapper on the host and in containers: a repo's
|
||||
`.bazelversion` wins, otherwise 8.7.0 (`USE_BAZEL_FALLBACK_VERSION`). Bazel 8.7.0
|
||||
is fetched once per node by Nix (official release, sha256-checked) and the wrapper
|
||||
enters it into each user's bazelisk cache as a link, so containers share that one
|
||||
read-only copy and never download it. Other versions download into each container's
|
||||
own `/root/.cache/bazelisk`. Upstream Bazel only runs on NixOS through
|
||||
`programs.nix-ld`, enabled on the host and in containers. Bump `bazelVersion` and
|
||||
`bazelSha256` together.
|
||||
- **`colibri.nix`** runs `coli serve` (colibrì's OpenAI-compatible API) as a native systemd
|
||||
service. `package` picks the backend (`pkgs.colibri`
|
||||
cpu / `pkgs.colibri-rocm`); `environmentFile` supplies `COLI_API_KEY` via agenix instead of
|
||||
|
||||
@@ -20,6 +20,34 @@ with lib;
|
||||
let
|
||||
cfg = config.skyg.nixos.server.services.bazel-build;
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
# Official Bazel release, checked against its sha256. Containers see
|
||||
# /nix/store read-only, so one copy per node serves all of them. Upstream
|
||||
# builds only run on NixOS through nix-ld. To bump: change both values.
|
||||
bazelVersion = "8.7.0";
|
||||
bazelSha256 = "d7606e679b78067c811096fb3d6cf135225b528835ca396e3a4dddf957859544";
|
||||
# Copied as-is: patching would break the archive appended to the binary.
|
||||
bazelRelease = pkgs.runCommand "bazel-${bazelVersion}-release" { } ''
|
||||
install -Dm755 ${pkgs.fetchurl {
|
||||
url = "https://releases.bazel.build/${bazelVersion}/release/bazel-${bazelVersion}-linux-x86_64";
|
||||
sha256 = bazelSha256;
|
||||
}} $out/bin/bazel
|
||||
'';
|
||||
|
||||
# `bazel` runs bazelisk on the host and in containers: a repo's
|
||||
# .bazelversion wins, otherwise bazelVersion. bazelVersion is pre-entered in
|
||||
# bazelisk's cache (format of bazelisk 1.28) as a link to bazelRelease, so it
|
||||
# is never downloaded. Other versions download into each user's own cache.
|
||||
bazel = pkgs.writeShellScriptBin "bazel" ''
|
||||
export USE_BAZEL_FALLBACK_VERSION=''${USE_BAZEL_FALLBACK_VERSION:-silent:${bazelVersion}}
|
||||
dl=''${BAZELISK_HOME_LINUX:-''${BAZELISK_HOME:-''${XDG_CACHE_HOME:-$HOME/.cache}/bazelisk}}/downloads
|
||||
meta=$dl/metadata/bazelbuild/bazel-${bazelVersion}-linux-x86_64
|
||||
if [ ! -e "$dl/sha256/${bazelSha256}/bin/bazel" ] || [ "$(cat "$meta" 2>/dev/null)" != ${bazelSha256} ]; then
|
||||
mkdir -p "$dl/sha256" "''${meta%/*}"
|
||||
ln -sfn ${bazelRelease} "$dl/sha256/${bazelSha256}"
|
||||
printf %s ${bazelSha256} > "$meta"
|
||||
fi
|
||||
exec ${pkgs.bazelisk}/bin/bazelisk "$@"
|
||||
'';
|
||||
|
||||
stateDir = "/var/lib/nativelink";
|
||||
handoffRoot = "/var/lib/build-handoff";
|
||||
@@ -386,7 +414,7 @@ let
|
||||
remote=--remote_executor=grpc://$gw:${toString cfg.port}
|
||||
rm -f $H/bep.json $H/result.json
|
||||
touch $H/started
|
||||
${pkgs.bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@"
|
||||
${bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@"
|
||||
rc=$?
|
||||
rm -f $H/started
|
||||
for _ in $(seq 600); do [ -f $H/result.json ] && break; sleep 0.1; done
|
||||
@@ -399,15 +427,22 @@ let
|
||||
# Rare: outputs were evicted before the host could link them. Bazel
|
||||
# finds them gone, rebuilds them, and downloads them over the network.
|
||||
echo "rbuild: rebuilding ''${#missing[@]} evicted target(s)" >&2
|
||||
${pkgs.bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$?
|
||||
${bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$?
|
||||
fi
|
||||
echo "rbuild: outputs are in /outputs" >&2
|
||||
exit $rc
|
||||
'';
|
||||
|
||||
remoteRc = "/run/bazel-remote.bazelrc";
|
||||
|
||||
bazelrc = ''
|
||||
# Builds run on the host, so the Bazel server in a container stays small.
|
||||
startup --host_jvm_args=-Xmx512m
|
||||
|
||||
build --remote_instance_name=main
|
||||
build --remote_download_minimal
|
||||
# Plain `bazel build` downloads final outputs; rbuild overrides this with
|
||||
# minimal and gets them through /outputs instead.
|
||||
build --remote_download_toplevel
|
||||
build --jobs=${toString (cfg.cpuCount * 4)}
|
||||
|
||||
# Rebuild instead of failing when the cache evicts something Bazel needs.
|
||||
@@ -416,13 +451,62 @@ let
|
||||
build --action_env=PATH=${toolchain}/bin
|
||||
build --host_action_env=PATH=${toolchain}/bin
|
||||
build --repo_env=CC=${toolchain}/bin/gcc
|
||||
|
||||
# Points plain `bazel` at the host (written at boot). Missing = local build.
|
||||
try-import ${remoteRc}
|
||||
'';
|
||||
|
||||
# Global goose instructions in each container.
|
||||
goosehints = ''
|
||||
# Bazel builds run on the host, never in this container
|
||||
|
||||
This container is small (about 2 GB RAM, 1 CPU). `bazel build/test/run` is
|
||||
set up to run every action on the host's build server, and `rbuild //target`
|
||||
does the same and puts outputs in /outputs.
|
||||
|
||||
A normal build ends with a line like
|
||||
`INFO: 120 processes: 110 remote, 10 internal.` (or "remote cache hit").
|
||||
If instead you see `linux-sandbox`, `local` or `processwrapper-sandbox`
|
||||
doing the work, or /run/bazel-remote.bazelrc is missing, or a build can't
|
||||
reach the remote executor, something is wrong:
|
||||
- stop the build (`bazel shutdown`), don't retry it locally,
|
||||
- don't remove or override the --remote_* flags,
|
||||
- tell the user what you saw, with the error lines.
|
||||
'';
|
||||
|
||||
# Added to slackerbot's agent system. Uses host packages so bazel and the
|
||||
# toolchain match the host exactly.
|
||||
agentModule = {
|
||||
environment.systemPackages = [ pkgs.bazel pkgs.git rbuild ];
|
||||
environment.systemPackages = [ bazel pkgs.bazelisk pkgs.go pkgs.git rbuild ];
|
||||
environment.etc."bazel.bazelrc".text = bazelrc;
|
||||
environment.etc."goose/goosehints".text = goosehints;
|
||||
programs.nix-ld.enable = true;
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /root/.config/goose 0755 root root -"
|
||||
"L+ /root/.config/goose/.goosehints - - - - /etc/goose/goosehints"
|
||||
];
|
||||
|
||||
# The host is the default gateway, whose address changes per container,
|
||||
# so write it for bazel.bazelrc once the route is up. Doesn't hold up boot.
|
||||
systemd.services.bazel-remote-rc = {
|
||||
description = "Point Bazel at the host build server";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
path = [ pkgs.iproute2 pkgs.gawk pkgs.coreutils ];
|
||||
script = ''
|
||||
for _ in $(seq 60); do
|
||||
gw=$(ip -4 route show default | awk '{print $3; exit}')
|
||||
if [ -n "$gw" ]; then
|
||||
echo "build --remote_executor=grpc://$gw:${toString cfg.port}" > ${remoteRc}
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "no default route after 60s; bazel will build locally" >&2
|
||||
exit 1
|
||||
'';
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
@@ -448,14 +532,14 @@ in
|
||||
};
|
||||
|
||||
agentSystem = mkOption {
|
||||
description = "slackerbot's agent system with bazel, bazelrc and rbuild added. Use for slackerbot's `system`.";
|
||||
description = "slackerbot's agent system with bazel, go, bazelrc and rbuild added. Use for slackerbot's `system`.";
|
||||
readOnly = true;
|
||||
type = types.package;
|
||||
default = (slackerbot.nixosConfigurations.agent.extendModules {
|
||||
modules = [ agentModule ];
|
||||
}).config.system.build.toplevel;
|
||||
# Keeps the manual from building it on every host.
|
||||
defaultText = literalExpression "slackerbot agent + bazel, bazelrc, rbuild";
|
||||
defaultText = literalExpression "slackerbot agent + bazel, go, bazelrc, rbuild";
|
||||
};
|
||||
|
||||
binds = mkOption {
|
||||
@@ -471,7 +555,8 @@ in
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
environment.systemPackages = [ pkgs.bazel ];
|
||||
environment.systemPackages = [ bazel pkgs.bazelisk ];
|
||||
programs.nix-ld.enable = true; # for the Bazel bazelisk downloads
|
||||
|
||||
systemd.services.nativelink = {
|
||||
description = "NativeLink remote cache and build service";
|
||||
|
||||
@@ -83,7 +83,7 @@ stdenv.mkDerivation {
|
||||
owner = "JustVugg";
|
||||
repo = "colibri";
|
||||
rev = "8579957378ca922ad4340a6392513485cbc50c2c"; # tag v1.5.0
|
||||
hash = "sha256-4gxlmTiB3RfWdeqQfkr6nstBrQktRA0jUP7TY29KpM0=";
|
||||
hash = "sha256-wuSLM50nAzcFWDC7fJb3dfAvuq1sc69quT1y0ANq9Pk=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [ makeWrapper ];
|
||||
|
||||
Reference in new issue
Block a user