bazel related limit issues
This commit is contained in:
1 parent
dd4965f030
commit
594a1c1901
7 files changed
+114
-16
No files matched your search
@@ -10,7 +10,7 @@ Key non-negotiable rules from that file:
|
|||||||
changes — no matter how small the task.
|
changes — no matter how small the task.
|
||||||
- **Never run builds, system switches, or remote installs** (`nixos-rebuild`,
|
- **Never run builds, system switches, or remote installs** (`nixos-rebuild`,
|
||||||
`nh os`, `skyg os`, `skyg remote install`, `nixos-anywhere`, `nix build`,
|
`nh os`, `skyg os`, `skyg remote install`, `nixos-anywhere`, `nix build`,
|
||||||
`nix flake check`, etc.) — zero exceptions, even if asked.
|
`nix flake check`, etc.). You may only run it after asking the user 3 times if the are sure about it!
|
||||||
- Never edit `flake.lock`, `*.hardware-configuration.nix`, or anything in
|
- Never edit `flake.lock`, `*.hardware-configuration.nix`, or anything in
|
||||||
`secrets/`.
|
`secrets/`.
|
||||||
- Never add `imports = [...]` to host files.
|
- Never add `imports = [...]` to host files.
|
||||||
|
|||||||
Generated
+4
-4
@@ -3434,11 +3434,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1793561072,
|
"lastModified": 1793566366,
|
||||||
"narHash": "sha256-5LTBU0P1Qbsu6SbO7iy73xt+djzpWn+qXtdJhdjdWqA=",
|
"narHash": "sha256-yytC8S47uJZNm7GraZQb84QH4YHA/tIOKIw6sis7sJA=",
|
||||||
"ref": "refs/heads/main",
|
"ref": "refs/heads/main",
|
||||||
"rev": "3ba96528bbb56b4b4b3bde5b112cd839510ce8c3",
|
"rev": "917a361370ab4da27ae10b232da34fcb40e04adf",
|
||||||
"revCount": 11,
|
"revCount": 19,
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://gitea.skyg.ca/ari/slacker-bot.git"
|
"url": "https://gitea.skyg.ca/ari/slacker-bot.git"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ let
|
|||||||
OPENROUTER_API_KEY = "$OPENROUTER_API_KEY";
|
OPENROUTER_API_KEY = "$OPENROUTER_API_KEY";
|
||||||
};
|
};
|
||||||
|
|
||||||
container = { memory_mb = 1024; cpus = 1.0; };
|
container = { memory_mb = 2048; cpus = 1.0; };
|
||||||
max_container = { memory_mb = 4096; cpus = 2.0; };
|
max_container = { memory_mb = 4096; cpus = 2.0; };
|
||||||
# Leave room for ollama, ds4, comfyui, hermes and the robots.
|
# Leave room for ollama, ds4, comfyui, hermes and the robots.
|
||||||
host_reserve = { memory_mb = 32768; cpus = 4.0; };
|
host_reserve = { memory_mb = 32768; cpus = 4.0; };
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ let
|
|||||||
allow_indefinite = true;
|
allow_indefinite = true;
|
||||||
boot_timeout = "2m";
|
boot_timeout = "2m";
|
||||||
|
|
||||||
container = { memory_mb = 1024; cpus = 1.0; };
|
container = { memory_mb = 2048; cpus = 1.0; };
|
||||||
max_container = { memory_mb = 4096; cpus = 2.0; };
|
max_container = { memory_mb = 4096; cpus = 2.0; };
|
||||||
host_reserve = { memory_mb = 8192; cpus = 2.0; };
|
host_reserve = { memory_mb = 8192; cpus = 2.0; };
|
||||||
storage = { min = "2G"; max = "5G"; limit = "20G"; };
|
storage = { min = "2G"; max = "5G"; limit = "20G"; };
|
||||||
|
|||||||
@@ -27,10 +27,23 @@ services/
|
|||||||
to v1.8.3 because its config format changes) plus `build-materializer`, which
|
to v1.8.3 because its config format changes) plus `build-materializer`, which
|
||||||
hardlinks finished outputs from the cache into `/var/lib/build-outputs/<container>`.
|
hardlinks finished outputs from the cache into `/var/lib/build-outputs/<container>`.
|
||||||
Containers run `rbuild //target` (finds the host as the default gateway) and read
|
Containers run `rbuild //target` (finds the host as the default gateway) and read
|
||||||
results at `/outputs`. It also exposes `agentSystem` (slackerbot's agent + bazel,
|
results at `/outputs`. Plain `bazel` builds remotely too: at boot,
|
||||||
|
`bazel-remote-rc` writes the gateway into `/run/bazel-remote.bazelrc`, which the
|
||||||
|
container's `/etc/bazel.bazelrc` try-imports (missing = local build). Plain builds
|
||||||
|
download final outputs (`--remote_download_toplevel`). The Bazel server is capped at
|
||||||
|
512M. Global goose hints (`/root/.config/goose/.goosehints` → `/etc/goose/goosehints`)
|
||||||
|
tell goose that a local build means something is wrong: stop and report it. It also exposes `agentSystem` (slackerbot's agent + bazel, go,
|
||||||
bazelrc, rbuild) and `binds` (slackerbot `ro_binds`/`rw_binds`). Used on hl-fws1
|
bazelrc, rbuild) and `binds` (slackerbot `ro_binds`/`rw_binds`). Used on hl-fws1
|
||||||
and hl-fwdesk. Slackerbot must create `ro_binds` source folders, or containers fail
|
and hl-fwdesk. Slackerbot must create `ro_binds` source folders, or containers fail
|
||||||
to start before their first build.
|
to start before their first build.
|
||||||
|
`bazel` is a bazelisk wrapper on the host and in containers: a repo's
|
||||||
|
`.bazelversion` wins, otherwise 8.7.0 (`USE_BAZEL_FALLBACK_VERSION`). Bazel 8.7.0
|
||||||
|
is fetched once per node by Nix (official release, sha256-checked) and the wrapper
|
||||||
|
enters it into each user's bazelisk cache as a link, so containers share that one
|
||||||
|
read-only copy and never download it. Other versions download into each container's
|
||||||
|
own `/root/.cache/bazelisk`. Upstream Bazel only runs on NixOS through
|
||||||
|
`programs.nix-ld`, enabled on the host and in containers. Bump `bazelVersion` and
|
||||||
|
`bazelSha256` together.
|
||||||
- **`colibri.nix`** runs `coli serve` (colibrì's OpenAI-compatible API) as a native systemd
|
- **`colibri.nix`** runs `coli serve` (colibrì's OpenAI-compatible API) as a native systemd
|
||||||
service. `package` picks the backend (`pkgs.colibri`
|
service. `package` picks the backend (`pkgs.colibri`
|
||||||
cpu / `pkgs.colibri-rocm`); `environmentFile` supplies `COLI_API_KEY` via agenix instead of
|
cpu / `pkgs.colibri-rocm`); `environmentFile` supplies `COLI_API_KEY` via agenix instead of
|
||||||
|
|||||||
@@ -20,6 +20,34 @@ with lib;
|
|||||||
let
|
let
|
||||||
cfg = config.skyg.nixos.server.services.bazel-build;
|
cfg = config.skyg.nixos.server.services.bazel-build;
|
||||||
system = pkgs.stdenv.hostPlatform.system;
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
|
# Official Bazel release, checked against its sha256. Containers see
|
||||||
|
# /nix/store read-only, so one copy per node serves all of them. Upstream
|
||||||
|
# builds only run on NixOS through nix-ld. To bump: change both values.
|
||||||
|
bazelVersion = "8.7.0";
|
||||||
|
bazelSha256 = "d7606e679b78067c811096fb3d6cf135225b528835ca396e3a4dddf957859544";
|
||||||
|
# Copied as-is: patching would break the archive appended to the binary.
|
||||||
|
bazelRelease = pkgs.runCommand "bazel-${bazelVersion}-release" { } ''
|
||||||
|
install -Dm755 ${pkgs.fetchurl {
|
||||||
|
url = "https://releases.bazel.build/${bazelVersion}/release/bazel-${bazelVersion}-linux-x86_64";
|
||||||
|
sha256 = bazelSha256;
|
||||||
|
}} $out/bin/bazel
|
||||||
|
'';
|
||||||
|
|
||||||
|
# `bazel` runs bazelisk on the host and in containers: a repo's
|
||||||
|
# .bazelversion wins, otherwise bazelVersion. bazelVersion is pre-entered in
|
||||||
|
# bazelisk's cache (format of bazelisk 1.28) as a link to bazelRelease, so it
|
||||||
|
# is never downloaded. Other versions download into each user's own cache.
|
||||||
|
bazel = pkgs.writeShellScriptBin "bazel" ''
|
||||||
|
export USE_BAZEL_FALLBACK_VERSION=''${USE_BAZEL_FALLBACK_VERSION:-silent:${bazelVersion}}
|
||||||
|
dl=''${BAZELISK_HOME_LINUX:-''${BAZELISK_HOME:-''${XDG_CACHE_HOME:-$HOME/.cache}/bazelisk}}/downloads
|
||||||
|
meta=$dl/metadata/bazelbuild/bazel-${bazelVersion}-linux-x86_64
|
||||||
|
if [ ! -e "$dl/sha256/${bazelSha256}/bin/bazel" ] || [ "$(cat "$meta" 2>/dev/null)" != ${bazelSha256} ]; then
|
||||||
|
mkdir -p "$dl/sha256" "''${meta%/*}"
|
||||||
|
ln -sfn ${bazelRelease} "$dl/sha256/${bazelSha256}"
|
||||||
|
printf %s ${bazelSha256} > "$meta"
|
||||||
|
fi
|
||||||
|
exec ${pkgs.bazelisk}/bin/bazelisk "$@"
|
||||||
|
'';
|
||||||
|
|
||||||
stateDir = "/var/lib/nativelink";
|
stateDir = "/var/lib/nativelink";
|
||||||
handoffRoot = "/var/lib/build-handoff";
|
handoffRoot = "/var/lib/build-handoff";
|
||||||
@@ -386,7 +414,7 @@ let
|
|||||||
remote=--remote_executor=grpc://$gw:${toString cfg.port}
|
remote=--remote_executor=grpc://$gw:${toString cfg.port}
|
||||||
rm -f $H/bep.json $H/result.json
|
rm -f $H/bep.json $H/result.json
|
||||||
touch $H/started
|
touch $H/started
|
||||||
${pkgs.bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@"
|
${bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@"
|
||||||
rc=$?
|
rc=$?
|
||||||
rm -f $H/started
|
rm -f $H/started
|
||||||
for _ in $(seq 600); do [ -f $H/result.json ] && break; sleep 0.1; done
|
for _ in $(seq 600); do [ -f $H/result.json ] && break; sleep 0.1; done
|
||||||
@@ -399,15 +427,22 @@ let
|
|||||||
# Rare: outputs were evicted before the host could link them. Bazel
|
# Rare: outputs were evicted before the host could link them. Bazel
|
||||||
# finds them gone, rebuilds them, and downloads them over the network.
|
# finds them gone, rebuilds them, and downloads them over the network.
|
||||||
echo "rbuild: rebuilding ''${#missing[@]} evicted target(s)" >&2
|
echo "rbuild: rebuilding ''${#missing[@]} evicted target(s)" >&2
|
||||||
${pkgs.bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$?
|
${bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$?
|
||||||
fi
|
fi
|
||||||
echo "rbuild: outputs are in /outputs" >&2
|
echo "rbuild: outputs are in /outputs" >&2
|
||||||
exit $rc
|
exit $rc
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
remoteRc = "/run/bazel-remote.bazelrc";
|
||||||
|
|
||||||
bazelrc = ''
|
bazelrc = ''
|
||||||
|
# Builds run on the host, so the Bazel server in a container stays small.
|
||||||
|
startup --host_jvm_args=-Xmx512m
|
||||||
|
|
||||||
build --remote_instance_name=main
|
build --remote_instance_name=main
|
||||||
build --remote_download_minimal
|
# Plain `bazel build` downloads final outputs; rbuild overrides this with
|
||||||
|
# minimal and gets them through /outputs instead.
|
||||||
|
build --remote_download_toplevel
|
||||||
build --jobs=${toString (cfg.cpuCount * 4)}
|
build --jobs=${toString (cfg.cpuCount * 4)}
|
||||||
|
|
||||||
# Rebuild instead of failing when the cache evicts something Bazel needs.
|
# Rebuild instead of failing when the cache evicts something Bazel needs.
|
||||||
@@ -416,13 +451,62 @@ let
|
|||||||
build --action_env=PATH=${toolchain}/bin
|
build --action_env=PATH=${toolchain}/bin
|
||||||
build --host_action_env=PATH=${toolchain}/bin
|
build --host_action_env=PATH=${toolchain}/bin
|
||||||
build --repo_env=CC=${toolchain}/bin/gcc
|
build --repo_env=CC=${toolchain}/bin/gcc
|
||||||
|
|
||||||
|
# Points plain `bazel` at the host (written at boot). Missing = local build.
|
||||||
|
try-import ${remoteRc}
|
||||||
|
'';
|
||||||
|
|
||||||
|
# Global goose instructions in each container.
|
||||||
|
goosehints = ''
|
||||||
|
# Bazel builds run on the host, never in this container
|
||||||
|
|
||||||
|
This container is small (about 2 GB RAM, 1 CPU). `bazel build/test/run` is
|
||||||
|
set up to run every action on the host's build server, and `rbuild //target`
|
||||||
|
does the same and puts outputs in /outputs.
|
||||||
|
|
||||||
|
A normal build ends with a line like
|
||||||
|
`INFO: 120 processes: 110 remote, 10 internal.` (or "remote cache hit").
|
||||||
|
If instead you see `linux-sandbox`, `local` or `processwrapper-sandbox`
|
||||||
|
doing the work, or /run/bazel-remote.bazelrc is missing, or a build can't
|
||||||
|
reach the remote executor, something is wrong:
|
||||||
|
- stop the build (`bazel shutdown`), don't retry it locally,
|
||||||
|
- don't remove or override the --remote_* flags,
|
||||||
|
- tell the user what you saw, with the error lines.
|
||||||
'';
|
'';
|
||||||
|
|
||||||
# Added to slackerbot's agent system. Uses host packages so bazel and the
|
# Added to slackerbot's agent system. Uses host packages so bazel and the
|
||||||
# toolchain match the host exactly.
|
# toolchain match the host exactly.
|
||||||
agentModule = {
|
agentModule = {
|
||||||
environment.systemPackages = [ pkgs.bazel pkgs.git rbuild ];
|
environment.systemPackages = [ bazel pkgs.bazelisk pkgs.go pkgs.git rbuild ];
|
||||||
environment.etc."bazel.bazelrc".text = bazelrc;
|
environment.etc."bazel.bazelrc".text = bazelrc;
|
||||||
|
environment.etc."goose/goosehints".text = goosehints;
|
||||||
|
programs.nix-ld.enable = true;
|
||||||
|
|
||||||
|
systemd.tmpfiles.rules = [
|
||||||
|
"d /root/.config/goose 0755 root root -"
|
||||||
|
"L+ /root/.config/goose/.goosehints - - - - /etc/goose/goosehints"
|
||||||
|
];
|
||||||
|
|
||||||
|
# The host is the default gateway, whose address changes per container,
|
||||||
|
# so write it for bazel.bazelrc once the route is up. Doesn't hold up boot.
|
||||||
|
systemd.services.bazel-remote-rc = {
|
||||||
|
description = "Point Bazel at the host build server";
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
after = [ "network.target" ];
|
||||||
|
path = [ pkgs.iproute2 pkgs.gawk pkgs.coreutils ];
|
||||||
|
script = ''
|
||||||
|
for _ in $(seq 60); do
|
||||||
|
gw=$(ip -4 route show default | awk '{print $3; exit}')
|
||||||
|
if [ -n "$gw" ]; then
|
||||||
|
echo "build --remote_executor=grpc://$gw:${toString cfg.port}" > ${remoteRc}
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "no default route after 60s; bazel will build locally" >&2
|
||||||
|
exit 1
|
||||||
|
'';
|
||||||
|
};
|
||||||
};
|
};
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
@@ -448,14 +532,14 @@ in
|
|||||||
};
|
};
|
||||||
|
|
||||||
agentSystem = mkOption {
|
agentSystem = mkOption {
|
||||||
description = "slackerbot's agent system with bazel, bazelrc and rbuild added. Use for slackerbot's `system`.";
|
description = "slackerbot's agent system with bazel, go, bazelrc and rbuild added. Use for slackerbot's `system`.";
|
||||||
readOnly = true;
|
readOnly = true;
|
||||||
type = types.package;
|
type = types.package;
|
||||||
default = (slackerbot.nixosConfigurations.agent.extendModules {
|
default = (slackerbot.nixosConfigurations.agent.extendModules {
|
||||||
modules = [ agentModule ];
|
modules = [ agentModule ];
|
||||||
}).config.system.build.toplevel;
|
}).config.system.build.toplevel;
|
||||||
# Keeps the manual from building it on every host.
|
# Keeps the manual from building it on every host.
|
||||||
defaultText = literalExpression "slackerbot agent + bazel, bazelrc, rbuild";
|
defaultText = literalExpression "slackerbot agent + bazel, go, bazelrc, rbuild";
|
||||||
};
|
};
|
||||||
|
|
||||||
binds = mkOption {
|
binds = mkOption {
|
||||||
@@ -471,7 +555,8 @@ in
|
|||||||
};
|
};
|
||||||
|
|
||||||
config = mkIf cfg.enable {
|
config = mkIf cfg.enable {
|
||||||
environment.systemPackages = [ pkgs.bazel ];
|
environment.systemPackages = [ bazel pkgs.bazelisk ];
|
||||||
|
programs.nix-ld.enable = true; # for the Bazel bazelisk downloads
|
||||||
|
|
||||||
systemd.services.nativelink = {
|
systemd.services.nativelink = {
|
||||||
description = "NativeLink remote cache and build service";
|
description = "NativeLink remote cache and build service";
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ stdenv.mkDerivation {
|
|||||||
owner = "JustVugg";
|
owner = "JustVugg";
|
||||||
repo = "colibri";
|
repo = "colibri";
|
||||||
rev = "8579957378ca922ad4340a6392513485cbc50c2c"; # tag v1.5.0
|
rev = "8579957378ca922ad4340a6392513485cbc50c2c"; # tag v1.5.0
|
||||||
hash = "sha256-4gxlmTiB3RfWdeqQfkr6nstBrQktRA0jUP7TY29KpM0=";
|
hash = "sha256-wuSLM50nAzcFWDC7fJb3dfAvuq1sc69quT1y0ANq9Pk=";
|
||||||
};
|
};
|
||||||
|
|
||||||
nativeBuildInputs = [ makeWrapper ];
|
nativeBuildInputs = [ makeWrapper ];
|
||||||
|
|||||||
Reference in new issue
Block a user