bazel
This commit is contained in:
1 parent
a12f10d964
commit
dd4965f030
8 files changed
+784
-19
No files matched your search
Generated
+220
-11
@@ -279,6 +279,21 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"crane_2": {
|
||||
"locked": {
|
||||
"lastModified": 1780029330,
|
||||
"narHash": "sha256-fWFKEKB5CP+NO8/3uOaEZIVbd03mvDJ//VsnKkXty08=",
|
||||
"owner": "ipetkov",
|
||||
"repo": "crane",
|
||||
"rev": "6823b493a0bc2142082075576efa6633b537197e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "ipetkov",
|
||||
"repo": "crane",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"crate2nix": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
@@ -614,6 +629,22 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-compat_10": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1767039857,
|
||||
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-compat_2": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
@@ -829,6 +860,24 @@
|
||||
}
|
||||
},
|
||||
"flake-parts_5": {
|
||||
"inputs": {
|
||||
"nixpkgs-lib": "nixpkgs-lib"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1769996383,
|
||||
"narHash": "sha256-AnYjnFWgS49RlqX7LrC4uA+sCCDBj0Ry/WOJ5XWAsa0=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"rev": "57928607ea566b5db3ad13af0e57e921e6b12381",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-parts_6": {
|
||||
"inputs": {
|
||||
"nixpkgs-lib": [
|
||||
"stylix",
|
||||
@@ -849,6 +898,21 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-root": {
|
||||
"locked": {
|
||||
"lastModified": 1723604017,
|
||||
"narHash": "sha256-rBtQ8gg+Dn4Sx/s+pvjdq3CB2wQNzx9XGFq/JVGCB6k=",
|
||||
"owner": "srid",
|
||||
"repo": "flake-root",
|
||||
"rev": "b759a56851e10cb13f6b8e5698af7b59c44be26e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "srid",
|
||||
"repo": "flake-root",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flox": {
|
||||
"inputs": {
|
||||
"crane": "crane",
|
||||
@@ -1019,6 +1083,29 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"git-hooks_4": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat_9",
|
||||
"gitignore": "gitignore_2",
|
||||
"nixpkgs": [
|
||||
"nativelink",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1772024342,
|
||||
"narHash": "sha256-+eXlIc4/7dE6EcPs9a2DaSY3fTA9AE526hGqkNID3Wg=",
|
||||
"owner": "cachix",
|
||||
"repo": "git-hooks.nix",
|
||||
"rev": "6e34e97ed9788b17796ee43ccdbaf871a5c2b476",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "cachix",
|
||||
"repo": "git-hooks.nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"gitignore": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
@@ -1041,6 +1128,28 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"gitignore_2": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nativelink",
|
||||
"git-hooks",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1709087332,
|
||||
"narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "gitignore.nix",
|
||||
"rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "hercules-ci",
|
||||
"repo": "gitignore.nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"gnome-shell": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
@@ -1579,7 +1688,7 @@
|
||||
"hyprutils": "hyprutils_3",
|
||||
"hyprwayland-scanner": "hyprwayland-scanner_3",
|
||||
"hyprwire": "hyprwire_3",
|
||||
"nixpkgs": "nixpkgs_11",
|
||||
"nixpkgs": "nixpkgs_12",
|
||||
"pre-commit-hooks": "pre-commit-hooks_3",
|
||||
"systems": "systems_3",
|
||||
"xdph": "xdph_2"
|
||||
@@ -2197,6 +2306,31 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nativelink": {
|
||||
"inputs": {
|
||||
"crane": "crane_2",
|
||||
"flake-parts": "flake-parts_5",
|
||||
"flake-root": "flake-root",
|
||||
"git-hooks": "git-hooks_4",
|
||||
"nix2container": "nix2container",
|
||||
"nixpkgs": "nixpkgs_11",
|
||||
"rust-overlay": "rust-overlay_3"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1793212176,
|
||||
"narHash": "sha256-k9Vzp5nOq2D/JnYfnG1vlIZrca8dt7F6qz6SwirXLec=",
|
||||
"owner": "TraceMachina",
|
||||
"repo": "nativelink",
|
||||
"rev": "bd336137d794a6a4e7ac32e561ab55e2d7e70ff3",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "TraceMachina",
|
||||
"ref": "v1.8.3",
|
||||
"repo": "nativelink",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix": {
|
||||
"inputs": {
|
||||
"bdwgc": "bdwgc",
|
||||
@@ -2313,6 +2447,28 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix2container": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nativelink",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1775487831,
|
||||
"narHash": "sha256-2lguQpLPQaxpQCJjXhmEEAfabwsAhkP29Z7fgLzHARA=",
|
||||
"owner": "nlewo",
|
||||
"repo": "nix2container",
|
||||
"rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nlewo",
|
||||
"repo": "nix2container",
|
||||
"rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix_2": {
|
||||
"inputs": {
|
||||
"flake-parts": "flake-parts_2",
|
||||
@@ -2531,7 +2687,7 @@
|
||||
},
|
||||
"nixos-hardware_2": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs_12"
|
||||
"nixpkgs": "nixpkgs_13"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1792491056,
|
||||
@@ -2596,6 +2752,21 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-lib": {
|
||||
"locked": {
|
||||
"lastModified": 1769909678,
|
||||
"narHash": "sha256-cBEymOf4/o3FD5AZnzC3J9hLbiZ+QDT/KDuyHXVJOpM=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nixpkgs.lib",
|
||||
"rev": "72716169fe93074c333e8d0173151350670b824c",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "nixpkgs.lib",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-regression": {
|
||||
"locked": {
|
||||
"lastModified": 1643052045,
|
||||
@@ -2675,6 +2846,22 @@
|
||||
}
|
||||
},
|
||||
"nixpkgs_11": {
|
||||
"locked": {
|
||||
"lastModified": 1787498568,
|
||||
"narHash": "sha256-9i/VTdusq/+NM/tz+J1Re+ojkMB8MBf0QshnYfzHz30=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "56c02bc00adcf003215cc4bd996d6efaf4cff188",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_12": {
|
||||
"locked": {
|
||||
"lastModified": 1788404924,
|
||||
"narHash": "sha256-lhEhY8X5EgkQ/eg6IFz4cc8jRuSYSvnxx1al7d1dvZ0=",
|
||||
@@ -2690,7 +2877,7 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_12": {
|
||||
"nixpkgs_13": {
|
||||
"locked": {
|
||||
"lastModified": 1789546076,
|
||||
"narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=",
|
||||
@@ -2703,7 +2890,7 @@
|
||||
"url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz"
|
||||
}
|
||||
},
|
||||
"nixpkgs_13": {
|
||||
"nixpkgs_14": {
|
||||
"locked": {
|
||||
"lastModified": 1792503283,
|
||||
"narHash": "sha256-vhe2jyvRCgufWPqDuDq1S3t3ypE+ka46IFutV02xKso=",
|
||||
@@ -2984,7 +3171,7 @@
|
||||
},
|
||||
"pre-commit-hooks_3": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat_9",
|
||||
"flake-compat": "flake-compat_10",
|
||||
"nixpkgs": [
|
||||
"nixarchy",
|
||||
"hyprland",
|
||||
@@ -3102,12 +3289,13 @@
|
||||
"hyprland": "hyprland",
|
||||
"hyprlauncher": "hyprlauncher",
|
||||
"my-nixpkgs": "my-nixpkgs",
|
||||
"nativelink": "nativelink",
|
||||
"nix-darwin": "nix-darwin",
|
||||
"nix-flatpak": "nix-flatpak",
|
||||
"nix-index-database": "nix-index-database",
|
||||
"nixarchy": "nixarchy",
|
||||
"nixos-hardware": "nixos-hardware_2",
|
||||
"nixpkgs": "nixpkgs_13",
|
||||
"nixpkgs": "nixpkgs_14",
|
||||
"nixpkgs-unstable": "nixpkgs-unstable",
|
||||
"noctalia": "noctalia",
|
||||
"robotnix": "robotnix",
|
||||
@@ -3178,6 +3366,27 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"rust-overlay_3": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nativelink",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1787713867,
|
||||
"narHash": "sha256-ZRYTEvDb8QZLCHRoMyIZle9V3Bvw905m0teeo1RvI7M=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "494680f9ffa2b4bf9f2df0f442c7096e44adfa00",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"skyg-linux-tracker": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
@@ -3225,11 +3434,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1793556114,
|
||||
"narHash": "sha256-TxDFlMNXyd9GXHa1FUCfZjauf5W//m73aMlhArE4vAQ=",
|
||||
"lastModified": 1793561072,
|
||||
"narHash": "sha256-5LTBU0P1Qbsu6SbO7iy73xt+djzpWn+qXtdJhdjdWqA=",
|
||||
"ref": "refs/heads/main",
|
||||
"rev": "0ac61b4eceb017ccf94f86c8cde30d1fb6db3d3e",
|
||||
"revCount": 2,
|
||||
"rev": "3ba96528bbb56b4b4b3bde5b112cd839510ce8c3",
|
||||
"revCount": 11,
|
||||
"type": "git",
|
||||
"url": "https://gitea.skyg.ca/ari/slacker-bot.git"
|
||||
},
|
||||
@@ -3283,7 +3492,7 @@
|
||||
"base16-helix": "base16-helix",
|
||||
"base16-vim": "base16-vim",
|
||||
"firefox-gnome-theme": "firefox-gnome-theme",
|
||||
"flake-parts": "flake-parts_5",
|
||||
"flake-parts": "flake-parts_6",
|
||||
"gnome-shell": "gnome-shell",
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
|
||||
@@ -25,6 +25,9 @@
|
||||
url = "git+https://gitea.skyg.ca/ari/slacker-bot.git";
|
||||
inputs.nixpkgs.follows = "nixpkgs-unstable";
|
||||
};
|
||||
# Bazel remote build server for slackerbot containers. Pinned: its config
|
||||
# format changes between releases (see bazel-build.nix).
|
||||
nativelink.url = "github:TraceMachina/nativelink/v1.8.3";
|
||||
# Headless Hyprland session streamed over a Unix socket.
|
||||
hhstream = {
|
||||
url = "git+ssh://gitea@gitea.app.internal:22/Hermenix/hyprland-headless-stream-plugin.git";
|
||||
@@ -126,6 +129,7 @@
|
||||
, robotnix
|
||||
, skyg-linux-tracker
|
||||
, slackerbot
|
||||
, nativelink
|
||||
, hhstream
|
||||
, nixpkgs-unstable
|
||||
, flox
|
||||
@@ -225,6 +229,7 @@
|
||||
robotnix
|
||||
skyg-linux-tracker
|
||||
slackerbot
|
||||
nativelink
|
||||
hhstream
|
||||
flox
|
||||
determinate
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
{ config, lib, pkgs, slackerbot, skyg-secrets, ... }:
|
||||
{ config, lib, pkgs, skyg-secrets, ... }:
|
||||
let
|
||||
ports = { cluster = 7946; };
|
||||
homes = "/data/slackerbot";
|
||||
secrets = [ "slackerbot-slack" "slackerbot-goose" "slackerbot-cluster" ];
|
||||
bazel = config.skyg.nixos.server.services.bazel-build;
|
||||
settings = {
|
||||
# The system each container boots; updated on every rebuild.
|
||||
system = "${slackerbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}";
|
||||
# The system each container boots (slackerbot's agent + bazel/rbuild);
|
||||
# updated on every rebuild.
|
||||
system = "${bazel.agentSystem}";
|
||||
# /handoff and /outputs for remote Bazel builds on this host.
|
||||
rw_binds = bazel.binds.rw;
|
||||
ro_binds = bazel.binds.ro;
|
||||
machines_dir = "/var/lib/machines";
|
||||
state_dir = "/var/lib/slackerbot";
|
||||
agent_cwd = "/root";
|
||||
@@ -56,6 +61,13 @@ in
|
||||
clusterPort = ports.cluster;
|
||||
configFile = (pkgs.formats.toml { }).generate "slackerbot.toml" settings;
|
||||
};
|
||||
# Bazel build server for the containers (16 cores / 32 threads).
|
||||
skyg.nixos.server.services.bazel-build = {
|
||||
enable = true;
|
||||
cpuCount = 32;
|
||||
cacheGB = 200;
|
||||
};
|
||||
|
||||
# The module takes one env file; these add to it.
|
||||
systemd.services.slackerbot.serviceConfig.EnvironmentFile =
|
||||
map (name: config.age.secrets.${name}.path) secrets;
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
{ config, pkgs, slackerbot, skyg-secrets, ... }:
|
||||
{ config, pkgs, skyg-secrets, ... }:
|
||||
let
|
||||
homes = "/var/lib/slackerbot/homes";
|
||||
bazel = config.skyg.nixos.server.services.bazel-build;
|
||||
settings = {
|
||||
# The system each container boots; updated on every rebuild.
|
||||
system = "${slackerbot.packages.${pkgs.stdenv.hostPlatform.system}.agent}";
|
||||
# The system each container boots (slackerbot's agent + bazel/rbuild);
|
||||
# updated on every rebuild.
|
||||
system = "${bazel.agentSystem}";
|
||||
# /handoff and /outputs for remote Bazel builds on this host.
|
||||
rw_binds = bazel.binds.rw;
|
||||
ro_binds = bazel.binds.ro;
|
||||
machines_dir = "/var/lib/machines";
|
||||
state_dir = "/var/lib/slackerbot";
|
||||
agent_cwd = "/root";
|
||||
@@ -43,6 +48,13 @@ in
|
||||
environmentFile = config.age.secrets.slackerbot-cluster.path;
|
||||
};
|
||||
|
||||
# Bazel build server for the containers (4 cores / 8 threads).
|
||||
skyg.nixos.server.services.bazel-build = {
|
||||
enable = true;
|
||||
cpuCount = 8;
|
||||
cacheGB = 50;
|
||||
};
|
||||
|
||||
# Container links belong to systemd-networkd.
|
||||
networking.networkmanager.unmanaged = [ "interface-name:ve-*" ];
|
||||
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
|
||||
|
||||
@@ -11,6 +11,7 @@ independent `skyg.nixos.server.services.<name>.enable` flag. Most run as OCI con
|
||||
services/
|
||||
├── default.nix # imports the services below
|
||||
├── ai-services.nix # skyg.nixos.server.services.ai — Wyoming (whisper/piper/wake)
|
||||
├── bazel-build.nix # skyg.nixos.server.services.bazel-build — NativeLink Bazel build server for slackerbot containers
|
||||
├── audiobookshelf.nix # skyg.nixos.server.services.audiobookshelf — audiobook/podcast server
|
||||
├── jellyfin.nix # skyg.nixos.server.services.jellyfin — media server (uid/gid 7777)
|
||||
├── signal-cli.nix # skyg.nixos.server.services.signal-cli — signal-cli HTTP daemon (Hermes Signal bridge)
|
||||
@@ -22,6 +23,14 @@ services/
|
||||
|
||||
- **`ai-services.nix`** openwakeword as containers and
|
||||
enables NixOS-native Wyoming faster-whisper + piper. Opens 11434/10200/10300/10400.
|
||||
- **`bazel-build.nix`** runs NativeLink (from the `nativelink` flake input, pinned
|
||||
to v1.8.3 because its config format changes) plus `build-materializer`, which
|
||||
hardlinks finished outputs from the cache into `/var/lib/build-outputs/<container>`.
|
||||
Containers run `rbuild //target` (finds the host as the default gateway) and read
|
||||
results at `/outputs`. It also exposes `agentSystem` (slackerbot's agent + bazel,
|
||||
bazelrc, rbuild) and `binds` (slackerbot `ro_binds`/`rw_binds`). Used on hl-fws1
|
||||
and hl-fwdesk. Slackerbot must create `ro_binds` source folders, or containers fail
|
||||
to start before their first build.
|
||||
- **`colibri.nix`** runs `coli serve` (colibrì's OpenAI-compatible API) as a native systemd
|
||||
service. `package` picks the backend (`pkgs.colibri`
|
||||
cpu / `pkgs.colibri-rocm`); `environmentFile` supplies `COLI_API_KEY` via agenix instead of
|
||||
@@ -32,6 +41,7 @@ services/
|
||||
```
|
||||
skyg.nixos.server.services.ai.enable
|
||||
skyg.nixos.server.services.audiobookshelf.{enable,package,configDir,dataDir}
|
||||
skyg.nixos.server.services.bazel-build.{enable,cpuCount,cacheGB,port,agentSystem,binds}
|
||||
skyg.nixos.server.services.jellyfin.enable
|
||||
skyg.nixos.server.services.signal-cli.{enable,host,port,configDir,account,environmentFile,openFirewall}
|
||||
skyg.nixos.server.services.colibri.{enable,package,model,host,port,environmentFile,openFirewall,...}
|
||||
|
||||
@@ -0,0 +1,516 @@
|
||||
# Bazel build server for slackerbot containers on the same host.
|
||||
# One NativeLink (cache + scheduler + worker) runs builds on the host CPUs.
|
||||
# Final outputs are not sent over the network: a host service hardlinks them
|
||||
# from NativeLink's cache into a folder each container sees at /outputs.
|
||||
#
|
||||
# container: rbuild //app ──grpc──▶ NativeLink (host) ──▶ host CPUs
|
||||
# │ writes build events to /handoff/bep.json
|
||||
# ▼
|
||||
# host: build-materializer follows bep.json and hardlinks each finished
|
||||
# output into /var/lib/build-outputs/<container>/
|
||||
# ▼
|
||||
# container: files appear at /outputs (read-only)
|
||||
#
|
||||
# slackerbot mounts the folders through `binds`. Containers run as root with
|
||||
# no private user IDs, so no uid mapping is needed.
|
||||
{ config, lib, pkgs, nativelink, slackerbot, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.skyg.nixos.server.services.bazel-build;
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
|
||||
stateDir = "/var/lib/nativelink";
|
||||
handoffRoot = "/var/lib/build-handoff";
|
||||
# Must be on the same disk as stateDir, or hardlinks fail (falls back to copies).
|
||||
outputsRoot = "/var/lib/build-outputs";
|
||||
workerPort = 50061; # host only
|
||||
|
||||
# Tools build actions run with. Same store paths on the host and in containers.
|
||||
toolchain = pkgs.buildEnv {
|
||||
name = "bazel-action-toolchain";
|
||||
paths = with pkgs; [ gcc binutils coreutils bash gnused gnugrep gawk findutils diffutils python3 ];
|
||||
};
|
||||
|
||||
# Matches NativeLink v1.8.3 (basic_cas example). Field names change between
|
||||
# releases; check them when bumping the flake input.
|
||||
instances = f: map (instance_name: { inherit instance_name; } // f) [ "" "main" ];
|
||||
nativelinkConfig = pkgs.writeText "nativelink.json" (builtins.toJSON {
|
||||
stores = [
|
||||
{
|
||||
# If any output of a cached result was evicted, report a miss so the
|
||||
# action is rebuilt instead of returning a result with missing files.
|
||||
name = "AC_MAIN_STORE";
|
||||
completeness_checking = {
|
||||
backend.filesystem = {
|
||||
content_path = "${stateDir}/ac/content";
|
||||
temp_path = "${stateDir}/ac/tmp";
|
||||
eviction_policy.max_bytes = 10 * 1000 * 1000 * 1000;
|
||||
};
|
||||
cas_store.ref_store.name = "WORKER_FAST_SLOW_STORE";
|
||||
};
|
||||
}
|
||||
{
|
||||
name = "WORKER_FAST_SLOW_STORE";
|
||||
fast_slow = {
|
||||
fast.filesystem = {
|
||||
content_path = "${stateDir}/cas/content"; # materializer links from here
|
||||
temp_path = "${stateDir}/cas/tmp";
|
||||
eviction_policy.max_bytes = cfg.cacheGB * 1000 * 1000 * 1000;
|
||||
};
|
||||
slow.noop = { };
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
schedulers = [{
|
||||
name = "MAIN_SCHEDULER";
|
||||
simple.supported_platform_properties = {
|
||||
cpu_count = "minimum";
|
||||
OSFamily = "priority";
|
||||
"container-image" = "priority";
|
||||
};
|
||||
}];
|
||||
|
||||
workers = [{
|
||||
local = {
|
||||
worker_api_endpoint.uri = "grpc://127.0.0.1:${toString workerPort}";
|
||||
cas_fast_slow_store = "WORKER_FAST_SLOW_STORE";
|
||||
upload_action_result.ac_store = "AC_MAIN_STORE";
|
||||
work_directory = "${stateDir}/work";
|
||||
platform_properties = {
|
||||
cpu_count.values = [ (toString cfg.cpuCount) ];
|
||||
OSFamily.values = [ "" ];
|
||||
"container-image".values = [ "" ];
|
||||
};
|
||||
};
|
||||
}];
|
||||
|
||||
servers = [
|
||||
{
|
||||
name = "public";
|
||||
# Only container links (ve-*) are trusted; the LAN port stays closed.
|
||||
listener.http.socket_address = "0.0.0.0:${toString cfg.port}";
|
||||
services = {
|
||||
cas = instances { cas_store = "WORKER_FAST_SLOW_STORE"; };
|
||||
ac = instances { ac_store = "AC_MAIN_STORE"; };
|
||||
execution = instances { cas_store = "WORKER_FAST_SLOW_STORE"; scheduler = "MAIN_SCHEDULER"; };
|
||||
capabilities = instances { remote_execution.scheduler = "MAIN_SCHEDULER"; };
|
||||
bytestream = instances { cas_store = "WORKER_FAST_SLOW_STORE"; };
|
||||
};
|
||||
}
|
||||
{
|
||||
name = "private_workers_servers";
|
||||
listener.http.socket_address = "127.0.0.1:${toString workerPort}";
|
||||
services = {
|
||||
worker_api.scheduler = "MAIN_SCHEDULER";
|
||||
admin = { };
|
||||
health = { };
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
global.max_open_files = 65536;
|
||||
});
|
||||
|
||||
# One long-running service for all containers. When a container drops a
|
||||
# "started" marker in its handoff folder, a thread follows its build event
|
||||
# file. As each target finishes, it opens each output in the cache and
|
||||
# hardlinks it into that container's outputs folder (opening first closes
|
||||
# the eviction race). Outputs already evicted are reported back so the
|
||||
# container can rebuild them.
|
||||
#
|
||||
# Handoff folders are written by the containers, so they are untrusted: no
|
||||
# symlinks are followed and every destination is checked to stay inside
|
||||
# that container's outputs folder.
|
||||
#
|
||||
# NativeLink v1.8 names cache files d2/<sha256>-<size>-<generation>. The
|
||||
# generation is a counter we can't guess, so the folder is indexed and
|
||||
# re-read when a file isn't found.
|
||||
materializer = pkgs.writeScript "build-materializer" ''
|
||||
#!${pkgs.python3}/bin/python3
|
||||
import ctypes, errno, fcntl, json, os, re, shutil, sys, threading, time
|
||||
|
||||
HANDOFF_ROOT = "${handoffRoot}"
|
||||
OUT_ROOT = "${outputsRoot}"
|
||||
DIGESTS = "${stateDir}/cas/content/d2"
|
||||
NAME_OK = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
||||
BLOB = re.compile(r"/blobs/([0-9a-f]{64})/(\d+)")
|
||||
FICLONE = 0x40049409
|
||||
LIBC = ctypes.CDLL(None, use_errno=True)
|
||||
AT_FDCWD, AT_EMPTY_PATH = -100, 0x1000
|
||||
|
||||
def log(*a):
|
||||
print(*a, file=sys.stderr, flush=True)
|
||||
|
||||
class CasIndex:
|
||||
# Maps "<sha256>-<size>" to the newest file name in the cache.
|
||||
def __init__(self):
|
||||
self.names, self.at, self.lock = {}, 0.0, threading.Lock()
|
||||
|
||||
def rescan(self):
|
||||
# Read the folder at most twice a second, however many threads ask.
|
||||
wait = self.at + 0.5 - time.monotonic()
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
names = {}
|
||||
try:
|
||||
with os.scandir(DIGESTS) as it:
|
||||
for e in it:
|
||||
key, _, gen = e.name.rpartition("-")
|
||||
if key and gen.isdigit() and int(gen) >= names.get(key, (-1, ""))[0]:
|
||||
names[key] = (int(gen), e.name)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
self.names, self.at = names, time.monotonic()
|
||||
|
||||
def lookup(self, h, size, refresh):
|
||||
key = f"{h}-{size}"
|
||||
with self.lock:
|
||||
if refresh or key not in self.names:
|
||||
self.rescan()
|
||||
hit = self.names.get(key)
|
||||
return os.path.join(DIGESTS, hit[1]) if hit else None
|
||||
|
||||
CAS = CasIndex()
|
||||
|
||||
def link_fd(fd, dest):
|
||||
# Hardlink the exact inode we hold open (root only). Fails with
|
||||
# ENOENT if the file was evicted after we opened it.
|
||||
if LIBC.linkat(fd, b"", AT_FDCWD, os.fsencode(dest), AT_EMPTY_PATH) != 0:
|
||||
err = ctypes.get_errno()
|
||||
raise OSError(err, os.strerror(err), dest)
|
||||
|
||||
def copy_fd(fd, dest):
|
||||
# Reflink on btrfs/XFS (no extra disk), plain copy otherwise.
|
||||
with open(dest, "wb") as out:
|
||||
try:
|
||||
fcntl.ioctl(out.fileno(), FICLONE, fd)
|
||||
except OSError:
|
||||
with os.fdopen(os.dup(fd), "rb") as src:
|
||||
src.seek(0)
|
||||
shutil.copyfileobj(src, out, 1 << 20)
|
||||
|
||||
def place(h, size, dest):
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
tmp = dest + ".materializing"
|
||||
if os.path.lexists(tmp):
|
||||
os.unlink(tmp)
|
||||
if size == 0:
|
||||
open(tmp, "wb").close()
|
||||
os.replace(tmp, dest)
|
||||
return True
|
||||
for refresh in (False, True):
|
||||
path = CAS.lookup(h, size, refresh)
|
||||
if path is None:
|
||||
continue
|
||||
try:
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
except FileNotFoundError:
|
||||
continue # index was stale; re-read it once
|
||||
try:
|
||||
head = os.pread(fd, 4, 0)
|
||||
if head.startswith(b"\x7fELF") or head.startswith(b"#!"):
|
||||
# Executables get their own copy so chmod never touches the cache.
|
||||
copy_fd(fd, tmp)
|
||||
os.chmod(tmp, 0o755)
|
||||
else:
|
||||
try:
|
||||
link_fd(fd, tmp)
|
||||
except OSError as e:
|
||||
if e.errno not in (errno.ENOENT, errno.EXDEV):
|
||||
raise
|
||||
# Evicted after we opened it (or another disk):
|
||||
# the open fd still has the data, so copy it.
|
||||
copy_fd(fd, tmp)
|
||||
os.replace(tmp, dest)
|
||||
# Renaming a hardlink onto the same file is a no-op that
|
||||
# leaves tmp behind (output unchanged since last build).
|
||||
if os.path.lexists(tmp):
|
||||
os.unlink(tmp)
|
||||
return True
|
||||
finally:
|
||||
os.close(fd)
|
||||
return False
|
||||
|
||||
class Job:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.handoff = os.path.join(HANDOFF_ROOT, name)
|
||||
self.out = os.path.join(OUT_ROOT, name)
|
||||
os.makedirs(self.out, exist_ok=True)
|
||||
self.out = os.path.realpath(self.out)
|
||||
self.started = os.path.join(self.handoff, "started")
|
||||
self.sets = {}
|
||||
self.missing = []
|
||||
|
||||
def safe_dest(self, parts):
|
||||
dest = os.path.normpath(os.path.join(self.out, *parts))
|
||||
if not dest.startswith(self.out + "/"):
|
||||
raise ValueError("path escapes outputs folder: " + dest)
|
||||
return dest
|
||||
|
||||
def files_in(self, set_id, seen):
|
||||
if set_id in seen or set_id not in self.sets:
|
||||
return []
|
||||
seen.add(set_id)
|
||||
s = self.sets[set_id]
|
||||
found = list(s.get("files", []))
|
||||
for sub in s.get("fileSets", []):
|
||||
found += self.files_in(sub["id"], seen)
|
||||
return found
|
||||
|
||||
@staticmethod
|
||||
def digest_of(f):
|
||||
if f.get("digest"):
|
||||
return f["digest"], int(f.get("length", 0))
|
||||
m = BLOB.search(f.get("uri", ""))
|
||||
return (m.group(1), int(m.group(2))) if m else (None, None)
|
||||
|
||||
def handle(self, ev):
|
||||
if "namedSetOfFiles" in ev:
|
||||
self.sets[ev["id"]["namedSet"]["id"]] = ev["namedSetOfFiles"]
|
||||
elif "completed" in ev and ev["completed"].get("success"):
|
||||
label = ev["id"]["targetCompleted"]["label"]
|
||||
ok, seen = True, set()
|
||||
for group in ev["completed"].get("outputGroup", []):
|
||||
for fs in group.get("fileSets", []):
|
||||
for f in self.files_in(fs["id"], seen):
|
||||
h, size = self.digest_of(f)
|
||||
if h is None:
|
||||
continue
|
||||
try:
|
||||
dest = self.safe_dest(f.get("pathPrefix", []) + [f["name"]])
|
||||
except ValueError as e:
|
||||
log(self.name, e)
|
||||
continue
|
||||
if not place(h, size, dest):
|
||||
ok = False
|
||||
if not ok:
|
||||
self.missing.append(label)
|
||||
|
||||
def open_bep(self):
|
||||
path = os.path.join(self.handoff, "bep.json")
|
||||
while os.path.lexists(self.started):
|
||||
try:
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
return os.fdopen(fd, "r")
|
||||
except FileNotFoundError:
|
||||
time.sleep(0.1)
|
||||
return None
|
||||
|
||||
def follow(self):
|
||||
fh = self.open_bep()
|
||||
if fh is None:
|
||||
return
|
||||
buf, finishing = "", False
|
||||
with fh:
|
||||
while True:
|
||||
chunk = fh.readline()
|
||||
if not chunk:
|
||||
if finishing:
|
||||
return # build over and file fully read
|
||||
if not os.path.lexists(self.started):
|
||||
finishing = True # one more pass for the last lines
|
||||
else:
|
||||
time.sleep(0.05)
|
||||
continue
|
||||
buf += chunk
|
||||
if not buf.endswith("\n"):
|
||||
continue
|
||||
line, buf = buf, ""
|
||||
if line.strip():
|
||||
self.handle(json.loads(line))
|
||||
|
||||
def write_result(self):
|
||||
# O_EXCL + O_NOFOLLOW: a container can't aim this at a host file.
|
||||
tmp = os.path.join(self.handoff, ".result.tmp")
|
||||
if os.path.lexists(tmp):
|
||||
os.unlink(tmp)
|
||||
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644)
|
||||
with os.fdopen(fd, "w") as fh:
|
||||
json.dump({"missing": self.missing}, fh)
|
||||
os.replace(tmp, os.path.join(self.handoff, "result.json"))
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
self.follow()
|
||||
except Exception as e:
|
||||
log(self.name, "build handoff failed:", e)
|
||||
self.missing.append("//...") # tell rbuild to fall back to a download
|
||||
while os.path.lexists(self.started):
|
||||
time.sleep(0.1)
|
||||
try:
|
||||
self.write_result()
|
||||
except Exception as e:
|
||||
log(self.name, "could not write result:", e)
|
||||
|
||||
active, lock = set(), threading.Lock()
|
||||
|
||||
def worker(name):
|
||||
try:
|
||||
Job(name).run()
|
||||
finally:
|
||||
with lock:
|
||||
active.discard(name)
|
||||
|
||||
os.makedirs(HANDOFF_ROOT, exist_ok=True)
|
||||
while True:
|
||||
for name in os.listdir(HANDOFF_ROOT):
|
||||
if not NAME_OK.match(name):
|
||||
continue
|
||||
d = os.path.join(HANDOFF_ROOT, name)
|
||||
if os.path.islink(d) or not os.path.isdir(d):
|
||||
continue
|
||||
if not os.path.lexists(os.path.join(d, "started")):
|
||||
continue
|
||||
with lock:
|
||||
if name in active:
|
||||
continue
|
||||
active.add(name)
|
||||
threading.Thread(target=worker, args=(name,), daemon=True).start()
|
||||
time.sleep(0.2)
|
||||
'';
|
||||
|
||||
# Container side: `rbuild //target ...` instead of `bazel build`.
|
||||
# The host is the container's default gateway (ve-* addresses are not fixed).
|
||||
rbuild = pkgs.writeShellScriptBin "rbuild" ''
|
||||
set -u
|
||||
H=/handoff
|
||||
gw=$(${pkgs.iproute2}/bin/ip -4 route show default | ${pkgs.gawk}/bin/awk '{print $3; exit}')
|
||||
if [ -z "$gw" ]; then
|
||||
echo "rbuild: no default route, can't find the build server" >&2
|
||||
exit 1
|
||||
fi
|
||||
remote=--remote_executor=grpc://$gw:${toString cfg.port}
|
||||
rm -f $H/bep.json $H/result.json
|
||||
touch $H/started
|
||||
${pkgs.bazel}/bin/bazel build "$remote" --remote_download_minimal --build_event_json_file=$H/bep.json "$@"
|
||||
rc=$?
|
||||
rm -f $H/started
|
||||
for _ in $(seq 600); do [ -f $H/result.json ] && break; sleep 0.1; done
|
||||
if [ ! -f $H/result.json ]; then
|
||||
echo "rbuild: host did not hand back outputs" >&2
|
||||
exit 1
|
||||
fi
|
||||
mapfile -t missing < <(${pkgs.jq}/bin/jq -r '.missing[]' $H/result.json)
|
||||
if [ ''${#missing[@]} -gt 0 ]; then
|
||||
# Rare: outputs were evicted before the host could link them. Bazel
|
||||
# finds them gone, rebuilds them, and downloads them over the network.
|
||||
echo "rbuild: rebuilding ''${#missing[@]} evicted target(s)" >&2
|
||||
${pkgs.bazel}/bin/bazel build "$remote" --remote_download_toplevel "''${missing[@]}" || rc=$?
|
||||
fi
|
||||
echo "rbuild: outputs are in /outputs" >&2
|
||||
exit $rc
|
||||
'';
|
||||
|
||||
bazelrc = ''
|
||||
build --remote_instance_name=main
|
||||
build --remote_download_minimal
|
||||
build --jobs=${toString (cfg.cpuCount * 4)}
|
||||
|
||||
# Rebuild instead of failing when the cache evicts something Bazel needs.
|
||||
build --experimental_remote_cache_eviction_retries=5
|
||||
|
||||
build --action_env=PATH=${toolchain}/bin
|
||||
build --host_action_env=PATH=${toolchain}/bin
|
||||
build --repo_env=CC=${toolchain}/bin/gcc
|
||||
'';
|
||||
|
||||
# Added to slackerbot's agent system. Uses host packages so bazel and the
|
||||
# toolchain match the host exactly.
|
||||
agentModule = {
|
||||
environment.systemPackages = [ pkgs.bazel pkgs.git rbuild ];
|
||||
environment.etc."bazel.bazelrc".text = bazelrc;
|
||||
};
|
||||
in
|
||||
{
|
||||
options = {
|
||||
skyg.nixos.server.services.bazel-build = {
|
||||
enable = mkEnableOption "Bazel build server (NativeLink) for slackerbot containers";
|
||||
|
||||
cpuCount = mkOption {
|
||||
description = "CPU threads the host worker offers to builds.";
|
||||
type = types.ints.positive;
|
||||
};
|
||||
|
||||
cacheGB = mkOption {
|
||||
description = "Size of the build cache in GB.";
|
||||
default = 100;
|
||||
type = types.ints.positive;
|
||||
};
|
||||
|
||||
port = mkOption {
|
||||
description = "Port containers reach NativeLink on (container links only).";
|
||||
default = 50051;
|
||||
type = types.port;
|
||||
};
|
||||
|
||||
agentSystem = mkOption {
|
||||
description = "slackerbot's agent system with bazel, bazelrc and rbuild added. Use for slackerbot's `system`.";
|
||||
readOnly = true;
|
||||
type = types.package;
|
||||
default = (slackerbot.nixosConfigurations.agent.extendModules {
|
||||
modules = [ agentModule ];
|
||||
}).config.system.build.toplevel;
|
||||
# Keeps the manual from building it on every host.
|
||||
defaultText = literalExpression "slackerbot agent + bazel, bazelrc, rbuild";
|
||||
};
|
||||
|
||||
binds = mkOption {
|
||||
description = "slackerbot ro_binds / rw_binds that give each container /handoff and /outputs.";
|
||||
readOnly = true;
|
||||
type = types.attrsOf (types.listOf types.str);
|
||||
default = {
|
||||
rw = [ "${handoffRoot}/{machine}:/handoff" ];
|
||||
ro = [ "${outputsRoot}/{machine}:/outputs" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
environment.systemPackages = [ pkgs.bazel ];
|
||||
|
||||
systemd.services.nativelink = {
|
||||
description = "NativeLink remote cache and build service";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
path = [ toolchain ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${nativelink.packages.${system}.nativelink}/bin/nativelink ${nativelinkConfig}";
|
||||
DynamicUser = true;
|
||||
StateDirectory = "nativelink";
|
||||
# Cached files must be readable by the materializer and containers.
|
||||
UMask = "0022";
|
||||
LimitNOFILE = 65536;
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.build-materializer = {
|
||||
description = "Hand Bazel outputs to slackerbot containers";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "nativelink.service" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${materializer}"; # root: needed to link cache files
|
||||
UMask = "0022";
|
||||
Restart = "always";
|
||||
};
|
||||
};
|
||||
|
||||
# Outputs older than 7 days are removed. Hardlinked files keep their data
|
||||
# alive after NativeLink evicts them, so this is what frees that space.
|
||||
# `e` ages what's inside each container's folder, never the folder itself
|
||||
# (it is a live bind mount source).
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${handoffRoot} 0755 root root -"
|
||||
"d ${outputsRoot} 0755 root root -"
|
||||
"e ${outputsRoot}/* - - - 7d"
|
||||
];
|
||||
|
||||
# Keep the toolchain from garbage collection.
|
||||
system.extraDependencies = [ toolchain ];
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
imports = [
|
||||
./audiobookshelf.nix
|
||||
./ai-services.nix
|
||||
./bazel-build.nix
|
||||
./colibri.nix
|
||||
./comfyui
|
||||
./jellyfin.nix
|
||||
|
||||
@@ -77,12 +77,12 @@ let
|
||||
in
|
||||
stdenv.mkDerivation {
|
||||
pname = "colibri" + lib.optionalString (backend == "rocm") "-rocm";
|
||||
version = "2.0.0";
|
||||
version = "2.1.0";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "JustVugg";
|
||||
repo = "colibri";
|
||||
rev = "a7f535a9b34ffa31a4eef18c8a0bb6775c973a6e"; # tag v1.5.0
|
||||
rev = "8579957378ca922ad4340a6392513485cbc50c2c"; # tag v1.5.0
|
||||
hash = "sha256-4gxlmTiB3RfWdeqQfkr6nstBrQktRA0jUP7TY29KpM0=";
|
||||
};
|
||||
|
||||
|
||||
Reference in new issue
Block a user