From 6dd651d1566eb29c73d4ae36921090607f686425 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Wed, 2 Sep 2026 13:37:09 -0400 Subject: [PATCH] bigbox2 as cache machine approach #1 --- hosts/ABOUTME.md | 1 + hosts/hl-bigbox2.nix | 13 ++++++---- modules/core/ABOUTME.md | 5 ++-- modules/core/nix-substituters.nix | 2 ++ modules/nixos/common/ABOUTME.md | 2 ++ modules/nixos/common/binary-cache.nix | 35 +++++++++++++++++++++++++++ modules/nixos/common/default.nix | 1 + 7 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 modules/nixos/common/binary-cache.nix diff --git a/hosts/ABOUTME.md b/hosts/ABOUTME.md index facaa00..1edd267 100644 --- a/hosts/ABOUTME.md +++ b/hosts/ABOUTME.md @@ -28,6 +28,7 @@ hosts/ | `hl-minipc1.nix` | hl-minipc1 | Core infra: Gitea, nix-serve, Docker registry | | `hl-minipc2.nix` | hl-minipc2 | Container/media server | | `hl-minipc3.nix` | hl-minipc3 | K3s worker node | +| `hl-pi1.nix` | hl-pi1 | Raspberry Pi CM4 (aarch64) — headless server, SSH only | | `hl-terra1.nix` | hl-terra1 | NFS storage + Jellyfin backup | | `iso.nix` | iso | Bootable NixOS installer image | diff --git a/hosts/hl-bigbox2.nix b/hosts/hl-bigbox2.nix index 03b80a3..696544b 100644 --- a/hosts/hl-bigbox2.nix +++ b/hosts/hl-bigbox2.nix @@ -21,11 +21,14 @@ in bigBox2.enable = false; }; }; - # services.nix-serve = { - # enable = true; - # secretKeyFile = "/home/ari/cache-keys/bigbox2.lab.internal.private"; - # port = ports.nixServe; - # }; + # Nix binary cache server (bigbox2 is the cache store for all builds) + services.nix-serve = { + enable = true; + secretKeyFile = "/home/ari/cache-keys/cache-priv-key.pem"; + port = ports.nixServe; + }; + # bigbox2 is the cache itself — don't push builds back to itself. + skyg.nixos.common.cachePush.enable = false; # firmware updater services.fwupd.enable = true; services.nfs.server = { diff --git a/modules/core/ABOUTME.md b/modules/core/ABOUTME.md index 5048ba2..9d3c0a6 100644 --- a/modules/core/ABOUTME.md +++ b/modules/core/ABOUTME.md @@ -27,8 +27,9 @@ core/ `skyg.home-manager.{version,extraImports}`. When `skyg.user.enable` is set it wires up Home Manager for both `root` and the named user using the functions in `modules/home`. - **Substituters** (`nix-substituters.nix`): the standing list of caches (local `minipc1` - cache, cache.nixos.org, cachix caches for cuda/cosmic/ai/noctalia, flox, devenv) and the - trusted public keys. Extra entries can be appended per-host via `skyg.core.substituters.{urls,keys}`. + and `bigbox2` caches, cache.nixos.org, cachix caches for cuda/cosmic/ai/noctalia, flox, + devenv) and the trusted public keys. Extra entries can be appended per-host via + `skyg.core.substituters.{urls,keys}`. ## Option Namespace diff --git a/modules/core/nix-substituters.nix b/modules/core/nix-substituters.nix index c040384..0b205f4 100644 --- a/modules/core/nix-substituters.nix +++ b/modules/core/nix-substituters.nix @@ -23,6 +23,7 @@ in config = { nix.settings.substituters = [ "http://minipc1.lab.internal:5000" + "http://bigbox2.lab.internal:5000" "https://cache.nixos.org" "https://cache.flox.dev" "https://devenv.cachix.org" @@ -40,6 +41,7 @@ in ] ++ cfg.urls; nix.settings.trusted-public-keys = [ "minipc1.lab.internal:buUlsyg+xRqkUk0MWACmIyRUtHIOPQQzg7nc4qZCc4E=" + "bigbox2.lab.internal:hQVdN2RYFVq4hUu0L9h7VVJ+tpqdqmtHKYsHO+1WATs=" "flox-cache-public-1:7F4OyH7ZCnFhcze3fJdfyXYLQw/aV7GEed86nQ7IsOs=" "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=" "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" diff --git a/modules/nixos/common/ABOUTME.md b/modules/nixos/common/ABOUTME.md index ac54023..cd47786 100644 --- a/modules/nixos/common/ABOUTME.md +++ b/modules/nixos/common/ABOUTME.md @@ -16,6 +16,7 @@ common/ ├── ssh-server.nix # skyg.nixos.common.ssh-server — hardened sshd + master pubkey ├── ssh-notify.nix # skyg.nixos.common.ssh-notify — forward notify-send from SSH sessions to a desktop client ├── qemu.nix # skyg.core.qemu — QEMU/quickemu + SPICE guest tooling +├── binary-cache.nix # skyg.nixos.common.cachePush — push finished builds to the bigbox2 cache ├── containers/ # Docker vs Podman runtime selection ├── hardware/ # GPU, fan control, audio, laptop power, Coral TPU udev └── pritunl/ # Pritunl VPN client service @@ -41,6 +42,7 @@ skyg.user.createSystemUser → user.nix skyg.core.qemu.* → qemu.nix skyg.nixos.common.ssh-server.* → ssh-server.nix skyg.nixos.common.ssh-notify.* → ssh-notify.nix +skyg.nixos.common.cachePush.* → binary-cache.nix skyg.nixos.common.containers.* → containers/ skyg.nixos.common.hardware.* → hardware/ skyg.nixos.common.pritunl.* → pritunl/ diff --git a/modules/nixos/common/binary-cache.nix b/modules/nixos/common/binary-cache.nix new file mode 100644 index 0000000..1418368 --- /dev/null +++ b/modules/nixos/common/binary-cache.nix @@ -0,0 +1,35 @@ +{ config, lib, pkgs, ... }: +let + cfg = config.skyg.nixos.common.cachePush; +in +{ + options.skyg.nixos.common.cachePush = { + enable = lib.mkEnableOption "push finished builds to the bigbox2 binary cache"; + url = lib.mkOption { + type = lib.types.str; + default = "http://bigbox2.lab.internal:5000"; + }; + }; + + config = lib.mkIf cfg.enable { + # Push every finished build to the cache right after it finishes. + nix.settings.post-build-hook = pkgs.writeShellScript "nix-post-build-hook" '' + set -u + if [ -n "''${OUT_PATHS:-}" ]; then + nix copy --to "${cfg.url}" $OUT_PATHS \ + || echo "post-build-hook: failed to push to ${cfg.url}" >&2 + fi + ''; + + # Manual/backstop push command, available on every host's PATH. + environment.systemPackages = [ + (pkgs.writeShellScriptBin "push-cache" '' + set -eu + if [ "''$#" -eq 0 ]; then + set -- /run/current-system + fi + exec nix copy --to "${cfg.url}" "''$@" + '') + ]; + }; +} diff --git a/modules/nixos/common/default.nix b/modules/nixos/common/default.nix index 4f540e5..6c1ac83 100644 --- a/modules/nixos/common/default.nix +++ b/modules/nixos/common/default.nix @@ -10,6 +10,7 @@ ./ssh-server.nix ./ssh-notify.nix ./qemu.nix + ./binary-cache.nix ./pritunl ]; config = {