From 10d85b2a91628ab4c64fe22be11761fea7b30766 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Thu, 12 Nov 2026 10:12:07 -0500 Subject: [PATCH] Push builds to bigbox2 over ssh-ng from a background queue bigbox2 runs nix-serve, which only serves its own store: every 'nix copy --to http://bigbox2.lab.internal:5000' from the post-build hook ended in HTTP 404 after compressing the whole closure, so nothing ever landed and the next build tried the same paths again. A post-build-hook also blocks the build until it returns, which held up every build on fwdesk (nbot's VM checks spent most of a cold CI run there). - cacheReceive (on bigbox2): nix.sshServe with write over ssh-ng, for fwdesk's ssh host key; nix-ssh is trusted, and nix-serve signs the pushed paths when it serves them. - cachePush: the hook only queues $OUT_PATHS. nix-cache-push.service drains the queue with nix copy to ssh-ng://nix-ssh@bigbox2.lab.internal as the host key, with bigbox2's host key pinned; a path unit starts it, and a failed push is retried every 10 minutes by a timer. Co-Authored-By: nBot --- hosts/hl-bigbox2/default.nix | 2 + modules/nixos/common/binary-cache.nix | 136 ++++++++++++++++++++++---- 2 files changed, 117 insertions(+), 21 deletions(-) diff --git a/hosts/hl-bigbox2/default.nix b/hosts/hl-bigbox2/default.nix index 8e8d68a..38a1f8a 100644 --- a/hosts/hl-bigbox2/default.nix +++ b/hosts/hl-bigbox2/default.nix @@ -20,6 +20,8 @@ parent = "enp3s0"; }; cachePush.enable = false; + # fwdesk pushes its builds here over ssh-ng; nix-serve serves them. + cacheReceive.enable = true; }; networkDrives = { diff --git a/modules/nixos/common/binary-cache.nix b/modules/nixos/common/binary-cache.nix index 1418368..2c3b1fc 100644 --- a/modules/nixos/common/binary-cache.nix +++ b/modules/nixos/common/binary-cache.nix @@ -1,35 +1,129 @@ { config, lib, pkgs, ... }: let cfg = config.skyg.nixos.common.cachePush; + recv = config.skyg.nixos.common.cacheReceive; + lab = import ./diagnix/hosts.nix; + + # The hook writes into tmp and moves into queue, which the path unit watches. + # A push that fails moves its files to retry, for the timer. + state = "/var/lib/nix-cache-push"; + queue = "${state}/queue"; + retry = "${state}/retry"; + + # nix-serve on bigbox2 only serves its own store and can't take uploads, so + # pushes go over ssh-ng into that store, as the host's root key. + sshOpts = "-i /etc/ssh/ssh_host_ed25519_key -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=10"; + + drain = pkgs.writeShellScript "nix-cache-push-drain" '' + set -u + shopt -s nullglob + new=(${queue}/*) + old=(${retry}/*) + files=("''${new[@]}" "''${old[@]}") + [ ''${#files[@]} -eq 0 ] && exit 0 + paths=() + for f in "''${files[@]}"; do + for p in $(cat "$f"); do + [ -e "$p" ] && paths+=("$p") + done + done + if [ ''${#paths[@]} -gt 0 ]; then + if ! NIX_SSHOPTS="${sshOpts}" ${config.nix.package}/bin/nix copy --to "${cfg.url}" "''${paths[@]}"; then + [ ''${#new[@]} -gt 0 ] && mv "''${new[@]}" ${retry}/ + exit 1 + fi + fi + rm -f "''${files[@]}" + ''; in { options.skyg.nixos.common.cachePush = { enable = lib.mkEnableOption "push finished builds to the bigbox2 binary cache"; url = lib.mkOption { type = lib.types.str; - default = "http://bigbox2.lab.internal:5000"; + default = "ssh-ng://nix-ssh@bigbox2.lab.internal"; + }; + host = lib.mkOption { + type = lib.types.str; + default = "bigbox2.lab.internal"; + description = "The cache's ssh host, whose host key is pinned from diagnix/hosts.nix."; }; }; - config = lib.mkIf cfg.enable { - # Push every finished build to the cache right after it finishes. - nix.settings.post-build-hook = pkgs.writeShellScript "nix-post-build-hook" '' - set -u - if [ -n "''${OUT_PATHS:-}" ]; then - nix copy --to "${cfg.url}" $OUT_PATHS \ - || echo "post-build-hook: failed to push to ${cfg.url}" >&2 - fi - ''; - - # Manual/backstop push command, available on every host's PATH. - environment.systemPackages = [ - (pkgs.writeShellScriptBin "push-cache" '' - set -eu - if [ "''$#" -eq 0 ]; then - set -- /run/current-system - fi - exec nix copy --to "${cfg.url}" "''$@" - '') - ]; + options.skyg.nixos.common.cacheReceive = { + enable = lib.mkEnableOption "accept pushes into this host's store over ssh-ng (nix-ssh)"; + pushers = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ "hl-fwdesk" ]; + description = "Hosts in diagnix/hosts.nix whose ssh host key may push."; + }; }; + + config = lib.mkMerge [ + (lib.mkIf cfg.enable { + # The hook only queues the paths: Nix waits for a post-build-hook before it + # goes on, so copying here held up every build for the whole upload. + nix.settings.post-build-hook = pkgs.writeShellScript "nix-post-build-hook" '' + set -u + if [ -n "''${OUT_PATHS:-}" ]; then + tmp=$(mktemp -p ${state}/tmp) || exit 0 + echo "$OUT_PATHS" > "$tmp" && mv "$tmp" "${queue}/$(date +%s%N)-$$" + fi + ''; + + systemd.tmpfiles.rules = map (d: "d ${d} 0700 root root -") [ state "${state}/tmp" queue retry ]; + + programs.ssh.knownHosts.nix-cache-push = { + hostNames = [ cfg.host ]; + publicKey = (lib.findFirst (h: h.address == cfg.host) null (lib.attrValues lab.hosts)).hostKey; + }; + + systemd.services.nix-cache-push = { + description = "Push queued builds to ${cfg.url}"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + path = [ config.programs.ssh.package ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = drain; + Nice = 10; + IOSchedulingClass = "idle"; + }; + }; + + # Runs when the hook queues a build, and every 10 minutes for the retries. + systemd.paths.nix-cache-push = { + wantedBy = [ "multi-user.target" ]; + pathConfig.DirectoryNotEmpty = queue; + }; + systemd.timers.nix-cache-push = { + wantedBy = [ "timers.target" ]; + timerConfig.OnUnitInactiveSec = "10min"; + timerConfig.OnBootSec = "5min"; + }; + + # Manual/backstop push command, available on every host's PATH. + environment.systemPackages = [ + (pkgs.writeShellScriptBin "push-cache" '' + set -eu + if [ "''$#" -eq 0 ]; then + set -- /run/current-system + fi + NIX_SSHOPTS="${sshOpts}" exec nix copy --to "${cfg.url}" "''$@" + '') + ]; + }) + + (lib.mkIf recv.enable { + nix.sshServe = { + enable = true; + write = true; + protocol = "ssh-ng"; + # Pushed paths aren't signed; nix-serve signs them with this host's key + # when it serves them. + trusted = true; + keys = map (h: lab.hosts.${h}.hostKey) recv.pushers; + }; + }) + ]; }