Files
ariandnBot 10d85b2a91 Push builds to bigbox2 over ssh-ng from a background queue
bigbox2 runs nix-serve, which only serves its own store: every
'nix copy --to http://bigbox2.lab.internal:5000' from the post-build hook
ended in HTTP 404 after compressing the whole closure, so nothing ever
landed and the next build tried the same paths again. A post-build-hook
also blocks the build until it returns, which held up every build on
fwdesk (nbot's VM checks spent most of a cold CI run there).

- cacheReceive (on bigbox2): nix.sshServe with write over ssh-ng, for
  fwdesk's ssh host key; nix-ssh is trusted, and nix-serve signs the
  pushed paths when it serves them.
- cachePush: the hook only queues $OUT_PATHS. nix-cache-push.service
  drains the queue with nix copy to ssh-ng://nix-ssh@bigbox2.lab.internal
  as the host key, with bigbox2's host key pinned; a path unit starts it,
  and a failed push is retried every 10 minutes by a timer.

Co-Authored-By: nBot
2026-11-12 10:12:07 -05:00

130 lines
4.3 KiB
Nix

{ config, lib, pkgs, ... }:
let
cfg = config.skyg.nixos.common.cachePush;
recv = config.skyg.nixos.common.cacheReceive;
lab = import ./diagnix/hosts.nix;
# The hook writes into tmp and moves into queue, which the path unit watches.
# A push that fails moves its files to retry, for the timer.
state = "/var/lib/nix-cache-push";
queue = "${state}/queue";
retry = "${state}/retry";
# nix-serve on bigbox2 only serves its own store and can't take uploads, so
# pushes go over ssh-ng into that store, as the host's root key.
sshOpts = "-i /etc/ssh/ssh_host_ed25519_key -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=10";
drain = pkgs.writeShellScript "nix-cache-push-drain" ''
set -u
shopt -s nullglob
new=(${queue}/*)
old=(${retry}/*)
files=("''${new[@]}" "''${old[@]}")
[ ''${#files[@]} -eq 0 ] && exit 0
paths=()
for f in "''${files[@]}"; do
for p in $(cat "$f"); do
[ -e "$p" ] && paths+=("$p")
done
done
if [ ''${#paths[@]} -gt 0 ]; then
if ! NIX_SSHOPTS="${sshOpts}" ${config.nix.package}/bin/nix copy --to "${cfg.url}" "''${paths[@]}"; then
[ ''${#new[@]} -gt 0 ] && mv "''${new[@]}" ${retry}/
exit 1
fi
fi
rm -f "''${files[@]}"
'';
in
{
options.skyg.nixos.common.cachePush = {
enable = lib.mkEnableOption "push finished builds to the bigbox2 binary cache";
url = lib.mkOption {
type = lib.types.str;
default = "ssh-ng://nix-ssh@bigbox2.lab.internal";
};
host = lib.mkOption {
type = lib.types.str;
default = "bigbox2.lab.internal";
description = "The cache's ssh host, whose host key is pinned from diagnix/hosts.nix.";
};
};
options.skyg.nixos.common.cacheReceive = {
enable = lib.mkEnableOption "accept pushes into this host's store over ssh-ng (nix-ssh)";
pushers = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ "hl-fwdesk" ];
description = "Hosts in diagnix/hosts.nix whose ssh host key may push.";
};
};
config = lib.mkMerge [
(lib.mkIf cfg.enable {
# The hook only queues the paths: Nix waits for a post-build-hook before it
# goes on, so copying here held up every build for the whole upload.
nix.settings.post-build-hook = pkgs.writeShellScript "nix-post-build-hook" ''
set -u
if [ -n "''${OUT_PATHS:-}" ]; then
tmp=$(mktemp -p ${state}/tmp) || exit 0
echo "$OUT_PATHS" > "$tmp" && mv "$tmp" "${queue}/$(date +%s%N)-$$"
fi
'';
systemd.tmpfiles.rules = map (d: "d ${d} 0700 root root -") [ state "${state}/tmp" queue retry ];
programs.ssh.knownHosts.nix-cache-push = {
hostNames = [ cfg.host ];
publicKey = (lib.findFirst (h: h.address == cfg.host) null (lib.attrValues lab.hosts)).hostKey;
};
systemd.services.nix-cache-push = {
description = "Push queued builds to ${cfg.url}";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
path = [ config.programs.ssh.package ];
serviceConfig = {
Type = "oneshot";
ExecStart = drain;
Nice = 10;
IOSchedulingClass = "idle";
};
};
# Runs when the hook queues a build, and every 10 minutes for the retries.
systemd.paths.nix-cache-push = {
wantedBy = [ "multi-user.target" ];
pathConfig.DirectoryNotEmpty = queue;
};
systemd.timers.nix-cache-push = {
wantedBy = [ "timers.target" ];
timerConfig.OnUnitInactiveSec = "10min";
timerConfig.OnBootSec = "5min";
};
# Manual/backstop push command, available on every host's PATH.
environment.systemPackages = [
(pkgs.writeShellScriptBin "push-cache" ''
set -eu
if [ "''$#" -eq 0 ]; then
set -- /run/current-system
fi
NIX_SSHOPTS="${sshOpts}" exec nix copy --to "${cfg.url}" "''$@"
'')
];
})
(lib.mkIf recv.enable {
nix.sshServe = {
enable = true;
write = true;
protocol = "ssh-ng";
# Pushed paths aren't signed; nix-serve signs them with this host's key
# when it serves them.
trusted = true;
keys = map (h: lab.hosts.${h}.hostKey) recv.pushers;
};
})
];
}