Files
hermenix/nu/host/setup.nu
T
2026-09-23 13:24:07 -04:00

410 lines
16 KiB
Nu

# hermenix setup — guided host-side provisioning.
#
# Walks through every prerequisite check, disk creation, cache signing key,
# local.nix, image build and image write. Safe to re-run; each step checks
# what already exists. Never runs nixos-rebuild.
#
# Depends on: lib/common.nu, lib/guest.nu.
# `truncate -s` treats a bare number as bytes, not gigabytes — the size
# prompts' own defaults ("80G", "20G") make gigabytes the obvious intent, so
# normalize a bare integer answer to that rather than silently creating a
# byte-sized disk. Anything with an existing suffix passes through as-is.
def normalize-size [s: string] {
try {
let n = $s | into int
$"($n)G"
} catch {
$s
}
}
# Run a command, sudo-prefixed unless already root (set once via
# $env.HERMENIX_NEED_SUDO at the top of `main setup`). Fails loudly, like any
# other external command, if the command fails.
def priv [...args: string] {
let need_sudo = $env.HERMENIX_NEED_SUDO? | default false
let argv = if $need_sudo { ["sudo"] ++ $args } else { $args }
^($argv | first) ...($argv | skip 1)
}
# Same as priv, but captures stdout/exit code instead of failing the script.
def priv-out [...args: string] {
let need_sudo = $env.HERMENIX_NEED_SUDO? | default false
let argv = if $need_sudo { ["sudo"] ++ $args } else { $args }
(^($argv | first) ...($argv | skip 1) | complete)
}
# Guided host-side setup: disks, cache signing key, /etc/hermenix/local.nix,
# build and write the bootstrap image. Safe to re-run — every step checks
# what's already done and offers to skip. Never runs nixos-rebuild. Works
# from any directory - no local checkout of this repo needed.
def "main setup" [
--flake: string = "git+ssh://gitea@minipc1.lab.internal:22/ari/hermenix.git"
--clean # stop the VM and delete existing disk images + UEFI vars first, for a fully fresh start
] {
$env.HERMENIX_NEED_SUDO = ((^id -u | str trim) != "0")
let local_path = "/etc/hermenix/local.nix"
let cache_name = "hermes-cache-1"
let secret_key = "/var/lib/secrets/hermes-cache.secret"
let public_key = "/var/lib/secrets/hermes-cache.pub"
let determinate_sub = "https://install.determinate.systems"
let determinate_key = "cache.flakehub.com-3:hJuILl5sVK4iKm86JzgdXW12Y2Hwd5G07qKtHTOcDCM="
print $"(ansi attr_bold) hermenix · guided setup(ansi reset)"
note "This prepares the host. It does not run nixos-rebuild — you'll paste a"
note "module snippet into your own configuration at the end."
step 1 "Checking prerequisites"
if (which nix | is-empty) { die "nix not found on PATH" }
if (do -i { ^nix flake --help } | complete).exit_code != 0 {
die "flakes are not enabled. Add 'experimental-features = nix-command flakes' to /etc/nix/nix.conf"
}
ok "nix with flakes"
if (which btrfs | is-empty) { die "btrfs-progs not found. Snapshots need it." }
if (which mkfs.ext4 | is-empty) { die "mkfs.ext4 not found (e2fsprogs)" }
ok "btrfs-progs"
if not ("/dev/kvm" | path exists) {
die "/dev/kvm is missing. KVM is required; check virtualisation is enabled in firmware."
}
ok "kvm"
if (which nixos-version | is-empty) {
warn "this host does not look like NixOS — the module expects NixOS"
} else {
let ver = (
(^nixos-version | complete).stdout
| str trim
| split row " "
| first
)
ok $"host is NixOS \(($ver)\)"
}
step 2 "Guest disks"
note "Two disks, deliberately:"
note " system - NixOS and the Nix store. Big, churny, rebuildable. Not backed up."
note " state - identity, memories, workspace, /etc/nixos. Small. Snapshotted."
let base = ask "Directory for the disk images" "/var/lib/hermes-vm"
if $clean {
warn "--clean: this stops the VM and deletes the existing system/state"
warn "disk images and UEFI vars under this directory. The btrfs"
warn "subvolumes and any snapshots are left alone."
if not (confirm-word "clean" "Type 'clean' to confirm:") { die "aborted" }
do -i { priv systemctl stop hermes-vm.service } | complete | ignore
do -i { priv rm "-f" $"($base)/system/system.raw" } | complete | ignore
do -i { priv rm "-f" $"($base)/state/state.raw" } | complete | ignore
do -i { priv rm "-f" $"($base)/OVMF_VARS.fd" } | complete | ignore
# A fresh state disk means a fresh SSH host key on next boot - drop the
# stale known_hosts entry now so the next connect doesn't fail on a
# host-key mismatch. Not run through priv: this is the invoking user's
# own ~/.ssh/known_hosts, not a root-owned path.
do -i { ^ssh-keygen -R hermes } | complete | ignore
ok "cleaned existing disk images and stale known_hosts entry"
}
let parent = $base | path dirname
let fstype = (do -i { ^stat -f -c "%T" $parent } | complete).stdout | str trim
let use_btrfs = if $fstype != "btrfs" {
warn $"($parent) is ($fstype), not btrfs"
warn "snapshots need btrfs. Everything else works; you'd rely on backups alone."
if not (confirm "Continue anyway?") { die "aborted" }
false
} else {
true
}
priv mkdir "-p" $base
def make-subvol [path: string, use_btrfs: bool] {
let exists = (
$use_btrfs and (do -i { priv btrfs subvolume show $path } | complete).exit_code == 0
)
if $use_btrfs and not $exists {
do -i { priv rmdir $path } | complete | ignore
priv btrfs subvolume create $path | ignore
priv chattr +C $path
do -i { priv btrfs property set $path compression none } | complete | ignore
priv chown "hermes-vm:hermes-vm" $path
ok $"subvolume ($path) \(nodatacow\)"
} else {
priv mkdir "-p" $path
priv chown "hermes-vm:hermes-vm" $path
}
}
make-subvol $"($base)/system" $use_btrfs
make-subvol $"($base)/state" $use_btrfs
priv mkdir "-p" $"($base)/snapshots"
let system_img = $"($base)/system/system.raw"
let state_img = $"($base)/state/state.raw"
let system_existed = ((priv-out test "-f" $system_img).exit_code == 0)
if $system_existed {
let sz = (
(priv-out du "-h" "--apparent-size" $system_img).stdout
| str trim
| split column "\t"
| get column0.0
)
priv chown "hermes-vm:hermes-vm" $system_img
ok $"system disk exists \(($sz)\)"
} else {
let syssize = (normalize-size (ask "System disk size" "80G"))
priv truncate "-s" $syssize $system_img
priv chown "hermes-vm:hermes-vm" $system_img
if $use_btrfs {
let attrs = (priv-out lsattr "-l" $system_img).stdout
if ($attrs | str contains "No_COW") {
ok "system disk created, nodatacow confirmed"
} else {
warn $"system disk created but nodatacow did NOT apply - check 'lsattr ($system_img)'"
}
} else {
ok "system disk created"
}
}
if (priv-out test "-f" $state_img).exit_code == 0 {
let sz = (
(priv-out du "-h" "--apparent-size" $state_img).stdout
| str trim
| split column "\t"
| get column0.0
)
priv chown "hermes-vm:hermes-vm" $state_img
ok $"state disk exists \(($sz)\)"
} else {
let statesize = (normalize-size (ask "State disk size" "20G"))
priv truncate "-s" $statesize $state_img
priv mkfs.ext4 "-q" "-L" hermes-state $state_img
priv chown "hermes-vm:hermes-vm" $state_img
ok "state disk created and formatted (label: hermes-state)"
}
step 3 "Binary cache signing key"
note "The host serves its store to the guest read-only, so the guest doesn't"
note "re-download from the internet what you already have."
let key_existed = (priv-out test "-f" $secret_key).exit_code == 0
if $key_existed {
ok "signing key already exists"
} else {
priv mkdir "-p" /var/lib/secrets
(priv
nix-store
"--generate-binary-cache-key"
$cache_name
$secret_key
$public_key
)
priv chmod 0400 $secret_key
ok $"generated ($cache_name)"
}
let cache_pub = (priv-out cat $public_key).stdout | str trim
say $"Public key: (ansi dark_gray)($cache_pub)(ansi reset)"
if (do -i { ^id nix-serve } | complete).exit_code == 0 {
priv chown nix-serve $secret_key
ok "key owned by nix-serve"
} else {
warn "the nix-serve user does not exist yet — that's normal on a first run"
note "after your nixos-rebuild, run:"
note $" sudo chown nix-serve ($secret_key)"
}
step 4 "SSH access to the guest"
let existing = if ($local_path | path exists) {
let evaluated = (
do -i { ^nix-instantiate --eval --json --strict -E $"import ($local_path)" }
| complete
)
if $evaluated.exit_code == 0 {
$evaluated.stdout | from json
} else {
warn $"($local_path) exists but failed to evaluate - treating as empty"
{
cachePublicKey: ""
authorizedKeys: []
}
}
} else {
{
cachePublicKey: ""
authorizedKeys: []
}
}
let existing_keys = $existing.authorizedKeys? | default []
let replace_keys = if ($existing_keys | length) > 0 {
ok $"($existing_keys | length) key\(s\) already configured in ($local_path)"
confirm "Replace them?"
} else {
true
}
mut selected_keys = $existing_keys
if $replace_keys {
mut picked = []
let found = (glob $"($env.HOME)/.ssh/*.pub")
if ($found | length) > 0 {
say "Public keys found in ~/.ssh:"
for pair in ($found | enumerate) {
let comment = (
open $pair.item
| str trim
| split row " "
| skip 2
| str join " "
)
note $" ($pair.index + 1)) ($pair.item | path basename) — ($comment)"
}
let pick = ask "Numbers to use (space separated, or 'n' to paste one)" "1"
if $pick != "n" {
for n in ($pick | split row " ") {
let idx = ($n | into int) - 1
if $idx < 0 or $idx >= ($found | length) {
die $"no such option: ($n)"
}
$picked = ($picked | append (open ($found | get $idx) | str trim))
}
}
}
if ($picked | is-empty) {
let pasted = ask "Paste a public key (blank to skip)"
if $pasted != "" { $picked = [$pasted] }
}
if ($picked | is-empty) {
warn "no SSH key set — you'll only be able to reach the guest with 'hermenix vm console'"
}
$selected_keys = $picked
}
step 5 $"Writing ($local_path)"
let key_lines = $selected_keys | each {|k| $" \"($k)\"" } | str join "\n"
let content = $"# Written by `hermenix setup`. Lives on the host, not in a repo checkout -
# see guest/local.nix.example in the hermenix repo for the reference shape.
\{
cachePublicKey = \"($cache_pub)\";
authorizedKeys = [
($key_lines)
];
\}
"
let tmp = (^mktemp | str trim)
$content | save -f $tmp
priv mkdir "-p" (($local_path | path dirname))
priv cp $tmp $local_path
priv chmod 0644 $local_path
rm $tmp
ok $"($local_path) written \(($selected_keys | length) key\(s\)\)"
step 6 "Building the bootstrap image"
note "The extra substituter flags stop you compiling Determinate Nix yourself."
note "--impure is needed to read /etc/hermenix/local.nix at eval time."
if (confirm "Build now? (first run can take a while)") {
^nix build --impure --no-write-lock-file --refresh $"($flake)#hermes-guest-image" --option extra-substituters $determinate_sub --option extra-trusted-public-keys $determinate_key
ok "built"
} else {
say "Skipped. Run manually:"
note $" nix build --impure --no-write-lock-file --refresh ($flake)#hermes-guest-image \\"
note $" --option extra-substituters ($determinate_sub) \\"
note $" --option extra-trusted-public-keys ($determinate_key)"
}
step 7 "Writing the bootstrap image to the system disk"
let images = (glob "result/*.img")
if ($images | is-empty) {
warn "no image found under ./result - skipping"
note "build it, then:"
note $" sudo dd if=result/nixos.img of=($system_img) bs=4M conv=notrunc,fsync status=progress"
} else {
let image = $images | first
let img_bytes = ls $image | get size.0 | into int
let vol_bytes = (priv-out stat "-c" "%s" $system_img).stdout | str trim | into int
say $"image: ($image) \((($img_bytes / 1024 / 1024) | math floor) MiB\)"
say $"target: ($system_img) \((($vol_bytes / 1024 / 1024 / 1024) | math floor) GiB\)"
if $img_bytes > $vol_bytes { die "image is larger than the system disk" }
if $system_existed {
warn "the system disk already existed. Writing the image replaces the"
warn "guest OS. Hermes's identity and memories live on the STATE disk and"
warn "are untouched, but anything it installed outside /var/lib/hermes is lost."
}
if (confirm-word "overwrite" "Type 'overwrite' to write the image:") {
(priv
dd
$"if=($image)"
$"of=($system_img)"
"bs=4M"
"conv=notrunc,fsync"
"status=progress"
)
ok "written"
} else {
say "Skipped."
}
}
step 8 "Next: add the module to your system flake"
let uplink = ((do -i { ^ip route get 1.1.1.1 } | complete).stdout
| parse -r '.* dev (?P<dev>\S+).*' | get dev | get 0? | default "eno1")
print $"
(ansi dark_gray)# flake.nix(ansi reset)
inputs.hermenix.url = \"($flake)\";
(ansi dark_gray)# then in your modules list(ansi reset)
hermenix.nixosModules.default
(ansi dark_gray)# configuration.nix(ansi reset)
services.hermenix = \{
enable = true;
disk.baseDir = \"($base)\";
network.externalInterface = \"($uplink)\";
vcpu = 4;
mem = 8192;
\};
"
note $"detected uplink: ($uplink) — check that's right before rebuilding"
note "network.bridged = true instead puts the guest on your LAN (own DHCP"
note "address, every opened port reachable from the LAN) - see the README."
print $"
Then:
sudo nixos-rebuild switch
hermenix vm console (ansi dark_gray)# watch it boot(ansi reset)
hermenix vm ssh (ansi dark_gray)# get in(ansi reset)
Inside the guest, in this order:
lsblk -f (ansi dark_gray)# CHECK LABELS match guest/hardware.nix(ansi reset)
determinate-nixd login (ansi dark_gray)# FlakeHub Cache(ansi reset)
sudo nix flake lock /etc/nixos
(ansi dark_gray)# hermes installs itself automatically - give it a few minutes(ansi reset)
hermes setup
hermes gateway install
Back on the host:
hermenix ctl units (ansi dark_gray)# confirm the gateway unit name(ansi reset)
hermenix ctl status
hermenix ctl soul
"
warn "Check the disk labels before the guest's first nixos-rebuild. If they"
warn "don't match guest/hardware.nix, the first self-rebuild produces a"
warn "system that won't boot."
print $"\n(ansi green)✓ host setup complete(ansi reset)\n"
}