clean up of the codebase

This commit is contained in:
ari committed 2026-11-01 13:01:54 -05:00
1 parent 7822529a38
commit 0ac61b4ece
52 files changed
+2545 -2274

No files matched your search

+60
View File
@@ -0,0 +1,60 @@
package local
import (
"bufio"
"fmt"
"os"
"runtime"
"strconv"
"strings"
"slackerbot/internal/model"
)
type Capacity struct {
Total model.Size
Usable model.Size
}
func DetectCapacity(reserve model.Size) (Capacity, error) {
memMB, err := totalMemoryMB()
if err != nil {
return Capacity{}, err
}
total := model.Size{MemoryMB: memMB, CPUs: float64(runtime.NumCPU())}
usable := total.Sub(reserve)
if usable.MemoryMB <= 0 || usable.CPUs <= 0 {
return Capacity{}, fmt.Errorf("host_reserve (%s) leaves nothing on this node (%s)", reserve, total)
}
return Capacity{Total: total, Usable: usable}, nil
}
func (c Capacity) Room(used, want model.Size) bool {
return used.Add(want).Fits(c.Usable)
}
func (c Capacity) Slots(size model.Size) int {
byMem := c.Usable.MemoryMB / size.MemoryMB
byCPU := int(c.Usable.CPUs/size.CPUs + 1e-9)
return min(byMem, byCPU)
}
func totalMemoryMB() (int, error) {
f, err := os.Open("/proc/meminfo")
if err != nil {
return 0, err
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
fields := strings.Fields(sc.Text())
if len(fields) >= 2 && fields[0] == "MemTotal:" {
kb, err := strconv.Atoi(fields[1])
if err != nil {
return 0, err
}
return kb / 1024, nil
}
}
return 0, fmt.Errorf("MemTotal not found in /proc/meminfo")
}
@@ -0,0 +1,24 @@
package local
import (
"testing"
"slackerbot/internal/model"
)
func TestCapacity(t *testing.T) {
c := Capacity{Usable: model.Size{MemoryMB: 6144, CPUs: 3}}
def := model.Size{MemoryMB: 2048, CPUs: 1}
if got := c.Slots(def); got != 3 {
t.Fatalf("slots = %d, want 3", got)
}
if !c.Room(model.Size{MemoryMB: 4096, CPUs: 2}, def) {
t.Fatal("third container should fit")
}
if c.Room(model.Size{MemoryMB: 6144, CPUs: 3}, def) {
t.Fatal("fourth container should not fit")
}
if c.Room(model.Size{MemoryMB: 2048, CPUs: 2.5}, def) {
t.Fatal("should be blocked by CPU")
}
}
+325
View File
@@ -0,0 +1,325 @@
package local
import (
"context"
"errors"
"fmt"
"math"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/godbus/dbus/v5"
"slackerbot/internal/config"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/storage"
)
const (
ServicePort = 1984
machinedName = "org.freedesktop.machine1"
machinedPath = "/org/freedesktop/machine1"
machinedMgr = "org.freedesktop.machine1.Manager"
sigPoweroff = 38 // SIGRTMIN+4, what `machinectl poweroff` sends
agentEnvInside = "/var/lib/slackerbot/agent.env"
)
type Local struct {
cfg *config.Config
name string
cap Capacity
bus *dbus.Conn
system string
exports transfers
imports transfers
}
func New(cfg *config.Config) (*Local, error) {
system, err := filepath.EvalSymlinks(cfg.System)
if err != nil {
return nil, err
}
if _, err := os.Stat(filepath.Join(system, "init")); err != nil {
return nil, fmt.Errorf("%s has no init. Is it a NixOS build with boot.isContainer = true?", system)
}
for _, d := range cfg.Drives {
if err := storage.PrepareDrive(d.Path); err != nil {
return nil, err
}
}
capacity, err := DetectCapacity(cfg.HostReserve)
if err != nil {
return nil, err
}
bus, err := dbus.ConnectSystemBus()
if err != nil {
return nil, fmt.Errorf("connecting to the system bus: %w", err)
}
return &Local{cfg: cfg, name: cfg.Cluster.Name, cap: capacity, bus: bus, system: system}, nil
}
func unitName(machine string) string { return "slackerbot-" + machine + ".service" }
func (l *Local) Name() string { return l.name }
func (l *Local) machined() dbus.BusObject { return l.bus.Object(machinedName, machinedPath) }
func (l *Local) Info(ctx context.Context) (node.NodeInfo, error) {
info := node.NodeInfo{Name: l.name, Total: l.cap.Total, Usable: l.cap.Usable}
for _, d := range l.cfg.Drives {
free, err := storage.FreeMB(d.Path)
if err != nil {
return info, err
}
di := node.DriveInfo{Path: d.Path, FreeMB: free, ReserveMB: d.ReserveMB, HomesMB: map[string]int64{}}
entries, err := os.ReadDir(d.Path)
if err != nil {
return info, err
}
for _, e := range entries {
if e.IsDir() && model.IsOurs(e.Name()) {
used, _ := storage.UsedMB(filepath.Join(d.Path, e.Name()))
di.HomesMB[e.Name()] = used
}
}
info.Drives = append(info.Drives, di)
}
running, err := l.Running(ctx)
info.Running = running
return info, err
}
func (l *Local) Running(ctx context.Context) ([]node.RunningMachine, error) {
var list []struct {
Name, Class, Service string
Path dbus.ObjectPath
}
if err := l.machined().Call(machinedMgr+".ListMachines", 0).Store(&list); err != nil {
return nil, err
}
var out []node.RunningMachine
for _, x := range list {
if !model.IsOurs(x.Name) {
continue
}
r := node.RunningMachine{Name: x.Name}
if v, err := l.bus.Object(machinedName, x.Path).GetProperty("org.freedesktop.machine1.Machine.Timestamp"); err == nil {
if us, ok := v.Value().(uint64); ok {
r.Since = time.UnixMicro(int64(us))
}
}
out = append(out, r)
}
return out, nil
}
func (l *Local) IsRunning(machine string) bool {
var p dbus.ObjectPath
return l.machined().Call(machinedMgr+".GetMachine", 0, machine).Store(&p) == nil
}
func (l *Local) drive(path string) (config.DriveConfig, error) {
for _, d := range l.cfg.Drives {
if d.Path == path {
return d, nil
}
}
return config.DriveConfig{}, fmt.Errorf("drive %s isn't configured on node %s", path, l.name)
}
func (l *Local) Start(ctx context.Context, spec node.StartSpec) error {
if !model.IsOurs(spec.Machine) {
return fmt.Errorf("unexpected machine name %q", spec.Machine)
}
if _, err := l.drive(spec.Drive); err != nil {
return err
}
home := filepath.Join(spec.Drive, spec.Machine)
if _, err := os.Stat(home); errors.Is(err, os.ErrNotExist) {
if err := storage.CreateSubvolume(home); err != nil {
return err
}
} else if err != nil {
return err
}
if err := storage.SetQuota(home, spec.StorageMaxMB); err != nil {
return err
}
// Only the home folder persists. NixOS rebuilds the rest on boot.
root := filepath.Join(l.cfg.MachinesDir, spec.Machine)
if err := os.RemoveAll(root); err != nil {
return fmt.Errorf("clearing old container files: %w", err)
}
if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil {
return err
}
if f, err := os.OpenFile(filepath.Join(root, "etc/os-release"), os.O_CREATE, 0o644); err == nil {
f.Close()
}
envFile, err := l.writeEnvFile(spec)
if err != nil {
return err
}
args := []string{
"--unit=" + unitName(spec.Machine),
"--collect",
"--quiet",
"--description=" + spec.Description,
// Same as systemd-nspawn@.service.
"--property=Type=notify",
"--property=KillMode=mixed",
"--property=Delegate=yes",
fmt.Sprintf("--property=MemoryMax=%dM", spec.Size.MemoryMB),
fmt.Sprintf("--property=CPUQuota=%d%%", int(math.Round(spec.Size.CPUs*100))),
"systemd-nspawn",
"--machine=" + spec.Machine,
"--directory=" + root,
"--keep-unit",
"--network-veth", // own network, so every container can use port 1984
"--bind-ro=/nix/store",
"--bind-ro=" + envFile + ":" + agentEnvInside,
"--bind=" + home + ":/root",
}
for _, b := range l.cfg.ROBinds {
args = append(args, "--bind-ro="+strings.ReplaceAll(b, "{machine}", spec.Machine))
}
for _, b := range l.cfg.RWBinds {
b = strings.ReplaceAll(b, "{machine}", spec.Machine)
src, _, _ := strings.Cut(b, ":")
if err := os.MkdirAll(src, 0o755); err != nil {
return err
}
args = append(args, "--bind="+b)
}
args = append(args, filepath.Join(l.system, "init"))
if out, err := exec.CommandContext(ctx, "systemd-run", args...).CombinedOutput(); err != nil {
return fmt.Errorf("systemd-run: %v: %s", err, strings.TrimSpace(string(out)))
}
// Wait until machined lists it, so the next capacity check counts it.
for i := 0; i < 50; i++ {
if l.IsRunning(spec.Machine) {
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("%s started but never registered with machined", spec.Machine)
}
func (l *Local) writeEnvFile(spec node.StartSpec) (string, error) {
dir := filepath.Join(l.cfg.StateDir, "env")
if err := os.MkdirAll(dir, 0o700); err != nil {
return "", err
}
var b strings.Builder
line := func(k, v string) {
v = strings.ReplaceAll(v, `\`, `\\`)
v = strings.ReplaceAll(v, `"`, `\"`)
fmt.Fprintf(&b, "%s=\"%s\"\n", k, v)
}
line("GOOSE_SERVER__SECRET_KEY", spec.Secret)
for k, v := range spec.Env {
line(k, v)
}
path := filepath.Join(dir, spec.Machine+".env")
return path, os.WriteFile(path, []byte(b.String()), 0o600)
}
func (l *Local) Poweroff(ctx context.Context, machine string) error {
return l.machined().CallWithContext(ctx, machinedMgr+".KillMachine", 0, machine, "leader", int32(sigPoweroff)).Err
}
func (l *Local) SetLimits(ctx context.Context, machine string, s model.Size) error {
type prop struct {
Name string
Value dbus.Variant
}
props := []prop{
{"MemoryMax", dbus.MakeVariant(uint64(s.MemoryMB) << 20)},
{"CPUQuotaPerSecUSec", dbus.MakeVariant(uint64(s.CPUs * 1e6))},
}
return l.bus.Object("org.freedesktop.systemd1", "/org/freedesktop/systemd1").
CallWithContext(ctx, "org.freedesktop.systemd1.Manager.SetUnitProperties", 0, unitName(machine), true, props).Err
}
func (l *Local) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
if _, err := l.drive(drive); err != nil {
return err
}
if !model.IsOurs(machine) {
return fmt.Errorf("unexpected machine name %q", machine)
}
return storage.SetQuota(filepath.Join(drive, machine), mb)
}
func (l *Local) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
addr, err := l.waitReady(ctx, machine)
if err != nil {
return nil, err
}
return node.DialAgentWS(ctx, addr, secret)
}
func (l *Local) address(machine string) (string, error) {
var addrs []struct {
Family int32
Addr []byte
}
if err := l.machined().Call(machinedMgr+".GetMachineAddresses", 0, machine).Store(&addrs); err != nil {
return "", err
}
for _, a := range addrs {
ip := net.IP(a.Addr)
if a.Family == syscall.AF_INET && !ip.IsLinkLocalUnicast() {
return net.JoinHostPort(ip.String(), strconv.Itoa(ServicePort)), nil
}
}
return "", fmt.Errorf("%s has no IPv4 address yet", machine)
}
func (l *Local) waitReady(ctx context.Context, machine string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, l.cfg.BootTimeout)
defer cancel()
client := http.Client{Timeout: 3 * time.Second}
var lastErr error
for {
addr, err := l.address(machine)
if err == nil {
var resp *http.Response
resp, err = client.Get("http://" + addr + "/status")
if err == nil {
resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return addr, nil
}
err = fmt.Errorf("/status returned %s", resp.Status)
}
}
lastErr = err
select {
case <-ctx.Done():
return "", fmt.Errorf("goose didn't answer on port %d within %s (last error: %v)", ServicePort, l.cfg.BootTimeout, lastErr)
case <-time.After(time.Second):
}
}
}
func (l *Local) Capacity() Capacity { return l.cap }
+265
View File
@@ -0,0 +1,265 @@
package local
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"slackerbot/internal/model"
"slackerbot/internal/storage"
"slackerbot/internal/util"
)
type transfer struct {
cmd *exec.Cmd
out io.ReadCloser
in io.WriteCloser
stderr bytes.Buffer
snapshot string
tmpDir string
drive string
machine string
seen map[string]bool
latest string
}
type transfers struct {
mu sync.Mutex
m map[string]*transfer
}
func (t *transfers) add(x *transfer) string {
id := util.RandomHex(8)
t.mu.Lock()
defer t.mu.Unlock()
if t.m == nil {
t.m = map[string]*transfer{}
}
t.m[id] = x
return id
}
func (t *transfers) get(id string) (*transfer, error) {
t.mu.Lock()
defer t.mu.Unlock()
if x := t.m[id]; x != nil {
return x, nil
}
return nil, fmt.Errorf("no transfer %s", id)
}
func (t *transfers) take(id string) *transfer {
t.mu.Lock()
defer t.mu.Unlock()
x := t.m[id]
delete(t.m, id)
return x
}
func (l *Local) home(drive, machine string) (string, error) {
if _, err := l.drive(drive); err != nil {
return "", err
}
if !model.IsOurs(machine) {
return "", fmt.Errorf("unexpected machine name %q", machine)
}
return filepath.Join(drive, machine), nil
}
func (l *Local) WaitStopped(ctx context.Context, machine string) error {
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
for l.IsRunning(machine) {
select {
case <-ctx.Done():
return fmt.Errorf("%s didn't stop in time", machine)
case <-time.After(500 * time.Millisecond):
}
}
return nil
}
// With a parent export, only the changes since the parent's snapshot are sent.
func (l *Local) ExportHome(ctx context.Context, drive, machine, parent string) (string, error) {
home, err := l.home(drive, machine)
if err != nil {
return "", err
}
args := []string{"send", "-q"}
if parent != "" {
p, err := l.exports.get(parent)
if err != nil {
return "", err
}
args = append(args, "-p", p.snapshot)
}
snapshot := filepath.Join(drive, ".move-"+machine+"-"+util.RandomHex(4))
if out, err := exec.Command("btrfs", "subvolume", "snapshot", "-r", home, snapshot).CombinedOutput(); err != nil {
return "", fmt.Errorf("snapshot: %v: %s", err, strings.TrimSpace(string(out)))
}
x := &transfer{snapshot: snapshot, drive: drive, machine: machine}
x.cmd = exec.Command("btrfs", append(args, snapshot)...)
x.cmd.Stderr = &x.stderr
if x.out, err = x.cmd.StdoutPipe(); err == nil {
err = x.cmd.Start()
}
if err != nil {
storage.DeleteSubvolume(snapshot)
return "", err
}
return l.exports.add(x), nil
}
func (l *Local) ReadExport(ctx context.Context, id string, max int) ([]byte, bool, error) {
x, err := l.exports.get(id)
if err != nil {
return nil, false, err
}
buf := make([]byte, max)
n, err := io.ReadFull(x.out, buf)
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
if werr := x.cmd.Wait(); werr != nil {
return nil, false, fmt.Errorf("btrfs send: %v: %s", werr, strings.TrimSpace(x.stderr.String()))
}
return buf[:n], true, nil
}
return buf[:n], false, err
}
func (l *Local) CloseExport(ctx context.Context, id string) error {
x := l.exports.take(id)
if x == nil {
return nil
}
if x.cmd.ProcessState == nil {
_ = x.cmd.Process.Kill()
_ = x.cmd.Wait()
}
return storage.DeleteSubvolume(x.snapshot)
}
func (l *Local) ImportHome(ctx context.Context, drive, machine string) (string, error) {
home, err := l.home(drive, machine)
if err != nil {
return "", err
}
if _, err := os.Stat(home); err == nil {
return "", fmt.Errorf("%s already has a home folder for %s; remove it first", l.name, machine)
}
x := &transfer{drive: drive, machine: machine, seen: map[string]bool{},
tmpDir: filepath.Join(drive, ".incoming-"+machine+"-"+util.RandomHex(4))}
if err := os.Mkdir(x.tmpDir, 0o700); err != nil {
return "", err
}
return l.imports.add(x), nil
}
func (l *Local) WriteImport(ctx context.Context, id string, data []byte) error {
x, err := l.imports.get(id)
if err != nil {
return err
}
if x.cmd == nil {
x.stderr.Reset()
x.cmd = exec.Command("btrfs", "receive", "-q", x.tmpDir)
x.cmd.Stderr = &x.stderr
if x.in, err = x.cmd.StdinPipe(); err == nil {
err = x.cmd.Start()
}
if err != nil {
x.cmd = nil
return err
}
}
if _, err := x.in.Write(data); err != nil {
return fmt.Errorf("btrfs receive: %v: %s", err, strings.TrimSpace(x.stderr.String()))
}
return nil
}
func (l *Local) EndReceive(ctx context.Context, id string) error {
x, err := l.imports.get(id)
if err != nil {
return err
}
if x.cmd == nil {
return errors.New("nothing was received")
}
x.in.Close()
err = x.cmd.Wait()
x.cmd = nil
if err != nil {
return fmt.Errorf("btrfs receive: %v: %s", err, strings.TrimSpace(x.stderr.String()))
}
entries, err := os.ReadDir(x.tmpDir)
if err != nil {
return err
}
for _, e := range entries {
if !x.seen[e.Name()] {
x.seen[e.Name()] = true
x.latest = e.Name()
}
}
return nil
}
// Received subvolumes are read-only; a writable snapshot of the latest one
// becomes the new home.
func (l *Local) FinishImport(ctx context.Context, id string, quotaMB int64) error {
x := l.imports.take(id)
if x == nil {
return fmt.Errorf("no transfer %s", id)
}
defer cleanupIncoming(x.tmpDir)
if x.latest == "" {
return errors.New("nothing was received")
}
home := filepath.Join(x.drive, x.machine)
if out, err := exec.Command("btrfs", "subvolume", "snapshot", filepath.Join(x.tmpDir, x.latest), home).CombinedOutput(); err != nil {
return fmt.Errorf("snapshot: %v: %s", err, strings.TrimSpace(string(out)))
}
return storage.SetQuota(home, quotaMB)
}
func (l *Local) AbortImport(ctx context.Context, id string) error {
x := l.imports.take(id)
if x == nil {
return nil
}
if x.cmd != nil {
x.in.Close()
_ = x.cmd.Process.Kill()
_ = x.cmd.Wait()
}
cleanupIncoming(x.tmpDir)
return nil
}
func (l *Local) DeleteHome(ctx context.Context, drive, machine string) error {
home, err := l.home(drive, machine)
if err != nil {
return err
}
if l.IsRunning(machine) {
return fmt.Errorf("%s is running", machine)
}
return storage.DeleteSubvolume(home)
}
func cleanupIncoming(dir string) {
entries, _ := os.ReadDir(dir)
for _, e := range entries {
_ = storage.DeleteSubvolume(filepath.Join(dir, e.Name()))
}
_ = os.RemoveAll(dir)
}