Files
slacker-bot/cluster_test.go
T
2026-11-01 11:47:41 -05:00

438 lines
13 KiB
Go

package main
import (
"context"
"errors"
"net"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
func TestPlacementBalancesAllThree(t *testing.T) {
drive := func(usable, committed int64) []DriveReport {
return []DriveReport{{Path: "/d", UsableMB: usable, CommittedMB: committed}}
}
want := Size{MemoryMB: 2048, CPUs: 1}
views := []NodeView{
{Name: "cpu-busy", Usable: Size{16384, 8}, Used: Size{2048, 6}, Drives: drive(100_000, 10_000)},
{Name: "ram-busy", Usable: Size{16384, 8}, Used: Size{12288, 1}, Drives: drive(100_000, 10_000)},
{Name: "disk-busy", Usable: Size{16384, 8}, Used: Size{2048, 1}, Drives: drive(100_000, 80_000)},
{Name: "calm", Usable: Size{16384, 8}, Used: Size{4096, 2}, Drives: drive(100_000, 20_000)},
}
node, d, err := choosePlacement(views, want, 5120)
if err != nil || node != "calm" || d != "/d" {
t.Fatalf("got %s %s %v, want calm", node, d, err)
}
}
func TestPlacementSpreadsAcrossEqualNodes(t *testing.T) {
views := []NodeView{
{Name: "a", Usable: Size{8192, 4}, Drives: []DriveReport{{Path: "/a", UsableMB: 50_000}}},
{Name: "b", Usable: Size{8192, 4}, Drives: []DriveReport{{Path: "/b", UsableMB: 50_000}}},
}
want := Size{2048, 1}
counts := map[string]int{}
for i := 0; i < 4; i++ {
node, _, err := choosePlacement(views, want, 5120)
if err != nil {
t.Fatal(err)
}
counts[node]++
for j := range views {
if views[j].Name == node {
views[j].Used = views[j].Used.Add(want)
views[j].Drives[0].CommittedMB += 5120
}
}
}
if counts["a"] != 2 || counts["b"] != 2 {
t.Fatalf("uneven spread: %v", counts)
}
}
func TestPlacementWhenFull(t *testing.T) {
full := []NodeView{{Name: "a", Usable: Size{4096, 2}, Used: Size{4096, 2}, Drives: []DriveReport{{Path: "/a", UsableMB: 50_000}}}}
if _, _, err := choosePlacement(full, Size{1024, 1}, 1024); !errors.Is(err, ErrMaxedOut) {
t.Fatalf("want maxed out, got %v", err)
}
noDisk := []NodeView{{Name: "a", Usable: Size{4096, 2}, Drives: []DriveReport{{Path: "/a", UsableMB: 1000}}}}
if _, _, err := choosePlacement(noDisk, Size{1024, 1}, 5120); !errors.Is(err, ErrStorageFull) {
t.Fatalf("want storage full, got %v", err)
}
}
func TestBuildView(t *testing.T) {
info := NodeInfo{
Name: "n1",
Usable: Size{16384, 8},
Running: []RunningMachine{{Name: "c-a"}, {Name: "u-stranger"}},
Drives: []DriveInfo{{Path: "/d", FreeMB: 90_000, ReserveMB: 10_000,
HomesMB: map[string]int64{"c-a": 8000, "c-b": 100, "u-stranger": 3000}}},
}
owners := []Owner{
{ID: "A", Machine: "c-a", Node: "n1", Drive: "/d", Size: Size{4096, 2}, StorageMinMB: 5120, StorageMaxMB: 10240},
{ID: "B", Machine: "c-b", Node: "n1", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
{ID: "C", Machine: "c-c", Node: "n1", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
{ID: "D", Machine: "c-d", Node: "other", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
}
v := buildView(info, owners, Size{1024, 1})
if v.Used != (Size{4096 + 1024, 3}) {
t.Fatalf("used = %v", v.Used)
}
d := v.Drives[0]
if d.UsableMB != 90_000+11_100-10_000 {
t.Fatalf("usable = %d", d.UsableMB)
}
if d.CommittedMB != 8000+5120+5120+3000 || d.BookedMB != 3*10240+3000 || d.Homes != 4 {
t.Fatalf("committed %d booked %d homes %d", d.CommittedMB, d.BookedMB, d.Homes)
}
}
type fakeNode struct {
name string
mu sync.Mutex
runs map[string]bool
last StartSpec
homes map[string][]byte
exports map[string]*fakeExport
imports map[string]*fakeImport
passes int
failWrites bool
usable Size
}
func newFakeNode(name string) *fakeNode {
return &fakeNode{name: name, runs: map[string]bool{}, homes: map[string][]byte{},
exports: map[string]*fakeExport{}, imports: map[string]*fakeImport{}, usable: Size{8192, 4}}
}
func (f *fakeNode) Name() string { return f.name }
func (f *fakeNode) Info(ctx context.Context) (NodeInfo, error) {
running, _ := f.Running(ctx)
f.mu.Lock()
defer f.mu.Unlock()
homes := map[string]int64{}
for k, v := range f.homes {
homes[strings.TrimPrefix(k, "/d/")] = int64(len(v)) >> 20
}
return NodeInfo{Name: f.name, Usable: f.usable, Running: running,
Drives: []DriveInfo{{Path: "/d", FreeMB: 50_000, HomesMB: homes}}}, nil
}
func (f *fakeNode) Running(context.Context) ([]RunningMachine, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []RunningMachine
for m := range f.runs {
out = append(out, RunningMachine{Name: m})
}
return out, nil
}
func (f *fakeNode) IsRunning(machine string) bool {
f.mu.Lock()
defer f.mu.Unlock()
return f.runs[machine]
}
func (f *fakeNode) Start(_ context.Context, spec StartSpec) error {
f.mu.Lock()
defer f.mu.Unlock()
f.last, f.runs[spec.Machine] = spec, true
return nil
}
func (f *fakeNode) Poweroff(_ context.Context, machine string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.runs, machine)
return nil
}
func (f *fakeNode) SetLimits(context.Context, string, Size) error { return nil }
func (f *fakeNode) SetQuota(context.Context, string, string, int64) error { return nil }
func (f *fakeNode) DialAgent(_ context.Context, machine, secret string) (msgConn, error) {
if secret != "agent-secret" {
return nil, errors.New("bad agent secret")
}
return newEcho(), nil
}
func (f *fakeNode) WaitStopped(_ context.Context, machine string) error {
if f.IsRunning(machine) {
return errors.New("still running")
}
return nil
}
func (f *fakeNode) ExportHome(_ context.Context, drive, machine, parent string) (string, error) {
f.mu.Lock()
defer f.mu.Unlock()
data, ok := f.homes[drive+"/"+machine]
if !ok {
return "", errors.New("no such home")
}
kind := "FULL:"
if parent != "" {
kind = "INCR:"
}
id := randomHex(4)
f.exports[id] = &fakeExport{data: append([]byte(kind), data...)}
return id, nil
}
func (f *fakeNode) ReadExport(_ context.Context, id string, max int) ([]byte, bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
x := f.exports[id]
if x == nil {
return nil, false, errors.New("no export")
}
n := min(max, len(x.data)-x.pos)
chunk := x.data[x.pos : x.pos+n]
x.pos += n
return chunk, x.pos == len(x.data), nil
}
func (f *fakeNode) CloseExport(_ context.Context, id string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.exports, id)
return nil
}
func (f *fakeNode) ImportHome(_ context.Context, drive, machine string) (string, error) {
f.mu.Lock()
defer f.mu.Unlock()
if _, exists := f.homes[drive+"/"+machine]; exists {
return "", errors.New("home already exists")
}
id := randomHex(4)
f.imports[id] = &fakeImport{key: drive + "/" + machine}
return id, nil
}
func (f *fakeNode) WriteImport(_ context.Context, id string, data []byte) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.failWrites {
return errors.New("disk on fire")
}
x := f.imports[id]
if x == nil {
return errors.New("no import")
}
x.cur = append(x.cur, data...)
return nil
}
func (f *fakeNode) EndReceive(_ context.Context, id string) error {
f.mu.Lock()
defer f.mu.Unlock()
x := f.imports[id]
if x == nil || len(x.cur) < 5 {
return errors.New("nothing received")
}
if string(x.cur[:5]) == "INCR:" && x.latest == nil {
return errors.New("incremental send without a parent")
}
x.latest, x.cur = x.cur[5:], nil
x.passes++
return nil
}
func (f *fakeNode) FinishImport(_ context.Context, id string, quotaMB int64) error {
f.mu.Lock()
defer f.mu.Unlock()
x := f.imports[id]
delete(f.imports, id)
f.homes[x.key] = x.latest
f.passes = x.passes
return nil
}
func (f *fakeNode) AbortImport(_ context.Context, id string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.imports, id)
return nil
}
func (f *fakeNode) DeleteHome(_ context.Context, drive, machine string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.homes, drive+"/"+machine)
return nil
}
type fakeExport struct {
data []byte
pos int
}
type fakeImport struct {
key string
cur, latest []byte
passes int
}
type echo struct {
ch chan []byte
closed chan struct{}
once sync.Once
}
func newEcho() *echo { return &echo{ch: make(chan []byte, 16), closed: make(chan struct{})} }
func (e *echo) Read(ctx context.Context) ([]byte, error) {
select {
case b := <-e.ch:
return b, nil
case <-e.closed:
return nil, errors.New("closed")
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (e *echo) Write(_ context.Context, b []byte) error {
e.ch <- append([]byte("echo: "), b...)
return nil
}
func (e *echo) Close() error { e.once.Do(func() { close(e.closed) }); return nil }
func freePort(t *testing.T) string {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer l.Close()
return l.Addr().String()
}
func waitFor(t *testing.T, what string, cond func() bool) {
deadline := time.Now().Add(10 * time.Second)
for !cond() {
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for %s", what)
}
time.Sleep(20 * time.Millisecond)
}
}
func TestMasterAndSlave(t *testing.T) {
addr := freePort(t)
secret := "cluster-secret-1234567890"
masterDir, slaveDir := t.TempDir(), t.TempDir()
masterCfg := &Config{StateDir: masterDir, Container: Size{1024, 1}}
masterCfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr}
store, err := OpenStore(filepath.Join(masterDir, "state.json"))
if err != nil {
t.Fatal(err)
}
mgr := NewManager(masterCfg, store, newFakeNode("master"))
backCh := make(chan string, 4)
mgr.onNodeBack = func(n string) { backCh <- n }
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go mgr.ServeCluster(ctx)
slaveCfg := &Config{StateDir: slaveDir}
slaveCfg.Cluster = ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr}
slave := newFakeNode("slave-1")
slaveCtx, stopSlave := context.WithCancel(ctx)
go RunSlave(slaveCtx, slaveCfg, slave)
waitFor(t, "slave to come online", func() bool { _, err := mgr.node("slave-1"); return err == nil })
if got := <-backCh; got != "slave-1" {
t.Fatalf("back callback for %q", got)
}
pinned, err := os.ReadFile(filepath.Join(slaveDir, "master.fingerprint"))
if err != nil || len(strings.TrimSpace(string(pinned))) != 64 {
t.Fatalf("slave should pin the master's certificate, got %q %v", pinned, err)
}
n, _ := mgr.node("slave-1")
spec := StartSpec{Machine: "c-abc", Secret: "agent-secret", Size: Size{1024, 1}, Drive: "/d"}
if err := n.Start(ctx, spec); err != nil {
t.Fatal(err)
}
if !slave.IsRunning("c-abc") || slave.last.Drive != "/d" {
t.Fatal("start didn't reach the slave")
}
if !n.IsRunning("c-abc") {
t.Fatal("master should count the container as running right away")
}
conn, err := n.DialAgent(ctx, "c-abc", "agent-secret")
if err != nil {
t.Fatal(err)
}
if err := conn.Write(ctx, []byte(`{"jsonrpc":"2.0"}`)); err != nil {
t.Fatal(err)
}
readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second)
got, err := conn.Read(readCtx)
readCancel()
if err != nil || string(got) != `echo: {"jsonrpc":"2.0"}` {
t.Fatalf("relay got %q %v", got, err)
}
conn.Close()
if _, err := n.DialAgent(ctx, "c-abc", "wrong"); err == nil {
t.Fatal("a bad agent secret should fail through the relay")
}
if err := n.Poweroff(ctx, "c-abc"); err != nil {
t.Fatal(err)
}
if slave.IsRunning("c-abc") || n.IsRunning("c-abc") {
t.Fatal("poweroff didn't take")
}
stopSlave()
waitFor(t, "slave to go offline", func() bool {
_, err := mgr.node("slave-1")
var off *NodeOfflineError
return errors.As(err, &off) && off.Node == "slave-1"
})
if off := mgr.OfflineNodes(); len(off) != 1 || off[0] != "slave-1" {
t.Fatalf("offline nodes = %v", off)
}
}
func TestSlaveRejectsWrongMasterOrSecret(t *testing.T) {
addr := freePort(t)
secret := "cluster-secret-1234567890"
masterCfg := &Config{StateDir: t.TempDir()}
masterCfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr}
store, _ := OpenStore(filepath.Join(masterCfg.StateDir, "state.json"))
mgr := NewManager(masterCfg, store, newFakeNode("master"))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go mgr.ServeCluster(ctx)
waitFor(t, "master to listen", func() bool {
c, err := net.Dial("tcp", addr)
if err == nil {
c.Close()
}
return err == nil
})
wrongPin := &Config{StateDir: t.TempDir()}
wrongPin.Cluster = ClusterConfig{Name: "s", Secret: secret, Master: "wss://" + addr, MasterFingerprint: strings.Repeat("ab", 32)}
if err := slaveSession(ctx, wrongPin, newFakeNode("s")); err == nil {
t.Fatal("a master with a different certificate should be refused")
}
wrongSecret := &Config{StateDir: t.TempDir()}
wrongSecret.Cluster = ClusterConfig{Name: "s", Secret: "not-the-secret-at-all", Master: "wss://" + addr}
if err := slaveSession(ctx, wrongSecret, newFakeNode("s")); err == nil {
t.Fatal("a slave with the wrong secret should be refused")
}
sameName := &Config{StateDir: t.TempDir()}
sameName.Cluster = ClusterConfig{Name: "master", Secret: secret, Master: "wss://" + addr}
if err := slaveSession(ctx, sameName, newFakeNode("master")); err == nil {
t.Fatal("a slave can't take the master's name")
}
}