package main import ( "context" "errors" "net" "os" "path/filepath" "strings" "sync" "testing" "time" ) func TestPlacementBalancesAllThree(t *testing.T) { drive := func(usable, committed int64) []DriveReport { return []DriveReport{{Path: "/d", UsableMB: usable, CommittedMB: committed}} } want := Size{MemoryMB: 2048, CPUs: 1} views := []NodeView{ {Name: "cpu-busy", Usable: Size{16384, 8}, Used: Size{2048, 6}, Drives: drive(100_000, 10_000)}, {Name: "ram-busy", Usable: Size{16384, 8}, Used: Size{12288, 1}, Drives: drive(100_000, 10_000)}, {Name: "disk-busy", Usable: Size{16384, 8}, Used: Size{2048, 1}, Drives: drive(100_000, 80_000)}, {Name: "calm", Usable: Size{16384, 8}, Used: Size{4096, 2}, Drives: drive(100_000, 20_000)}, } node, d, err := choosePlacement(views, want, 5120) if err != nil || node != "calm" || d != "/d" { t.Fatalf("got %s %s %v, want calm", node, d, err) } } func TestPlacementSpreadsAcrossEqualNodes(t *testing.T) { views := []NodeView{ {Name: "a", Usable: Size{8192, 4}, Drives: []DriveReport{{Path: "/a", UsableMB: 50_000}}}, {Name: "b", Usable: Size{8192, 4}, Drives: []DriveReport{{Path: "/b", UsableMB: 50_000}}}, } want := Size{2048, 1} counts := map[string]int{} for i := 0; i < 4; i++ { node, _, err := choosePlacement(views, want, 5120) if err != nil { t.Fatal(err) } counts[node]++ for j := range views { if views[j].Name == node { views[j].Used = views[j].Used.Add(want) views[j].Drives[0].CommittedMB += 5120 } } } if counts["a"] != 2 || counts["b"] != 2 { t.Fatalf("uneven spread: %v", counts) } } func TestPlacementWhenFull(t *testing.T) { full := []NodeView{{Name: "a", Usable: Size{4096, 2}, Used: Size{4096, 2}, Drives: []DriveReport{{Path: "/a", UsableMB: 50_000}}}} if _, _, err := choosePlacement(full, Size{1024, 1}, 1024); !errors.Is(err, ErrMaxedOut) { t.Fatalf("want maxed out, got %v", err) } noDisk := []NodeView{{Name: "a", Usable: Size{4096, 2}, Drives: []DriveReport{{Path: "/a", UsableMB: 1000}}}} if _, _, err := choosePlacement(noDisk, Size{1024, 1}, 5120); !errors.Is(err, ErrStorageFull) { t.Fatalf("want storage full, got %v", err) } } func TestBuildView(t *testing.T) { info := NodeInfo{ Name: "n1", Usable: Size{16384, 8}, Running: []RunningMachine{{Name: "c-a"}, {Name: "u-stranger"}}, Drives: []DriveInfo{{Path: "/d", FreeMB: 90_000, ReserveMB: 10_000, HomesMB: map[string]int64{"c-a": 8000, "c-b": 100, "u-stranger": 3000}}}, } owners := []Owner{ {ID: "A", Machine: "c-a", Node: "n1", Drive: "/d", Size: Size{4096, 2}, StorageMinMB: 5120, StorageMaxMB: 10240}, {ID: "B", Machine: "c-b", Node: "n1", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240}, {ID: "C", Machine: "c-c", Node: "n1", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240}, {ID: "D", Machine: "c-d", Node: "other", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240}, } v := buildView(info, owners, Size{1024, 1}) if v.Used != (Size{4096 + 1024, 3}) { t.Fatalf("used = %v", v.Used) } d := v.Drives[0] if d.UsableMB != 90_000+11_100-10_000 { t.Fatalf("usable = %d", d.UsableMB) } if d.CommittedMB != 8000+5120+5120+3000 || d.BookedMB != 3*10240+3000 || d.Homes != 4 { t.Fatalf("committed %d booked %d homes %d", d.CommittedMB, d.BookedMB, d.Homes) } } type fakeNode struct { name string mu sync.Mutex runs map[string]bool last StartSpec homes map[string][]byte exports map[string]*fakeExport imports map[string]*fakeImport passes int failWrites bool usable Size } func newFakeNode(name string) *fakeNode { return &fakeNode{name: name, runs: map[string]bool{}, homes: map[string][]byte{}, exports: map[string]*fakeExport{}, imports: map[string]*fakeImport{}, usable: Size{8192, 4}} } func (f *fakeNode) Name() string { return f.name } func (f *fakeNode) Info(ctx context.Context) (NodeInfo, error) { running, _ := f.Running(ctx) f.mu.Lock() defer f.mu.Unlock() homes := map[string]int64{} for k, v := range f.homes { homes[strings.TrimPrefix(k, "/d/")] = int64(len(v)) >> 20 } return NodeInfo{Name: f.name, Usable: f.usable, Running: running, Drives: []DriveInfo{{Path: "/d", FreeMB: 50_000, HomesMB: homes}}}, nil } func (f *fakeNode) Running(context.Context) ([]RunningMachine, error) { f.mu.Lock() defer f.mu.Unlock() var out []RunningMachine for m := range f.runs { out = append(out, RunningMachine{Name: m}) } return out, nil } func (f *fakeNode) IsRunning(machine string) bool { f.mu.Lock() defer f.mu.Unlock() return f.runs[machine] } func (f *fakeNode) Start(_ context.Context, spec StartSpec) error { f.mu.Lock() defer f.mu.Unlock() f.last, f.runs[spec.Machine] = spec, true return nil } func (f *fakeNode) Poweroff(_ context.Context, machine string) error { f.mu.Lock() defer f.mu.Unlock() delete(f.runs, machine) return nil } func (f *fakeNode) SetLimits(context.Context, string, Size) error { return nil } func (f *fakeNode) SetQuota(context.Context, string, string, int64) error { return nil } func (f *fakeNode) DialAgent(_ context.Context, machine, secret string) (msgConn, error) { if secret != "agent-secret" { return nil, errors.New("bad agent secret") } return newEcho(), nil } func (f *fakeNode) WaitStopped(_ context.Context, machine string) error { if f.IsRunning(machine) { return errors.New("still running") } return nil } func (f *fakeNode) ExportHome(_ context.Context, drive, machine, parent string) (string, error) { f.mu.Lock() defer f.mu.Unlock() data, ok := f.homes[drive+"/"+machine] if !ok { return "", errors.New("no such home") } kind := "FULL:" if parent != "" { kind = "INCR:" } id := randomHex(4) f.exports[id] = &fakeExport{data: append([]byte(kind), data...)} return id, nil } func (f *fakeNode) ReadExport(_ context.Context, id string, max int) ([]byte, bool, error) { f.mu.Lock() defer f.mu.Unlock() x := f.exports[id] if x == nil { return nil, false, errors.New("no export") } n := min(max, len(x.data)-x.pos) chunk := x.data[x.pos : x.pos+n] x.pos += n return chunk, x.pos == len(x.data), nil } func (f *fakeNode) CloseExport(_ context.Context, id string) error { f.mu.Lock() defer f.mu.Unlock() delete(f.exports, id) return nil } func (f *fakeNode) ImportHome(_ context.Context, drive, machine string) (string, error) { f.mu.Lock() defer f.mu.Unlock() if _, exists := f.homes[drive+"/"+machine]; exists { return "", errors.New("home already exists") } id := randomHex(4) f.imports[id] = &fakeImport{key: drive + "/" + machine} return id, nil } func (f *fakeNode) WriteImport(_ context.Context, id string, data []byte) error { f.mu.Lock() defer f.mu.Unlock() if f.failWrites { return errors.New("disk on fire") } x := f.imports[id] if x == nil { return errors.New("no import") } x.cur = append(x.cur, data...) return nil } func (f *fakeNode) EndReceive(_ context.Context, id string) error { f.mu.Lock() defer f.mu.Unlock() x := f.imports[id] if x == nil || len(x.cur) < 5 { return errors.New("nothing received") } if string(x.cur[:5]) == "INCR:" && x.latest == nil { return errors.New("incremental send without a parent") } x.latest, x.cur = x.cur[5:], nil x.passes++ return nil } func (f *fakeNode) FinishImport(_ context.Context, id string, quotaMB int64) error { f.mu.Lock() defer f.mu.Unlock() x := f.imports[id] delete(f.imports, id) f.homes[x.key] = x.latest f.passes = x.passes return nil } func (f *fakeNode) AbortImport(_ context.Context, id string) error { f.mu.Lock() defer f.mu.Unlock() delete(f.imports, id) return nil } func (f *fakeNode) DeleteHome(_ context.Context, drive, machine string) error { f.mu.Lock() defer f.mu.Unlock() delete(f.homes, drive+"/"+machine) return nil } type fakeExport struct { data []byte pos int } type fakeImport struct { key string cur, latest []byte passes int } type echo struct { ch chan []byte closed chan struct{} once sync.Once } func newEcho() *echo { return &echo{ch: make(chan []byte, 16), closed: make(chan struct{})} } func (e *echo) Read(ctx context.Context) ([]byte, error) { select { case b := <-e.ch: return b, nil case <-e.closed: return nil, errors.New("closed") case <-ctx.Done(): return nil, ctx.Err() } } func (e *echo) Write(_ context.Context, b []byte) error { e.ch <- append([]byte("echo: "), b...) return nil } func (e *echo) Close() error { e.once.Do(func() { close(e.closed) }); return nil } func freePort(t *testing.T) string { l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer l.Close() return l.Addr().String() } func waitFor(t *testing.T, what string, cond func() bool) { deadline := time.Now().Add(10 * time.Second) for !cond() { if time.Now().After(deadline) { t.Fatalf("timed out waiting for %s", what) } time.Sleep(20 * time.Millisecond) } } func TestMasterAndSlave(t *testing.T) { addr := freePort(t) secret := "cluster-secret-1234567890" masterDir, slaveDir := t.TempDir(), t.TempDir() masterCfg := &Config{StateDir: masterDir, Container: Size{1024, 1}} masterCfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr} store, err := OpenStore(filepath.Join(masterDir, "state.json")) if err != nil { t.Fatal(err) } mgr := NewManager(masterCfg, store, newFakeNode("master")) backCh := make(chan string, 4) mgr.onNodeBack = func(n string) { backCh <- n } ctx, cancel := context.WithCancel(context.Background()) defer cancel() go mgr.ServeCluster(ctx) slaveCfg := &Config{StateDir: slaveDir} slaveCfg.Cluster = ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr} slave := newFakeNode("slave-1") slaveCtx, stopSlave := context.WithCancel(ctx) go RunSlave(slaveCtx, slaveCfg, slave) waitFor(t, "slave to come online", func() bool { _, err := mgr.node("slave-1"); return err == nil }) if got := <-backCh; got != "slave-1" { t.Fatalf("back callback for %q", got) } pinned, err := os.ReadFile(filepath.Join(slaveDir, "master.fingerprint")) if err != nil || len(strings.TrimSpace(string(pinned))) != 64 { t.Fatalf("slave should pin the master's certificate, got %q %v", pinned, err) } n, _ := mgr.node("slave-1") spec := StartSpec{Machine: "c-abc", Secret: "agent-secret", Size: Size{1024, 1}, Drive: "/d"} if err := n.Start(ctx, spec); err != nil { t.Fatal(err) } if !slave.IsRunning("c-abc") || slave.last.Drive != "/d" { t.Fatal("start didn't reach the slave") } if !n.IsRunning("c-abc") { t.Fatal("master should count the container as running right away") } conn, err := n.DialAgent(ctx, "c-abc", "agent-secret") if err != nil { t.Fatal(err) } if err := conn.Write(ctx, []byte(`{"jsonrpc":"2.0"}`)); err != nil { t.Fatal(err) } readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second) got, err := conn.Read(readCtx) readCancel() if err != nil || string(got) != `echo: {"jsonrpc":"2.0"}` { t.Fatalf("relay got %q %v", got, err) } conn.Close() if _, err := n.DialAgent(ctx, "c-abc", "wrong"); err == nil { t.Fatal("a bad agent secret should fail through the relay") } if err := n.Poweroff(ctx, "c-abc"); err != nil { t.Fatal(err) } if slave.IsRunning("c-abc") || n.IsRunning("c-abc") { t.Fatal("poweroff didn't take") } stopSlave() waitFor(t, "slave to go offline", func() bool { _, err := mgr.node("slave-1") var off *NodeOfflineError return errors.As(err, &off) && off.Node == "slave-1" }) if off := mgr.OfflineNodes(); len(off) != 1 || off[0] != "slave-1" { t.Fatalf("offline nodes = %v", off) } } func TestSlaveRejectsWrongMasterOrSecret(t *testing.T) { addr := freePort(t) secret := "cluster-secret-1234567890" masterCfg := &Config{StateDir: t.TempDir()} masterCfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr} store, _ := OpenStore(filepath.Join(masterCfg.StateDir, "state.json")) mgr := NewManager(masterCfg, store, newFakeNode("master")) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go mgr.ServeCluster(ctx) waitFor(t, "master to listen", func() bool { c, err := net.Dial("tcp", addr) if err == nil { c.Close() } return err == nil }) wrongPin := &Config{StateDir: t.TempDir()} wrongPin.Cluster = ClusterConfig{Name: "s", Secret: secret, Master: "wss://" + addr, MasterFingerprint: strings.Repeat("ab", 32)} if err := slaveSession(ctx, wrongPin, newFakeNode("s")); err == nil { t.Fatal("a master with a different certificate should be refused") } wrongSecret := &Config{StateDir: t.TempDir()} wrongSecret.Cluster = ClusterConfig{Name: "s", Secret: "not-the-secret-at-all", Master: "wss://" + addr} if err := slaveSession(ctx, wrongSecret, newFakeNode("s")); err == nil { t.Fatal("a slave with the wrong secret should be refused") } sameName := &Config{StateDir: t.TempDir()} sameName.Cluster = ClusterConfig{Name: "master", Secret: secret, Master: "wss://" + addr} if err := slaveSession(ctx, sameName, newFakeNode("master")); err == nil { t.Fatal("a slave can't take the master's name") } }