Files
slacker-bot/migrate_test.go
T
2026-11-01 11:47:41 -05:00

286 lines
8.5 KiB
Go

package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"github.com/slack-go/slack"
)
type testCluster struct {
mgr *Manager
store *Store
master, slave *fakeNode
ctx context.Context
}
func newTestCluster(t *testing.T) *testCluster {
addr := freePort(t)
secret := "cluster-secret-1234567890"
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
cfg := &Config{StateDir: t.TempDir(), Container: Size{1024, 1}}
cfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr}
store, err := OpenStore(filepath.Join(cfg.StateDir, "state.json"))
if err != nil {
t.Fatal(err)
}
c := &testCluster{store: store, master: newFakeNode("master"), slave: newFakeNode("slave-1"), ctx: ctx}
c.mgr = NewManager(cfg, store, c.master)
go c.mgr.ServeCluster(ctx)
slaveCfg := &Config{StateDir: t.TempDir()}
slaveCfg.Cluster = ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr}
go RunSlave(ctx, slaveCfg, c.slave)
waitFor(t, "slave to come online", func() bool { _, err := c.mgr.node("slave-1"); return err == nil })
return c
}
func (c *testCluster) addOwner(t *testing.T, data []byte, running bool) {
u := &Owner{ID: "C1", Kind: OwnerChannel, Machine: "c-c1", Secret: "s", Size: Size{1024, 1},
Node: "master", Drive: "/d", StorageMinMB: 1024, StorageMaxMB: 2048}
if err := c.store.PutOwner(u); err != nil {
t.Fatal(err)
}
c.master.mu.Lock()
defer c.master.mu.Unlock()
c.master.homes["/d/c-c1"] = data
if running {
c.master.runs["c-c1"] = true
}
}
func homeData() []byte { return bytes.Repeat([]byte("0123456789abcdef"), 600_000) }
func TestMoveLiveContainer(t *testing.T) {
c := newTestCluster(t)
c.addOwner(t, homeData(), true)
var phases []MovePhase
res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{
Target: "slave-1",
WaitIdle: func(context.Context) {
c.master.mu.Lock()
c.master.homes["/d/c-c1"] = append(c.master.homes["/d/c-c1"], []byte("written during the bulk copy")...)
c.master.mu.Unlock()
},
Progress: func(p MoveProgress) {
if len(phases) == 0 || phases[len(phases)-1] != p.Phase {
phases = append(phases, p.Phase)
}
},
})
if err != nil {
t.Fatal(err)
}
want := append(homeData(), []byte("written during the bulk copy")...)
if !bytes.Equal(c.slave.homes["/d/c-c1"], want) {
t.Fatal("the late write didn't make it to the new node")
}
if c.slave.passes != 2 {
t.Fatalf("a live move should copy twice (bulk + final sync), got %d", c.slave.passes)
}
if _, still := c.master.homes["/d/c-c1"]; still {
t.Fatal("old home should be deleted after a good move")
}
if !c.slave.IsRunning("c-c1") || c.master.IsRunning("c-c1") || !res.Started {
t.Fatal("container should be running on the new node only")
}
if u := c.store.Owner("C1"); u.Node != "slave-1" || u.Drive != "/d" {
t.Fatalf("placement not switched: %+v", u)
}
if len(c.master.exports) != 0 {
t.Fatal("snapshots on the old node should be cleaned up")
}
wantPhases := []MovePhase{PhaseCopy, PhaseFinal, PhaseCleanup, PhaseStart}
if len(phases) != len(wantPhases) {
t.Fatalf("phases %v, want %v", phases, wantPhases)
}
}
func TestMoveAutoStoppedContainer(t *testing.T) {
c := newTestCluster(t)
c.addOwner(t, homeData(), false)
res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{})
if err != nil {
t.Fatal(err)
}
if res.To != "slave-1" || res.Started || c.slave.IsRunning("c-c1") {
t.Fatalf("a stopped container should move without starting: %+v", res)
}
if c.slave.passes != 1 {
t.Fatalf("a stopped container needs one copy, got %d", c.slave.passes)
}
}
func TestMoveFullNodeStartsElsewhere(t *testing.T) {
c := newTestCluster(t)
c.addOwner(t, homeData(), false)
c.master.mu.Lock()
c.master.usable = Size{512, 0.5}
c.master.mu.Unlock()
_, err := c.mgr.Ensure(c.ctx, "C1", OwnerChannel)
var full *NodeFullError
if !errors.As(err, &full) || full.Node != "master" {
t.Fatalf("want node full on master, got %v", err)
}
res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{StartAfter: true})
if err != nil || !res.Started || !c.slave.IsRunning("c-c1") {
t.Fatalf("auto-move should start it on the new node: %+v %v", res, err)
}
}
func TestMoveFailureRollsBack(t *testing.T) {
c := newTestCluster(t)
c.addOwner(t, homeData(), true)
_, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{
Target: "slave-1",
WaitIdle: func(context.Context) {
c.slave.mu.Lock()
c.slave.failWrites = true
c.slave.mu.Unlock()
},
})
if err == nil {
t.Fatal("expected the final sync to fail")
}
if u := c.store.Owner("C1"); u.Node != "master" {
t.Fatalf("container should stay on its old node, got %s", u.Node)
}
if !c.master.IsRunning("c-c1") {
t.Fatal("container should be restarted on its old node")
}
if _, ok := c.master.homes["/d/c-c1"]; !ok {
t.Fatal("old home must survive a failed move")
}
if _, ok := c.slave.homes["/d/c-c1"]; ok || len(c.slave.imports) != 0 {
t.Fatal("half-copied home on the target should be cleaned up")
}
}
func TestMoveRefusals(t *testing.T) {
c := newTestCluster(t)
c.addOwner(t, homeData(), false)
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "master"}); !errors.Is(err, ErrSameNode) {
t.Fatalf("same node: %v", err)
}
var unknown *UnknownNodeError
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "nope"}); !errors.As(err, &unknown) {
t.Fatalf("unknown node: %v", err)
}
if _, err := c.mgr.Migrate(c.ctx, "nobody", MoveOptions{}); !errors.Is(err, ErrNoContainer) {
t.Fatalf("no container: %v", err)
}
c.slave.mu.Lock()
c.slave.usable = Size{512, 0.5}
c.slave.mu.Unlock()
c.master.mu.Lock()
c.master.runs["c-c1"] = true
c.master.mu.Unlock()
slave, _ := c.mgr.node("slave-1")
waitFor(t, "slave report", func() bool {
info, _ := slave.Info(c.ctx)
return info.Usable.MemoryMB == 512
})
var full *NodeFullError
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "slave-1"}); !errors.As(err, &full) {
t.Fatalf("full target: %v", err)
}
}
type slackLog struct {
mu sync.Mutex
n int
text map[string]string
under map[string]string
order []string
}
func (s *slackLog) ServeHTTP(w http.ResponseWriter, r *http.Request) {
_ = r.ParseForm()
s.mu.Lock()
defer s.mu.Unlock()
ts := r.Form.Get("ts")
if strings.HasSuffix(r.URL.Path, "chat.postMessage") {
s.n++
ts = fmt.Sprintf("%d.0", s.n)
s.order = append(s.order, ts)
s.under[ts] = r.Form.Get("thread_ts")
}
if !strings.HasSuffix(r.URL.Path, "chat.getPermalink") {
s.text[ts] = r.Form.Get("text")
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "C1", "ts": ts, "permalink": "https://slack.example/" + ts})
}
func TestEveryMoveStartsAnAnnouncedThread(t *testing.T) {
c := newTestCluster(t)
c.addOwner(t, homeData(), false)
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
srv := httptest.NewServer(log)
defer srv.Close()
b := &Bot{cfg: c.mgr.cfg, store: c.store, mgr: c.mgr, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/")),
acps: map[string]*ACP{}, busy: map[string]int{}}
b.moveFor(c.ctx, "C1", "C1", "slave-1", "<@U1>", "100.0")
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "master"}); err != nil {
t.Fatal(err)
}
b.runMove(c.ctx, "C1", "C1", MoveOptions{}, "auto process (master was full), for <@U2>'s message", nil)
log.mu.Lock()
defer log.mu.Unlock()
var roots []string
for _, ts := range log.order {
if log.under[ts] == "" {
roots = append(roots, ts)
}
}
linked := 0
for ts, parent := range log.under {
if parent == "100.0" && strings.Contains(log.text[ts], "Follow the move here") {
linked++
}
}
if linked != 1 {
t.Fatalf("the thread where the move started should get one link to the move thread, got %d", linked)
}
if len(roots) != 2 {
t.Fatalf("want one new top-level message per move, got %d", len(roots))
}
checks := []string{"started by <@U1>: master → slave-1", "started by auto process (master was full), for <@U2>'s message"}
for i, root := range roots {
if !strings.Contains(log.text[root], checks[i]) || !strings.Contains(log.text[root], ":white_check_mark: done") {
t.Fatalf("root %d = %q", i, log.text[root])
}
replies := 0
for ts, parent := range log.under {
if parent == root {
replies++
if !strings.Contains(log.text[ts], "Moved from") {
t.Fatalf("progress reply = %q", log.text[ts])
}
}
}
if replies != 1 {
t.Fatalf("want progress in one thread reply, got %d", replies)
}
}
}