package main import ( "bytes" "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "path/filepath" "strings" "sync" "testing" "github.com/slack-go/slack" ) type testCluster struct { mgr *Manager store *Store master, slave *fakeNode ctx context.Context } func newTestCluster(t *testing.T) *testCluster { addr := freePort(t) secret := "cluster-secret-1234567890" ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) cfg := &Config{StateDir: t.TempDir(), Container: Size{1024, 1}} cfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr} store, err := OpenStore(filepath.Join(cfg.StateDir, "state.json")) if err != nil { t.Fatal(err) } c := &testCluster{store: store, master: newFakeNode("master"), slave: newFakeNode("slave-1"), ctx: ctx} c.mgr = NewManager(cfg, store, c.master) go c.mgr.ServeCluster(ctx) slaveCfg := &Config{StateDir: t.TempDir()} slaveCfg.Cluster = ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr} go RunSlave(ctx, slaveCfg, c.slave) waitFor(t, "slave to come online", func() bool { _, err := c.mgr.node("slave-1"); return err == nil }) return c } func (c *testCluster) addOwner(t *testing.T, data []byte, running bool) { u := &Owner{ID: "C1", Kind: OwnerChannel, Machine: "c-c1", Secret: "s", Size: Size{1024, 1}, Node: "master", Drive: "/d", StorageMinMB: 1024, StorageMaxMB: 2048} if err := c.store.PutOwner(u); err != nil { t.Fatal(err) } c.master.mu.Lock() defer c.master.mu.Unlock() c.master.homes["/d/c-c1"] = data if running { c.master.runs["c-c1"] = true } } func homeData() []byte { return bytes.Repeat([]byte("0123456789abcdef"), 600_000) } func TestMoveLiveContainer(t *testing.T) { c := newTestCluster(t) c.addOwner(t, homeData(), true) var phases []MovePhase res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{ Target: "slave-1", WaitIdle: func(context.Context) { c.master.mu.Lock() c.master.homes["/d/c-c1"] = append(c.master.homes["/d/c-c1"], []byte("written during the bulk copy")...) c.master.mu.Unlock() }, Progress: func(p MoveProgress) { if len(phases) == 0 || phases[len(phases)-1] != p.Phase { phases = append(phases, p.Phase) } }, }) if err != nil { t.Fatal(err) } want := append(homeData(), []byte("written during the bulk copy")...) if !bytes.Equal(c.slave.homes["/d/c-c1"], want) { t.Fatal("the late write didn't make it to the new node") } if c.slave.passes != 2 { t.Fatalf("a live move should copy twice (bulk + final sync), got %d", c.slave.passes) } if _, still := c.master.homes["/d/c-c1"]; still { t.Fatal("old home should be deleted after a good move") } if !c.slave.IsRunning("c-c1") || c.master.IsRunning("c-c1") || !res.Started { t.Fatal("container should be running on the new node only") } if u := c.store.Owner("C1"); u.Node != "slave-1" || u.Drive != "/d" { t.Fatalf("placement not switched: %+v", u) } if len(c.master.exports) != 0 { t.Fatal("snapshots on the old node should be cleaned up") } wantPhases := []MovePhase{PhaseCopy, PhaseFinal, PhaseCleanup, PhaseStart} if len(phases) != len(wantPhases) { t.Fatalf("phases %v, want %v", phases, wantPhases) } } func TestMoveAutoStoppedContainer(t *testing.T) { c := newTestCluster(t) c.addOwner(t, homeData(), false) res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{}) if err != nil { t.Fatal(err) } if res.To != "slave-1" || res.Started || c.slave.IsRunning("c-c1") { t.Fatalf("a stopped container should move without starting: %+v", res) } if c.slave.passes != 1 { t.Fatalf("a stopped container needs one copy, got %d", c.slave.passes) } } func TestMoveFullNodeStartsElsewhere(t *testing.T) { c := newTestCluster(t) c.addOwner(t, homeData(), false) c.master.mu.Lock() c.master.usable = Size{512, 0.5} c.master.mu.Unlock() _, err := c.mgr.Ensure(c.ctx, "C1", OwnerChannel) var full *NodeFullError if !errors.As(err, &full) || full.Node != "master" { t.Fatalf("want node full on master, got %v", err) } res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{StartAfter: true}) if err != nil || !res.Started || !c.slave.IsRunning("c-c1") { t.Fatalf("auto-move should start it on the new node: %+v %v", res, err) } } func TestMoveFailureRollsBack(t *testing.T) { c := newTestCluster(t) c.addOwner(t, homeData(), true) _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{ Target: "slave-1", WaitIdle: func(context.Context) { c.slave.mu.Lock() c.slave.failWrites = true c.slave.mu.Unlock() }, }) if err == nil { t.Fatal("expected the final sync to fail") } if u := c.store.Owner("C1"); u.Node != "master" { t.Fatalf("container should stay on its old node, got %s", u.Node) } if !c.master.IsRunning("c-c1") { t.Fatal("container should be restarted on its old node") } if _, ok := c.master.homes["/d/c-c1"]; !ok { t.Fatal("old home must survive a failed move") } if _, ok := c.slave.homes["/d/c-c1"]; ok || len(c.slave.imports) != 0 { t.Fatal("half-copied home on the target should be cleaned up") } } func TestMoveRefusals(t *testing.T) { c := newTestCluster(t) c.addOwner(t, homeData(), false) if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "master"}); !errors.Is(err, ErrSameNode) { t.Fatalf("same node: %v", err) } var unknown *UnknownNodeError if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "nope"}); !errors.As(err, &unknown) { t.Fatalf("unknown node: %v", err) } if _, err := c.mgr.Migrate(c.ctx, "nobody", MoveOptions{}); !errors.Is(err, ErrNoContainer) { t.Fatalf("no container: %v", err) } c.slave.mu.Lock() c.slave.usable = Size{512, 0.5} c.slave.mu.Unlock() c.master.mu.Lock() c.master.runs["c-c1"] = true c.master.mu.Unlock() slave, _ := c.mgr.node("slave-1") waitFor(t, "slave report", func() bool { info, _ := slave.Info(c.ctx) return info.Usable.MemoryMB == 512 }) var full *NodeFullError if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "slave-1"}); !errors.As(err, &full) { t.Fatalf("full target: %v", err) } } type slackLog struct { mu sync.Mutex n int text map[string]string under map[string]string order []string } func (s *slackLog) ServeHTTP(w http.ResponseWriter, r *http.Request) { _ = r.ParseForm() s.mu.Lock() defer s.mu.Unlock() ts := r.Form.Get("ts") if strings.HasSuffix(r.URL.Path, "chat.postMessage") { s.n++ ts = fmt.Sprintf("%d.0", s.n) s.order = append(s.order, ts) s.under[ts] = r.Form.Get("thread_ts") } if !strings.HasSuffix(r.URL.Path, "chat.getPermalink") { s.text[ts] = r.Form.Get("text") } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "C1", "ts": ts, "permalink": "https://slack.example/" + ts}) } func TestEveryMoveStartsAnAnnouncedThread(t *testing.T) { c := newTestCluster(t) c.addOwner(t, homeData(), false) log := &slackLog{text: map[string]string{}, under: map[string]string{}} srv := httptest.NewServer(log) defer srv.Close() b := &Bot{cfg: c.mgr.cfg, store: c.store, mgr: c.mgr, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/")), acps: map[string]*ACP{}, busy: map[string]int{}} b.moveFor(c.ctx, "C1", "C1", "slave-1", "<@U1>", "100.0") if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "master"}); err != nil { t.Fatal(err) } b.runMove(c.ctx, "C1", "C1", MoveOptions{}, "auto process (master was full), for <@U2>'s message", nil) log.mu.Lock() defer log.mu.Unlock() var roots []string for _, ts := range log.order { if log.under[ts] == "" { roots = append(roots, ts) } } linked := 0 for ts, parent := range log.under { if parent == "100.0" && strings.Contains(log.text[ts], "Follow the move here") { linked++ } } if linked != 1 { t.Fatalf("the thread where the move started should get one link to the move thread, got %d", linked) } if len(roots) != 2 { t.Fatalf("want one new top-level message per move, got %d", len(roots)) } checks := []string{"started by <@U1>: master → slave-1", "started by auto process (master was full), for <@U2>'s message"} for i, root := range roots { if !strings.Contains(log.text[root], checks[i]) || !strings.Contains(log.text[root], ":white_check_mark: done") { t.Fatalf("root %d = %q", i, log.text[root]) } replies := 0 for ts, parent := range log.under { if parent == root { replies++ if !strings.Contains(log.text[ts], "Moved from") { t.Fatalf("progress reply = %q", log.text[ts]) } } } if replies != 1 { t.Fatalf("want progress in one thread reply, got %d", replies) } } }