286 lines
8.5 KiB
Go
286 lines
8.5 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/slack-go/slack"
|
|
)
|
|
|
|
type testCluster struct {
|
|
mgr *Manager
|
|
store *Store
|
|
master, slave *fakeNode
|
|
ctx context.Context
|
|
}
|
|
|
|
func newTestCluster(t *testing.T) *testCluster {
|
|
addr := freePort(t)
|
|
secret := "cluster-secret-1234567890"
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
t.Cleanup(cancel)
|
|
|
|
cfg := &Config{StateDir: t.TempDir(), Container: Size{1024, 1}}
|
|
cfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr}
|
|
store, err := OpenStore(filepath.Join(cfg.StateDir, "state.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c := &testCluster{store: store, master: newFakeNode("master"), slave: newFakeNode("slave-1"), ctx: ctx}
|
|
c.mgr = NewManager(cfg, store, c.master)
|
|
go c.mgr.ServeCluster(ctx)
|
|
|
|
slaveCfg := &Config{StateDir: t.TempDir()}
|
|
slaveCfg.Cluster = ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr}
|
|
go RunSlave(ctx, slaveCfg, c.slave)
|
|
waitFor(t, "slave to come online", func() bool { _, err := c.mgr.node("slave-1"); return err == nil })
|
|
return c
|
|
}
|
|
|
|
func (c *testCluster) addOwner(t *testing.T, data []byte, running bool) {
|
|
u := &Owner{ID: "C1", Kind: OwnerChannel, Machine: "c-c1", Secret: "s", Size: Size{1024, 1},
|
|
Node: "master", Drive: "/d", StorageMinMB: 1024, StorageMaxMB: 2048}
|
|
if err := c.store.PutOwner(u); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c.master.mu.Lock()
|
|
defer c.master.mu.Unlock()
|
|
c.master.homes["/d/c-c1"] = data
|
|
if running {
|
|
c.master.runs["c-c1"] = true
|
|
}
|
|
}
|
|
|
|
func homeData() []byte { return bytes.Repeat([]byte("0123456789abcdef"), 600_000) }
|
|
|
|
func TestMoveLiveContainer(t *testing.T) {
|
|
c := newTestCluster(t)
|
|
c.addOwner(t, homeData(), true)
|
|
|
|
var phases []MovePhase
|
|
res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{
|
|
Target: "slave-1",
|
|
WaitIdle: func(context.Context) {
|
|
c.master.mu.Lock()
|
|
c.master.homes["/d/c-c1"] = append(c.master.homes["/d/c-c1"], []byte("written during the bulk copy")...)
|
|
c.master.mu.Unlock()
|
|
},
|
|
Progress: func(p MoveProgress) {
|
|
if len(phases) == 0 || phases[len(phases)-1] != p.Phase {
|
|
phases = append(phases, p.Phase)
|
|
}
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
want := append(homeData(), []byte("written during the bulk copy")...)
|
|
if !bytes.Equal(c.slave.homes["/d/c-c1"], want) {
|
|
t.Fatal("the late write didn't make it to the new node")
|
|
}
|
|
if c.slave.passes != 2 {
|
|
t.Fatalf("a live move should copy twice (bulk + final sync), got %d", c.slave.passes)
|
|
}
|
|
if _, still := c.master.homes["/d/c-c1"]; still {
|
|
t.Fatal("old home should be deleted after a good move")
|
|
}
|
|
if !c.slave.IsRunning("c-c1") || c.master.IsRunning("c-c1") || !res.Started {
|
|
t.Fatal("container should be running on the new node only")
|
|
}
|
|
if u := c.store.Owner("C1"); u.Node != "slave-1" || u.Drive != "/d" {
|
|
t.Fatalf("placement not switched: %+v", u)
|
|
}
|
|
if len(c.master.exports) != 0 {
|
|
t.Fatal("snapshots on the old node should be cleaned up")
|
|
}
|
|
wantPhases := []MovePhase{PhaseCopy, PhaseFinal, PhaseCleanup, PhaseStart}
|
|
if len(phases) != len(wantPhases) {
|
|
t.Fatalf("phases %v, want %v", phases, wantPhases)
|
|
}
|
|
}
|
|
|
|
func TestMoveAutoStoppedContainer(t *testing.T) {
|
|
c := newTestCluster(t)
|
|
c.addOwner(t, homeData(), false)
|
|
|
|
res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if res.To != "slave-1" || res.Started || c.slave.IsRunning("c-c1") {
|
|
t.Fatalf("a stopped container should move without starting: %+v", res)
|
|
}
|
|
if c.slave.passes != 1 {
|
|
t.Fatalf("a stopped container needs one copy, got %d", c.slave.passes)
|
|
}
|
|
}
|
|
|
|
func TestMoveFullNodeStartsElsewhere(t *testing.T) {
|
|
c := newTestCluster(t)
|
|
c.addOwner(t, homeData(), false)
|
|
c.master.mu.Lock()
|
|
c.master.usable = Size{512, 0.5}
|
|
c.master.mu.Unlock()
|
|
|
|
_, err := c.mgr.Ensure(c.ctx, "C1", OwnerChannel)
|
|
var full *NodeFullError
|
|
if !errors.As(err, &full) || full.Node != "master" {
|
|
t.Fatalf("want node full on master, got %v", err)
|
|
}
|
|
res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{StartAfter: true})
|
|
if err != nil || !res.Started || !c.slave.IsRunning("c-c1") {
|
|
t.Fatalf("auto-move should start it on the new node: %+v %v", res, err)
|
|
}
|
|
}
|
|
|
|
func TestMoveFailureRollsBack(t *testing.T) {
|
|
c := newTestCluster(t)
|
|
c.addOwner(t, homeData(), true)
|
|
|
|
_, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{
|
|
Target: "slave-1",
|
|
WaitIdle: func(context.Context) {
|
|
c.slave.mu.Lock()
|
|
c.slave.failWrites = true
|
|
c.slave.mu.Unlock()
|
|
},
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected the final sync to fail")
|
|
}
|
|
if u := c.store.Owner("C1"); u.Node != "master" {
|
|
t.Fatalf("container should stay on its old node, got %s", u.Node)
|
|
}
|
|
if !c.master.IsRunning("c-c1") {
|
|
t.Fatal("container should be restarted on its old node")
|
|
}
|
|
if _, ok := c.master.homes["/d/c-c1"]; !ok {
|
|
t.Fatal("old home must survive a failed move")
|
|
}
|
|
if _, ok := c.slave.homes["/d/c-c1"]; ok || len(c.slave.imports) != 0 {
|
|
t.Fatal("half-copied home on the target should be cleaned up")
|
|
}
|
|
}
|
|
|
|
func TestMoveRefusals(t *testing.T) {
|
|
c := newTestCluster(t)
|
|
c.addOwner(t, homeData(), false)
|
|
|
|
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "master"}); !errors.Is(err, ErrSameNode) {
|
|
t.Fatalf("same node: %v", err)
|
|
}
|
|
var unknown *UnknownNodeError
|
|
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "nope"}); !errors.As(err, &unknown) {
|
|
t.Fatalf("unknown node: %v", err)
|
|
}
|
|
if _, err := c.mgr.Migrate(c.ctx, "nobody", MoveOptions{}); !errors.Is(err, ErrNoContainer) {
|
|
t.Fatalf("no container: %v", err)
|
|
}
|
|
c.slave.mu.Lock()
|
|
c.slave.usable = Size{512, 0.5}
|
|
c.slave.mu.Unlock()
|
|
c.master.mu.Lock()
|
|
c.master.runs["c-c1"] = true
|
|
c.master.mu.Unlock()
|
|
slave, _ := c.mgr.node("slave-1")
|
|
waitFor(t, "slave report", func() bool {
|
|
info, _ := slave.Info(c.ctx)
|
|
return info.Usable.MemoryMB == 512
|
|
})
|
|
var full *NodeFullError
|
|
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "slave-1"}); !errors.As(err, &full) {
|
|
t.Fatalf("full target: %v", err)
|
|
}
|
|
}
|
|
|
|
type slackLog struct {
|
|
mu sync.Mutex
|
|
n int
|
|
text map[string]string
|
|
under map[string]string
|
|
order []string
|
|
}
|
|
|
|
func (s *slackLog) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
_ = r.ParseForm()
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
ts := r.Form.Get("ts")
|
|
if strings.HasSuffix(r.URL.Path, "chat.postMessage") {
|
|
s.n++
|
|
ts = fmt.Sprintf("%d.0", s.n)
|
|
s.order = append(s.order, ts)
|
|
s.under[ts] = r.Form.Get("thread_ts")
|
|
}
|
|
if !strings.HasSuffix(r.URL.Path, "chat.getPermalink") {
|
|
s.text[ts] = r.Form.Get("text")
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "C1", "ts": ts, "permalink": "https://slack.example/" + ts})
|
|
}
|
|
|
|
func TestEveryMoveStartsAnAnnouncedThread(t *testing.T) {
|
|
c := newTestCluster(t)
|
|
c.addOwner(t, homeData(), false)
|
|
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
|
|
srv := httptest.NewServer(log)
|
|
defer srv.Close()
|
|
b := &Bot{cfg: c.mgr.cfg, store: c.store, mgr: c.mgr, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/")),
|
|
acps: map[string]*ACP{}, busy: map[string]int{}}
|
|
|
|
b.moveFor(c.ctx, "C1", "C1", "slave-1", "<@U1>", "100.0")
|
|
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "master"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b.runMove(c.ctx, "C1", "C1", MoveOptions{}, "auto process (master was full), for <@U2>'s message", nil)
|
|
|
|
log.mu.Lock()
|
|
defer log.mu.Unlock()
|
|
var roots []string
|
|
for _, ts := range log.order {
|
|
if log.under[ts] == "" {
|
|
roots = append(roots, ts)
|
|
}
|
|
}
|
|
linked := 0
|
|
for ts, parent := range log.under {
|
|
if parent == "100.0" && strings.Contains(log.text[ts], "Follow the move here") {
|
|
linked++
|
|
}
|
|
}
|
|
if linked != 1 {
|
|
t.Fatalf("the thread where the move started should get one link to the move thread, got %d", linked)
|
|
}
|
|
if len(roots) != 2 {
|
|
t.Fatalf("want one new top-level message per move, got %d", len(roots))
|
|
}
|
|
checks := []string{"started by <@U1>: master → slave-1", "started by auto process (master was full), for <@U2>'s message"}
|
|
for i, root := range roots {
|
|
if !strings.Contains(log.text[root], checks[i]) || !strings.Contains(log.text[root], ":white_check_mark: done") {
|
|
t.Fatalf("root %d = %q", i, log.text[root])
|
|
}
|
|
replies := 0
|
|
for ts, parent := range log.under {
|
|
if parent == root {
|
|
replies++
|
|
if !strings.Contains(log.text[ts], "Moved from") {
|
|
t.Fatalf("progress reply = %q", log.text[ts])
|
|
}
|
|
}
|
|
}
|
|
if replies != 1 {
|
|
t.Fatalf("want progress in one thread reply, got %d", replies)
|
|
}
|
|
}
|
|
}
|