first pass
This commit is contained in:
commit
7822529a38
31 files changed
+7050
No files matched your search
+285
@@ -0,0 +1,285 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/slack-go/slack"
|
||||
)
|
||||
|
||||
type testCluster struct {
|
||||
mgr *Manager
|
||||
store *Store
|
||||
master, slave *fakeNode
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func newTestCluster(t *testing.T) *testCluster {
|
||||
addr := freePort(t)
|
||||
secret := "cluster-secret-1234567890"
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
cfg := &Config{StateDir: t.TempDir(), Container: Size{1024, 1}}
|
||||
cfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr}
|
||||
store, err := OpenStore(filepath.Join(cfg.StateDir, "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := &testCluster{store: store, master: newFakeNode("master"), slave: newFakeNode("slave-1"), ctx: ctx}
|
||||
c.mgr = NewManager(cfg, store, c.master)
|
||||
go c.mgr.ServeCluster(ctx)
|
||||
|
||||
slaveCfg := &Config{StateDir: t.TempDir()}
|
||||
slaveCfg.Cluster = ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr}
|
||||
go RunSlave(ctx, slaveCfg, c.slave)
|
||||
waitFor(t, "slave to come online", func() bool { _, err := c.mgr.node("slave-1"); return err == nil })
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *testCluster) addOwner(t *testing.T, data []byte, running bool) {
|
||||
u := &Owner{ID: "C1", Kind: OwnerChannel, Machine: "c-c1", Secret: "s", Size: Size{1024, 1},
|
||||
Node: "master", Drive: "/d", StorageMinMB: 1024, StorageMaxMB: 2048}
|
||||
if err := c.store.PutOwner(u); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.master.mu.Lock()
|
||||
defer c.master.mu.Unlock()
|
||||
c.master.homes["/d/c-c1"] = data
|
||||
if running {
|
||||
c.master.runs["c-c1"] = true
|
||||
}
|
||||
}
|
||||
|
||||
func homeData() []byte { return bytes.Repeat([]byte("0123456789abcdef"), 600_000) }
|
||||
|
||||
func TestMoveLiveContainer(t *testing.T) {
|
||||
c := newTestCluster(t)
|
||||
c.addOwner(t, homeData(), true)
|
||||
|
||||
var phases []MovePhase
|
||||
res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{
|
||||
Target: "slave-1",
|
||||
WaitIdle: func(context.Context) {
|
||||
c.master.mu.Lock()
|
||||
c.master.homes["/d/c-c1"] = append(c.master.homes["/d/c-c1"], []byte("written during the bulk copy")...)
|
||||
c.master.mu.Unlock()
|
||||
},
|
||||
Progress: func(p MoveProgress) {
|
||||
if len(phases) == 0 || phases[len(phases)-1] != p.Phase {
|
||||
phases = append(phases, p.Phase)
|
||||
}
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := append(homeData(), []byte("written during the bulk copy")...)
|
||||
if !bytes.Equal(c.slave.homes["/d/c-c1"], want) {
|
||||
t.Fatal("the late write didn't make it to the new node")
|
||||
}
|
||||
if c.slave.passes != 2 {
|
||||
t.Fatalf("a live move should copy twice (bulk + final sync), got %d", c.slave.passes)
|
||||
}
|
||||
if _, still := c.master.homes["/d/c-c1"]; still {
|
||||
t.Fatal("old home should be deleted after a good move")
|
||||
}
|
||||
if !c.slave.IsRunning("c-c1") || c.master.IsRunning("c-c1") || !res.Started {
|
||||
t.Fatal("container should be running on the new node only")
|
||||
}
|
||||
if u := c.store.Owner("C1"); u.Node != "slave-1" || u.Drive != "/d" {
|
||||
t.Fatalf("placement not switched: %+v", u)
|
||||
}
|
||||
if len(c.master.exports) != 0 {
|
||||
t.Fatal("snapshots on the old node should be cleaned up")
|
||||
}
|
||||
wantPhases := []MovePhase{PhaseCopy, PhaseFinal, PhaseCleanup, PhaseStart}
|
||||
if len(phases) != len(wantPhases) {
|
||||
t.Fatalf("phases %v, want %v", phases, wantPhases)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveAutoStoppedContainer(t *testing.T) {
|
||||
c := newTestCluster(t)
|
||||
c.addOwner(t, homeData(), false)
|
||||
|
||||
res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.To != "slave-1" || res.Started || c.slave.IsRunning("c-c1") {
|
||||
t.Fatalf("a stopped container should move without starting: %+v", res)
|
||||
}
|
||||
if c.slave.passes != 1 {
|
||||
t.Fatalf("a stopped container needs one copy, got %d", c.slave.passes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveFullNodeStartsElsewhere(t *testing.T) {
|
||||
c := newTestCluster(t)
|
||||
c.addOwner(t, homeData(), false)
|
||||
c.master.mu.Lock()
|
||||
c.master.usable = Size{512, 0.5}
|
||||
c.master.mu.Unlock()
|
||||
|
||||
_, err := c.mgr.Ensure(c.ctx, "C1", OwnerChannel)
|
||||
var full *NodeFullError
|
||||
if !errors.As(err, &full) || full.Node != "master" {
|
||||
t.Fatalf("want node full on master, got %v", err)
|
||||
}
|
||||
res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{StartAfter: true})
|
||||
if err != nil || !res.Started || !c.slave.IsRunning("c-c1") {
|
||||
t.Fatalf("auto-move should start it on the new node: %+v %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveFailureRollsBack(t *testing.T) {
|
||||
c := newTestCluster(t)
|
||||
c.addOwner(t, homeData(), true)
|
||||
|
||||
_, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{
|
||||
Target: "slave-1",
|
||||
WaitIdle: func(context.Context) {
|
||||
c.slave.mu.Lock()
|
||||
c.slave.failWrites = true
|
||||
c.slave.mu.Unlock()
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected the final sync to fail")
|
||||
}
|
||||
if u := c.store.Owner("C1"); u.Node != "master" {
|
||||
t.Fatalf("container should stay on its old node, got %s", u.Node)
|
||||
}
|
||||
if !c.master.IsRunning("c-c1") {
|
||||
t.Fatal("container should be restarted on its old node")
|
||||
}
|
||||
if _, ok := c.master.homes["/d/c-c1"]; !ok {
|
||||
t.Fatal("old home must survive a failed move")
|
||||
}
|
||||
if _, ok := c.slave.homes["/d/c-c1"]; ok || len(c.slave.imports) != 0 {
|
||||
t.Fatal("half-copied home on the target should be cleaned up")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveRefusals(t *testing.T) {
|
||||
c := newTestCluster(t)
|
||||
c.addOwner(t, homeData(), false)
|
||||
|
||||
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "master"}); !errors.Is(err, ErrSameNode) {
|
||||
t.Fatalf("same node: %v", err)
|
||||
}
|
||||
var unknown *UnknownNodeError
|
||||
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "nope"}); !errors.As(err, &unknown) {
|
||||
t.Fatalf("unknown node: %v", err)
|
||||
}
|
||||
if _, err := c.mgr.Migrate(c.ctx, "nobody", MoveOptions{}); !errors.Is(err, ErrNoContainer) {
|
||||
t.Fatalf("no container: %v", err)
|
||||
}
|
||||
c.slave.mu.Lock()
|
||||
c.slave.usable = Size{512, 0.5}
|
||||
c.slave.mu.Unlock()
|
||||
c.master.mu.Lock()
|
||||
c.master.runs["c-c1"] = true
|
||||
c.master.mu.Unlock()
|
||||
slave, _ := c.mgr.node("slave-1")
|
||||
waitFor(t, "slave report", func() bool {
|
||||
info, _ := slave.Info(c.ctx)
|
||||
return info.Usable.MemoryMB == 512
|
||||
})
|
||||
var full *NodeFullError
|
||||
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "slave-1"}); !errors.As(err, &full) {
|
||||
t.Fatalf("full target: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type slackLog struct {
|
||||
mu sync.Mutex
|
||||
n int
|
||||
text map[string]string
|
||||
under map[string]string
|
||||
order []string
|
||||
}
|
||||
|
||||
func (s *slackLog) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
_ = r.ParseForm()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ts := r.Form.Get("ts")
|
||||
if strings.HasSuffix(r.URL.Path, "chat.postMessage") {
|
||||
s.n++
|
||||
ts = fmt.Sprintf("%d.0", s.n)
|
||||
s.order = append(s.order, ts)
|
||||
s.under[ts] = r.Form.Get("thread_ts")
|
||||
}
|
||||
if !strings.HasSuffix(r.URL.Path, "chat.getPermalink") {
|
||||
s.text[ts] = r.Form.Get("text")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "C1", "ts": ts, "permalink": "https://slack.example/" + ts})
|
||||
}
|
||||
|
||||
func TestEveryMoveStartsAnAnnouncedThread(t *testing.T) {
|
||||
c := newTestCluster(t)
|
||||
c.addOwner(t, homeData(), false)
|
||||
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
|
||||
srv := httptest.NewServer(log)
|
||||
defer srv.Close()
|
||||
b := &Bot{cfg: c.mgr.cfg, store: c.store, mgr: c.mgr, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/")),
|
||||
acps: map[string]*ACP{}, busy: map[string]int{}}
|
||||
|
||||
b.moveFor(c.ctx, "C1", "C1", "slave-1", "<@U1>", "100.0")
|
||||
if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "master"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.runMove(c.ctx, "C1", "C1", MoveOptions{}, "auto process (master was full), for <@U2>'s message", nil)
|
||||
|
||||
log.mu.Lock()
|
||||
defer log.mu.Unlock()
|
||||
var roots []string
|
||||
for _, ts := range log.order {
|
||||
if log.under[ts] == "" {
|
||||
roots = append(roots, ts)
|
||||
}
|
||||
}
|
||||
linked := 0
|
||||
for ts, parent := range log.under {
|
||||
if parent == "100.0" && strings.Contains(log.text[ts], "Follow the move here") {
|
||||
linked++
|
||||
}
|
||||
}
|
||||
if linked != 1 {
|
||||
t.Fatalf("the thread where the move started should get one link to the move thread, got %d", linked)
|
||||
}
|
||||
if len(roots) != 2 {
|
||||
t.Fatalf("want one new top-level message per move, got %d", len(roots))
|
||||
}
|
||||
checks := []string{"started by <@U1>: master → slave-1", "started by auto process (master was full), for <@U2>'s message"}
|
||||
for i, root := range roots {
|
||||
if !strings.Contains(log.text[root], checks[i]) || !strings.Contains(log.text[root], ":white_check_mark: done") {
|
||||
t.Fatalf("root %d = %q", i, log.text[root])
|
||||
}
|
||||
replies := 0
|
||||
for ts, parent := range log.under {
|
||||
if parent == root {
|
||||
replies++
|
||||
if !strings.Contains(log.text[ts], "Moved from") {
|
||||
t.Fatalf("progress reply = %q", log.text[ts])
|
||||
}
|
||||
}
|
||||
}
|
||||
if replies != 1 {
|
||||
t.Fatalf("want progress in one thread reply, got %d", replies)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user