438 lines
13 KiB
Go
438 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestPlacementBalancesAllThree(t *testing.T) {
|
|
drive := func(usable, committed int64) []DriveReport {
|
|
return []DriveReport{{Path: "/d", UsableMB: usable, CommittedMB: committed}}
|
|
}
|
|
want := Size{MemoryMB: 2048, CPUs: 1}
|
|
views := []NodeView{
|
|
{Name: "cpu-busy", Usable: Size{16384, 8}, Used: Size{2048, 6}, Drives: drive(100_000, 10_000)},
|
|
{Name: "ram-busy", Usable: Size{16384, 8}, Used: Size{12288, 1}, Drives: drive(100_000, 10_000)},
|
|
{Name: "disk-busy", Usable: Size{16384, 8}, Used: Size{2048, 1}, Drives: drive(100_000, 80_000)},
|
|
{Name: "calm", Usable: Size{16384, 8}, Used: Size{4096, 2}, Drives: drive(100_000, 20_000)},
|
|
}
|
|
node, d, err := choosePlacement(views, want, 5120)
|
|
if err != nil || node != "calm" || d != "/d" {
|
|
t.Fatalf("got %s %s %v, want calm", node, d, err)
|
|
}
|
|
}
|
|
|
|
func TestPlacementSpreadsAcrossEqualNodes(t *testing.T) {
|
|
views := []NodeView{
|
|
{Name: "a", Usable: Size{8192, 4}, Drives: []DriveReport{{Path: "/a", UsableMB: 50_000}}},
|
|
{Name: "b", Usable: Size{8192, 4}, Drives: []DriveReport{{Path: "/b", UsableMB: 50_000}}},
|
|
}
|
|
want := Size{2048, 1}
|
|
counts := map[string]int{}
|
|
for i := 0; i < 4; i++ {
|
|
node, _, err := choosePlacement(views, want, 5120)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
counts[node]++
|
|
for j := range views {
|
|
if views[j].Name == node {
|
|
views[j].Used = views[j].Used.Add(want)
|
|
views[j].Drives[0].CommittedMB += 5120
|
|
}
|
|
}
|
|
}
|
|
if counts["a"] != 2 || counts["b"] != 2 {
|
|
t.Fatalf("uneven spread: %v", counts)
|
|
}
|
|
}
|
|
|
|
func TestPlacementWhenFull(t *testing.T) {
|
|
full := []NodeView{{Name: "a", Usable: Size{4096, 2}, Used: Size{4096, 2}, Drives: []DriveReport{{Path: "/a", UsableMB: 50_000}}}}
|
|
if _, _, err := choosePlacement(full, Size{1024, 1}, 1024); !errors.Is(err, ErrMaxedOut) {
|
|
t.Fatalf("want maxed out, got %v", err)
|
|
}
|
|
noDisk := []NodeView{{Name: "a", Usable: Size{4096, 2}, Drives: []DriveReport{{Path: "/a", UsableMB: 1000}}}}
|
|
if _, _, err := choosePlacement(noDisk, Size{1024, 1}, 5120); !errors.Is(err, ErrStorageFull) {
|
|
t.Fatalf("want storage full, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestBuildView(t *testing.T) {
|
|
info := NodeInfo{
|
|
Name: "n1",
|
|
Usable: Size{16384, 8},
|
|
Running: []RunningMachine{{Name: "c-a"}, {Name: "u-stranger"}},
|
|
Drives: []DriveInfo{{Path: "/d", FreeMB: 90_000, ReserveMB: 10_000,
|
|
HomesMB: map[string]int64{"c-a": 8000, "c-b": 100, "u-stranger": 3000}}},
|
|
}
|
|
owners := []Owner{
|
|
{ID: "A", Machine: "c-a", Node: "n1", Drive: "/d", Size: Size{4096, 2}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
|
{ID: "B", Machine: "c-b", Node: "n1", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
|
{ID: "C", Machine: "c-c", Node: "n1", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
|
{ID: "D", Machine: "c-d", Node: "other", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
|
}
|
|
v := buildView(info, owners, Size{1024, 1})
|
|
if v.Used != (Size{4096 + 1024, 3}) {
|
|
t.Fatalf("used = %v", v.Used)
|
|
}
|
|
d := v.Drives[0]
|
|
if d.UsableMB != 90_000+11_100-10_000 {
|
|
t.Fatalf("usable = %d", d.UsableMB)
|
|
}
|
|
if d.CommittedMB != 8000+5120+5120+3000 || d.BookedMB != 3*10240+3000 || d.Homes != 4 {
|
|
t.Fatalf("committed %d booked %d homes %d", d.CommittedMB, d.BookedMB, d.Homes)
|
|
}
|
|
}
|
|
|
|
type fakeNode struct {
|
|
name string
|
|
mu sync.Mutex
|
|
runs map[string]bool
|
|
last StartSpec
|
|
homes map[string][]byte
|
|
exports map[string]*fakeExport
|
|
imports map[string]*fakeImport
|
|
passes int
|
|
failWrites bool
|
|
usable Size
|
|
}
|
|
|
|
func newFakeNode(name string) *fakeNode {
|
|
return &fakeNode{name: name, runs: map[string]bool{}, homes: map[string][]byte{},
|
|
exports: map[string]*fakeExport{}, imports: map[string]*fakeImport{}, usable: Size{8192, 4}}
|
|
}
|
|
|
|
func (f *fakeNode) Name() string { return f.name }
|
|
func (f *fakeNode) Info(ctx context.Context) (NodeInfo, error) {
|
|
running, _ := f.Running(ctx)
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
homes := map[string]int64{}
|
|
for k, v := range f.homes {
|
|
homes[strings.TrimPrefix(k, "/d/")] = int64(len(v)) >> 20
|
|
}
|
|
return NodeInfo{Name: f.name, Usable: f.usable, Running: running,
|
|
Drives: []DriveInfo{{Path: "/d", FreeMB: 50_000, HomesMB: homes}}}, nil
|
|
}
|
|
func (f *fakeNode) Running(context.Context) ([]RunningMachine, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
var out []RunningMachine
|
|
for m := range f.runs {
|
|
out = append(out, RunningMachine{Name: m})
|
|
}
|
|
return out, nil
|
|
}
|
|
func (f *fakeNode) IsRunning(machine string) bool {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.runs[machine]
|
|
}
|
|
func (f *fakeNode) Start(_ context.Context, spec StartSpec) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.last, f.runs[spec.Machine] = spec, true
|
|
return nil
|
|
}
|
|
func (f *fakeNode) Poweroff(_ context.Context, machine string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
delete(f.runs, machine)
|
|
return nil
|
|
}
|
|
func (f *fakeNode) SetLimits(context.Context, string, Size) error { return nil }
|
|
func (f *fakeNode) SetQuota(context.Context, string, string, int64) error { return nil }
|
|
func (f *fakeNode) DialAgent(_ context.Context, machine, secret string) (msgConn, error) {
|
|
if secret != "agent-secret" {
|
|
return nil, errors.New("bad agent secret")
|
|
}
|
|
return newEcho(), nil
|
|
}
|
|
|
|
func (f *fakeNode) WaitStopped(_ context.Context, machine string) error {
|
|
if f.IsRunning(machine) {
|
|
return errors.New("still running")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeNode) ExportHome(_ context.Context, drive, machine, parent string) (string, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
data, ok := f.homes[drive+"/"+machine]
|
|
if !ok {
|
|
return "", errors.New("no such home")
|
|
}
|
|
kind := "FULL:"
|
|
if parent != "" {
|
|
kind = "INCR:"
|
|
}
|
|
id := randomHex(4)
|
|
f.exports[id] = &fakeExport{data: append([]byte(kind), data...)}
|
|
return id, nil
|
|
}
|
|
|
|
func (f *fakeNode) ReadExport(_ context.Context, id string, max int) ([]byte, bool, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
x := f.exports[id]
|
|
if x == nil {
|
|
return nil, false, errors.New("no export")
|
|
}
|
|
n := min(max, len(x.data)-x.pos)
|
|
chunk := x.data[x.pos : x.pos+n]
|
|
x.pos += n
|
|
return chunk, x.pos == len(x.data), nil
|
|
}
|
|
|
|
func (f *fakeNode) CloseExport(_ context.Context, id string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
delete(f.exports, id)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeNode) ImportHome(_ context.Context, drive, machine string) (string, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if _, exists := f.homes[drive+"/"+machine]; exists {
|
|
return "", errors.New("home already exists")
|
|
}
|
|
id := randomHex(4)
|
|
f.imports[id] = &fakeImport{key: drive + "/" + machine}
|
|
return id, nil
|
|
}
|
|
|
|
func (f *fakeNode) WriteImport(_ context.Context, id string, data []byte) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.failWrites {
|
|
return errors.New("disk on fire")
|
|
}
|
|
x := f.imports[id]
|
|
if x == nil {
|
|
return errors.New("no import")
|
|
}
|
|
x.cur = append(x.cur, data...)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeNode) EndReceive(_ context.Context, id string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
x := f.imports[id]
|
|
if x == nil || len(x.cur) < 5 {
|
|
return errors.New("nothing received")
|
|
}
|
|
if string(x.cur[:5]) == "INCR:" && x.latest == nil {
|
|
return errors.New("incremental send without a parent")
|
|
}
|
|
x.latest, x.cur = x.cur[5:], nil
|
|
x.passes++
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeNode) FinishImport(_ context.Context, id string, quotaMB int64) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
x := f.imports[id]
|
|
delete(f.imports, id)
|
|
f.homes[x.key] = x.latest
|
|
f.passes = x.passes
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeNode) AbortImport(_ context.Context, id string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
delete(f.imports, id)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeNode) DeleteHome(_ context.Context, drive, machine string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
delete(f.homes, drive+"/"+machine)
|
|
return nil
|
|
}
|
|
|
|
type fakeExport struct {
|
|
data []byte
|
|
pos int
|
|
}
|
|
|
|
type fakeImport struct {
|
|
key string
|
|
cur, latest []byte
|
|
passes int
|
|
}
|
|
|
|
type echo struct {
|
|
ch chan []byte
|
|
closed chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
func newEcho() *echo { return &echo{ch: make(chan []byte, 16), closed: make(chan struct{})} }
|
|
func (e *echo) Read(ctx context.Context) ([]byte, error) {
|
|
select {
|
|
case b := <-e.ch:
|
|
return b, nil
|
|
case <-e.closed:
|
|
return nil, errors.New("closed")
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
func (e *echo) Write(_ context.Context, b []byte) error {
|
|
e.ch <- append([]byte("echo: "), b...)
|
|
return nil
|
|
}
|
|
func (e *echo) Close() error { e.once.Do(func() { close(e.closed) }); return nil }
|
|
|
|
func freePort(t *testing.T) string {
|
|
l, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer l.Close()
|
|
return l.Addr().String()
|
|
}
|
|
|
|
func waitFor(t *testing.T, what string, cond func() bool) {
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
for !cond() {
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("timed out waiting for %s", what)
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestMasterAndSlave(t *testing.T) {
|
|
addr := freePort(t)
|
|
secret := "cluster-secret-1234567890"
|
|
masterDir, slaveDir := t.TempDir(), t.TempDir()
|
|
|
|
masterCfg := &Config{StateDir: masterDir, Container: Size{1024, 1}}
|
|
masterCfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr}
|
|
store, err := OpenStore(filepath.Join(masterDir, "state.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mgr := NewManager(masterCfg, store, newFakeNode("master"))
|
|
backCh := make(chan string, 4)
|
|
mgr.onNodeBack = func(n string) { backCh <- n }
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go mgr.ServeCluster(ctx)
|
|
|
|
slaveCfg := &Config{StateDir: slaveDir}
|
|
slaveCfg.Cluster = ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr}
|
|
slave := newFakeNode("slave-1")
|
|
slaveCtx, stopSlave := context.WithCancel(ctx)
|
|
go RunSlave(slaveCtx, slaveCfg, slave)
|
|
|
|
waitFor(t, "slave to come online", func() bool { _, err := mgr.node("slave-1"); return err == nil })
|
|
if got := <-backCh; got != "slave-1" {
|
|
t.Fatalf("back callback for %q", got)
|
|
}
|
|
pinned, err := os.ReadFile(filepath.Join(slaveDir, "master.fingerprint"))
|
|
if err != nil || len(strings.TrimSpace(string(pinned))) != 64 {
|
|
t.Fatalf("slave should pin the master's certificate, got %q %v", pinned, err)
|
|
}
|
|
|
|
n, _ := mgr.node("slave-1")
|
|
spec := StartSpec{Machine: "c-abc", Secret: "agent-secret", Size: Size{1024, 1}, Drive: "/d"}
|
|
if err := n.Start(ctx, spec); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !slave.IsRunning("c-abc") || slave.last.Drive != "/d" {
|
|
t.Fatal("start didn't reach the slave")
|
|
}
|
|
if !n.IsRunning("c-abc") {
|
|
t.Fatal("master should count the container as running right away")
|
|
}
|
|
|
|
conn, err := n.DialAgent(ctx, "c-abc", "agent-secret")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := conn.Write(ctx, []byte(`{"jsonrpc":"2.0"}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second)
|
|
got, err := conn.Read(readCtx)
|
|
readCancel()
|
|
if err != nil || string(got) != `echo: {"jsonrpc":"2.0"}` {
|
|
t.Fatalf("relay got %q %v", got, err)
|
|
}
|
|
conn.Close()
|
|
if _, err := n.DialAgent(ctx, "c-abc", "wrong"); err == nil {
|
|
t.Fatal("a bad agent secret should fail through the relay")
|
|
}
|
|
|
|
if err := n.Poweroff(ctx, "c-abc"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if slave.IsRunning("c-abc") || n.IsRunning("c-abc") {
|
|
t.Fatal("poweroff didn't take")
|
|
}
|
|
|
|
stopSlave()
|
|
waitFor(t, "slave to go offline", func() bool {
|
|
_, err := mgr.node("slave-1")
|
|
var off *NodeOfflineError
|
|
return errors.As(err, &off) && off.Node == "slave-1"
|
|
})
|
|
if off := mgr.OfflineNodes(); len(off) != 1 || off[0] != "slave-1" {
|
|
t.Fatalf("offline nodes = %v", off)
|
|
}
|
|
}
|
|
|
|
func TestSlaveRejectsWrongMasterOrSecret(t *testing.T) {
|
|
addr := freePort(t)
|
|
secret := "cluster-secret-1234567890"
|
|
masterCfg := &Config{StateDir: t.TempDir()}
|
|
masterCfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr}
|
|
store, _ := OpenStore(filepath.Join(masterCfg.StateDir, "state.json"))
|
|
mgr := NewManager(masterCfg, store, newFakeNode("master"))
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go mgr.ServeCluster(ctx)
|
|
waitFor(t, "master to listen", func() bool {
|
|
c, err := net.Dial("tcp", addr)
|
|
if err == nil {
|
|
c.Close()
|
|
}
|
|
return err == nil
|
|
})
|
|
|
|
wrongPin := &Config{StateDir: t.TempDir()}
|
|
wrongPin.Cluster = ClusterConfig{Name: "s", Secret: secret, Master: "wss://" + addr, MasterFingerprint: strings.Repeat("ab", 32)}
|
|
if err := slaveSession(ctx, wrongPin, newFakeNode("s")); err == nil {
|
|
t.Fatal("a master with a different certificate should be refused")
|
|
}
|
|
|
|
wrongSecret := &Config{StateDir: t.TempDir()}
|
|
wrongSecret.Cluster = ClusterConfig{Name: "s", Secret: "not-the-secret-at-all", Master: "wss://" + addr}
|
|
if err := slaveSession(ctx, wrongSecret, newFakeNode("s")); err == nil {
|
|
t.Fatal("a slave with the wrong secret should be refused")
|
|
}
|
|
|
|
sameName := &Config{StateDir: t.TempDir()}
|
|
sameName.Cluster = ClusterConfig{Name: "master", Secret: secret, Master: "wss://" + addr}
|
|
if err := slaveSession(ctx, sameName, newFakeNode("master")); err == nil {
|
|
t.Fatal("a slave can't take the master's name")
|
|
}
|
|
}
|