first pass
This commit is contained in:
commit
7822529a38
31 files changed
+7050
No files matched your search
+437
@@ -0,0 +1,437 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPlacementBalancesAllThree(t *testing.T) {
|
||||
drive := func(usable, committed int64) []DriveReport {
|
||||
return []DriveReport{{Path: "/d", UsableMB: usable, CommittedMB: committed}}
|
||||
}
|
||||
want := Size{MemoryMB: 2048, CPUs: 1}
|
||||
views := []NodeView{
|
||||
{Name: "cpu-busy", Usable: Size{16384, 8}, Used: Size{2048, 6}, Drives: drive(100_000, 10_000)},
|
||||
{Name: "ram-busy", Usable: Size{16384, 8}, Used: Size{12288, 1}, Drives: drive(100_000, 10_000)},
|
||||
{Name: "disk-busy", Usable: Size{16384, 8}, Used: Size{2048, 1}, Drives: drive(100_000, 80_000)},
|
||||
{Name: "calm", Usable: Size{16384, 8}, Used: Size{4096, 2}, Drives: drive(100_000, 20_000)},
|
||||
}
|
||||
node, d, err := choosePlacement(views, want, 5120)
|
||||
if err != nil || node != "calm" || d != "/d" {
|
||||
t.Fatalf("got %s %s %v, want calm", node, d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlacementSpreadsAcrossEqualNodes(t *testing.T) {
|
||||
views := []NodeView{
|
||||
{Name: "a", Usable: Size{8192, 4}, Drives: []DriveReport{{Path: "/a", UsableMB: 50_000}}},
|
||||
{Name: "b", Usable: Size{8192, 4}, Drives: []DriveReport{{Path: "/b", UsableMB: 50_000}}},
|
||||
}
|
||||
want := Size{2048, 1}
|
||||
counts := map[string]int{}
|
||||
for i := 0; i < 4; i++ {
|
||||
node, _, err := choosePlacement(views, want, 5120)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counts[node]++
|
||||
for j := range views {
|
||||
if views[j].Name == node {
|
||||
views[j].Used = views[j].Used.Add(want)
|
||||
views[j].Drives[0].CommittedMB += 5120
|
||||
}
|
||||
}
|
||||
}
|
||||
if counts["a"] != 2 || counts["b"] != 2 {
|
||||
t.Fatalf("uneven spread: %v", counts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlacementWhenFull(t *testing.T) {
|
||||
full := []NodeView{{Name: "a", Usable: Size{4096, 2}, Used: Size{4096, 2}, Drives: []DriveReport{{Path: "/a", UsableMB: 50_000}}}}
|
||||
if _, _, err := choosePlacement(full, Size{1024, 1}, 1024); !errors.Is(err, ErrMaxedOut) {
|
||||
t.Fatalf("want maxed out, got %v", err)
|
||||
}
|
||||
noDisk := []NodeView{{Name: "a", Usable: Size{4096, 2}, Drives: []DriveReport{{Path: "/a", UsableMB: 1000}}}}
|
||||
if _, _, err := choosePlacement(noDisk, Size{1024, 1}, 5120); !errors.Is(err, ErrStorageFull) {
|
||||
t.Fatalf("want storage full, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildView(t *testing.T) {
|
||||
info := NodeInfo{
|
||||
Name: "n1",
|
||||
Usable: Size{16384, 8},
|
||||
Running: []RunningMachine{{Name: "c-a"}, {Name: "u-stranger"}},
|
||||
Drives: []DriveInfo{{Path: "/d", FreeMB: 90_000, ReserveMB: 10_000,
|
||||
HomesMB: map[string]int64{"c-a": 8000, "c-b": 100, "u-stranger": 3000}}},
|
||||
}
|
||||
owners := []Owner{
|
||||
{ID: "A", Machine: "c-a", Node: "n1", Drive: "/d", Size: Size{4096, 2}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
||||
{ID: "B", Machine: "c-b", Node: "n1", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
||||
{ID: "C", Machine: "c-c", Node: "n1", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
||||
{ID: "D", Machine: "c-d", Node: "other", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
||||
}
|
||||
v := buildView(info, owners, Size{1024, 1})
|
||||
if v.Used != (Size{4096 + 1024, 3}) {
|
||||
t.Fatalf("used = %v", v.Used)
|
||||
}
|
||||
d := v.Drives[0]
|
||||
if d.UsableMB != 90_000+11_100-10_000 {
|
||||
t.Fatalf("usable = %d", d.UsableMB)
|
||||
}
|
||||
if d.CommittedMB != 8000+5120+5120+3000 || d.BookedMB != 3*10240+3000 || d.Homes != 4 {
|
||||
t.Fatalf("committed %d booked %d homes %d", d.CommittedMB, d.BookedMB, d.Homes)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeNode struct {
|
||||
name string
|
||||
mu sync.Mutex
|
||||
runs map[string]bool
|
||||
last StartSpec
|
||||
homes map[string][]byte
|
||||
exports map[string]*fakeExport
|
||||
imports map[string]*fakeImport
|
||||
passes int
|
||||
failWrites bool
|
||||
usable Size
|
||||
}
|
||||
|
||||
func newFakeNode(name string) *fakeNode {
|
||||
return &fakeNode{name: name, runs: map[string]bool{}, homes: map[string][]byte{},
|
||||
exports: map[string]*fakeExport{}, imports: map[string]*fakeImport{}, usable: Size{8192, 4}}
|
||||
}
|
||||
|
||||
func (f *fakeNode) Name() string { return f.name }
|
||||
func (f *fakeNode) Info(ctx context.Context) (NodeInfo, error) {
|
||||
running, _ := f.Running(ctx)
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
homes := map[string]int64{}
|
||||
for k, v := range f.homes {
|
||||
homes[strings.TrimPrefix(k, "/d/")] = int64(len(v)) >> 20
|
||||
}
|
||||
return NodeInfo{Name: f.name, Usable: f.usable, Running: running,
|
||||
Drives: []DriveInfo{{Path: "/d", FreeMB: 50_000, HomesMB: homes}}}, nil
|
||||
}
|
||||
func (f *fakeNode) Running(context.Context) ([]RunningMachine, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []RunningMachine
|
||||
for m := range f.runs {
|
||||
out = append(out, RunningMachine{Name: m})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (f *fakeNode) IsRunning(machine string) bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.runs[machine]
|
||||
}
|
||||
func (f *fakeNode) Start(_ context.Context, spec StartSpec) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.last, f.runs[spec.Machine] = spec, true
|
||||
return nil
|
||||
}
|
||||
func (f *fakeNode) Poweroff(_ context.Context, machine string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.runs, machine)
|
||||
return nil
|
||||
}
|
||||
func (f *fakeNode) SetLimits(context.Context, string, Size) error { return nil }
|
||||
func (f *fakeNode) SetQuota(context.Context, string, string, int64) error { return nil }
|
||||
func (f *fakeNode) DialAgent(_ context.Context, machine, secret string) (msgConn, error) {
|
||||
if secret != "agent-secret" {
|
||||
return nil, errors.New("bad agent secret")
|
||||
}
|
||||
return newEcho(), nil
|
||||
}
|
||||
|
||||
func (f *fakeNode) WaitStopped(_ context.Context, machine string) error {
|
||||
if f.IsRunning(machine) {
|
||||
return errors.New("still running")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeNode) ExportHome(_ context.Context, drive, machine, parent string) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
data, ok := f.homes[drive+"/"+machine]
|
||||
if !ok {
|
||||
return "", errors.New("no such home")
|
||||
}
|
||||
kind := "FULL:"
|
||||
if parent != "" {
|
||||
kind = "INCR:"
|
||||
}
|
||||
id := randomHex(4)
|
||||
f.exports[id] = &fakeExport{data: append([]byte(kind), data...)}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (f *fakeNode) ReadExport(_ context.Context, id string, max int) ([]byte, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
x := f.exports[id]
|
||||
if x == nil {
|
||||
return nil, false, errors.New("no export")
|
||||
}
|
||||
n := min(max, len(x.data)-x.pos)
|
||||
chunk := x.data[x.pos : x.pos+n]
|
||||
x.pos += n
|
||||
return chunk, x.pos == len(x.data), nil
|
||||
}
|
||||
|
||||
func (f *fakeNode) CloseExport(_ context.Context, id string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.exports, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeNode) ImportHome(_ context.Context, drive, machine string) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, exists := f.homes[drive+"/"+machine]; exists {
|
||||
return "", errors.New("home already exists")
|
||||
}
|
||||
id := randomHex(4)
|
||||
f.imports[id] = &fakeImport{key: drive + "/" + machine}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (f *fakeNode) WriteImport(_ context.Context, id string, data []byte) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.failWrites {
|
||||
return errors.New("disk on fire")
|
||||
}
|
||||
x := f.imports[id]
|
||||
if x == nil {
|
||||
return errors.New("no import")
|
||||
}
|
||||
x.cur = append(x.cur, data...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeNode) EndReceive(_ context.Context, id string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
x := f.imports[id]
|
||||
if x == nil || len(x.cur) < 5 {
|
||||
return errors.New("nothing received")
|
||||
}
|
||||
if string(x.cur[:5]) == "INCR:" && x.latest == nil {
|
||||
return errors.New("incremental send without a parent")
|
||||
}
|
||||
x.latest, x.cur = x.cur[5:], nil
|
||||
x.passes++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeNode) FinishImport(_ context.Context, id string, quotaMB int64) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
x := f.imports[id]
|
||||
delete(f.imports, id)
|
||||
f.homes[x.key] = x.latest
|
||||
f.passes = x.passes
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeNode) AbortImport(_ context.Context, id string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.imports, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeNode) DeleteHome(_ context.Context, drive, machine string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.homes, drive+"/"+machine)
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeExport struct {
|
||||
data []byte
|
||||
pos int
|
||||
}
|
||||
|
||||
type fakeImport struct {
|
||||
key string
|
||||
cur, latest []byte
|
||||
passes int
|
||||
}
|
||||
|
||||
type echo struct {
|
||||
ch chan []byte
|
||||
closed chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func newEcho() *echo { return &echo{ch: make(chan []byte, 16), closed: make(chan struct{})} }
|
||||
func (e *echo) Read(ctx context.Context) ([]byte, error) {
|
||||
select {
|
||||
case b := <-e.ch:
|
||||
return b, nil
|
||||
case <-e.closed:
|
||||
return nil, errors.New("closed")
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
func (e *echo) Write(_ context.Context, b []byte) error {
|
||||
e.ch <- append([]byte("echo: "), b...)
|
||||
return nil
|
||||
}
|
||||
func (e *echo) Close() error { e.once.Do(func() { close(e.closed) }); return nil }
|
||||
|
||||
func freePort(t *testing.T) string {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer l.Close()
|
||||
return l.Addr().String()
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, what string, cond func() bool) {
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for !cond() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for %s", what)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMasterAndSlave(t *testing.T) {
|
||||
addr := freePort(t)
|
||||
secret := "cluster-secret-1234567890"
|
||||
masterDir, slaveDir := t.TempDir(), t.TempDir()
|
||||
|
||||
masterCfg := &Config{StateDir: masterDir, Container: Size{1024, 1}}
|
||||
masterCfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr}
|
||||
store, err := OpenStore(filepath.Join(masterDir, "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mgr := NewManager(masterCfg, store, newFakeNode("master"))
|
||||
backCh := make(chan string, 4)
|
||||
mgr.onNodeBack = func(n string) { backCh <- n }
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go mgr.ServeCluster(ctx)
|
||||
|
||||
slaveCfg := &Config{StateDir: slaveDir}
|
||||
slaveCfg.Cluster = ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr}
|
||||
slave := newFakeNode("slave-1")
|
||||
slaveCtx, stopSlave := context.WithCancel(ctx)
|
||||
go RunSlave(slaveCtx, slaveCfg, slave)
|
||||
|
||||
waitFor(t, "slave to come online", func() bool { _, err := mgr.node("slave-1"); return err == nil })
|
||||
if got := <-backCh; got != "slave-1" {
|
||||
t.Fatalf("back callback for %q", got)
|
||||
}
|
||||
pinned, err := os.ReadFile(filepath.Join(slaveDir, "master.fingerprint"))
|
||||
if err != nil || len(strings.TrimSpace(string(pinned))) != 64 {
|
||||
t.Fatalf("slave should pin the master's certificate, got %q %v", pinned, err)
|
||||
}
|
||||
|
||||
n, _ := mgr.node("slave-1")
|
||||
spec := StartSpec{Machine: "c-abc", Secret: "agent-secret", Size: Size{1024, 1}, Drive: "/d"}
|
||||
if err := n.Start(ctx, spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slave.IsRunning("c-abc") || slave.last.Drive != "/d" {
|
||||
t.Fatal("start didn't reach the slave")
|
||||
}
|
||||
if !n.IsRunning("c-abc") {
|
||||
t.Fatal("master should count the container as running right away")
|
||||
}
|
||||
|
||||
conn, err := n.DialAgent(ctx, "c-abc", "agent-secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.Write(ctx, []byte(`{"jsonrpc":"2.0"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
got, err := conn.Read(readCtx)
|
||||
readCancel()
|
||||
if err != nil || string(got) != `echo: {"jsonrpc":"2.0"}` {
|
||||
t.Fatalf("relay got %q %v", got, err)
|
||||
}
|
||||
conn.Close()
|
||||
if _, err := n.DialAgent(ctx, "c-abc", "wrong"); err == nil {
|
||||
t.Fatal("a bad agent secret should fail through the relay")
|
||||
}
|
||||
|
||||
if err := n.Poweroff(ctx, "c-abc"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if slave.IsRunning("c-abc") || n.IsRunning("c-abc") {
|
||||
t.Fatal("poweroff didn't take")
|
||||
}
|
||||
|
||||
stopSlave()
|
||||
waitFor(t, "slave to go offline", func() bool {
|
||||
_, err := mgr.node("slave-1")
|
||||
var off *NodeOfflineError
|
||||
return errors.As(err, &off) && off.Node == "slave-1"
|
||||
})
|
||||
if off := mgr.OfflineNodes(); len(off) != 1 || off[0] != "slave-1" {
|
||||
t.Fatalf("offline nodes = %v", off)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlaveRejectsWrongMasterOrSecret(t *testing.T) {
|
||||
addr := freePort(t)
|
||||
secret := "cluster-secret-1234567890"
|
||||
masterCfg := &Config{StateDir: t.TempDir()}
|
||||
masterCfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr}
|
||||
store, _ := OpenStore(filepath.Join(masterCfg.StateDir, "state.json"))
|
||||
mgr := NewManager(masterCfg, store, newFakeNode("master"))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go mgr.ServeCluster(ctx)
|
||||
waitFor(t, "master to listen", func() bool {
|
||||
c, err := net.Dial("tcp", addr)
|
||||
if err == nil {
|
||||
c.Close()
|
||||
}
|
||||
return err == nil
|
||||
})
|
||||
|
||||
wrongPin := &Config{StateDir: t.TempDir()}
|
||||
wrongPin.Cluster = ClusterConfig{Name: "s", Secret: secret, Master: "wss://" + addr, MasterFingerprint: strings.Repeat("ab", 32)}
|
||||
if err := slaveSession(ctx, wrongPin, newFakeNode("s")); err == nil {
|
||||
t.Fatal("a master with a different certificate should be refused")
|
||||
}
|
||||
|
||||
wrongSecret := &Config{StateDir: t.TempDir()}
|
||||
wrongSecret.Cluster = ClusterConfig{Name: "s", Secret: "not-the-secret-at-all", Master: "wss://" + addr}
|
||||
if err := slaveSession(ctx, wrongSecret, newFakeNode("s")); err == nil {
|
||||
t.Fatal("a slave with the wrong secret should be refused")
|
||||
}
|
||||
|
||||
sameName := &Config{StateDir: t.TempDir()}
|
||||
sameName.Cluster = ClusterConfig{Name: "master", Secret: secret, Master: "wss://" + addr}
|
||||
if err := slaveSession(ctx, sameName, newFakeNode("master")); err == nil {
|
||||
t.Fatal("a slave can't take the master's name")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user