clean up of the codebase

This commit is contained in:
ari committed 2026-11-01 13:01:54 -05:00
1 parent 7822529a38
commit 0ac61b4ece
52 files changed
+2545 -2274

No files matched your search

+787
View File
@@ -0,0 +1,787 @@
package manager
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/hex"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"log"
"math/big"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/coder/websocket"
"slackerbot/internal/config"
"slackerbot/internal/model"
"slackerbot/internal/node"
)
const (
reportEvery = 5 * time.Second
offlineAfter = 20 * time.Second
callTimeout = 3 * time.Minute
pendingTimeout = 30 * time.Second
streamBuffer = 4096
)
type frame struct {
Type string `json:"t"`
ID int64 `json:"id,omitempty"`
Method string `json:"m,omitempty"`
Params json.RawMessage `json:"p,omitempty"`
Result json.RawMessage `json:"r,omitempty"`
Error string `json:"e,omitempty"`
Data string `json:"d,omitempty"`
}
type link struct {
conn *websocket.Conn
writeMu sync.Mutex
nextID atomic.Int64
mu sync.Mutex
pending map[int64]chan frame
streams map[int64]*stream
onCall func(f frame)
onReport func(info node.NodeInfo)
done chan struct{}
err error
}
func newLink(conn *websocket.Conn) *link {
conn.SetReadLimit(64 << 20)
return &link{
conn: conn,
pending: map[int64]chan frame{},
streams: map[int64]*stream{},
done: make(chan struct{}),
}
}
func (l *link) Done() <-chan struct{} { return l.done }
func (l *link) Close() { l.conn.Close(websocket.StatusNormalClosure, "") }
func (l *link) send(ctx context.Context, f frame) error {
b, err := json.Marshal(f)
if err != nil {
return err
}
l.writeMu.Lock()
defer l.writeMu.Unlock()
return l.conn.Write(ctx, websocket.MessageText, b)
}
func (l *link) run() {
defer func() {
l.mu.Lock()
streams := l.streams
l.streams = map[int64]*stream{}
l.mu.Unlock()
for _, s := range streams {
s.closeLocal()
}
close(l.done)
}()
for {
_, data, err := l.conn.Read(context.Background())
if err != nil {
l.err = err
return
}
var f frame
if json.Unmarshal(data, &f) != nil {
continue
}
switch f.Type {
case "call":
if l.onCall != nil {
go l.onCall(f)
}
case "reply":
l.mu.Lock()
ch := l.pending[f.ID]
l.mu.Unlock()
if ch != nil {
ch <- f
}
case "report":
var info node.NodeInfo
if l.onReport != nil && json.Unmarshal(f.Params, &info) == nil {
l.onReport(info)
}
case "data":
if s := l.stream(f.ID); s != nil {
select {
case s.in <- []byte(f.Data):
default:
s.Close()
}
}
case "close":
if s := l.stream(f.ID); s != nil {
s.closeLocal()
}
}
}
}
func (l *link) call(ctx context.Context, method string, params, result any) error {
return l.callWithID(ctx, l.nextID.Add(1), method, params, result)
}
func (l *link) callWithID(ctx context.Context, id int64, method string, params, result any) error {
ch := make(chan frame, 1)
l.mu.Lock()
l.pending[id] = ch
l.mu.Unlock()
defer func() {
l.mu.Lock()
delete(l.pending, id)
l.mu.Unlock()
}()
p, err := json.Marshal(params)
if err != nil {
return err
}
if err := l.send(ctx, frame{Type: "call", ID: id, Method: method, Params: p}); err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case <-l.done:
return fmt.Errorf("link closed: %v", l.err)
case f := <-ch:
if f.Error != "" {
return errors.New(f.Error)
}
if result != nil && len(f.Result) > 0 {
return json.Unmarshal(f.Result, result)
}
return nil
}
}
func (l *link) reply(id int64, result any, err error) {
f := frame{Type: "reply", ID: id}
if err != nil {
f.Error = err.Error()
} else if result != nil {
f.Result, _ = json.Marshal(result)
}
_ = l.send(context.Background(), f)
}
func (l *link) report(ctx context.Context, info node.NodeInfo) error {
p, err := json.Marshal(info)
if err != nil {
return err
}
return l.send(ctx, frame{Type: "report", Params: p})
}
func (l *link) stream(id int64) *stream {
l.mu.Lock()
defer l.mu.Unlock()
return l.streams[id]
}
func (l *link) addStream(id int64) *stream {
s := &stream{l: l, id: id, in: make(chan []byte, streamBuffer), closed: make(chan struct{})}
l.mu.Lock()
l.streams[id] = s
l.mu.Unlock()
return s
}
func (l *link) dropStream(id int64) {
l.mu.Lock()
delete(l.streams, id)
l.mu.Unlock()
}
// The stream shares its ID with the "open_agent" call, so it's registered
// before the call goes out and no early data is lost.
func (l *link) openAgent(ctx context.Context, machine, secret string) (*stream, error) {
id := l.nextID.Add(1)
s := l.addStream(id)
err := l.callWithID(ctx, id, "open_agent", map[string]string{"machine": machine, "secret": secret}, nil)
if err != nil {
l.dropStream(id)
return nil, err
}
return s, nil
}
type stream struct {
l *link
id int64
in chan []byte
closed chan struct{}
once sync.Once
}
func (s *stream) Read(ctx context.Context) ([]byte, error) {
select {
case b := <-s.in:
return b, nil
default:
}
select {
case b := <-s.in:
return b, nil
case <-s.closed:
return nil, errors.New("stream closed")
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (s *stream) Write(ctx context.Context, b []byte) error {
select {
case <-s.closed:
return errors.New("stream closed")
default:
}
return s.l.send(ctx, frame{Type: "data", ID: s.id, Data: string(b)})
}
func (s *stream) Close() error {
s.once.Do(func() {
_ = s.l.send(context.Background(), frame{Type: "close", ID: s.id})
s.l.dropStream(s.id)
close(s.closed)
})
return nil
}
func (s *stream) closeLocal() {
s.once.Do(func() {
s.l.dropStream(s.id)
close(s.closed)
})
}
type Remote struct {
name string
mu sync.Mutex
link *link
info node.NodeInfo
infoAt time.Time
starting map[string]time.Time
stopping map[string]time.Time
}
func (r *Remote) Name() string { return r.name }
func (r *Remote) Online() bool {
r.mu.Lock()
defer r.mu.Unlock()
if r.link == nil || time.Since(r.infoAt) > offlineAfter {
return false
}
select {
case <-r.link.Done():
return false
default:
return true
}
}
func (r *Remote) currentLink() (*link, error) {
if !r.Online() {
return nil, &node.NodeOfflineError{Node: r.name}
}
r.mu.Lock()
defer r.mu.Unlock()
return r.link, nil
}
func (r *Remote) setInfo(info node.NodeInfo) {
r.mu.Lock()
defer r.mu.Unlock()
info.Name = r.name
r.info, r.infoAt = info, time.Now()
for _, m := range info.Running {
delete(r.starting, m.Name)
}
for m := range r.stopping {
if !containsMachine(info.Running, m) {
delete(r.stopping, m)
}
}
}
func containsMachine(list []node.RunningMachine, name string) bool {
for _, m := range list {
if m.Name == name {
return true
}
}
return false
}
// Reports can lag behind a start or stop the master just asked for,
// so recent requests override them for a short while.
func (r *Remote) runningLocked() []node.RunningMachine {
var out []node.RunningMachine
for _, m := range r.info.Running {
if t, ok := r.stopping[m.Name]; ok && time.Since(t) < pendingTimeout {
continue
}
out = append(out, m)
}
for m, t := range r.starting {
if time.Since(t) < pendingTimeout && !containsMachine(out, m) {
out = append(out, node.RunningMachine{Name: m, Since: t})
}
}
return out
}
func (r *Remote) Info(ctx context.Context) (node.NodeInfo, error) {
if !r.Online() {
return node.NodeInfo{}, &node.NodeOfflineError{Node: r.name}
}
r.mu.Lock()
defer r.mu.Unlock()
info := r.info
info.Running = r.runningLocked()
return info, nil
}
func (r *Remote) Running(ctx context.Context) ([]node.RunningMachine, error) {
info, err := r.Info(ctx)
return info.Running, err
}
func (r *Remote) IsRunning(machine string) bool {
r.mu.Lock()
defer r.mu.Unlock()
return containsMachine(r.runningLocked(), machine)
}
func (r *Remote) callNode(ctx context.Context, method string, params, result any) error {
l, err := r.currentLink()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, callTimeout)
defer cancel()
return l.call(ctx, method, params, result)
}
func (r *Remote) Start(ctx context.Context, spec node.StartSpec) error {
if err := r.callNode(ctx, "start", spec, nil); err != nil {
return err
}
r.mu.Lock()
r.starting[spec.Machine] = time.Now()
delete(r.stopping, spec.Machine)
r.mu.Unlock()
return nil
}
func (r *Remote) Poweroff(ctx context.Context, machine string) error {
if err := r.callNode(ctx, "poweroff", map[string]string{"machine": machine}, nil); err != nil {
return err
}
r.mu.Lock()
r.stopping[machine] = time.Now()
delete(r.starting, machine)
r.mu.Unlock()
return nil
}
func (r *Remote) SetLimits(ctx context.Context, machine string, size model.Size) error {
return r.callNode(ctx, "set_limits", callParams{Machine: machine, Size: size}, nil)
}
func (r *Remote) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
return r.callNode(ctx, "set_quota", callParams{Drive: drive, Machine: machine, MB: mb}, nil)
}
func (r *Remote) WaitStopped(ctx context.Context, machine string) error {
return r.callNode(ctx, "wait_stopped", callParams{Machine: machine}, nil)
}
func (r *Remote) ExportHome(ctx context.Context, drive, machine, parent string) (string, error) {
var res callParams
err := r.callNode(ctx, "export_home", callParams{Drive: drive, Machine: machine, Parent: parent}, &res)
return res.ID, err
}
func (r *Remote) ReadExport(ctx context.Context, id string, max int) ([]byte, bool, error) {
var res callParams
err := r.callNode(ctx, "read_export", callParams{ID: id, Max: max}, &res)
return res.Data, res.EOF, err
}
func (r *Remote) CloseExport(ctx context.Context, id string) error {
return r.callNode(ctx, "close_export", callParams{ID: id}, nil)
}
func (r *Remote) ImportHome(ctx context.Context, drive, machine string) (string, error) {
var res callParams
err := r.callNode(ctx, "import_home", callParams{Drive: drive, Machine: machine}, &res)
return res.ID, err
}
func (r *Remote) WriteImport(ctx context.Context, id string, data []byte) error {
return r.callNode(ctx, "write_import", callParams{ID: id, Data: data}, nil)
}
func (r *Remote) EndReceive(ctx context.Context, id string) error {
return r.callNode(ctx, "end_receive", callParams{ID: id}, nil)
}
func (r *Remote) FinishImport(ctx context.Context, id string, quotaMB int64) error {
return r.callNode(ctx, "finish_import", callParams{ID: id, MB: quotaMB}, nil)
}
func (r *Remote) AbortImport(ctx context.Context, id string) error {
return r.callNode(ctx, "abort_import", callParams{ID: id}, nil)
}
func (r *Remote) DeleteHome(ctx context.Context, drive, machine string) error {
return r.callNode(ctx, "delete_home", callParams{Drive: drive, Machine: machine}, nil)
}
func (r *Remote) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
l, err := r.currentLink()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(ctx, callTimeout)
defer cancel()
return l.openAgent(ctx, machine, secret)
}
func (m *Manager) attach(name string, l *link) (*Remote, error) {
if name == "" || name == m.local.Name() {
return nil, fmt.Errorf("node name %q is empty or taken by the master", name)
}
m.mu.Lock()
r := m.remotes[name]
if r == nil {
r = &Remote{name: name, starting: map[string]time.Time{}, stopping: map[string]time.Time{}}
m.remotes[name] = r
}
m.mu.Unlock()
r.mu.Lock()
old := r.link
r.link = l
r.infoAt = time.Time{}
r.mu.Unlock()
if old != nil {
old.Close()
}
return r, nil
}
func (m *Manager) ServeCluster(ctx context.Context) error {
cc := m.cfg.Cluster
cert, fingerprint, err := loadOrCreateCert(m.cfg.StateDir)
if err != nil {
return err
}
log.Printf("cluster: listening on %s, certificate fingerprint %s", cc.Listen, fingerprint)
mux := http.NewServeMux()
mux.HandleFunc("/cluster", func(w http.ResponseWriter, req *http.Request) {
if subtle.ConstantTimeCompare([]byte(req.Header.Get("X-Cluster-Secret")), []byte(cc.Secret)) != 1 {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
name := req.Header.Get("X-Node-Name")
conn, err := websocket.Accept(w, req, nil)
if err != nil {
return
}
l := newLink(conn)
r, err := m.attach(name, l)
if err != nil {
conn.Close(websocket.StatusPolicyViolation, err.Error())
return
}
back := false
l.onReport = func(info node.NodeInfo) {
r.setInfo(info)
if !back {
back = true
log.Printf("cluster: node %s connected", name)
if m.onNodeBack != nil {
go m.onNodeBack(name)
}
}
}
l.onCall = func(f frame) { l.reply(f.ID, nil, errors.New("the master takes no calls")) }
l.run()
log.Printf("cluster: node %s disconnected: %v", name, l.err)
})
srv := &http.Server{
Addr: cc.Listen,
Handler: mux,
TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13},
}
go func() {
<-ctx.Done()
srv.Close()
}()
if err := srv.ListenAndServeTLS("", ""); !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}
func loadOrCreateCert(dir string) (tls.Certificate, string, error) {
certPath, keyPath := filepath.Join(dir, "cluster.crt"), filepath.Join(dir, "cluster.key")
if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
return cert, fingerprintOf(cert.Certificate[0]), nil
}
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return tls.Certificate{}, "", err
}
serial, _ := rand.Int(rand.Reader, big.NewInt(1<<62))
tmpl := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: "slackerbot master"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().AddDate(20, 0, 0),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
return tls.Certificate{}, "", err
}
keyDER, err := x509.MarshalECPrivateKey(key)
if err != nil {
return tls.Certificate{}, "", err
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return tls.Certificate{}, "", err
}
if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
return tls.Certificate{}, "", err
}
if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
return tls.Certificate{}, "", err
}
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
return cert, fingerprintOf(der), err
}
func fingerprintOf(der []byte) string {
sum := sha256.Sum256(der)
return hex.EncodeToString(sum[:])
}
func normalizeFingerprint(s string) string {
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), ":", ""))
}
func RunSlave(ctx context.Context, cfg *config.Config, local node.Node) error {
backoff := time.Second
for {
started := time.Now()
err := slaveSession(ctx, cfg, local)
if ctx.Err() != nil {
return nil
}
if time.Since(started) > time.Minute {
backoff = time.Second
}
log.Printf("cluster: link to master lost (%v), retrying in %s", err, backoff)
select {
case <-ctx.Done():
return nil
case <-time.After(backoff):
}
backoff = min(backoff*2, 30*time.Second)
}
}
func pinnedTLS(cfg *config.Config) *tls.Config {
pinned := normalizeFingerprint(cfg.Cluster.MasterFingerprint)
savedPath := filepath.Join(cfg.StateDir, "master.fingerprint")
if pinned == "" {
if b, err := os.ReadFile(savedPath); err == nil {
pinned = normalizeFingerprint(string(b))
}
}
return &tls.Config{
MinVersion: tls.VersionTLS13,
// Not verified against a CA: the master's self-signed certificate is pinned instead.
InsecureSkipVerify: true,
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
if len(raw) == 0 {
return errors.New("master sent no certificate")
}
got := fingerprintOf(raw[0])
if pinned == "" {
log.Printf("cluster: trusting master certificate %s from now on", got)
pinned = got
_ = os.MkdirAll(cfg.StateDir, 0o700)
return os.WriteFile(savedPath, []byte(got+"\n"), 0o600)
}
if got != pinned {
return fmt.Errorf("master certificate %s doesn't match the pinned %s", got, pinned)
}
return nil
},
}
}
func slaveSession(ctx context.Context, cfg *config.Config, local node.Node) error {
client := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(cfg), Proxy: http.ProxyFromEnvironment}}
dialCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
conn, _, err := websocket.Dial(dialCtx, strings.TrimSuffix(cfg.Cluster.Master, "/")+"/cluster", &websocket.DialOptions{
HTTPClient: client,
HTTPHeader: http.Header{
"X-Cluster-Secret": []string{cfg.Cluster.Secret},
"X-Node-Name": []string{cfg.Cluster.Name},
},
})
cancel()
if err != nil {
return err
}
l := newLink(conn)
defer l.Close()
l.onCall = func(f frame) { serveCall(ctx, l, local, f) }
go l.run()
log.Printf("cluster: connected to master %s as %s", cfg.Cluster.Master, cfg.Cluster.Name)
t := time.NewTicker(reportEvery)
defer t.Stop()
for {
info, err := local.Info(ctx)
if err != nil {
log.Printf("cluster: reading node info: %v", err)
} else if err := l.report(ctx, info); err != nil {
return err
}
select {
case <-ctx.Done():
return nil
case <-l.Done():
return l.err
case <-t.C:
}
}
}
type callParams struct {
Machine string `json:"machine,omitempty"`
Secret string `json:"secret,omitempty"`
Drive string `json:"drive,omitempty"`
MB int64 `json:"mb,omitempty"`
Size model.Size `json:"size"`
ID string `json:"id,omitempty"`
Parent string `json:"parent,omitempty"`
Max int `json:"max,omitempty"`
Data []byte `json:"data,omitempty"`
EOF bool `json:"eof,omitempty"`
}
func serveCall(ctx context.Context, l *link, local node.Node, f frame) {
var p callParams
_ = json.Unmarshal(f.Params, &p)
switch f.Method {
case "start":
var spec node.StartSpec
if err := json.Unmarshal(f.Params, &spec); err != nil {
l.reply(f.ID, nil, err)
return
}
l.reply(f.ID, nil, local.Start(ctx, spec))
case "poweroff":
l.reply(f.ID, nil, local.Poweroff(ctx, p.Machine))
case "set_limits":
l.reply(f.ID, nil, local.SetLimits(ctx, p.Machine, p.Size))
case "set_quota":
l.reply(f.ID, nil, local.SetQuota(ctx, p.Drive, p.Machine, p.MB))
case "wait_stopped":
l.reply(f.ID, nil, local.WaitStopped(ctx, p.Machine))
case "export_home":
id, err := local.ExportHome(ctx, p.Drive, p.Machine, p.Parent)
l.reply(f.ID, callParams{ID: id}, err)
case "read_export":
data, eof, err := local.ReadExport(ctx, p.ID, min(p.Max, transferChunk))
l.reply(f.ID, callParams{Data: data, EOF: eof}, err)
case "close_export":
l.reply(f.ID, nil, local.CloseExport(ctx, p.ID))
case "import_home":
id, err := local.ImportHome(ctx, p.Drive, p.Machine)
l.reply(f.ID, callParams{ID: id}, err)
case "write_import":
l.reply(f.ID, nil, local.WriteImport(ctx, p.ID, p.Data))
case "end_receive":
l.reply(f.ID, nil, local.EndReceive(ctx, p.ID))
case "finish_import":
l.reply(f.ID, nil, local.FinishImport(ctx, p.ID, p.MB))
case "abort_import":
l.reply(f.ID, nil, local.AbortImport(ctx, p.ID))
case "delete_home":
l.reply(f.ID, nil, local.DeleteHome(ctx, p.Drive, p.Machine))
case "open_agent":
agent, err := local.DialAgent(ctx, p.Machine, p.Secret)
if err != nil {
l.reply(f.ID, nil, err)
return
}
s := l.addStream(f.ID)
l.reply(f.ID, nil, nil)
pipe(ctx, agent, s)
default:
l.reply(f.ID, nil, fmt.Errorf("unknown call %q", f.Method))
}
}
func pipe(ctx context.Context, a, b node.MsgConn) {
ctx, cancel := context.WithCancel(ctx)
copyTo := func(dst, src node.MsgConn) {
defer cancel()
for {
msg, err := src.Read(ctx)
if err != nil {
return
}
if err := dst.Write(ctx, msg); err != nil {
return
}
}
}
go copyTo(a, b)
copyTo(b, a)
<-ctx.Done()
a.Close()
b.Close()
}
+137
View File
@@ -0,0 +1,137 @@
package manager
import (
"context"
"errors"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
"slackerbot/internal/config"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/state"
"slackerbot/internal/testutil"
)
func TestMasterAndSlave(t *testing.T) {
addr := testutil.FreePort(t)
secret := "cluster-secret-1234567890"
masterDir, slaveDir := t.TempDir(), t.TempDir()
masterCfg := &config.Config{StateDir: masterDir, Container: model.Size{MemoryMB: 1024, CPUs: 1}}
masterCfg.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Listen: addr}
store, err := state.Open(filepath.Join(masterDir, "state.json"))
if err != nil {
t.Fatal(err)
}
mgr := New(masterCfg, store, testutil.NewFakeNode("master"))
backCh := make(chan string, 4)
mgr.onNodeBack = func(n string) { backCh <- n }
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go mgr.ServeCluster(ctx)
slaveCfg := &config.Config{StateDir: slaveDir}
slaveCfg.Cluster = config.ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr}
slave := testutil.NewFakeNode("slave-1")
slaveCtx, stopSlave := context.WithCancel(ctx)
go RunSlave(slaveCtx, slaveCfg, slave)
testutil.WaitFor(t, "slave to come online", func() bool { _, err := mgr.nodeByName("slave-1"); return err == nil })
if got := <-backCh; got != "slave-1" {
t.Fatalf("back callback for %q", got)
}
pinned, err := os.ReadFile(filepath.Join(slaveDir, "master.fingerprint"))
if err != nil || len(strings.TrimSpace(string(pinned))) != 64 {
t.Fatalf("slave should pin the master's certificate, got %q %v", pinned, err)
}
n, _ := mgr.nodeByName("slave-1")
spec := node.StartSpec{Machine: "c-abc", Secret: "agent-secret", Size: model.Size{MemoryMB: 1024, CPUs: 1}, Drive: "/d"}
if err := n.Start(ctx, spec); err != nil {
t.Fatal(err)
}
if !slave.IsRunning("c-abc") || slave.Last.Drive != "/d" {
t.Fatal("start didn't reach the slave")
}
if !n.IsRunning("c-abc") {
t.Fatal("master should count the container as running right away")
}
conn, err := n.DialAgent(ctx, "c-abc", "agent-secret")
if err != nil {
t.Fatal(err)
}
if err := conn.Write(ctx, []byte(`{"jsonrpc":"2.0"}`)); err != nil {
t.Fatal(err)
}
readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second)
got, err := conn.Read(readCtx)
readCancel()
if err != nil || string(got) != `echo: {"jsonrpc":"2.0"}` {
t.Fatalf("relay got %q %v", got, err)
}
conn.Close()
if _, err := n.DialAgent(ctx, "c-abc", "wrong"); err == nil {
t.Fatal("a bad agent secret should fail through the relay")
}
if err := n.Poweroff(ctx, "c-abc"); err != nil {
t.Fatal(err)
}
if slave.IsRunning("c-abc") || n.IsRunning("c-abc") {
t.Fatal("poweroff didn't take")
}
stopSlave()
testutil.WaitFor(t, "slave to go offline", func() bool {
_, err := mgr.nodeByName("slave-1")
var off *node.NodeOfflineError
return errors.As(err, &off) && off.Node == "slave-1"
})
if off := mgr.OfflineNodes(); len(off) != 1 || off[0] != "slave-1" {
t.Fatalf("offline nodes = %v", off)
}
}
func TestSlaveRejectsWrongMasterOrSecret(t *testing.T) {
addr := testutil.FreePort(t)
secret := "cluster-secret-1234567890"
masterCfg := &config.Config{StateDir: t.TempDir()}
masterCfg.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Listen: addr}
store, _ := state.Open(filepath.Join(masterCfg.StateDir, "state.json"))
mgr := New(masterCfg, store, testutil.NewFakeNode("master"))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go mgr.ServeCluster(ctx)
testutil.WaitFor(t, "master to listen", func() bool {
c, err := net.Dial("tcp", addr)
if err == nil {
c.Close()
}
return err == nil
})
wrongPin := &config.Config{StateDir: t.TempDir()}
wrongPin.Cluster = config.ClusterConfig{Name: "s", Secret: secret, Master: "wss://" + addr, MasterFingerprint: strings.Repeat("ab", 32)}
if err := slaveSession(ctx, wrongPin, testutil.NewFakeNode("s")); err == nil {
t.Fatal("a master with a different certificate should be refused")
}
wrongSecret := &config.Config{StateDir: t.TempDir()}
wrongSecret.Cluster = config.ClusterConfig{Name: "s", Secret: "not-the-secret-at-all", Master: "wss://" + addr}
if err := slaveSession(ctx, wrongSecret, testutil.NewFakeNode("s")); err == nil {
t.Fatal("a slave with the wrong secret should be refused")
}
sameName := &config.Config{StateDir: t.TempDir()}
sameName.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Master: "wss://" + addr}
if err := slaveSession(ctx, sameName, testutil.NewFakeNode("master")); err == nil {
t.Fatal("a slave can't take the master's name")
}
}
+427
View File
@@ -0,0 +1,427 @@
package manager
import (
"context"
"errors"
"fmt"
"math"
"sort"
"sync"
"time"
"slackerbot/internal/balance"
"slackerbot/internal/config"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/state"
"slackerbot/internal/storage"
"slackerbot/internal/util"
)
var (
ErrTooBig = errors.New("bigger than max_container allows")
ErrTooSmall = errors.New("smaller than the minimum size")
MinContainer = model.Size{MemoryMB: 256, CPUs: 0.25}
)
type State string
const (
StateRunning State = "running"
StateStopped State = "stopped"
StateNone State = "not created"
StateOffline State = "on an offline node"
)
type Manager struct {
cfg *config.Config
store *state.Store
local node.Node
mu sync.RWMutex
remotes map[string]*Remote
onNodeBack func(node string)
startMu sync.Mutex
ownerMu util.KeyedMutex
migrating sync.Map
}
func New(cfg *config.Config, store *state.Store, local node.Node) *Manager {
return &Manager{cfg: cfg, store: store, local: local, remotes: map[string]*Remote{}}
}
func (m *Manager) SetNodeBackHandler(fn func(node string)) { m.onNodeBack = fn }
func (m *Manager) nodeByName(name string) (node.Node, error) {
if name == m.local.Name() {
return m.local, nil
}
m.mu.RLock()
r := m.remotes[name]
m.mu.RUnlock()
if r != nil && r.Online() {
return r, nil
}
return nil, &node.NodeOfflineError{Node: name}
}
func (m *Manager) onlineNodes() []node.Node {
nodes := []node.Node{m.local}
m.mu.RLock()
defer m.mu.RUnlock()
for _, r := range m.remotes {
if r.Online() {
nodes = append(nodes, r)
}
}
return nodes
}
func (m *Manager) OfflineNodes() []string {
m.mu.RLock()
defer m.mu.RUnlock()
var out []string
for name, r := range m.remotes {
if !r.Online() {
out = append(out, name)
}
}
sort.Strings(out)
return out
}
func (m *Manager) Views(ctx context.Context) ([]balance.NodeView, error) {
owners := m.store.Owners()
var views []balance.NodeView
for _, n := range m.onlineNodes() {
info, err := n.Info(ctx)
if err != nil {
if n == m.local {
return nil, err
}
continue
}
views = append(views, balance.BuildView(info, owners, m.cfg.Container))
}
sort.Slice(views, func(i, j int) bool { return views[i].Name < views[j].Name })
return views, nil
}
func (m *Manager) view(ctx context.Context, n node.Node) (balance.NodeView, node.NodeInfo, error) {
info, err := n.Info(ctx)
if err != nil {
return balance.NodeView{}, info, err
}
return balance.BuildView(info, m.store.Owners(), m.cfg.Container), info, nil
}
func (m *Manager) Totals(ctx context.Context) (used, usable model.Size) {
views, _ := m.Views(ctx)
for _, v := range views {
used, usable = used.Add(v.Used), usable.Add(v.Usable)
}
return
}
func (m *Manager) ownerRecord(id string, kind model.OwnerKind) (*model.Owner, error) {
if u := m.store.Owner(id); u != nil {
return u, nil
}
u := &model.Owner{
ID: id,
Kind: kind,
Machine: model.MachineName(id, kind),
Secret: util.RandomHex(32),
Size: m.cfg.Container,
StorageMinMB: m.cfg.Storage.MinMB,
StorageMaxMB: m.cfg.Storage.MaxMB,
LastActive: time.Now(),
}
return u, m.store.PutOwner(u)
}
func (m *Manager) Owner(id string, kind model.OwnerKind) (*model.Owner, error) {
unlock := m.ownerMu.Lock(id)
defer unlock()
return m.ownerRecord(id, kind)
}
func (m *Manager) spec(u *model.Owner) node.StartSpec {
return node.StartSpec{
Machine: u.Machine,
Description: "slackerbot container for " + string(u.Kind) + " " + u.ID,
Secret: u.Secret,
Size: u.Size,
Drive: u.Drive,
StorageMaxMB: u.StorageMaxMB,
Env: m.cfg.AgentEnv,
}
}
func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (*model.Owner, error) {
if m.IsMigrating(id) {
return nil, ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
u, err := m.ownerRecord(id, kind)
if err != nil {
return nil, err
}
if u.Node != "" {
n, err := m.nodeByName(u.Node)
if err != nil {
return u, err
}
if n.IsRunning(u.Machine) {
return u, nil
}
}
m.startMu.Lock()
defer m.startMu.Unlock()
if u.Node == "" {
views, err := m.Views(ctx)
if err != nil {
return u, err
}
target, drive, err := balance.ChoosePlacement(views, u.Size, u.StorageMinMB)
if err != nil {
return u, err
}
u.Node, u.Drive = target, drive
if err := m.store.SetPlacement(u.ID, target, drive); err != nil {
return u, err
}
}
n, err := m.nodeByName(u.Node)
if err != nil {
return u, err
}
v, _, err := m.view(ctx, n)
if err != nil {
return u, err
}
if !v.Used.Add(u.Size).Fits(v.Usable) {
return u, &node.NodeFullError{Node: u.Node}
}
return u, n.Start(ctx, m.spec(u))
}
// The old home folder stays on its node, untouched.
func (m *Manager) Recreate(id string) error {
if m.IsMigrating(id) {
return ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
if err := m.store.SetPlacement(id, "", ""); err != nil {
return err
}
return m.store.DeleteSessions(id)
}
func (m *Manager) DialAgent(ctx context.Context, u *model.Owner) (node.MsgConn, error) {
n, err := m.nodeByName(u.Node)
if err != nil {
return nil, err
}
return n.DialAgent(ctx, u.Machine, u.Secret)
}
func (m *Manager) State(u *model.Owner) State {
if u == nil || u.Node == "" {
return StateNone
}
n, err := m.nodeByName(u.Node)
if err != nil {
return StateOffline
}
if n.IsRunning(u.Machine) {
return StateRunning
}
return StateStopped
}
func (m *Manager) Poweroff(ctx context.Context, u *model.Owner) error {
n, err := m.nodeByName(u.Node)
if err != nil {
return err
}
return n.Poweroff(ctx, u.Machine)
}
func (m *Manager) RunningOwners(ctx context.Context) []model.Owner {
byNodeMachine := map[string]model.Owner{}
for _, u := range m.store.Owners() {
byNodeMachine[u.Node+"/"+u.Machine] = u
}
var out []model.Owner
for _, n := range m.onlineNodes() {
running, err := n.Running(ctx)
if err != nil {
continue
}
for _, r := range running {
if u, ok := byNodeMachine[n.Name()+"/"+r.Name]; ok {
out = append(out, u)
}
}
}
return out
}
func (m *Manager) Resize(ctx context.Context, id string, kind model.OwnerKind, change func(model.Size) (model.Size, error)) (old, next model.Size, err error) {
if m.IsMigrating(id) {
return old, old, ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
m.startMu.Lock()
defer m.startMu.Unlock()
u, err := m.ownerRecord(id, kind)
if err != nil {
return model.Size{}, model.Size{}, err
}
old = u.Size
if next, err = change(old); err != nil {
return old, old, err
}
next.CPUs = math.Round(next.CPUs*100) / 100
switch {
case !MinContainer.Fits(next):
return old, old, ErrTooSmall
case !next.Fits(m.cfg.MaxContainer):
return old, old, ErrTooBig
case next == old:
return old, old, nil
}
if u.Node != "" {
n, err := m.nodeByName(u.Node)
if err != nil {
return old, old, err
}
if n.IsRunning(u.Machine) {
v, _, err := m.view(ctx, n)
if err != nil {
return old, old, err
}
growth := model.Size{MemoryMB: max(0, next.MemoryMB-old.MemoryMB), CPUs: math.Max(0, next.CPUs-old.CPUs)}
if !v.Used.Add(growth).Fits(v.Usable) {
return old, old, &node.NodeFullError{Node: u.Node}
}
if err := n.SetLimits(ctx, u.Machine, next); err != nil {
return old, old, err
}
}
}
return old, next, m.store.SetSize(id, next)
}
type StorageChange struct {
OldMin, OldMax, NewMin, NewMax int64
UsedMB int64
Drive *storage.DriveReport
}
func (m *Manager) SetStorage(ctx context.Context, id string, kind model.OwnerKind, change func(min, max int64) (int64, int64, error)) (*StorageChange, error) {
if m.IsMigrating(id) {
return &StorageChange{}, ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
m.startMu.Lock()
defer m.startMu.Unlock()
u, err := m.ownerRecord(id, kind)
if err != nil {
return nil, err
}
c := &StorageChange{OldMin: u.StorageMinMB, OldMax: u.StorageMaxMB}
if c.NewMin, c.NewMax, err = change(c.OldMin, c.OldMax); err != nil {
return c, err
}
switch {
case c.NewMin < config.MinStorageMB || c.NewMax < config.MinStorageMB:
return c, storage.ErrStorageTooSmall
case c.NewMin > c.NewMax:
return c, storage.ErrMinOverMax
case c.NewMax > m.cfg.Storage.LimitMB:
return c, storage.ErrStorageTooBig
}
if u.Node == "" {
views, err := m.Views(ctx)
if err != nil {
return c, err
}
if _, _, err := balance.ChoosePlacement(views, u.Size, c.NewMin); err != nil {
return c, err
}
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
}
n, err := m.nodeByName(u.Node)
if err != nil {
return c, err
}
v, info, err := m.view(ctx, n)
if err != nil {
return c, err
}
for _, d := range info.Drives {
if d.Path == u.Drive {
c.UsedMB = d.HomesMB[u.Machine]
}
}
if c.NewMax < c.UsedMB {
return c, storage.ErrBelowUsage
}
var r *storage.DriveReport
for i := range v.Drives {
if v.Drives[i].Path == u.Drive {
r = &v.Drives[i]
}
}
if r == nil {
return c, fmt.Errorf("drive %s is no longer configured on %s", u.Drive, u.Node)
}
extra := storage.Committed(c.NewMin, c.UsedMB) - storage.Committed(c.OldMin, c.UsedMB)
if extra > 0 && extra > r.HeadroomMB() {
return c, storage.ErrStorageFull
}
if err := n.SetQuota(ctx, u.Drive, u.Machine, c.NewMax); err != nil {
return c, err
}
r.CommittedMB += extra
r.BookedMB += c.NewMax - c.OldMax
c.Drive = r
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
}
func (m *Manager) StorageUsed(ctx context.Context, u *model.Owner) (int64, bool) {
if u == nil || u.Node == "" {
return 0, false
}
n, err := m.nodeByName(u.Node)
if err != nil {
return 0, false
}
info, err := n.Info(ctx)
if err != nil {
return 0, false
}
for _, d := range info.Drives {
if used, ok := d.HomesMB[u.Machine]; ok && d.Path == u.Drive {
return used, true
}
}
return 0, false
}
+237
View File
@@ -0,0 +1,237 @@
package manager
import (
"context"
"errors"
"fmt"
"log"
"slackerbot/internal/balance"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/storage"
)
const transferChunk = 4 << 20
var (
ErrMigrating = errors.New("this container is being moved")
ErrNoContainer = errors.New("there's no container here yet")
ErrSameNode = errors.New("the container is already on that node")
)
type UnknownNodeError struct{ Node string }
func (e *UnknownNodeError) Error() string { return fmt.Sprintf("there's no node called %s", e.Node) }
type MovePhase string
const (
PhaseCopy MovePhase = "copy"
PhaseFinal MovePhase = "final"
PhaseStart MovePhase = "start"
PhaseCleanup MovePhase = "cleanup"
)
type MoveProgress struct {
To string
Phase MovePhase
CopiedMB int64
TotalMB int64
}
type MoveOptions struct {
Target string
StartAfter bool
WaitIdle func(ctx context.Context)
Progress func(MoveProgress)
}
type MoveResult struct {
From, To string
WasRunning bool
Started bool
CopiedMB int64
LeftOld error
}
func (m *Manager) IsMigrating(id string) bool {
_, ok := m.migrating.Load(id)
return ok
}
// Migrate copies the owner's home folder to another node and switches the
// container over. A running container keeps running through the bulk copy
// and is only stopped for a final incremental sync.
func (m *Manager) Migrate(ctx context.Context, id string, opts MoveOptions) (*MoveResult, error) {
if _, busy := m.migrating.LoadOrStore(id, true); busy {
return nil, ErrMigrating
}
defer m.migrating.Delete(id)
unlock := m.ownerMu.Lock(id)
defer unlock()
u := m.store.Owner(id)
if u == nil || u.Node == "" {
return nil, ErrNoContainer
}
src, err := m.nodeByName(u.Node)
if err != nil {
return nil, err
}
res := &MoveResult{From: u.Node, WasRunning: src.IsRunning(u.Machine)}
needStart := res.WasRunning || opts.StartAfter
usedMB, _ := m.StorageUsed(ctx, u)
target, drive, err := m.pickTarget(ctx, u, opts.Target, needStart, storage.Committed(u.StorageMinMB, usedMB))
if err != nil {
return nil, err
}
res.To = target
dst, err := m.nodeByName(target)
if err != nil {
return nil, err
}
progress := func(p MovePhase, copied int64) {
if opts.Progress != nil {
opts.Progress(MoveProgress{To: target, Phase: p, CopiedMB: copied >> 20, TotalMB: usedMB})
}
}
imp, err := dst.ImportHome(ctx, drive, u.Machine)
if err != nil {
return nil, err
}
var exports []string
stopped := false
rollback := func(cause error) (*MoveResult, error) {
_ = dst.AbortImport(context.Background(), imp)
for _, e := range exports {
_ = src.CloseExport(context.Background(), e)
}
if stopped {
if err := src.Start(context.Background(), m.spec(u)); err != nil {
return res, fmt.Errorf("%w (and restarting it on %s failed: %v)", cause, u.Node, err)
}
}
return res, cause
}
var copied int64
send := func(parent string, phase MovePhase) error {
exp, err := src.ExportHome(ctx, u.Drive, u.Machine, parent)
if err != nil {
return err
}
exports = append(exports, exp)
for {
data, eof, err := src.ReadExport(ctx, exp, transferChunk)
if err != nil {
return err
}
if len(data) > 0 {
if err := dst.WriteImport(ctx, imp, data); err != nil {
return err
}
copied += int64(len(data))
progress(phase, copied)
}
if eof {
return dst.EndReceive(ctx, imp)
}
}
}
progress(PhaseCopy, 0)
if err := send("", PhaseCopy); err != nil {
return rollback(err)
}
if res.WasRunning {
progress(PhaseFinal, copied)
if opts.WaitIdle != nil {
opts.WaitIdle(ctx)
}
if err := src.Poweroff(ctx, u.Machine); err != nil {
return rollback(err)
}
stopped = true
if err := src.WaitStopped(ctx, u.Machine); err != nil {
return rollback(err)
}
if err := send(exports[0], PhaseFinal); err != nil {
return rollback(err)
}
}
if err := dst.FinishImport(ctx, imp, u.StorageMaxMB); err != nil {
return rollback(err)
}
res.CopiedMB = copied >> 20
oldNode, oldDrive := u.Node, u.Drive
if err := m.store.SetPlacement(id, target, drive); err != nil {
return rollback(err)
}
u.Node, u.Drive = target, drive
progress(PhaseCleanup, copied)
for _, e := range exports {
_ = src.CloseExport(ctx, e)
}
if res.LeftOld = src.DeleteHome(ctx, oldDrive, u.Machine); res.LeftOld != nil {
log.Printf("move: old home of %s left on %s: %v", u.Machine, oldNode, res.LeftOld)
}
if !needStart {
return res, nil
}
progress(PhaseStart, copied)
if err := dst.Start(ctx, m.spec(u)); err != nil {
return res, fmt.Errorf("moved to %s, but starting it there failed: %w", target, err)
}
res.Started = true
return res, nil
}
func (m *Manager) pickTarget(ctx context.Context, u *model.Owner, want string, needStart bool, storageMB int64) (target, drive string, err error) {
if want == u.Node {
return "", "", ErrSameNode
}
m.startMu.Lock()
defer m.startMu.Unlock()
views, err := m.Views(ctx)
if err != nil {
return "", "", err
}
compute := model.Size{}
if needStart {
compute = u.Size
}
var candidates []balance.NodeView
for _, v := range views {
if v.Name != u.Node && (want == "" || v.Name == want) {
candidates = append(candidates, v)
}
}
if want != "" && len(candidates) == 0 {
if m.KnownNode(want) {
return "", "", &node.NodeOfflineError{Node: want}
}
return "", "", &UnknownNodeError{Node: want}
}
target, drive, err = balance.ChoosePlacement(candidates, compute, storageMB)
if err != nil && want != "" {
return "", "", &node.NodeFullError{Node: want}
}
return target, drive, err
}
func (m *Manager) KnownNode(name string) bool {
if name == m.local.Name() {
return true
}
m.mu.RLock()
defer m.mu.RUnlock()
_, ok := m.remotes[name]
return ok
}
+159
View File
@@ -0,0 +1,159 @@
package manager_test
import (
"bytes"
"context"
"errors"
"testing"
"slackerbot/internal/manager"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/testutil"
"slackerbot/internal/testutil/clustertest"
)
func TestMoveLiveContainer(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), true)
var phases []manager.MovePhase
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{
Target: "slave-1",
WaitIdle: func(context.Context) {
c.Master.Mu.Lock()
c.Master.Homes["/d/c-c1"] = append(c.Master.Homes["/d/c-c1"], []byte("written during the bulk copy")...)
c.Master.Mu.Unlock()
},
Progress: func(p manager.MoveProgress) {
if len(phases) == 0 || phases[len(phases)-1] != p.Phase {
phases = append(phases, p.Phase)
}
},
})
if err != nil {
t.Fatal(err)
}
want := append(testutil.HomeData(), []byte("written during the bulk copy")...)
if !bytes.Equal(c.Slave.Homes["/d/c-c1"], want) {
t.Fatal("the late write didn't make it to the new node")
}
if c.Slave.Passes != 2 {
t.Fatalf("a live move should copy twice (bulk + final sync), got %d", c.Slave.Passes)
}
if _, still := c.Master.Homes["/d/c-c1"]; still {
t.Fatal("old home should be deleted after a good move")
}
if !c.Slave.IsRunning("c-c1") || c.Master.IsRunning("c-c1") || !res.Started {
t.Fatal("container should be running on the new node only")
}
if u := c.Store.Owner("C1"); u.Node != "slave-1" || u.Drive != "/d" {
t.Fatalf("placement not switched: %+v", u)
}
if len(c.Master.Exports) != 0 {
t.Fatal("snapshots on the old node should be cleaned up")
}
wantPhases := []manager.MovePhase{manager.PhaseCopy, manager.PhaseFinal, manager.PhaseCleanup, manager.PhaseStart}
if len(phases) != len(wantPhases) {
t.Fatalf("phases %v, want %v", phases, wantPhases)
}
}
func TestMoveAutoStoppedContainer(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), false)
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{})
if err != nil {
t.Fatal(err)
}
if res.To != "slave-1" || res.Started || c.Slave.IsRunning("c-c1") {
t.Fatalf("a stopped container should move without starting: %+v", res)
}
if c.Slave.Passes != 1 {
t.Fatalf("a stopped container needs one copy, got %d", c.Slave.Passes)
}
}
func TestMoveFullNodeStartsElsewhere(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), false)
c.Master.Mu.Lock()
c.Master.Usable = model.Size{MemoryMB: 512, CPUs: 0.5}
c.Master.Mu.Unlock()
_, err := c.Mgr.Ensure(c.Ctx, "C1", model.OwnerChannel)
var full *node.NodeFullError
if !errors.As(err, &full) || full.Node != "master" {
t.Fatalf("want node full on master, got %v", err)
}
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{StartAfter: true})
if err != nil || !res.Started || !c.Slave.IsRunning("c-c1") {
t.Fatalf("auto-move should start it on the new node: %+v %v", res, err)
}
}
func TestMoveFailureRollsBack(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), true)
_, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{
Target: "slave-1",
WaitIdle: func(context.Context) {
c.Slave.Mu.Lock()
c.Slave.FailWrites = true
c.Slave.Mu.Unlock()
},
})
if err == nil {
t.Fatal("expected the final sync to fail")
}
if u := c.Store.Owner("C1"); u.Node != "master" {
t.Fatalf("container should stay on its old node, got %s", u.Node)
}
if !c.Master.IsRunning("c-c1") {
t.Fatal("container should be restarted on its old node")
}
if _, ok := c.Master.Homes["/d/c-c1"]; !ok {
t.Fatal("old home must survive a failed move")
}
if _, ok := c.Slave.Homes["/d/c-c1"]; ok || len(c.Slave.Imports) != 0 {
t.Fatal("half-copied home on the target should be cleaned up")
}
}
func TestMoveRefusals(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), false)
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "master"}); !errors.Is(err, manager.ErrSameNode) {
t.Fatalf("same node: %v", err)
}
var unknown *manager.UnknownNodeError
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "nope"}); !errors.As(err, &unknown) {
t.Fatalf("unknown node: %v", err)
}
if _, err := c.Mgr.Migrate(c.Ctx, "nobody", manager.MoveOptions{}); !errors.Is(err, manager.ErrNoContainer) {
t.Fatalf("no container: %v", err)
}
c.Slave.Mu.Lock()
c.Slave.Usable = model.Size{MemoryMB: 512, CPUs: 0.5}
c.Slave.Mu.Unlock()
c.Master.Mu.Lock()
c.Master.Runs["c-c1"] = true
c.Master.Mu.Unlock()
testutil.WaitFor(t, "slave report", func() bool {
views, _ := c.Mgr.Views(c.Ctx)
for _, v := range views {
if v.Name == "slave-1" {
return v.Usable.MemoryMB == 512
}
}
return false
})
var full *node.NodeFullError
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "slave-1"}); !errors.As(err, &full) {
t.Fatalf("full target: %v", err)
}
}