clean up of the codebase
This commit is contained in:
1 parent
7822529a38
commit
0ac61b4ece
52 files changed
+2545
-2274
No files matched your search
@@ -0,0 +1,787 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
)
|
||||
|
||||
const (
|
||||
reportEvery = 5 * time.Second
|
||||
offlineAfter = 20 * time.Second
|
||||
callTimeout = 3 * time.Minute
|
||||
pendingTimeout = 30 * time.Second
|
||||
streamBuffer = 4096
|
||||
)
|
||||
|
||||
type frame struct {
|
||||
Type string `json:"t"`
|
||||
ID int64 `json:"id,omitempty"`
|
||||
Method string `json:"m,omitempty"`
|
||||
Params json.RawMessage `json:"p,omitempty"`
|
||||
Result json.RawMessage `json:"r,omitempty"`
|
||||
Error string `json:"e,omitempty"`
|
||||
Data string `json:"d,omitempty"`
|
||||
}
|
||||
|
||||
type link struct {
|
||||
conn *websocket.Conn
|
||||
writeMu sync.Mutex
|
||||
nextID atomic.Int64
|
||||
|
||||
mu sync.Mutex
|
||||
pending map[int64]chan frame
|
||||
streams map[int64]*stream
|
||||
|
||||
onCall func(f frame)
|
||||
onReport func(info node.NodeInfo)
|
||||
|
||||
done chan struct{}
|
||||
err error
|
||||
}
|
||||
|
||||
func newLink(conn *websocket.Conn) *link {
|
||||
conn.SetReadLimit(64 << 20)
|
||||
return &link{
|
||||
conn: conn,
|
||||
pending: map[int64]chan frame{},
|
||||
streams: map[int64]*stream{},
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *link) Done() <-chan struct{} { return l.done }
|
||||
|
||||
func (l *link) Close() { l.conn.Close(websocket.StatusNormalClosure, "") }
|
||||
|
||||
func (l *link) send(ctx context.Context, f frame) error {
|
||||
b, err := json.Marshal(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l.writeMu.Lock()
|
||||
defer l.writeMu.Unlock()
|
||||
return l.conn.Write(ctx, websocket.MessageText, b)
|
||||
}
|
||||
|
||||
func (l *link) run() {
|
||||
defer func() {
|
||||
l.mu.Lock()
|
||||
streams := l.streams
|
||||
l.streams = map[int64]*stream{}
|
||||
l.mu.Unlock()
|
||||
for _, s := range streams {
|
||||
s.closeLocal()
|
||||
}
|
||||
close(l.done)
|
||||
}()
|
||||
for {
|
||||
_, data, err := l.conn.Read(context.Background())
|
||||
if err != nil {
|
||||
l.err = err
|
||||
return
|
||||
}
|
||||
var f frame
|
||||
if json.Unmarshal(data, &f) != nil {
|
||||
continue
|
||||
}
|
||||
switch f.Type {
|
||||
case "call":
|
||||
if l.onCall != nil {
|
||||
go l.onCall(f)
|
||||
}
|
||||
case "reply":
|
||||
l.mu.Lock()
|
||||
ch := l.pending[f.ID]
|
||||
l.mu.Unlock()
|
||||
if ch != nil {
|
||||
ch <- f
|
||||
}
|
||||
case "report":
|
||||
var info node.NodeInfo
|
||||
if l.onReport != nil && json.Unmarshal(f.Params, &info) == nil {
|
||||
l.onReport(info)
|
||||
}
|
||||
case "data":
|
||||
if s := l.stream(f.ID); s != nil {
|
||||
select {
|
||||
case s.in <- []byte(f.Data):
|
||||
default:
|
||||
s.Close()
|
||||
}
|
||||
}
|
||||
case "close":
|
||||
if s := l.stream(f.ID); s != nil {
|
||||
s.closeLocal()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *link) call(ctx context.Context, method string, params, result any) error {
|
||||
return l.callWithID(ctx, l.nextID.Add(1), method, params, result)
|
||||
}
|
||||
|
||||
func (l *link) callWithID(ctx context.Context, id int64, method string, params, result any) error {
|
||||
ch := make(chan frame, 1)
|
||||
l.mu.Lock()
|
||||
l.pending[id] = ch
|
||||
l.mu.Unlock()
|
||||
defer func() {
|
||||
l.mu.Lock()
|
||||
delete(l.pending, id)
|
||||
l.mu.Unlock()
|
||||
}()
|
||||
|
||||
p, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := l.send(ctx, frame{Type: "call", ID: id, Method: method, Params: p}); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-l.done:
|
||||
return fmt.Errorf("link closed: %v", l.err)
|
||||
case f := <-ch:
|
||||
if f.Error != "" {
|
||||
return errors.New(f.Error)
|
||||
}
|
||||
if result != nil && len(f.Result) > 0 {
|
||||
return json.Unmarshal(f.Result, result)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (l *link) reply(id int64, result any, err error) {
|
||||
f := frame{Type: "reply", ID: id}
|
||||
if err != nil {
|
||||
f.Error = err.Error()
|
||||
} else if result != nil {
|
||||
f.Result, _ = json.Marshal(result)
|
||||
}
|
||||
_ = l.send(context.Background(), f)
|
||||
}
|
||||
|
||||
func (l *link) report(ctx context.Context, info node.NodeInfo) error {
|
||||
p, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return l.send(ctx, frame{Type: "report", Params: p})
|
||||
}
|
||||
|
||||
func (l *link) stream(id int64) *stream {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.streams[id]
|
||||
}
|
||||
|
||||
func (l *link) addStream(id int64) *stream {
|
||||
s := &stream{l: l, id: id, in: make(chan []byte, streamBuffer), closed: make(chan struct{})}
|
||||
l.mu.Lock()
|
||||
l.streams[id] = s
|
||||
l.mu.Unlock()
|
||||
return s
|
||||
}
|
||||
|
||||
func (l *link) dropStream(id int64) {
|
||||
l.mu.Lock()
|
||||
delete(l.streams, id)
|
||||
l.mu.Unlock()
|
||||
}
|
||||
|
||||
// The stream shares its ID with the "open_agent" call, so it's registered
|
||||
// before the call goes out and no early data is lost.
|
||||
func (l *link) openAgent(ctx context.Context, machine, secret string) (*stream, error) {
|
||||
id := l.nextID.Add(1)
|
||||
s := l.addStream(id)
|
||||
err := l.callWithID(ctx, id, "open_agent", map[string]string{"machine": machine, "secret": secret}, nil)
|
||||
if err != nil {
|
||||
l.dropStream(id)
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
l *link
|
||||
id int64
|
||||
in chan []byte
|
||||
closed chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (s *stream) Read(ctx context.Context) ([]byte, error) {
|
||||
select {
|
||||
case b := <-s.in:
|
||||
return b, nil
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case b := <-s.in:
|
||||
return b, nil
|
||||
case <-s.closed:
|
||||
return nil, errors.New("stream closed")
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *stream) Write(ctx context.Context, b []byte) error {
|
||||
select {
|
||||
case <-s.closed:
|
||||
return errors.New("stream closed")
|
||||
default:
|
||||
}
|
||||
return s.l.send(ctx, frame{Type: "data", ID: s.id, Data: string(b)})
|
||||
}
|
||||
|
||||
func (s *stream) Close() error {
|
||||
s.once.Do(func() {
|
||||
_ = s.l.send(context.Background(), frame{Type: "close", ID: s.id})
|
||||
s.l.dropStream(s.id)
|
||||
close(s.closed)
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stream) closeLocal() {
|
||||
s.once.Do(func() {
|
||||
s.l.dropStream(s.id)
|
||||
close(s.closed)
|
||||
})
|
||||
}
|
||||
|
||||
type Remote struct {
|
||||
name string
|
||||
|
||||
mu sync.Mutex
|
||||
link *link
|
||||
info node.NodeInfo
|
||||
infoAt time.Time
|
||||
starting map[string]time.Time
|
||||
stopping map[string]time.Time
|
||||
}
|
||||
|
||||
func (r *Remote) Name() string { return r.name }
|
||||
|
||||
func (r *Remote) Online() bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.link == nil || time.Since(r.infoAt) > offlineAfter {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case <-r.link.Done():
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Remote) currentLink() (*link, error) {
|
||||
if !r.Online() {
|
||||
return nil, &node.NodeOfflineError{Node: r.name}
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.link, nil
|
||||
}
|
||||
|
||||
func (r *Remote) setInfo(info node.NodeInfo) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
info.Name = r.name
|
||||
r.info, r.infoAt = info, time.Now()
|
||||
for _, m := range info.Running {
|
||||
delete(r.starting, m.Name)
|
||||
}
|
||||
for m := range r.stopping {
|
||||
if !containsMachine(info.Running, m) {
|
||||
delete(r.stopping, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func containsMachine(list []node.RunningMachine, name string) bool {
|
||||
for _, m := range list {
|
||||
if m.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Reports can lag behind a start or stop the master just asked for,
|
||||
// so recent requests override them for a short while.
|
||||
func (r *Remote) runningLocked() []node.RunningMachine {
|
||||
var out []node.RunningMachine
|
||||
for _, m := range r.info.Running {
|
||||
if t, ok := r.stopping[m.Name]; ok && time.Since(t) < pendingTimeout {
|
||||
continue
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
for m, t := range r.starting {
|
||||
if time.Since(t) < pendingTimeout && !containsMachine(out, m) {
|
||||
out = append(out, node.RunningMachine{Name: m, Since: t})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *Remote) Info(ctx context.Context) (node.NodeInfo, error) {
|
||||
if !r.Online() {
|
||||
return node.NodeInfo{}, &node.NodeOfflineError{Node: r.name}
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
info := r.info
|
||||
info.Running = r.runningLocked()
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (r *Remote) Running(ctx context.Context) ([]node.RunningMachine, error) {
|
||||
info, err := r.Info(ctx)
|
||||
return info.Running, err
|
||||
}
|
||||
|
||||
func (r *Remote) IsRunning(machine string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return containsMachine(r.runningLocked(), machine)
|
||||
}
|
||||
|
||||
func (r *Remote) callNode(ctx context.Context, method string, params, result any) error {
|
||||
l, err := r.currentLink()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
return l.call(ctx, method, params, result)
|
||||
}
|
||||
|
||||
func (r *Remote) Start(ctx context.Context, spec node.StartSpec) error {
|
||||
if err := r.callNode(ctx, "start", spec, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.starting[spec.Machine] = time.Now()
|
||||
delete(r.stopping, spec.Machine)
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Remote) Poweroff(ctx context.Context, machine string) error {
|
||||
if err := r.callNode(ctx, "poweroff", map[string]string{"machine": machine}, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.stopping[machine] = time.Now()
|
||||
delete(r.starting, machine)
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Remote) SetLimits(ctx context.Context, machine string, size model.Size) error {
|
||||
return r.callNode(ctx, "set_limits", callParams{Machine: machine, Size: size}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
|
||||
return r.callNode(ctx, "set_quota", callParams{Drive: drive, Machine: machine, MB: mb}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) WaitStopped(ctx context.Context, machine string) error {
|
||||
return r.callNode(ctx, "wait_stopped", callParams{Machine: machine}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) ExportHome(ctx context.Context, drive, machine, parent string) (string, error) {
|
||||
var res callParams
|
||||
err := r.callNode(ctx, "export_home", callParams{Drive: drive, Machine: machine, Parent: parent}, &res)
|
||||
return res.ID, err
|
||||
}
|
||||
|
||||
func (r *Remote) ReadExport(ctx context.Context, id string, max int) ([]byte, bool, error) {
|
||||
var res callParams
|
||||
err := r.callNode(ctx, "read_export", callParams{ID: id, Max: max}, &res)
|
||||
return res.Data, res.EOF, err
|
||||
}
|
||||
|
||||
func (r *Remote) CloseExport(ctx context.Context, id string) error {
|
||||
return r.callNode(ctx, "close_export", callParams{ID: id}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) ImportHome(ctx context.Context, drive, machine string) (string, error) {
|
||||
var res callParams
|
||||
err := r.callNode(ctx, "import_home", callParams{Drive: drive, Machine: machine}, &res)
|
||||
return res.ID, err
|
||||
}
|
||||
|
||||
func (r *Remote) WriteImport(ctx context.Context, id string, data []byte) error {
|
||||
return r.callNode(ctx, "write_import", callParams{ID: id, Data: data}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) EndReceive(ctx context.Context, id string) error {
|
||||
return r.callNode(ctx, "end_receive", callParams{ID: id}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) FinishImport(ctx context.Context, id string, quotaMB int64) error {
|
||||
return r.callNode(ctx, "finish_import", callParams{ID: id, MB: quotaMB}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) AbortImport(ctx context.Context, id string) error {
|
||||
return r.callNode(ctx, "abort_import", callParams{ID: id}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) DeleteHome(ctx context.Context, drive, machine string) error {
|
||||
return r.callNode(ctx, "delete_home", callParams{Drive: drive, Machine: machine}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
|
||||
l, err := r.currentLink()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
return l.openAgent(ctx, machine, secret)
|
||||
}
|
||||
|
||||
func (m *Manager) attach(name string, l *link) (*Remote, error) {
|
||||
if name == "" || name == m.local.Name() {
|
||||
return nil, fmt.Errorf("node name %q is empty or taken by the master", name)
|
||||
}
|
||||
m.mu.Lock()
|
||||
r := m.remotes[name]
|
||||
if r == nil {
|
||||
r = &Remote{name: name, starting: map[string]time.Time{}, stopping: map[string]time.Time{}}
|
||||
m.remotes[name] = r
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
r.mu.Lock()
|
||||
old := r.link
|
||||
r.link = l
|
||||
r.infoAt = time.Time{}
|
||||
r.mu.Unlock()
|
||||
if old != nil {
|
||||
old.Close()
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (m *Manager) ServeCluster(ctx context.Context) error {
|
||||
cc := m.cfg.Cluster
|
||||
cert, fingerprint, err := loadOrCreateCert(m.cfg.StateDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("cluster: listening on %s, certificate fingerprint %s", cc.Listen, fingerprint)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/cluster", func(w http.ResponseWriter, req *http.Request) {
|
||||
if subtle.ConstantTimeCompare([]byte(req.Header.Get("X-Cluster-Secret")), []byte(cc.Secret)) != 1 {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
name := req.Header.Get("X-Node-Name")
|
||||
conn, err := websocket.Accept(w, req, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
l := newLink(conn)
|
||||
r, err := m.attach(name, l)
|
||||
if err != nil {
|
||||
conn.Close(websocket.StatusPolicyViolation, err.Error())
|
||||
return
|
||||
}
|
||||
back := false
|
||||
l.onReport = func(info node.NodeInfo) {
|
||||
r.setInfo(info)
|
||||
if !back {
|
||||
back = true
|
||||
log.Printf("cluster: node %s connected", name)
|
||||
if m.onNodeBack != nil {
|
||||
go m.onNodeBack(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
l.onCall = func(f frame) { l.reply(f.ID, nil, errors.New("the master takes no calls")) }
|
||||
l.run()
|
||||
log.Printf("cluster: node %s disconnected: %v", name, l.err)
|
||||
})
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: cc.Listen,
|
||||
Handler: mux,
|
||||
TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13},
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
srv.Close()
|
||||
}()
|
||||
if err := srv.ListenAndServeTLS("", ""); !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadOrCreateCert(dir string) (tls.Certificate, string, error) {
|
||||
certPath, keyPath := filepath.Join(dir, "cluster.crt"), filepath.Join(dir, "cluster.key")
|
||||
if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
|
||||
return cert, fingerprintOf(cert.Certificate[0]), nil
|
||||
}
|
||||
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
serial, _ := rand.Int(rand.Reader, big.NewInt(1<<62))
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: "slackerbot master"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().AddDate(20, 0, 0),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(key)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
return cert, fingerprintOf(der), err
|
||||
}
|
||||
|
||||
func fingerprintOf(der []byte) string {
|
||||
sum := sha256.Sum256(der)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func normalizeFingerprint(s string) string {
|
||||
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), ":", ""))
|
||||
}
|
||||
|
||||
func RunSlave(ctx context.Context, cfg *config.Config, local node.Node) error {
|
||||
backoff := time.Second
|
||||
for {
|
||||
started := time.Now()
|
||||
err := slaveSession(ctx, cfg, local)
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if time.Since(started) > time.Minute {
|
||||
backoff = time.Second
|
||||
}
|
||||
log.Printf("cluster: link to master lost (%v), retrying in %s", err, backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
backoff = min(backoff*2, 30*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func pinnedTLS(cfg *config.Config) *tls.Config {
|
||||
pinned := normalizeFingerprint(cfg.Cluster.MasterFingerprint)
|
||||
savedPath := filepath.Join(cfg.StateDir, "master.fingerprint")
|
||||
if pinned == "" {
|
||||
if b, err := os.ReadFile(savedPath); err == nil {
|
||||
pinned = normalizeFingerprint(string(b))
|
||||
}
|
||||
}
|
||||
return &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
// Not verified against a CA: the master's self-signed certificate is pinned instead.
|
||||
InsecureSkipVerify: true,
|
||||
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(raw) == 0 {
|
||||
return errors.New("master sent no certificate")
|
||||
}
|
||||
got := fingerprintOf(raw[0])
|
||||
if pinned == "" {
|
||||
log.Printf("cluster: trusting master certificate %s from now on", got)
|
||||
pinned = got
|
||||
_ = os.MkdirAll(cfg.StateDir, 0o700)
|
||||
return os.WriteFile(savedPath, []byte(got+"\n"), 0o600)
|
||||
}
|
||||
if got != pinned {
|
||||
return fmt.Errorf("master certificate %s doesn't match the pinned %s", got, pinned)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func slaveSession(ctx context.Context, cfg *config.Config, local node.Node) error {
|
||||
client := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(cfg), Proxy: http.ProxyFromEnvironment}}
|
||||
dialCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
conn, _, err := websocket.Dial(dialCtx, strings.TrimSuffix(cfg.Cluster.Master, "/")+"/cluster", &websocket.DialOptions{
|
||||
HTTPClient: client,
|
||||
HTTPHeader: http.Header{
|
||||
"X-Cluster-Secret": []string{cfg.Cluster.Secret},
|
||||
"X-Node-Name": []string{cfg.Cluster.Name},
|
||||
},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l := newLink(conn)
|
||||
defer l.Close()
|
||||
l.onCall = func(f frame) { serveCall(ctx, l, local, f) }
|
||||
go l.run()
|
||||
log.Printf("cluster: connected to master %s as %s", cfg.Cluster.Master, cfg.Cluster.Name)
|
||||
|
||||
t := time.NewTicker(reportEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
info, err := local.Info(ctx)
|
||||
if err != nil {
|
||||
log.Printf("cluster: reading node info: %v", err)
|
||||
} else if err := l.report(ctx, info); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-l.Done():
|
||||
return l.err
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type callParams struct {
|
||||
Machine string `json:"machine,omitempty"`
|
||||
Secret string `json:"secret,omitempty"`
|
||||
Drive string `json:"drive,omitempty"`
|
||||
MB int64 `json:"mb,omitempty"`
|
||||
Size model.Size `json:"size"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Parent string `json:"parent,omitempty"`
|
||||
Max int `json:"max,omitempty"`
|
||||
Data []byte `json:"data,omitempty"`
|
||||
EOF bool `json:"eof,omitempty"`
|
||||
}
|
||||
|
||||
func serveCall(ctx context.Context, l *link, local node.Node, f frame) {
|
||||
var p callParams
|
||||
_ = json.Unmarshal(f.Params, &p)
|
||||
|
||||
switch f.Method {
|
||||
case "start":
|
||||
var spec node.StartSpec
|
||||
if err := json.Unmarshal(f.Params, &spec); err != nil {
|
||||
l.reply(f.ID, nil, err)
|
||||
return
|
||||
}
|
||||
l.reply(f.ID, nil, local.Start(ctx, spec))
|
||||
case "poweroff":
|
||||
l.reply(f.ID, nil, local.Poweroff(ctx, p.Machine))
|
||||
case "set_limits":
|
||||
l.reply(f.ID, nil, local.SetLimits(ctx, p.Machine, p.Size))
|
||||
case "set_quota":
|
||||
l.reply(f.ID, nil, local.SetQuota(ctx, p.Drive, p.Machine, p.MB))
|
||||
case "wait_stopped":
|
||||
l.reply(f.ID, nil, local.WaitStopped(ctx, p.Machine))
|
||||
case "export_home":
|
||||
id, err := local.ExportHome(ctx, p.Drive, p.Machine, p.Parent)
|
||||
l.reply(f.ID, callParams{ID: id}, err)
|
||||
case "read_export":
|
||||
data, eof, err := local.ReadExport(ctx, p.ID, min(p.Max, transferChunk))
|
||||
l.reply(f.ID, callParams{Data: data, EOF: eof}, err)
|
||||
case "close_export":
|
||||
l.reply(f.ID, nil, local.CloseExport(ctx, p.ID))
|
||||
case "import_home":
|
||||
id, err := local.ImportHome(ctx, p.Drive, p.Machine)
|
||||
l.reply(f.ID, callParams{ID: id}, err)
|
||||
case "write_import":
|
||||
l.reply(f.ID, nil, local.WriteImport(ctx, p.ID, p.Data))
|
||||
case "end_receive":
|
||||
l.reply(f.ID, nil, local.EndReceive(ctx, p.ID))
|
||||
case "finish_import":
|
||||
l.reply(f.ID, nil, local.FinishImport(ctx, p.ID, p.MB))
|
||||
case "abort_import":
|
||||
l.reply(f.ID, nil, local.AbortImport(ctx, p.ID))
|
||||
case "delete_home":
|
||||
l.reply(f.ID, nil, local.DeleteHome(ctx, p.Drive, p.Machine))
|
||||
case "open_agent":
|
||||
agent, err := local.DialAgent(ctx, p.Machine, p.Secret)
|
||||
if err != nil {
|
||||
l.reply(f.ID, nil, err)
|
||||
return
|
||||
}
|
||||
s := l.addStream(f.ID)
|
||||
l.reply(f.ID, nil, nil)
|
||||
pipe(ctx, agent, s)
|
||||
default:
|
||||
l.reply(f.ID, nil, fmt.Errorf("unknown call %q", f.Method))
|
||||
}
|
||||
}
|
||||
|
||||
func pipe(ctx context.Context, a, b node.MsgConn) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
copyTo := func(dst, src node.MsgConn) {
|
||||
defer cancel()
|
||||
for {
|
||||
msg, err := src.Read(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := dst.Write(ctx, msg); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
go copyTo(a, b)
|
||||
copyTo(b, a)
|
||||
<-ctx.Done()
|
||||
a.Close()
|
||||
b.Close()
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/state"
|
||||
"slackerbot/internal/testutil"
|
||||
)
|
||||
|
||||
func TestMasterAndSlave(t *testing.T) {
|
||||
addr := testutil.FreePort(t)
|
||||
secret := "cluster-secret-1234567890"
|
||||
masterDir, slaveDir := t.TempDir(), t.TempDir()
|
||||
|
||||
masterCfg := &config.Config{StateDir: masterDir, Container: model.Size{MemoryMB: 1024, CPUs: 1}}
|
||||
masterCfg.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Listen: addr}
|
||||
store, err := state.Open(filepath.Join(masterDir, "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mgr := New(masterCfg, store, testutil.NewFakeNode("master"))
|
||||
backCh := make(chan string, 4)
|
||||
mgr.onNodeBack = func(n string) { backCh <- n }
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go mgr.ServeCluster(ctx)
|
||||
|
||||
slaveCfg := &config.Config{StateDir: slaveDir}
|
||||
slaveCfg.Cluster = config.ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr}
|
||||
slave := testutil.NewFakeNode("slave-1")
|
||||
slaveCtx, stopSlave := context.WithCancel(ctx)
|
||||
go RunSlave(slaveCtx, slaveCfg, slave)
|
||||
|
||||
testutil.WaitFor(t, "slave to come online", func() bool { _, err := mgr.nodeByName("slave-1"); return err == nil })
|
||||
if got := <-backCh; got != "slave-1" {
|
||||
t.Fatalf("back callback for %q", got)
|
||||
}
|
||||
pinned, err := os.ReadFile(filepath.Join(slaveDir, "master.fingerprint"))
|
||||
if err != nil || len(strings.TrimSpace(string(pinned))) != 64 {
|
||||
t.Fatalf("slave should pin the master's certificate, got %q %v", pinned, err)
|
||||
}
|
||||
|
||||
n, _ := mgr.nodeByName("slave-1")
|
||||
spec := node.StartSpec{Machine: "c-abc", Secret: "agent-secret", Size: model.Size{MemoryMB: 1024, CPUs: 1}, Drive: "/d"}
|
||||
if err := n.Start(ctx, spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slave.IsRunning("c-abc") || slave.Last.Drive != "/d" {
|
||||
t.Fatal("start didn't reach the slave")
|
||||
}
|
||||
if !n.IsRunning("c-abc") {
|
||||
t.Fatal("master should count the container as running right away")
|
||||
}
|
||||
|
||||
conn, err := n.DialAgent(ctx, "c-abc", "agent-secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.Write(ctx, []byte(`{"jsonrpc":"2.0"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
got, err := conn.Read(readCtx)
|
||||
readCancel()
|
||||
if err != nil || string(got) != `echo: {"jsonrpc":"2.0"}` {
|
||||
t.Fatalf("relay got %q %v", got, err)
|
||||
}
|
||||
conn.Close()
|
||||
if _, err := n.DialAgent(ctx, "c-abc", "wrong"); err == nil {
|
||||
t.Fatal("a bad agent secret should fail through the relay")
|
||||
}
|
||||
|
||||
if err := n.Poweroff(ctx, "c-abc"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if slave.IsRunning("c-abc") || n.IsRunning("c-abc") {
|
||||
t.Fatal("poweroff didn't take")
|
||||
}
|
||||
|
||||
stopSlave()
|
||||
testutil.WaitFor(t, "slave to go offline", func() bool {
|
||||
_, err := mgr.nodeByName("slave-1")
|
||||
var off *node.NodeOfflineError
|
||||
return errors.As(err, &off) && off.Node == "slave-1"
|
||||
})
|
||||
if off := mgr.OfflineNodes(); len(off) != 1 || off[0] != "slave-1" {
|
||||
t.Fatalf("offline nodes = %v", off)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlaveRejectsWrongMasterOrSecret(t *testing.T) {
|
||||
addr := testutil.FreePort(t)
|
||||
secret := "cluster-secret-1234567890"
|
||||
masterCfg := &config.Config{StateDir: t.TempDir()}
|
||||
masterCfg.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Listen: addr}
|
||||
store, _ := state.Open(filepath.Join(masterCfg.StateDir, "state.json"))
|
||||
mgr := New(masterCfg, store, testutil.NewFakeNode("master"))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go mgr.ServeCluster(ctx)
|
||||
testutil.WaitFor(t, "master to listen", func() bool {
|
||||
c, err := net.Dial("tcp", addr)
|
||||
if err == nil {
|
||||
c.Close()
|
||||
}
|
||||
return err == nil
|
||||
})
|
||||
|
||||
wrongPin := &config.Config{StateDir: t.TempDir()}
|
||||
wrongPin.Cluster = config.ClusterConfig{Name: "s", Secret: secret, Master: "wss://" + addr, MasterFingerprint: strings.Repeat("ab", 32)}
|
||||
if err := slaveSession(ctx, wrongPin, testutil.NewFakeNode("s")); err == nil {
|
||||
t.Fatal("a master with a different certificate should be refused")
|
||||
}
|
||||
|
||||
wrongSecret := &config.Config{StateDir: t.TempDir()}
|
||||
wrongSecret.Cluster = config.ClusterConfig{Name: "s", Secret: "not-the-secret-at-all", Master: "wss://" + addr}
|
||||
if err := slaveSession(ctx, wrongSecret, testutil.NewFakeNode("s")); err == nil {
|
||||
t.Fatal("a slave with the wrong secret should be refused")
|
||||
}
|
||||
|
||||
sameName := &config.Config{StateDir: t.TempDir()}
|
||||
sameName.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Master: "wss://" + addr}
|
||||
if err := slaveSession(ctx, sameName, testutil.NewFakeNode("master")); err == nil {
|
||||
t.Fatal("a slave can't take the master's name")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"slackerbot/internal/balance"
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/state"
|
||||
"slackerbot/internal/storage"
|
||||
"slackerbot/internal/util"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrTooBig = errors.New("bigger than max_container allows")
|
||||
ErrTooSmall = errors.New("smaller than the minimum size")
|
||||
|
||||
MinContainer = model.Size{MemoryMB: 256, CPUs: 0.25}
|
||||
)
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateRunning State = "running"
|
||||
StateStopped State = "stopped"
|
||||
StateNone State = "not created"
|
||||
StateOffline State = "on an offline node"
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
cfg *config.Config
|
||||
store *state.Store
|
||||
local node.Node
|
||||
|
||||
mu sync.RWMutex
|
||||
remotes map[string]*Remote
|
||||
onNodeBack func(node string)
|
||||
|
||||
startMu sync.Mutex
|
||||
ownerMu util.KeyedMutex
|
||||
migrating sync.Map
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, store *state.Store, local node.Node) *Manager {
|
||||
return &Manager{cfg: cfg, store: store, local: local, remotes: map[string]*Remote{}}
|
||||
}
|
||||
|
||||
func (m *Manager) SetNodeBackHandler(fn func(node string)) { m.onNodeBack = fn }
|
||||
|
||||
func (m *Manager) nodeByName(name string) (node.Node, error) {
|
||||
if name == m.local.Name() {
|
||||
return m.local, nil
|
||||
}
|
||||
m.mu.RLock()
|
||||
r := m.remotes[name]
|
||||
m.mu.RUnlock()
|
||||
if r != nil && r.Online() {
|
||||
return r, nil
|
||||
}
|
||||
return nil, &node.NodeOfflineError{Node: name}
|
||||
}
|
||||
|
||||
func (m *Manager) onlineNodes() []node.Node {
|
||||
nodes := []node.Node{m.local}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
for _, r := range m.remotes {
|
||||
if r.Online() {
|
||||
nodes = append(nodes, r)
|
||||
}
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
func (m *Manager) OfflineNodes() []string {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
var out []string
|
||||
for name, r := range m.remotes {
|
||||
if !r.Online() {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Manager) Views(ctx context.Context) ([]balance.NodeView, error) {
|
||||
owners := m.store.Owners()
|
||||
var views []balance.NodeView
|
||||
for _, n := range m.onlineNodes() {
|
||||
info, err := n.Info(ctx)
|
||||
if err != nil {
|
||||
if n == m.local {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
views = append(views, balance.BuildView(info, owners, m.cfg.Container))
|
||||
}
|
||||
sort.Slice(views, func(i, j int) bool { return views[i].Name < views[j].Name })
|
||||
return views, nil
|
||||
}
|
||||
|
||||
func (m *Manager) view(ctx context.Context, n node.Node) (balance.NodeView, node.NodeInfo, error) {
|
||||
info, err := n.Info(ctx)
|
||||
if err != nil {
|
||||
return balance.NodeView{}, info, err
|
||||
}
|
||||
return balance.BuildView(info, m.store.Owners(), m.cfg.Container), info, nil
|
||||
}
|
||||
|
||||
func (m *Manager) Totals(ctx context.Context) (used, usable model.Size) {
|
||||
views, _ := m.Views(ctx)
|
||||
for _, v := range views {
|
||||
used, usable = used.Add(v.Used), usable.Add(v.Usable)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (m *Manager) ownerRecord(id string, kind model.OwnerKind) (*model.Owner, error) {
|
||||
if u := m.store.Owner(id); u != nil {
|
||||
return u, nil
|
||||
}
|
||||
u := &model.Owner{
|
||||
ID: id,
|
||||
Kind: kind,
|
||||
Machine: model.MachineName(id, kind),
|
||||
Secret: util.RandomHex(32),
|
||||
Size: m.cfg.Container,
|
||||
StorageMinMB: m.cfg.Storage.MinMB,
|
||||
StorageMaxMB: m.cfg.Storage.MaxMB,
|
||||
LastActive: time.Now(),
|
||||
}
|
||||
return u, m.store.PutOwner(u)
|
||||
}
|
||||
|
||||
func (m *Manager) Owner(id string, kind model.OwnerKind) (*model.Owner, error) {
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
return m.ownerRecord(id, kind)
|
||||
}
|
||||
|
||||
func (m *Manager) spec(u *model.Owner) node.StartSpec {
|
||||
return node.StartSpec{
|
||||
Machine: u.Machine,
|
||||
Description: "slackerbot container for " + string(u.Kind) + " " + u.ID,
|
||||
Secret: u.Secret,
|
||||
Size: u.Size,
|
||||
Drive: u.Drive,
|
||||
StorageMaxMB: u.StorageMaxMB,
|
||||
Env: m.cfg.AgentEnv,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (*model.Owner, error) {
|
||||
if m.IsMigrating(id) {
|
||||
return nil, ErrMigrating
|
||||
}
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
|
||||
u, err := m.ownerRecord(id, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if u.Node != "" {
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return u, err
|
||||
}
|
||||
if n.IsRunning(u.Machine) {
|
||||
return u, nil
|
||||
}
|
||||
}
|
||||
|
||||
m.startMu.Lock()
|
||||
defer m.startMu.Unlock()
|
||||
|
||||
if u.Node == "" {
|
||||
views, err := m.Views(ctx)
|
||||
if err != nil {
|
||||
return u, err
|
||||
}
|
||||
target, drive, err := balance.ChoosePlacement(views, u.Size, u.StorageMinMB)
|
||||
if err != nil {
|
||||
return u, err
|
||||
}
|
||||
u.Node, u.Drive = target, drive
|
||||
if err := m.store.SetPlacement(u.ID, target, drive); err != nil {
|
||||
return u, err
|
||||
}
|
||||
}
|
||||
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return u, err
|
||||
}
|
||||
v, _, err := m.view(ctx, n)
|
||||
if err != nil {
|
||||
return u, err
|
||||
}
|
||||
if !v.Used.Add(u.Size).Fits(v.Usable) {
|
||||
return u, &node.NodeFullError{Node: u.Node}
|
||||
}
|
||||
return u, n.Start(ctx, m.spec(u))
|
||||
}
|
||||
|
||||
// The old home folder stays on its node, untouched.
|
||||
func (m *Manager) Recreate(id string) error {
|
||||
if m.IsMigrating(id) {
|
||||
return ErrMigrating
|
||||
}
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
if err := m.store.SetPlacement(id, "", ""); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.store.DeleteSessions(id)
|
||||
}
|
||||
|
||||
func (m *Manager) DialAgent(ctx context.Context, u *model.Owner) (node.MsgConn, error) {
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return n.DialAgent(ctx, u.Machine, u.Secret)
|
||||
}
|
||||
|
||||
func (m *Manager) State(u *model.Owner) State {
|
||||
if u == nil || u.Node == "" {
|
||||
return StateNone
|
||||
}
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return StateOffline
|
||||
}
|
||||
if n.IsRunning(u.Machine) {
|
||||
return StateRunning
|
||||
}
|
||||
return StateStopped
|
||||
}
|
||||
|
||||
func (m *Manager) Poweroff(ctx context.Context, u *model.Owner) error {
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return n.Poweroff(ctx, u.Machine)
|
||||
}
|
||||
|
||||
func (m *Manager) RunningOwners(ctx context.Context) []model.Owner {
|
||||
byNodeMachine := map[string]model.Owner{}
|
||||
for _, u := range m.store.Owners() {
|
||||
byNodeMachine[u.Node+"/"+u.Machine] = u
|
||||
}
|
||||
var out []model.Owner
|
||||
for _, n := range m.onlineNodes() {
|
||||
running, err := n.Running(ctx)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, r := range running {
|
||||
if u, ok := byNodeMachine[n.Name()+"/"+r.Name]; ok {
|
||||
out = append(out, u)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Manager) Resize(ctx context.Context, id string, kind model.OwnerKind, change func(model.Size) (model.Size, error)) (old, next model.Size, err error) {
|
||||
if m.IsMigrating(id) {
|
||||
return old, old, ErrMigrating
|
||||
}
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
m.startMu.Lock()
|
||||
defer m.startMu.Unlock()
|
||||
|
||||
u, err := m.ownerRecord(id, kind)
|
||||
if err != nil {
|
||||
return model.Size{}, model.Size{}, err
|
||||
}
|
||||
old = u.Size
|
||||
if next, err = change(old); err != nil {
|
||||
return old, old, err
|
||||
}
|
||||
next.CPUs = math.Round(next.CPUs*100) / 100
|
||||
switch {
|
||||
case !MinContainer.Fits(next):
|
||||
return old, old, ErrTooSmall
|
||||
case !next.Fits(m.cfg.MaxContainer):
|
||||
return old, old, ErrTooBig
|
||||
case next == old:
|
||||
return old, old, nil
|
||||
}
|
||||
|
||||
if u.Node != "" {
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return old, old, err
|
||||
}
|
||||
if n.IsRunning(u.Machine) {
|
||||
v, _, err := m.view(ctx, n)
|
||||
if err != nil {
|
||||
return old, old, err
|
||||
}
|
||||
growth := model.Size{MemoryMB: max(0, next.MemoryMB-old.MemoryMB), CPUs: math.Max(0, next.CPUs-old.CPUs)}
|
||||
if !v.Used.Add(growth).Fits(v.Usable) {
|
||||
return old, old, &node.NodeFullError{Node: u.Node}
|
||||
}
|
||||
if err := n.SetLimits(ctx, u.Machine, next); err != nil {
|
||||
return old, old, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return old, next, m.store.SetSize(id, next)
|
||||
}
|
||||
|
||||
type StorageChange struct {
|
||||
OldMin, OldMax, NewMin, NewMax int64
|
||||
UsedMB int64
|
||||
Drive *storage.DriveReport
|
||||
}
|
||||
|
||||
func (m *Manager) SetStorage(ctx context.Context, id string, kind model.OwnerKind, change func(min, max int64) (int64, int64, error)) (*StorageChange, error) {
|
||||
if m.IsMigrating(id) {
|
||||
return &StorageChange{}, ErrMigrating
|
||||
}
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
m.startMu.Lock()
|
||||
defer m.startMu.Unlock()
|
||||
|
||||
u, err := m.ownerRecord(id, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := &StorageChange{OldMin: u.StorageMinMB, OldMax: u.StorageMaxMB}
|
||||
if c.NewMin, c.NewMax, err = change(c.OldMin, c.OldMax); err != nil {
|
||||
return c, err
|
||||
}
|
||||
switch {
|
||||
case c.NewMin < config.MinStorageMB || c.NewMax < config.MinStorageMB:
|
||||
return c, storage.ErrStorageTooSmall
|
||||
case c.NewMin > c.NewMax:
|
||||
return c, storage.ErrMinOverMax
|
||||
case c.NewMax > m.cfg.Storage.LimitMB:
|
||||
return c, storage.ErrStorageTooBig
|
||||
}
|
||||
|
||||
if u.Node == "" {
|
||||
views, err := m.Views(ctx)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
if _, _, err := balance.ChoosePlacement(views, u.Size, c.NewMin); err != nil {
|
||||
return c, err
|
||||
}
|
||||
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
|
||||
}
|
||||
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
v, info, err := m.view(ctx, n)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
for _, d := range info.Drives {
|
||||
if d.Path == u.Drive {
|
||||
c.UsedMB = d.HomesMB[u.Machine]
|
||||
}
|
||||
}
|
||||
if c.NewMax < c.UsedMB {
|
||||
return c, storage.ErrBelowUsage
|
||||
}
|
||||
var r *storage.DriveReport
|
||||
for i := range v.Drives {
|
||||
if v.Drives[i].Path == u.Drive {
|
||||
r = &v.Drives[i]
|
||||
}
|
||||
}
|
||||
if r == nil {
|
||||
return c, fmt.Errorf("drive %s is no longer configured on %s", u.Drive, u.Node)
|
||||
}
|
||||
extra := storage.Committed(c.NewMin, c.UsedMB) - storage.Committed(c.OldMin, c.UsedMB)
|
||||
if extra > 0 && extra > r.HeadroomMB() {
|
||||
return c, storage.ErrStorageFull
|
||||
}
|
||||
if err := n.SetQuota(ctx, u.Drive, u.Machine, c.NewMax); err != nil {
|
||||
return c, err
|
||||
}
|
||||
r.CommittedMB += extra
|
||||
r.BookedMB += c.NewMax - c.OldMax
|
||||
c.Drive = r
|
||||
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
|
||||
}
|
||||
|
||||
func (m *Manager) StorageUsed(ctx context.Context, u *model.Owner) (int64, bool) {
|
||||
if u == nil || u.Node == "" {
|
||||
return 0, false
|
||||
}
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
info, err := n.Info(ctx)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
for _, d := range info.Drives {
|
||||
if used, ok := d.HomesMB[u.Machine]; ok && d.Path == u.Drive {
|
||||
return used, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"slackerbot/internal/balance"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/storage"
|
||||
)
|
||||
|
||||
const transferChunk = 4 << 20
|
||||
|
||||
var (
|
||||
ErrMigrating = errors.New("this container is being moved")
|
||||
ErrNoContainer = errors.New("there's no container here yet")
|
||||
ErrSameNode = errors.New("the container is already on that node")
|
||||
)
|
||||
|
||||
type UnknownNodeError struct{ Node string }
|
||||
|
||||
func (e *UnknownNodeError) Error() string { return fmt.Sprintf("there's no node called %s", e.Node) }
|
||||
|
||||
type MovePhase string
|
||||
|
||||
const (
|
||||
PhaseCopy MovePhase = "copy"
|
||||
PhaseFinal MovePhase = "final"
|
||||
PhaseStart MovePhase = "start"
|
||||
PhaseCleanup MovePhase = "cleanup"
|
||||
)
|
||||
|
||||
type MoveProgress struct {
|
||||
To string
|
||||
Phase MovePhase
|
||||
CopiedMB int64
|
||||
TotalMB int64
|
||||
}
|
||||
|
||||
type MoveOptions struct {
|
||||
Target string
|
||||
StartAfter bool
|
||||
WaitIdle func(ctx context.Context)
|
||||
Progress func(MoveProgress)
|
||||
}
|
||||
|
||||
type MoveResult struct {
|
||||
From, To string
|
||||
WasRunning bool
|
||||
Started bool
|
||||
CopiedMB int64
|
||||
LeftOld error
|
||||
}
|
||||
|
||||
func (m *Manager) IsMigrating(id string) bool {
|
||||
_, ok := m.migrating.Load(id)
|
||||
return ok
|
||||
}
|
||||
|
||||
// Migrate copies the owner's home folder to another node and switches the
|
||||
// container over. A running container keeps running through the bulk copy
|
||||
// and is only stopped for a final incremental sync.
|
||||
func (m *Manager) Migrate(ctx context.Context, id string, opts MoveOptions) (*MoveResult, error) {
|
||||
if _, busy := m.migrating.LoadOrStore(id, true); busy {
|
||||
return nil, ErrMigrating
|
||||
}
|
||||
defer m.migrating.Delete(id)
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
|
||||
u := m.store.Owner(id)
|
||||
if u == nil || u.Node == "" {
|
||||
return nil, ErrNoContainer
|
||||
}
|
||||
src, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &MoveResult{From: u.Node, WasRunning: src.IsRunning(u.Machine)}
|
||||
needStart := res.WasRunning || opts.StartAfter
|
||||
|
||||
usedMB, _ := m.StorageUsed(ctx, u)
|
||||
target, drive, err := m.pickTarget(ctx, u, opts.Target, needStart, storage.Committed(u.StorageMinMB, usedMB))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.To = target
|
||||
dst, err := m.nodeByName(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
progress := func(p MovePhase, copied int64) {
|
||||
if opts.Progress != nil {
|
||||
opts.Progress(MoveProgress{To: target, Phase: p, CopiedMB: copied >> 20, TotalMB: usedMB})
|
||||
}
|
||||
}
|
||||
|
||||
imp, err := dst.ImportHome(ctx, drive, u.Machine)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var exports []string
|
||||
stopped := false
|
||||
rollback := func(cause error) (*MoveResult, error) {
|
||||
_ = dst.AbortImport(context.Background(), imp)
|
||||
for _, e := range exports {
|
||||
_ = src.CloseExport(context.Background(), e)
|
||||
}
|
||||
if stopped {
|
||||
if err := src.Start(context.Background(), m.spec(u)); err != nil {
|
||||
return res, fmt.Errorf("%w (and restarting it on %s failed: %v)", cause, u.Node, err)
|
||||
}
|
||||
}
|
||||
return res, cause
|
||||
}
|
||||
|
||||
var copied int64
|
||||
send := func(parent string, phase MovePhase) error {
|
||||
exp, err := src.ExportHome(ctx, u.Drive, u.Machine, parent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
exports = append(exports, exp)
|
||||
for {
|
||||
data, eof, err := src.ReadExport(ctx, exp, transferChunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(data) > 0 {
|
||||
if err := dst.WriteImport(ctx, imp, data); err != nil {
|
||||
return err
|
||||
}
|
||||
copied += int64(len(data))
|
||||
progress(phase, copied)
|
||||
}
|
||||
if eof {
|
||||
return dst.EndReceive(ctx, imp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
progress(PhaseCopy, 0)
|
||||
if err := send("", PhaseCopy); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
if res.WasRunning {
|
||||
progress(PhaseFinal, copied)
|
||||
if opts.WaitIdle != nil {
|
||||
opts.WaitIdle(ctx)
|
||||
}
|
||||
if err := src.Poweroff(ctx, u.Machine); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
stopped = true
|
||||
if err := src.WaitStopped(ctx, u.Machine); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
if err := send(exports[0], PhaseFinal); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
}
|
||||
if err := dst.FinishImport(ctx, imp, u.StorageMaxMB); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
res.CopiedMB = copied >> 20
|
||||
|
||||
oldNode, oldDrive := u.Node, u.Drive
|
||||
if err := m.store.SetPlacement(id, target, drive); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
u.Node, u.Drive = target, drive
|
||||
|
||||
progress(PhaseCleanup, copied)
|
||||
for _, e := range exports {
|
||||
_ = src.CloseExport(ctx, e)
|
||||
}
|
||||
if res.LeftOld = src.DeleteHome(ctx, oldDrive, u.Machine); res.LeftOld != nil {
|
||||
log.Printf("move: old home of %s left on %s: %v", u.Machine, oldNode, res.LeftOld)
|
||||
}
|
||||
|
||||
if !needStart {
|
||||
return res, nil
|
||||
}
|
||||
progress(PhaseStart, copied)
|
||||
if err := dst.Start(ctx, m.spec(u)); err != nil {
|
||||
return res, fmt.Errorf("moved to %s, but starting it there failed: %w", target, err)
|
||||
}
|
||||
res.Started = true
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (m *Manager) pickTarget(ctx context.Context, u *model.Owner, want string, needStart bool, storageMB int64) (target, drive string, err error) {
|
||||
if want == u.Node {
|
||||
return "", "", ErrSameNode
|
||||
}
|
||||
m.startMu.Lock()
|
||||
defer m.startMu.Unlock()
|
||||
|
||||
views, err := m.Views(ctx)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
compute := model.Size{}
|
||||
if needStart {
|
||||
compute = u.Size
|
||||
}
|
||||
var candidates []balance.NodeView
|
||||
for _, v := range views {
|
||||
if v.Name != u.Node && (want == "" || v.Name == want) {
|
||||
candidates = append(candidates, v)
|
||||
}
|
||||
}
|
||||
if want != "" && len(candidates) == 0 {
|
||||
if m.KnownNode(want) {
|
||||
return "", "", &node.NodeOfflineError{Node: want}
|
||||
}
|
||||
return "", "", &UnknownNodeError{Node: want}
|
||||
}
|
||||
target, drive, err = balance.ChoosePlacement(candidates, compute, storageMB)
|
||||
if err != nil && want != "" {
|
||||
return "", "", &node.NodeFullError{Node: want}
|
||||
}
|
||||
return target, drive, err
|
||||
}
|
||||
|
||||
func (m *Manager) KnownNode(name string) bool {
|
||||
if name == m.local.Name() {
|
||||
return true
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
_, ok := m.remotes[name]
|
||||
return ok
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package manager_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"slackerbot/internal/manager"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/testutil"
|
||||
"slackerbot/internal/testutil/clustertest"
|
||||
)
|
||||
|
||||
func TestMoveLiveContainer(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), true)
|
||||
|
||||
var phases []manager.MovePhase
|
||||
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{
|
||||
Target: "slave-1",
|
||||
WaitIdle: func(context.Context) {
|
||||
c.Master.Mu.Lock()
|
||||
c.Master.Homes["/d/c-c1"] = append(c.Master.Homes["/d/c-c1"], []byte("written during the bulk copy")...)
|
||||
c.Master.Mu.Unlock()
|
||||
},
|
||||
Progress: func(p manager.MoveProgress) {
|
||||
if len(phases) == 0 || phases[len(phases)-1] != p.Phase {
|
||||
phases = append(phases, p.Phase)
|
||||
}
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := append(testutil.HomeData(), []byte("written during the bulk copy")...)
|
||||
if !bytes.Equal(c.Slave.Homes["/d/c-c1"], want) {
|
||||
t.Fatal("the late write didn't make it to the new node")
|
||||
}
|
||||
if c.Slave.Passes != 2 {
|
||||
t.Fatalf("a live move should copy twice (bulk + final sync), got %d", c.Slave.Passes)
|
||||
}
|
||||
if _, still := c.Master.Homes["/d/c-c1"]; still {
|
||||
t.Fatal("old home should be deleted after a good move")
|
||||
}
|
||||
if !c.Slave.IsRunning("c-c1") || c.Master.IsRunning("c-c1") || !res.Started {
|
||||
t.Fatal("container should be running on the new node only")
|
||||
}
|
||||
if u := c.Store.Owner("C1"); u.Node != "slave-1" || u.Drive != "/d" {
|
||||
t.Fatalf("placement not switched: %+v", u)
|
||||
}
|
||||
if len(c.Master.Exports) != 0 {
|
||||
t.Fatal("snapshots on the old node should be cleaned up")
|
||||
}
|
||||
wantPhases := []manager.MovePhase{manager.PhaseCopy, manager.PhaseFinal, manager.PhaseCleanup, manager.PhaseStart}
|
||||
if len(phases) != len(wantPhases) {
|
||||
t.Fatalf("phases %v, want %v", phases, wantPhases)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveAutoStoppedContainer(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), false)
|
||||
|
||||
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.To != "slave-1" || res.Started || c.Slave.IsRunning("c-c1") {
|
||||
t.Fatalf("a stopped container should move without starting: %+v", res)
|
||||
}
|
||||
if c.Slave.Passes != 1 {
|
||||
t.Fatalf("a stopped container needs one copy, got %d", c.Slave.Passes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveFullNodeStartsElsewhere(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), false)
|
||||
c.Master.Mu.Lock()
|
||||
c.Master.Usable = model.Size{MemoryMB: 512, CPUs: 0.5}
|
||||
c.Master.Mu.Unlock()
|
||||
|
||||
_, err := c.Mgr.Ensure(c.Ctx, "C1", model.OwnerChannel)
|
||||
var full *node.NodeFullError
|
||||
if !errors.As(err, &full) || full.Node != "master" {
|
||||
t.Fatalf("want node full on master, got %v", err)
|
||||
}
|
||||
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{StartAfter: true})
|
||||
if err != nil || !res.Started || !c.Slave.IsRunning("c-c1") {
|
||||
t.Fatalf("auto-move should start it on the new node: %+v %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveFailureRollsBack(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), true)
|
||||
|
||||
_, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{
|
||||
Target: "slave-1",
|
||||
WaitIdle: func(context.Context) {
|
||||
c.Slave.Mu.Lock()
|
||||
c.Slave.FailWrites = true
|
||||
c.Slave.Mu.Unlock()
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected the final sync to fail")
|
||||
}
|
||||
if u := c.Store.Owner("C1"); u.Node != "master" {
|
||||
t.Fatalf("container should stay on its old node, got %s", u.Node)
|
||||
}
|
||||
if !c.Master.IsRunning("c-c1") {
|
||||
t.Fatal("container should be restarted on its old node")
|
||||
}
|
||||
if _, ok := c.Master.Homes["/d/c-c1"]; !ok {
|
||||
t.Fatal("old home must survive a failed move")
|
||||
}
|
||||
if _, ok := c.Slave.Homes["/d/c-c1"]; ok || len(c.Slave.Imports) != 0 {
|
||||
t.Fatal("half-copied home on the target should be cleaned up")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveRefusals(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), false)
|
||||
|
||||
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "master"}); !errors.Is(err, manager.ErrSameNode) {
|
||||
t.Fatalf("same node: %v", err)
|
||||
}
|
||||
var unknown *manager.UnknownNodeError
|
||||
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "nope"}); !errors.As(err, &unknown) {
|
||||
t.Fatalf("unknown node: %v", err)
|
||||
}
|
||||
if _, err := c.Mgr.Migrate(c.Ctx, "nobody", manager.MoveOptions{}); !errors.Is(err, manager.ErrNoContainer) {
|
||||
t.Fatalf("no container: %v", err)
|
||||
}
|
||||
c.Slave.Mu.Lock()
|
||||
c.Slave.Usable = model.Size{MemoryMB: 512, CPUs: 0.5}
|
||||
c.Slave.Mu.Unlock()
|
||||
c.Master.Mu.Lock()
|
||||
c.Master.Runs["c-c1"] = true
|
||||
c.Master.Mu.Unlock()
|
||||
testutil.WaitFor(t, "slave report", func() bool {
|
||||
views, _ := c.Mgr.Views(c.Ctx)
|
||||
for _, v := range views {
|
||||
if v.Name == "slave-1" {
|
||||
return v.Usable.MemoryMB == 512
|
||||
}
|
||||
}
|
||||
return false
|
||||
})
|
||||
var full *node.NodeFullError
|
||||
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "slave-1"}); !errors.As(err, &full) {
|
||||
t.Fatalf("full target: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user