Files
slacker-bot/slackerbot/internal/manager/cluster.go
T
2026-11-01 14:55:33 -05:00

814 lines
21 KiB
Go

package manager
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/hex"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"log"
"math/big"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/coder/websocket"
"slackerbot/internal/config"
"slackerbot/internal/model"
"slackerbot/internal/node"
)
const (
reportEvery = 5 * time.Second
offlineAfter = 20 * time.Second
callTimeout = 3 * time.Minute
pendingTimeout = 30 * time.Second
streamBuffer = 4096
)
type frame struct {
Type string `json:"t"`
ID int64 `json:"id,omitempty"`
Method string `json:"m,omitempty"`
Params json.RawMessage `json:"p,omitempty"`
Result json.RawMessage `json:"r,omitempty"`
Error string `json:"e,omitempty"`
Data string `json:"d,omitempty"`
}
type link struct {
conn *websocket.Conn
writeMu sync.Mutex
nextID atomic.Int64
mu sync.Mutex
pending map[int64]chan frame
streams map[int64]*stream
onCall func(f frame)
onReport func(info node.NodeInfo)
done chan struct{}
err error
}
func newLink(conn *websocket.Conn) *link {
conn.SetReadLimit(64 << 20)
return &link{
conn: conn,
pending: map[int64]chan frame{},
streams: map[int64]*stream{},
done: make(chan struct{}),
}
}
func (l *link) Done() <-chan struct{} { return l.done }
func (l *link) Close() { l.conn.Close(websocket.StatusNormalClosure, "") }
func (l *link) send(ctx context.Context, f frame) error {
b, err := json.Marshal(f)
if err != nil {
return err
}
l.writeMu.Lock()
defer l.writeMu.Unlock()
return l.conn.Write(ctx, websocket.MessageText, b)
}
func (l *link) run() {
defer func() {
l.mu.Lock()
streams := l.streams
l.streams = map[int64]*stream{}
l.mu.Unlock()
for _, s := range streams {
s.closeLocal()
}
close(l.done)
}()
for {
_, data, err := l.conn.Read(context.Background())
if err != nil {
l.err = err
return
}
var f frame
if json.Unmarshal(data, &f) != nil {
continue
}
switch f.Type {
case "call":
if l.onCall != nil {
go l.onCall(f)
}
case "reply":
l.mu.Lock()
ch := l.pending[f.ID]
l.mu.Unlock()
if ch != nil {
ch <- f
}
case "report":
var info node.NodeInfo
if l.onReport != nil && json.Unmarshal(f.Params, &info) == nil {
l.onReport(info)
}
case "data":
if s := l.stream(f.ID); s != nil {
select {
case s.in <- []byte(f.Data):
default:
s.Close()
}
}
case "close":
if s := l.stream(f.ID); s != nil {
s.closeLocal()
}
}
}
}
func (l *link) call(ctx context.Context, method string, params, result any) error {
return l.callWithID(ctx, l.nextID.Add(1), method, params, result)
}
func (l *link) callWithID(ctx context.Context, id int64, method string, params, result any) error {
ch := make(chan frame, 1)
l.mu.Lock()
l.pending[id] = ch
l.mu.Unlock()
defer func() {
l.mu.Lock()
delete(l.pending, id)
l.mu.Unlock()
}()
p, err := json.Marshal(params)
if err != nil {
return err
}
if err := l.send(ctx, frame{Type: "call", ID: id, Method: method, Params: p}); err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case <-l.done:
return fmt.Errorf("link closed: %v", l.err)
case f := <-ch:
if f.Error != "" {
return errors.New(f.Error)
}
if result != nil && len(f.Result) > 0 {
return json.Unmarshal(f.Result, result)
}
return nil
}
}
func (l *link) reply(id int64, result any, err error) {
f := frame{Type: "reply", ID: id}
if err != nil {
f.Error = err.Error()
} else if result != nil {
f.Result, _ = json.Marshal(result)
}
_ = l.send(context.Background(), f)
}
func (l *link) report(ctx context.Context, info node.NodeInfo) error {
p, err := json.Marshal(info)
if err != nil {
return err
}
return l.send(ctx, frame{Type: "report", Params: p})
}
func (l *link) stream(id int64) *stream {
l.mu.Lock()
defer l.mu.Unlock()
return l.streams[id]
}
func (l *link) addStream(id int64) *stream {
s := &stream{l: l, id: id, in: make(chan []byte, streamBuffer), closed: make(chan struct{})}
l.mu.Lock()
l.streams[id] = s
l.mu.Unlock()
return s
}
func (l *link) dropStream(id int64) {
l.mu.Lock()
delete(l.streams, id)
l.mu.Unlock()
}
// The stream shares its ID with the "open_agent" call, so it's registered
// before the call goes out and no early data is lost.
func (l *link) openAgent(ctx context.Context, machine, secret string) (*stream, error) {
id := l.nextID.Add(1)
s := l.addStream(id)
err := l.callWithID(ctx, id, "open_agent", map[string]string{"machine": machine, "secret": secret}, nil)
if err != nil {
l.dropStream(id)
return nil, err
}
return s, nil
}
type stream struct {
l *link
id int64
in chan []byte
closed chan struct{}
once sync.Once
}
func (s *stream) Read(ctx context.Context) ([]byte, error) {
select {
case b := <-s.in:
return b, nil
default:
}
select {
case b := <-s.in:
return b, nil
case <-s.closed:
return nil, errors.New("stream closed")
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (s *stream) Write(ctx context.Context, b []byte) error {
select {
case <-s.closed:
return errors.New("stream closed")
default:
}
return s.l.send(ctx, frame{Type: "data", ID: s.id, Data: string(b)})
}
func (s *stream) Close() error {
s.once.Do(func() {
_ = s.l.send(context.Background(), frame{Type: "close", ID: s.id})
s.l.dropStream(s.id)
close(s.closed)
})
return nil
}
func (s *stream) closeLocal() {
s.once.Do(func() {
s.l.dropStream(s.id)
close(s.closed)
})
}
type Remote struct {
name string
mu sync.Mutex
link *link
info node.NodeInfo
infoAt time.Time
starting map[string]time.Time
stopping map[string]time.Time
}
func (r *Remote) Name() string { return r.name }
func (r *Remote) Online() bool {
r.mu.Lock()
defer r.mu.Unlock()
if r.link == nil || time.Since(r.infoAt) > offlineAfter {
return false
}
select {
case <-r.link.Done():
return false
default:
return true
}
}
func (r *Remote) currentLink() (*link, error) {
if !r.Online() {
return nil, &node.NodeOfflineError{Node: r.name}
}
r.mu.Lock()
defer r.mu.Unlock()
return r.link, nil
}
func (r *Remote) setInfo(info node.NodeInfo) {
r.mu.Lock()
defer r.mu.Unlock()
info.Name = r.name
r.info, r.infoAt = info, time.Now()
for _, m := range info.Running {
delete(r.starting, m.Name)
}
for m := range r.stopping {
if !containsMachine(info.Running, m) {
delete(r.stopping, m)
}
}
}
func containsMachine(list []node.RunningMachine, name string) bool {
for _, m := range list {
if m.Name == name {
return true
}
}
return false
}
// Reports can lag behind a start or stop the master just asked for,
// so recent requests override them for a short while.
func (r *Remote) runningLocked() []node.RunningMachine {
var out []node.RunningMachine
for _, m := range r.info.Running {
if t, ok := r.stopping[m.Name]; ok && time.Since(t) < pendingTimeout {
continue
}
out = append(out, m)
}
for m, t := range r.starting {
if time.Since(t) < pendingTimeout && !containsMachine(out, m) {
out = append(out, node.RunningMachine{Name: m, Since: t})
}
}
return out
}
func (r *Remote) Info(ctx context.Context) (node.NodeInfo, error) {
if !r.Online() {
return node.NodeInfo{}, &node.NodeOfflineError{Node: r.name}
}
r.mu.Lock()
defer r.mu.Unlock()
info := r.info
info.Running = r.runningLocked()
return info, nil
}
func (r *Remote) Running(ctx context.Context) ([]node.RunningMachine, error) {
info, err := r.Info(ctx)
return info.Running, err
}
func (r *Remote) IsRunning(machine string) bool {
r.mu.Lock()
defer r.mu.Unlock()
return containsMachine(r.runningLocked(), machine)
}
func (r *Remote) callNode(ctx context.Context, method string, params, result any) error {
l, err := r.currentLink()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, callTimeout)
defer cancel()
return l.call(ctx, method, params, result)
}
func (r *Remote) Start(ctx context.Context, spec node.StartSpec) error {
if err := r.callNode(ctx, "start", spec, nil); err != nil {
return err
}
r.mu.Lock()
r.starting[spec.Machine] = time.Now()
delete(r.stopping, spec.Machine)
r.mu.Unlock()
return nil
}
func (r *Remote) Poweroff(ctx context.Context, machine string) error {
if err := r.callNode(ctx, "poweroff", map[string]string{"machine": machine}, nil); err != nil {
return err
}
r.mu.Lock()
r.stopping[machine] = time.Now()
delete(r.starting, machine)
r.mu.Unlock()
return nil
}
func (r *Remote) SetLimits(ctx context.Context, machine string, size model.Size) error {
return r.callNode(ctx, "set_limits", callParams{Machine: machine, Size: size}, nil)
}
func (r *Remote) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
return r.callNode(ctx, "set_quota", callParams{Drive: drive, Machine: machine, MB: mb}, nil)
}
func (r *Remote) WaitStopped(ctx context.Context, machine string) error {
return r.callNode(ctx, "wait_stopped", callParams{Machine: machine}, nil)
}
func (r *Remote) ExportHome(ctx context.Context, drive, machine, parent string) (string, error) {
var res callParams
err := r.callNode(ctx, "export_home", callParams{Drive: drive, Machine: machine, Parent: parent}, &res)
return res.ID, err
}
func (r *Remote) ReadExport(ctx context.Context, id string, max int) ([]byte, bool, error) {
var res callParams
err := r.callNode(ctx, "read_export", callParams{ID: id, Max: max}, &res)
return res.Data, res.EOF, err
}
func (r *Remote) CloseExport(ctx context.Context, id string) error {
return r.callNode(ctx, "close_export", callParams{ID: id}, nil)
}
func (r *Remote) ImportHome(ctx context.Context, drive, machine string) (string, error) {
var res callParams
err := r.callNode(ctx, "import_home", callParams{Drive: drive, Machine: machine}, &res)
return res.ID, err
}
func (r *Remote) WriteImport(ctx context.Context, id string, data []byte) error {
return r.callNode(ctx, "write_import", callParams{ID: id, Data: data}, nil)
}
func (r *Remote) EndReceive(ctx context.Context, id string) error {
return r.callNode(ctx, "end_receive", callParams{ID: id}, nil)
}
func (r *Remote) FinishImport(ctx context.Context, id string, quotaMB int64) error {
return r.callNode(ctx, "finish_import", callParams{ID: id, MB: quotaMB}, nil)
}
func (r *Remote) AbortImport(ctx context.Context, id string) error {
return r.callNode(ctx, "abort_import", callParams{ID: id}, nil)
}
func (r *Remote) DeleteHome(ctx context.Context, drive, machine string) error {
return r.callNode(ctx, "delete_home", callParams{Drive: drive, Machine: machine}, nil)
}
func (r *Remote) ReadFile(ctx context.Context, drive, machine, path string, offset int64, limit int) ([]byte, int64, error) {
var res callParams
err := r.callNode(ctx, "read_file", callParams{Drive: drive, Machine: machine, Path: path, Offset: offset, Max: limit}, &res)
return res.Data, res.FileSize, tooOld(err)
}
func (r *Remote) WriteFile(ctx context.Context, drive, machine, path string, offset int64, data []byte) error {
return tooOld(r.callNode(ctx, "write_file", callParams{Drive: drive, Machine: machine, Path: path, Offset: offset, Data: data}, nil))
}
func tooOld(err error) error {
if err != nil && strings.HasPrefix(err.Error(), "unknown call ") {
return node.ErrTooOld
}
return err
}
func (r *Remote) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
l, err := r.currentLink()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(ctx, callTimeout)
defer cancel()
return l.openAgent(ctx, machine, secret)
}
func (m *Manager) attach(name string, l *link) (*Remote, error) {
if name == "" || name == m.local.Name() {
return nil, fmt.Errorf("node name %q is empty or taken by the master", name)
}
m.mu.Lock()
r := m.remotes[name]
if r == nil {
r = &Remote{name: name, starting: map[string]time.Time{}, stopping: map[string]time.Time{}}
m.remotes[name] = r
}
m.mu.Unlock()
r.mu.Lock()
old := r.link
r.link = l
r.infoAt = time.Time{}
r.mu.Unlock()
if old != nil {
old.Close()
}
return r, nil
}
func (m *Manager) ServeCluster(ctx context.Context) error {
cc := m.cfg.Cluster
cert, fingerprint, err := loadOrCreateCert(m.cfg.StateDir)
if err != nil {
return err
}
log.Printf("cluster: listening on %s, certificate fingerprint %s", cc.Listen, fingerprint)
mux := http.NewServeMux()
mux.HandleFunc("/cluster", func(w http.ResponseWriter, req *http.Request) {
if subtle.ConstantTimeCompare([]byte(req.Header.Get("X-Cluster-Secret")), []byte(cc.Secret)) != 1 {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
name := req.Header.Get("X-Node-Name")
conn, err := websocket.Accept(w, req, nil)
if err != nil {
return
}
l := newLink(conn)
r, err := m.attach(name, l)
if err != nil {
conn.Close(websocket.StatusPolicyViolation, err.Error())
return
}
back := false
l.onReport = func(info node.NodeInfo) {
r.setInfo(info)
if !back {
back = true
log.Printf("cluster: node %s connected", name)
if m.onNodeBack != nil {
go m.onNodeBack(name)
}
}
}
l.onCall = func(f frame) { l.reply(f.ID, nil, errors.New("the master takes no calls")) }
l.run()
log.Printf("cluster: node %s disconnected: %v", name, l.err)
})
srv := &http.Server{
Addr: cc.Listen,
Handler: mux,
TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13},
}
go func() {
<-ctx.Done()
srv.Close()
}()
if err := srv.ListenAndServeTLS("", ""); !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}
func loadOrCreateCert(dir string) (tls.Certificate, string, error) {
certPath, keyPath := filepath.Join(dir, "cluster.crt"), filepath.Join(dir, "cluster.key")
if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
return cert, fingerprintOf(cert.Certificate[0]), nil
}
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return tls.Certificate{}, "", err
}
serial, _ := rand.Int(rand.Reader, big.NewInt(1<<62))
tmpl := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: "slackerbot master"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().AddDate(20, 0, 0),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
return tls.Certificate{}, "", err
}
keyDER, err := x509.MarshalECPrivateKey(key)
if err != nil {
return tls.Certificate{}, "", err
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return tls.Certificate{}, "", err
}
if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
return tls.Certificate{}, "", err
}
if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
return tls.Certificate{}, "", err
}
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
return cert, fingerprintOf(der), err
}
func fingerprintOf(der []byte) string {
sum := sha256.Sum256(der)
return hex.EncodeToString(sum[:])
}
func normalizeFingerprint(s string) string {
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), ":", ""))
}
func RunSlave(ctx context.Context, cfg *config.Config, local node.Node) error {
backoff := time.Second
for {
started := time.Now()
err := slaveSession(ctx, cfg, local)
if ctx.Err() != nil {
return nil
}
if time.Since(started) > time.Minute {
backoff = time.Second
}
log.Printf("cluster: link to master lost (%v), retrying in %s", err, backoff)
select {
case <-ctx.Done():
return nil
case <-time.After(backoff):
}
backoff = min(backoff*2, 30*time.Second)
}
}
func pinnedTLS(cfg *config.Config) *tls.Config {
pinned := normalizeFingerprint(cfg.Cluster.MasterFingerprint)
savedPath := filepath.Join(cfg.StateDir, "master.fingerprint")
if pinned == "" {
if b, err := os.ReadFile(savedPath); err == nil {
pinned = normalizeFingerprint(string(b))
}
}
return &tls.Config{
MinVersion: tls.VersionTLS13,
// Not verified against a CA: the master's self-signed certificate is pinned instead.
InsecureSkipVerify: true,
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
if len(raw) == 0 {
return errors.New("master sent no certificate")
}
got := fingerprintOf(raw[0])
if pinned == "" {
log.Printf("cluster: trusting master certificate %s from now on", got)
pinned = got
_ = os.MkdirAll(cfg.StateDir, 0o700)
return os.WriteFile(savedPath, []byte(got+"\n"), 0o600)
}
if got != pinned {
return fmt.Errorf("master certificate %s doesn't match the pinned %s", got, pinned)
}
return nil
},
}
}
func slaveSession(ctx context.Context, cfg *config.Config, local node.Node) error {
client := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(cfg), Proxy: http.ProxyFromEnvironment}}
dialCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
conn, _, err := websocket.Dial(dialCtx, strings.TrimSuffix(cfg.Cluster.Master, "/")+"/cluster", &websocket.DialOptions{
HTTPClient: client,
HTTPHeader: http.Header{
"X-Cluster-Secret": []string{cfg.Cluster.Secret},
"X-Node-Name": []string{cfg.Cluster.Name},
},
})
cancel()
if err != nil {
return err
}
l := newLink(conn)
defer l.Close()
l.onCall = func(f frame) { serveCall(ctx, l, local, f) }
go l.run()
log.Printf("cluster: connected to master %s as %s", cfg.Cluster.Master, cfg.Cluster.Name)
t := time.NewTicker(reportEvery)
defer t.Stop()
for {
info, err := local.Info(ctx)
if err != nil {
log.Printf("cluster: reading node info: %v", err)
} else if err := l.report(ctx, info); err != nil {
return err
}
select {
case <-ctx.Done():
return nil
case <-l.Done():
return l.err
case <-t.C:
}
}
}
type callParams struct {
Machine string `json:"machine,omitempty"`
Secret string `json:"secret,omitempty"`
Drive string `json:"drive,omitempty"`
MB int64 `json:"mb,omitempty"`
Size model.Size `json:"size"`
ID string `json:"id,omitempty"`
Parent string `json:"parent,omitempty"`
Max int `json:"max,omitempty"`
Data []byte `json:"data,omitempty"`
EOF bool `json:"eof,omitempty"`
Path string `json:"path,omitempty"`
Offset int64 `json:"offset,omitempty"`
FileSize int64 `json:"file_size,omitempty"`
}
func serveCall(ctx context.Context, l *link, local node.Node, f frame) {
var p callParams
_ = json.Unmarshal(f.Params, &p)
switch f.Method {
case "start":
var spec node.StartSpec
if err := json.Unmarshal(f.Params, &spec); err != nil {
l.reply(f.ID, nil, err)
return
}
l.reply(f.ID, nil, local.Start(ctx, spec))
case "poweroff":
l.reply(f.ID, nil, local.Poweroff(ctx, p.Machine))
case "set_limits":
l.reply(f.ID, nil, local.SetLimits(ctx, p.Machine, p.Size))
case "set_quota":
l.reply(f.ID, nil, local.SetQuota(ctx, p.Drive, p.Machine, p.MB))
case "wait_stopped":
l.reply(f.ID, nil, local.WaitStopped(ctx, p.Machine))
case "export_home":
id, err := local.ExportHome(ctx, p.Drive, p.Machine, p.Parent)
l.reply(f.ID, callParams{ID: id}, err)
case "read_export":
data, eof, err := local.ReadExport(ctx, p.ID, min(p.Max, transferChunk))
l.reply(f.ID, callParams{Data: data, EOF: eof}, err)
case "close_export":
l.reply(f.ID, nil, local.CloseExport(ctx, p.ID))
case "import_home":
id, err := local.ImportHome(ctx, p.Drive, p.Machine)
l.reply(f.ID, callParams{ID: id}, err)
case "write_import":
l.reply(f.ID, nil, local.WriteImport(ctx, p.ID, p.Data))
case "end_receive":
l.reply(f.ID, nil, local.EndReceive(ctx, p.ID))
case "finish_import":
l.reply(f.ID, nil, local.FinishImport(ctx, p.ID, p.MB))
case "abort_import":
l.reply(f.ID, nil, local.AbortImport(ctx, p.ID))
case "delete_home":
l.reply(f.ID, nil, local.DeleteHome(ctx, p.Drive, p.Machine))
case "read_file":
data, size, err := local.ReadFile(ctx, p.Drive, p.Machine, p.Path, p.Offset, min(p.Max, transferChunk))
l.reply(f.ID, callParams{Data: data, FileSize: size}, err)
case "write_file":
l.reply(f.ID, nil, local.WriteFile(ctx, p.Drive, p.Machine, p.Path, p.Offset, p.Data))
case "open_agent":
agent, err := local.DialAgent(ctx, p.Machine, p.Secret)
if err != nil {
l.reply(f.ID, nil, err)
return
}
s := l.addStream(f.ID)
l.reply(f.ID, nil, nil)
pipe(ctx, agent, s)
default:
l.reply(f.ID, nil, fmt.Errorf("unknown call %q", f.Method))
}
}
func pipe(ctx context.Context, a, b node.MsgConn) {
ctx, cancel := context.WithCancel(ctx)
copyTo := func(dst, src node.MsgConn) {
defer cancel()
for {
msg, err := src.Read(ctx)
if err != nil {
return
}
if err := dst.Write(ctx, msg); err != nil {
return
}
}
}
go copyTo(a, b)
copyTo(b, a)
<-ctx.Done()
a.Close()
b.Close()
}