package manager import ( "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/sha256" "crypto/subtle" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/hex" "encoding/json" "encoding/pem" "errors" "fmt" "log" "math/big" "net/http" "os" "path/filepath" "strings" "sync" "sync/atomic" "time" "github.com/coder/websocket" "slackerbot/internal/config" "slackerbot/internal/model" "slackerbot/internal/node" ) const ( reportEvery = 5 * time.Second offlineAfter = 20 * time.Second callTimeout = 3 * time.Minute pendingTimeout = 30 * time.Second streamBuffer = 4096 ) type frame struct { Type string `json:"t"` ID int64 `json:"id,omitempty"` Method string `json:"m,omitempty"` Params json.RawMessage `json:"p,omitempty"` Result json.RawMessage `json:"r,omitempty"` Error string `json:"e,omitempty"` Data string `json:"d,omitempty"` } type link struct { conn *websocket.Conn writeMu sync.Mutex nextID atomic.Int64 mu sync.Mutex pending map[int64]chan frame streams map[int64]*stream onCall func(f frame) onReport func(info node.NodeInfo) done chan struct{} err error } func newLink(conn *websocket.Conn) *link { conn.SetReadLimit(64 << 20) return &link{ conn: conn, pending: map[int64]chan frame{}, streams: map[int64]*stream{}, done: make(chan struct{}), } } func (l *link) Done() <-chan struct{} { return l.done } func (l *link) Close() { l.conn.Close(websocket.StatusNormalClosure, "") } func (l *link) send(ctx context.Context, f frame) error { b, err := json.Marshal(f) if err != nil { return err } l.writeMu.Lock() defer l.writeMu.Unlock() return l.conn.Write(ctx, websocket.MessageText, b) } func (l *link) run() { defer func() { l.mu.Lock() streams := l.streams l.streams = map[int64]*stream{} l.mu.Unlock() for _, s := range streams { s.closeLocal() } close(l.done) }() for { _, data, err := l.conn.Read(context.Background()) if err != nil { l.err = err return } var f frame if json.Unmarshal(data, &f) != nil { continue } switch f.Type { case "call": if l.onCall != nil { go l.onCall(f) } case "reply": l.mu.Lock() ch := l.pending[f.ID] l.mu.Unlock() if ch != nil { ch <- f } case "report": var info node.NodeInfo if l.onReport != nil && json.Unmarshal(f.Params, &info) == nil { l.onReport(info) } case "data": if s := l.stream(f.ID); s != nil { select { case s.in <- []byte(f.Data): default: s.Close() } } case "close": if s := l.stream(f.ID); s != nil { s.closeLocal() } } } } func (l *link) call(ctx context.Context, method string, params, result any) error { return l.callWithID(ctx, l.nextID.Add(1), method, params, result) } func (l *link) callWithID(ctx context.Context, id int64, method string, params, result any) error { ch := make(chan frame, 1) l.mu.Lock() l.pending[id] = ch l.mu.Unlock() defer func() { l.mu.Lock() delete(l.pending, id) l.mu.Unlock() }() p, err := json.Marshal(params) if err != nil { return err } if err := l.send(ctx, frame{Type: "call", ID: id, Method: method, Params: p}); err != nil { return err } select { case <-ctx.Done(): return ctx.Err() case <-l.done: return fmt.Errorf("link closed: %v", l.err) case f := <-ch: if f.Error != "" { return errors.New(f.Error) } if result != nil && len(f.Result) > 0 { return json.Unmarshal(f.Result, result) } return nil } } func (l *link) reply(id int64, result any, err error) { f := frame{Type: "reply", ID: id} if err != nil { f.Error = err.Error() } else if result != nil { f.Result, _ = json.Marshal(result) } _ = l.send(context.Background(), f) } func (l *link) report(ctx context.Context, info node.NodeInfo) error { p, err := json.Marshal(info) if err != nil { return err } return l.send(ctx, frame{Type: "report", Params: p}) } func (l *link) stream(id int64) *stream { l.mu.Lock() defer l.mu.Unlock() return l.streams[id] } func (l *link) addStream(id int64) *stream { s := &stream{l: l, id: id, in: make(chan []byte, streamBuffer), closed: make(chan struct{})} l.mu.Lock() l.streams[id] = s l.mu.Unlock() return s } func (l *link) dropStream(id int64) { l.mu.Lock() delete(l.streams, id) l.mu.Unlock() } // The stream shares its ID with the "open_agent" call, so it's registered // before the call goes out and no early data is lost. func (l *link) openAgent(ctx context.Context, machine, secret string) (*stream, error) { id := l.nextID.Add(1) s := l.addStream(id) err := l.callWithID(ctx, id, "open_agent", map[string]string{"machine": machine, "secret": secret}, nil) if err != nil { l.dropStream(id) return nil, err } return s, nil } type stream struct { l *link id int64 in chan []byte closed chan struct{} once sync.Once } func (s *stream) Read(ctx context.Context) ([]byte, error) { select { case b := <-s.in: return b, nil default: } select { case b := <-s.in: return b, nil case <-s.closed: return nil, errors.New("stream closed") case <-ctx.Done(): return nil, ctx.Err() } } func (s *stream) Write(ctx context.Context, b []byte) error { select { case <-s.closed: return errors.New("stream closed") default: } return s.l.send(ctx, frame{Type: "data", ID: s.id, Data: string(b)}) } func (s *stream) Close() error { s.once.Do(func() { _ = s.l.send(context.Background(), frame{Type: "close", ID: s.id}) s.l.dropStream(s.id) close(s.closed) }) return nil } func (s *stream) closeLocal() { s.once.Do(func() { s.l.dropStream(s.id) close(s.closed) }) } type Remote struct { name string mu sync.Mutex link *link info node.NodeInfo infoAt time.Time starting map[string]time.Time stopping map[string]time.Time } func (r *Remote) Name() string { return r.name } func (r *Remote) Online() bool { r.mu.Lock() defer r.mu.Unlock() if r.link == nil || time.Since(r.infoAt) > offlineAfter { return false } select { case <-r.link.Done(): return false default: return true } } func (r *Remote) currentLink() (*link, error) { if !r.Online() { return nil, &node.NodeOfflineError{Node: r.name} } r.mu.Lock() defer r.mu.Unlock() return r.link, nil } func (r *Remote) setInfo(info node.NodeInfo) { r.mu.Lock() defer r.mu.Unlock() info.Name = r.name r.info, r.infoAt = info, time.Now() for _, m := range info.Running { delete(r.starting, m.Name) } for m := range r.stopping { if !containsMachine(info.Running, m) { delete(r.stopping, m) } } } func containsMachine(list []node.RunningMachine, name string) bool { for _, m := range list { if m.Name == name { return true } } return false } // Reports can lag behind a start or stop the master just asked for, // so recent requests override them for a short while. func (r *Remote) runningLocked() []node.RunningMachine { var out []node.RunningMachine for _, m := range r.info.Running { if t, ok := r.stopping[m.Name]; ok && time.Since(t) < pendingTimeout { continue } out = append(out, m) } for m, t := range r.starting { if time.Since(t) < pendingTimeout && !containsMachine(out, m) { out = append(out, node.RunningMachine{Name: m, Since: t}) } } return out } func (r *Remote) Info(ctx context.Context) (node.NodeInfo, error) { if !r.Online() { return node.NodeInfo{}, &node.NodeOfflineError{Node: r.name} } r.mu.Lock() defer r.mu.Unlock() info := r.info info.Running = r.runningLocked() return info, nil } func (r *Remote) Running(ctx context.Context) ([]node.RunningMachine, error) { info, err := r.Info(ctx) return info.Running, err } func (r *Remote) IsRunning(machine string) bool { r.mu.Lock() defer r.mu.Unlock() return containsMachine(r.runningLocked(), machine) } func (r *Remote) callNode(ctx context.Context, method string, params, result any) error { l, err := r.currentLink() if err != nil { return err } ctx, cancel := context.WithTimeout(ctx, callTimeout) defer cancel() return l.call(ctx, method, params, result) } func (r *Remote) Start(ctx context.Context, spec node.StartSpec) error { if err := r.callNode(ctx, "start", spec, nil); err != nil { return err } r.mu.Lock() r.starting[spec.Machine] = time.Now() delete(r.stopping, spec.Machine) r.mu.Unlock() return nil } func (r *Remote) Poweroff(ctx context.Context, machine string) error { if err := r.callNode(ctx, "poweroff", map[string]string{"machine": machine}, nil); err != nil { return err } r.mu.Lock() r.stopping[machine] = time.Now() delete(r.starting, machine) r.mu.Unlock() return nil } func (r *Remote) SetLimits(ctx context.Context, machine string, size model.Size) error { return r.callNode(ctx, "set_limits", callParams{Machine: machine, Size: size}, nil) } func (r *Remote) SetQuota(ctx context.Context, drive, machine string, mb int64) error { return r.callNode(ctx, "set_quota", callParams{Drive: drive, Machine: machine, MB: mb}, nil) } func (r *Remote) WaitStopped(ctx context.Context, machine string) error { return r.callNode(ctx, "wait_stopped", callParams{Machine: machine}, nil) } func (r *Remote) ExportHome(ctx context.Context, drive, machine, parent string) (string, error) { var res callParams err := r.callNode(ctx, "export_home", callParams{Drive: drive, Machine: machine, Parent: parent}, &res) return res.ID, err } func (r *Remote) ReadExport(ctx context.Context, id string, max int) ([]byte, bool, error) { var res callParams err := r.callNode(ctx, "read_export", callParams{ID: id, Max: max}, &res) return res.Data, res.EOF, err } func (r *Remote) CloseExport(ctx context.Context, id string) error { return r.callNode(ctx, "close_export", callParams{ID: id}, nil) } func (r *Remote) ImportHome(ctx context.Context, drive, machine string) (string, error) { var res callParams err := r.callNode(ctx, "import_home", callParams{Drive: drive, Machine: machine}, &res) return res.ID, err } func (r *Remote) WriteImport(ctx context.Context, id string, data []byte) error { return r.callNode(ctx, "write_import", callParams{ID: id, Data: data}, nil) } func (r *Remote) EndReceive(ctx context.Context, id string) error { return r.callNode(ctx, "end_receive", callParams{ID: id}, nil) } func (r *Remote) FinishImport(ctx context.Context, id string, quotaMB int64) error { return r.callNode(ctx, "finish_import", callParams{ID: id, MB: quotaMB}, nil) } func (r *Remote) AbortImport(ctx context.Context, id string) error { return r.callNode(ctx, "abort_import", callParams{ID: id}, nil) } func (r *Remote) DeleteHome(ctx context.Context, drive, machine string) error { return r.callNode(ctx, "delete_home", callParams{Drive: drive, Machine: machine}, nil) } func (r *Remote) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) { l, err := r.currentLink() if err != nil { return nil, err } ctx, cancel := context.WithTimeout(ctx, callTimeout) defer cancel() return l.openAgent(ctx, machine, secret) } func (m *Manager) attach(name string, l *link) (*Remote, error) { if name == "" || name == m.local.Name() { return nil, fmt.Errorf("node name %q is empty or taken by the master", name) } m.mu.Lock() r := m.remotes[name] if r == nil { r = &Remote{name: name, starting: map[string]time.Time{}, stopping: map[string]time.Time{}} m.remotes[name] = r } m.mu.Unlock() r.mu.Lock() old := r.link r.link = l r.infoAt = time.Time{} r.mu.Unlock() if old != nil { old.Close() } return r, nil } func (m *Manager) ServeCluster(ctx context.Context) error { cc := m.cfg.Cluster cert, fingerprint, err := loadOrCreateCert(m.cfg.StateDir) if err != nil { return err } log.Printf("cluster: listening on %s, certificate fingerprint %s", cc.Listen, fingerprint) mux := http.NewServeMux() mux.HandleFunc("/cluster", func(w http.ResponseWriter, req *http.Request) { if subtle.ConstantTimeCompare([]byte(req.Header.Get("X-Cluster-Secret")), []byte(cc.Secret)) != 1 { http.Error(w, "unauthorized", http.StatusUnauthorized) return } name := req.Header.Get("X-Node-Name") conn, err := websocket.Accept(w, req, nil) if err != nil { return } l := newLink(conn) r, err := m.attach(name, l) if err != nil { conn.Close(websocket.StatusPolicyViolation, err.Error()) return } back := false l.onReport = func(info node.NodeInfo) { r.setInfo(info) if !back { back = true log.Printf("cluster: node %s connected", name) if m.onNodeBack != nil { go m.onNodeBack(name) } } } l.onCall = func(f frame) { l.reply(f.ID, nil, errors.New("the master takes no calls")) } l.run() log.Printf("cluster: node %s disconnected: %v", name, l.err) }) srv := &http.Server{ Addr: cc.Listen, Handler: mux, TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13}, } go func() { <-ctx.Done() srv.Close() }() if err := srv.ListenAndServeTLS("", ""); !errors.Is(err, http.ErrServerClosed) { return err } return nil } func loadOrCreateCert(dir string) (tls.Certificate, string, error) { certPath, keyPath := filepath.Join(dir, "cluster.crt"), filepath.Join(dir, "cluster.key") if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil { return cert, fingerprintOf(cert.Certificate[0]), nil } key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { return tls.Certificate{}, "", err } serial, _ := rand.Int(rand.Reader, big.NewInt(1<<62)) tmpl := &x509.Certificate{ SerialNumber: serial, Subject: pkix.Name{CommonName: "slackerbot master"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().AddDate(20, 0, 0), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, } der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) if err != nil { return tls.Certificate{}, "", err } keyDER, err := x509.MarshalECPrivateKey(key) if err != nil { return tls.Certificate{}, "", err } if err := os.MkdirAll(dir, 0o700); err != nil { return tls.Certificate{}, "", err } if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil { return tls.Certificate{}, "", err } if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil { return tls.Certificate{}, "", err } cert, err := tls.LoadX509KeyPair(certPath, keyPath) return cert, fingerprintOf(der), err } func fingerprintOf(der []byte) string { sum := sha256.Sum256(der) return hex.EncodeToString(sum[:]) } func normalizeFingerprint(s string) string { return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), ":", "")) } func RunSlave(ctx context.Context, cfg *config.Config, local node.Node) error { backoff := time.Second for { started := time.Now() err := slaveSession(ctx, cfg, local) if ctx.Err() != nil { return nil } if time.Since(started) > time.Minute { backoff = time.Second } log.Printf("cluster: link to master lost (%v), retrying in %s", err, backoff) select { case <-ctx.Done(): return nil case <-time.After(backoff): } backoff = min(backoff*2, 30*time.Second) } } func pinnedTLS(cfg *config.Config) *tls.Config { pinned := normalizeFingerprint(cfg.Cluster.MasterFingerprint) savedPath := filepath.Join(cfg.StateDir, "master.fingerprint") if pinned == "" { if b, err := os.ReadFile(savedPath); err == nil { pinned = normalizeFingerprint(string(b)) } } return &tls.Config{ MinVersion: tls.VersionTLS13, // Not verified against a CA: the master's self-signed certificate is pinned instead. InsecureSkipVerify: true, VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error { if len(raw) == 0 { return errors.New("master sent no certificate") } got := fingerprintOf(raw[0]) if pinned == "" { log.Printf("cluster: trusting master certificate %s from now on", got) pinned = got _ = os.MkdirAll(cfg.StateDir, 0o700) return os.WriteFile(savedPath, []byte(got+"\n"), 0o600) } if got != pinned { return fmt.Errorf("master certificate %s doesn't match the pinned %s", got, pinned) } return nil }, } } func slaveSession(ctx context.Context, cfg *config.Config, local node.Node) error { client := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(cfg), Proxy: http.ProxyFromEnvironment}} dialCtx, cancel := context.WithTimeout(ctx, 30*time.Second) conn, _, err := websocket.Dial(dialCtx, strings.TrimSuffix(cfg.Cluster.Master, "/")+"/cluster", &websocket.DialOptions{ HTTPClient: client, HTTPHeader: http.Header{ "X-Cluster-Secret": []string{cfg.Cluster.Secret}, "X-Node-Name": []string{cfg.Cluster.Name}, }, }) cancel() if err != nil { return err } l := newLink(conn) defer l.Close() l.onCall = func(f frame) { serveCall(ctx, l, local, f) } go l.run() log.Printf("cluster: connected to master %s as %s", cfg.Cluster.Master, cfg.Cluster.Name) t := time.NewTicker(reportEvery) defer t.Stop() for { info, err := local.Info(ctx) if err != nil { log.Printf("cluster: reading node info: %v", err) } else if err := l.report(ctx, info); err != nil { return err } select { case <-ctx.Done(): return nil case <-l.Done(): return l.err case <-t.C: } } } type callParams struct { Machine string `json:"machine,omitempty"` Secret string `json:"secret,omitempty"` Drive string `json:"drive,omitempty"` MB int64 `json:"mb,omitempty"` Size model.Size `json:"size"` ID string `json:"id,omitempty"` Parent string `json:"parent,omitempty"` Max int `json:"max,omitempty"` Data []byte `json:"data,omitempty"` EOF bool `json:"eof,omitempty"` } func serveCall(ctx context.Context, l *link, local node.Node, f frame) { var p callParams _ = json.Unmarshal(f.Params, &p) switch f.Method { case "start": var spec node.StartSpec if err := json.Unmarshal(f.Params, &spec); err != nil { l.reply(f.ID, nil, err) return } l.reply(f.ID, nil, local.Start(ctx, spec)) case "poweroff": l.reply(f.ID, nil, local.Poweroff(ctx, p.Machine)) case "set_limits": l.reply(f.ID, nil, local.SetLimits(ctx, p.Machine, p.Size)) case "set_quota": l.reply(f.ID, nil, local.SetQuota(ctx, p.Drive, p.Machine, p.MB)) case "wait_stopped": l.reply(f.ID, nil, local.WaitStopped(ctx, p.Machine)) case "export_home": id, err := local.ExportHome(ctx, p.Drive, p.Machine, p.Parent) l.reply(f.ID, callParams{ID: id}, err) case "read_export": data, eof, err := local.ReadExport(ctx, p.ID, min(p.Max, transferChunk)) l.reply(f.ID, callParams{Data: data, EOF: eof}, err) case "close_export": l.reply(f.ID, nil, local.CloseExport(ctx, p.ID)) case "import_home": id, err := local.ImportHome(ctx, p.Drive, p.Machine) l.reply(f.ID, callParams{ID: id}, err) case "write_import": l.reply(f.ID, nil, local.WriteImport(ctx, p.ID, p.Data)) case "end_receive": l.reply(f.ID, nil, local.EndReceive(ctx, p.ID)) case "finish_import": l.reply(f.ID, nil, local.FinishImport(ctx, p.ID, p.MB)) case "abort_import": l.reply(f.ID, nil, local.AbortImport(ctx, p.ID)) case "delete_home": l.reply(f.ID, nil, local.DeleteHome(ctx, p.Drive, p.Machine)) case "open_agent": agent, err := local.DialAgent(ctx, p.Machine, p.Secret) if err != nil { l.reply(f.ID, nil, err) return } s := l.addStream(f.ID) l.reply(f.ID, nil, nil) pipe(ctx, agent, s) default: l.reply(f.ID, nil, fmt.Errorf("unknown call %q", f.Method)) } } func pipe(ctx context.Context, a, b node.MsgConn) { ctx, cancel := context.WithCancel(ctx) copyTo := func(dst, src node.MsgConn) { defer cancel() for { msg, err := src.Read(ctx) if err != nil { return } if err := dst.Write(ctx, msg); err != nil { return } } } go copyTo(a, b) copyTo(b, a) <-ctx.Done() a.Close() b.Close() }