814 lines
21 KiB
Go
814 lines
21 KiB
Go
package manager
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ecdsa"
|
|
"crypto/elliptic"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"crypto/subtle"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"crypto/x509/pkix"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"encoding/pem"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"math/big"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/coder/websocket"
|
|
|
|
"slackerbot/internal/config"
|
|
"slackerbot/internal/model"
|
|
"slackerbot/internal/node"
|
|
)
|
|
|
|
const (
|
|
reportEvery = 5 * time.Second
|
|
offlineAfter = 20 * time.Second
|
|
callTimeout = 3 * time.Minute
|
|
pendingTimeout = 30 * time.Second
|
|
streamBuffer = 4096
|
|
)
|
|
|
|
type frame struct {
|
|
Type string `json:"t"`
|
|
ID int64 `json:"id,omitempty"`
|
|
Method string `json:"m,omitempty"`
|
|
Params json.RawMessage `json:"p,omitempty"`
|
|
Result json.RawMessage `json:"r,omitempty"`
|
|
Error string `json:"e,omitempty"`
|
|
Data string `json:"d,omitempty"`
|
|
}
|
|
|
|
type link struct {
|
|
conn *websocket.Conn
|
|
writeMu sync.Mutex
|
|
nextID atomic.Int64
|
|
|
|
mu sync.Mutex
|
|
pending map[int64]chan frame
|
|
streams map[int64]*stream
|
|
|
|
onCall func(f frame)
|
|
onReport func(info node.NodeInfo)
|
|
|
|
done chan struct{}
|
|
err error
|
|
}
|
|
|
|
func newLink(conn *websocket.Conn) *link {
|
|
conn.SetReadLimit(64 << 20)
|
|
return &link{
|
|
conn: conn,
|
|
pending: map[int64]chan frame{},
|
|
streams: map[int64]*stream{},
|
|
done: make(chan struct{}),
|
|
}
|
|
}
|
|
|
|
func (l *link) Done() <-chan struct{} { return l.done }
|
|
|
|
func (l *link) Close() { l.conn.Close(websocket.StatusNormalClosure, "") }
|
|
|
|
func (l *link) send(ctx context.Context, f frame) error {
|
|
b, err := json.Marshal(f)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
l.writeMu.Lock()
|
|
defer l.writeMu.Unlock()
|
|
return l.conn.Write(ctx, websocket.MessageText, b)
|
|
}
|
|
|
|
func (l *link) run() {
|
|
defer func() {
|
|
l.mu.Lock()
|
|
streams := l.streams
|
|
l.streams = map[int64]*stream{}
|
|
l.mu.Unlock()
|
|
for _, s := range streams {
|
|
s.closeLocal()
|
|
}
|
|
close(l.done)
|
|
}()
|
|
for {
|
|
_, data, err := l.conn.Read(context.Background())
|
|
if err != nil {
|
|
l.err = err
|
|
return
|
|
}
|
|
var f frame
|
|
if json.Unmarshal(data, &f) != nil {
|
|
continue
|
|
}
|
|
switch f.Type {
|
|
case "call":
|
|
if l.onCall != nil {
|
|
go l.onCall(f)
|
|
}
|
|
case "reply":
|
|
l.mu.Lock()
|
|
ch := l.pending[f.ID]
|
|
l.mu.Unlock()
|
|
if ch != nil {
|
|
ch <- f
|
|
}
|
|
case "report":
|
|
var info node.NodeInfo
|
|
if l.onReport != nil && json.Unmarshal(f.Params, &info) == nil {
|
|
l.onReport(info)
|
|
}
|
|
case "data":
|
|
if s := l.stream(f.ID); s != nil {
|
|
select {
|
|
case s.in <- []byte(f.Data):
|
|
default:
|
|
s.Close()
|
|
}
|
|
}
|
|
case "close":
|
|
if s := l.stream(f.ID); s != nil {
|
|
s.closeLocal()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (l *link) call(ctx context.Context, method string, params, result any) error {
|
|
return l.callWithID(ctx, l.nextID.Add(1), method, params, result)
|
|
}
|
|
|
|
func (l *link) callWithID(ctx context.Context, id int64, method string, params, result any) error {
|
|
ch := make(chan frame, 1)
|
|
l.mu.Lock()
|
|
l.pending[id] = ch
|
|
l.mu.Unlock()
|
|
defer func() {
|
|
l.mu.Lock()
|
|
delete(l.pending, id)
|
|
l.mu.Unlock()
|
|
}()
|
|
|
|
p, err := json.Marshal(params)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := l.send(ctx, frame{Type: "call", ID: id, Method: method, Params: p}); err != nil {
|
|
return err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-l.done:
|
|
return fmt.Errorf("link closed: %v", l.err)
|
|
case f := <-ch:
|
|
if f.Error != "" {
|
|
return errors.New(f.Error)
|
|
}
|
|
if result != nil && len(f.Result) > 0 {
|
|
return json.Unmarshal(f.Result, result)
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func (l *link) reply(id int64, result any, err error) {
|
|
f := frame{Type: "reply", ID: id}
|
|
if err != nil {
|
|
f.Error = err.Error()
|
|
} else if result != nil {
|
|
f.Result, _ = json.Marshal(result)
|
|
}
|
|
_ = l.send(context.Background(), f)
|
|
}
|
|
|
|
func (l *link) report(ctx context.Context, info node.NodeInfo) error {
|
|
p, err := json.Marshal(info)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return l.send(ctx, frame{Type: "report", Params: p})
|
|
}
|
|
|
|
func (l *link) stream(id int64) *stream {
|
|
l.mu.Lock()
|
|
defer l.mu.Unlock()
|
|
return l.streams[id]
|
|
}
|
|
|
|
func (l *link) addStream(id int64) *stream {
|
|
s := &stream{l: l, id: id, in: make(chan []byte, streamBuffer), closed: make(chan struct{})}
|
|
l.mu.Lock()
|
|
l.streams[id] = s
|
|
l.mu.Unlock()
|
|
return s
|
|
}
|
|
|
|
func (l *link) dropStream(id int64) {
|
|
l.mu.Lock()
|
|
delete(l.streams, id)
|
|
l.mu.Unlock()
|
|
}
|
|
|
|
// The stream shares its ID with the "open_agent" call, so it's registered
|
|
// before the call goes out and no early data is lost.
|
|
func (l *link) openAgent(ctx context.Context, machine, secret string) (*stream, error) {
|
|
id := l.nextID.Add(1)
|
|
s := l.addStream(id)
|
|
err := l.callWithID(ctx, id, "open_agent", map[string]string{"machine": machine, "secret": secret}, nil)
|
|
if err != nil {
|
|
l.dropStream(id)
|
|
return nil, err
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
type stream struct {
|
|
l *link
|
|
id int64
|
|
in chan []byte
|
|
closed chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
func (s *stream) Read(ctx context.Context) ([]byte, error) {
|
|
select {
|
|
case b := <-s.in:
|
|
return b, nil
|
|
default:
|
|
}
|
|
select {
|
|
case b := <-s.in:
|
|
return b, nil
|
|
case <-s.closed:
|
|
return nil, errors.New("stream closed")
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (s *stream) Write(ctx context.Context, b []byte) error {
|
|
select {
|
|
case <-s.closed:
|
|
return errors.New("stream closed")
|
|
default:
|
|
}
|
|
return s.l.send(ctx, frame{Type: "data", ID: s.id, Data: string(b)})
|
|
}
|
|
|
|
func (s *stream) Close() error {
|
|
s.once.Do(func() {
|
|
_ = s.l.send(context.Background(), frame{Type: "close", ID: s.id})
|
|
s.l.dropStream(s.id)
|
|
close(s.closed)
|
|
})
|
|
return nil
|
|
}
|
|
|
|
func (s *stream) closeLocal() {
|
|
s.once.Do(func() {
|
|
s.l.dropStream(s.id)
|
|
close(s.closed)
|
|
})
|
|
}
|
|
|
|
type Remote struct {
|
|
name string
|
|
|
|
mu sync.Mutex
|
|
link *link
|
|
info node.NodeInfo
|
|
infoAt time.Time
|
|
starting map[string]time.Time
|
|
stopping map[string]time.Time
|
|
}
|
|
|
|
func (r *Remote) Name() string { return r.name }
|
|
|
|
func (r *Remote) Online() bool {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.link == nil || time.Since(r.infoAt) > offlineAfter {
|
|
return false
|
|
}
|
|
select {
|
|
case <-r.link.Done():
|
|
return false
|
|
default:
|
|
return true
|
|
}
|
|
}
|
|
|
|
func (r *Remote) currentLink() (*link, error) {
|
|
if !r.Online() {
|
|
return nil, &node.NodeOfflineError{Node: r.name}
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return r.link, nil
|
|
}
|
|
|
|
func (r *Remote) setInfo(info node.NodeInfo) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
info.Name = r.name
|
|
r.info, r.infoAt = info, time.Now()
|
|
for _, m := range info.Running {
|
|
delete(r.starting, m.Name)
|
|
}
|
|
for m := range r.stopping {
|
|
if !containsMachine(info.Running, m) {
|
|
delete(r.stopping, m)
|
|
}
|
|
}
|
|
}
|
|
|
|
func containsMachine(list []node.RunningMachine, name string) bool {
|
|
for _, m := range list {
|
|
if m.Name == name {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Reports can lag behind a start or stop the master just asked for,
|
|
// so recent requests override them for a short while.
|
|
func (r *Remote) runningLocked() []node.RunningMachine {
|
|
var out []node.RunningMachine
|
|
for _, m := range r.info.Running {
|
|
if t, ok := r.stopping[m.Name]; ok && time.Since(t) < pendingTimeout {
|
|
continue
|
|
}
|
|
out = append(out, m)
|
|
}
|
|
for m, t := range r.starting {
|
|
if time.Since(t) < pendingTimeout && !containsMachine(out, m) {
|
|
out = append(out, node.RunningMachine{Name: m, Since: t})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (r *Remote) Info(ctx context.Context) (node.NodeInfo, error) {
|
|
if !r.Online() {
|
|
return node.NodeInfo{}, &node.NodeOfflineError{Node: r.name}
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
info := r.info
|
|
info.Running = r.runningLocked()
|
|
return info, nil
|
|
}
|
|
|
|
func (r *Remote) Running(ctx context.Context) ([]node.RunningMachine, error) {
|
|
info, err := r.Info(ctx)
|
|
return info.Running, err
|
|
}
|
|
|
|
func (r *Remote) IsRunning(machine string) bool {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return containsMachine(r.runningLocked(), machine)
|
|
}
|
|
|
|
func (r *Remote) callNode(ctx context.Context, method string, params, result any) error {
|
|
l, err := r.currentLink()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
|
defer cancel()
|
|
return l.call(ctx, method, params, result)
|
|
}
|
|
|
|
func (r *Remote) Start(ctx context.Context, spec node.StartSpec) error {
|
|
if err := r.callNode(ctx, "start", spec, nil); err != nil {
|
|
return err
|
|
}
|
|
r.mu.Lock()
|
|
r.starting[spec.Machine] = time.Now()
|
|
delete(r.stopping, spec.Machine)
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
func (r *Remote) Poweroff(ctx context.Context, machine string) error {
|
|
if err := r.callNode(ctx, "poweroff", map[string]string{"machine": machine}, nil); err != nil {
|
|
return err
|
|
}
|
|
r.mu.Lock()
|
|
r.stopping[machine] = time.Now()
|
|
delete(r.starting, machine)
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
func (r *Remote) SetLimits(ctx context.Context, machine string, size model.Size) error {
|
|
return r.callNode(ctx, "set_limits", callParams{Machine: machine, Size: size}, nil)
|
|
}
|
|
|
|
func (r *Remote) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
|
|
return r.callNode(ctx, "set_quota", callParams{Drive: drive, Machine: machine, MB: mb}, nil)
|
|
}
|
|
|
|
func (r *Remote) WaitStopped(ctx context.Context, machine string) error {
|
|
return r.callNode(ctx, "wait_stopped", callParams{Machine: machine}, nil)
|
|
}
|
|
|
|
func (r *Remote) ExportHome(ctx context.Context, drive, machine, parent string) (string, error) {
|
|
var res callParams
|
|
err := r.callNode(ctx, "export_home", callParams{Drive: drive, Machine: machine, Parent: parent}, &res)
|
|
return res.ID, err
|
|
}
|
|
|
|
func (r *Remote) ReadExport(ctx context.Context, id string, max int) ([]byte, bool, error) {
|
|
var res callParams
|
|
err := r.callNode(ctx, "read_export", callParams{ID: id, Max: max}, &res)
|
|
return res.Data, res.EOF, err
|
|
}
|
|
|
|
func (r *Remote) CloseExport(ctx context.Context, id string) error {
|
|
return r.callNode(ctx, "close_export", callParams{ID: id}, nil)
|
|
}
|
|
|
|
func (r *Remote) ImportHome(ctx context.Context, drive, machine string) (string, error) {
|
|
var res callParams
|
|
err := r.callNode(ctx, "import_home", callParams{Drive: drive, Machine: machine}, &res)
|
|
return res.ID, err
|
|
}
|
|
|
|
func (r *Remote) WriteImport(ctx context.Context, id string, data []byte) error {
|
|
return r.callNode(ctx, "write_import", callParams{ID: id, Data: data}, nil)
|
|
}
|
|
|
|
func (r *Remote) EndReceive(ctx context.Context, id string) error {
|
|
return r.callNode(ctx, "end_receive", callParams{ID: id}, nil)
|
|
}
|
|
|
|
func (r *Remote) FinishImport(ctx context.Context, id string, quotaMB int64) error {
|
|
return r.callNode(ctx, "finish_import", callParams{ID: id, MB: quotaMB}, nil)
|
|
}
|
|
|
|
func (r *Remote) AbortImport(ctx context.Context, id string) error {
|
|
return r.callNode(ctx, "abort_import", callParams{ID: id}, nil)
|
|
}
|
|
|
|
func (r *Remote) DeleteHome(ctx context.Context, drive, machine string) error {
|
|
return r.callNode(ctx, "delete_home", callParams{Drive: drive, Machine: machine}, nil)
|
|
}
|
|
|
|
func (r *Remote) ReadFile(ctx context.Context, drive, machine, path string, offset int64, limit int) ([]byte, int64, error) {
|
|
var res callParams
|
|
err := r.callNode(ctx, "read_file", callParams{Drive: drive, Machine: machine, Path: path, Offset: offset, Max: limit}, &res)
|
|
return res.Data, res.FileSize, tooOld(err)
|
|
}
|
|
|
|
func (r *Remote) WriteFile(ctx context.Context, drive, machine, path string, offset int64, data []byte) error {
|
|
return tooOld(r.callNode(ctx, "write_file", callParams{Drive: drive, Machine: machine, Path: path, Offset: offset, Data: data}, nil))
|
|
}
|
|
|
|
func tooOld(err error) error {
|
|
if err != nil && strings.HasPrefix(err.Error(), "unknown call ") {
|
|
return node.ErrTooOld
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (r *Remote) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
|
|
l, err := r.currentLink()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
|
defer cancel()
|
|
return l.openAgent(ctx, machine, secret)
|
|
}
|
|
|
|
func (m *Manager) attach(name string, l *link) (*Remote, error) {
|
|
if name == "" || name == m.local.Name() {
|
|
return nil, fmt.Errorf("node name %q is empty or taken by the master", name)
|
|
}
|
|
m.mu.Lock()
|
|
r := m.remotes[name]
|
|
if r == nil {
|
|
r = &Remote{name: name, starting: map[string]time.Time{}, stopping: map[string]time.Time{}}
|
|
m.remotes[name] = r
|
|
}
|
|
m.mu.Unlock()
|
|
|
|
r.mu.Lock()
|
|
old := r.link
|
|
r.link = l
|
|
r.infoAt = time.Time{}
|
|
r.mu.Unlock()
|
|
if old != nil {
|
|
old.Close()
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
func (m *Manager) ServeCluster(ctx context.Context) error {
|
|
cc := m.cfg.Cluster
|
|
cert, fingerprint, err := loadOrCreateCert(m.cfg.StateDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Printf("cluster: listening on %s, certificate fingerprint %s", cc.Listen, fingerprint)
|
|
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/cluster", func(w http.ResponseWriter, req *http.Request) {
|
|
if subtle.ConstantTimeCompare([]byte(req.Header.Get("X-Cluster-Secret")), []byte(cc.Secret)) != 1 {
|
|
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
name := req.Header.Get("X-Node-Name")
|
|
conn, err := websocket.Accept(w, req, nil)
|
|
if err != nil {
|
|
return
|
|
}
|
|
l := newLink(conn)
|
|
r, err := m.attach(name, l)
|
|
if err != nil {
|
|
conn.Close(websocket.StatusPolicyViolation, err.Error())
|
|
return
|
|
}
|
|
back := false
|
|
l.onReport = func(info node.NodeInfo) {
|
|
r.setInfo(info)
|
|
if !back {
|
|
back = true
|
|
log.Printf("cluster: node %s connected", name)
|
|
if m.onNodeBack != nil {
|
|
go m.onNodeBack(name)
|
|
}
|
|
}
|
|
}
|
|
l.onCall = func(f frame) { l.reply(f.ID, nil, errors.New("the master takes no calls")) }
|
|
l.run()
|
|
log.Printf("cluster: node %s disconnected: %v", name, l.err)
|
|
})
|
|
|
|
srv := &http.Server{
|
|
Addr: cc.Listen,
|
|
Handler: mux,
|
|
TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13},
|
|
}
|
|
go func() {
|
|
<-ctx.Done()
|
|
srv.Close()
|
|
}()
|
|
if err := srv.ListenAndServeTLS("", ""); !errors.Is(err, http.ErrServerClosed) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func loadOrCreateCert(dir string) (tls.Certificate, string, error) {
|
|
certPath, keyPath := filepath.Join(dir, "cluster.crt"), filepath.Join(dir, "cluster.key")
|
|
if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
|
|
return cert, fingerprintOf(cert.Certificate[0]), nil
|
|
}
|
|
|
|
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
if err != nil {
|
|
return tls.Certificate{}, "", err
|
|
}
|
|
serial, _ := rand.Int(rand.Reader, big.NewInt(1<<62))
|
|
tmpl := &x509.Certificate{
|
|
SerialNumber: serial,
|
|
Subject: pkix.Name{CommonName: "slackerbot master"},
|
|
NotBefore: time.Now().Add(-time.Hour),
|
|
NotAfter: time.Now().AddDate(20, 0, 0),
|
|
KeyUsage: x509.KeyUsageDigitalSignature,
|
|
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
|
}
|
|
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
|
if err != nil {
|
|
return tls.Certificate{}, "", err
|
|
}
|
|
keyDER, err := x509.MarshalECPrivateKey(key)
|
|
if err != nil {
|
|
return tls.Certificate{}, "", err
|
|
}
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return tls.Certificate{}, "", err
|
|
}
|
|
if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
|
|
return tls.Certificate{}, "", err
|
|
}
|
|
if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
|
|
return tls.Certificate{}, "", err
|
|
}
|
|
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
|
return cert, fingerprintOf(der), err
|
|
}
|
|
|
|
func fingerprintOf(der []byte) string {
|
|
sum := sha256.Sum256(der)
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func normalizeFingerprint(s string) string {
|
|
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), ":", ""))
|
|
}
|
|
|
|
func RunSlave(ctx context.Context, cfg *config.Config, local node.Node) error {
|
|
backoff := time.Second
|
|
for {
|
|
started := time.Now()
|
|
err := slaveSession(ctx, cfg, local)
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
if time.Since(started) > time.Minute {
|
|
backoff = time.Second
|
|
}
|
|
log.Printf("cluster: link to master lost (%v), retrying in %s", err, backoff)
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case <-time.After(backoff):
|
|
}
|
|
backoff = min(backoff*2, 30*time.Second)
|
|
}
|
|
}
|
|
|
|
func pinnedTLS(cfg *config.Config) *tls.Config {
|
|
pinned := normalizeFingerprint(cfg.Cluster.MasterFingerprint)
|
|
savedPath := filepath.Join(cfg.StateDir, "master.fingerprint")
|
|
if pinned == "" {
|
|
if b, err := os.ReadFile(savedPath); err == nil {
|
|
pinned = normalizeFingerprint(string(b))
|
|
}
|
|
}
|
|
return &tls.Config{
|
|
MinVersion: tls.VersionTLS13,
|
|
// Not verified against a CA: the master's self-signed certificate is pinned instead.
|
|
InsecureSkipVerify: true,
|
|
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
|
|
if len(raw) == 0 {
|
|
return errors.New("master sent no certificate")
|
|
}
|
|
got := fingerprintOf(raw[0])
|
|
if pinned == "" {
|
|
log.Printf("cluster: trusting master certificate %s from now on", got)
|
|
pinned = got
|
|
_ = os.MkdirAll(cfg.StateDir, 0o700)
|
|
return os.WriteFile(savedPath, []byte(got+"\n"), 0o600)
|
|
}
|
|
if got != pinned {
|
|
return fmt.Errorf("master certificate %s doesn't match the pinned %s", got, pinned)
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
}
|
|
|
|
func slaveSession(ctx context.Context, cfg *config.Config, local node.Node) error {
|
|
client := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(cfg), Proxy: http.ProxyFromEnvironment}}
|
|
dialCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
conn, _, err := websocket.Dial(dialCtx, strings.TrimSuffix(cfg.Cluster.Master, "/")+"/cluster", &websocket.DialOptions{
|
|
HTTPClient: client,
|
|
HTTPHeader: http.Header{
|
|
"X-Cluster-Secret": []string{cfg.Cluster.Secret},
|
|
"X-Node-Name": []string{cfg.Cluster.Name},
|
|
},
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
l := newLink(conn)
|
|
defer l.Close()
|
|
l.onCall = func(f frame) { serveCall(ctx, l, local, f) }
|
|
go l.run()
|
|
log.Printf("cluster: connected to master %s as %s", cfg.Cluster.Master, cfg.Cluster.Name)
|
|
|
|
t := time.NewTicker(reportEvery)
|
|
defer t.Stop()
|
|
for {
|
|
info, err := local.Info(ctx)
|
|
if err != nil {
|
|
log.Printf("cluster: reading node info: %v", err)
|
|
} else if err := l.report(ctx, info); err != nil {
|
|
return err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case <-l.Done():
|
|
return l.err
|
|
case <-t.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
type callParams struct {
|
|
Machine string `json:"machine,omitempty"`
|
|
Secret string `json:"secret,omitempty"`
|
|
Drive string `json:"drive,omitempty"`
|
|
MB int64 `json:"mb,omitempty"`
|
|
Size model.Size `json:"size"`
|
|
ID string `json:"id,omitempty"`
|
|
Parent string `json:"parent,omitempty"`
|
|
Max int `json:"max,omitempty"`
|
|
Data []byte `json:"data,omitempty"`
|
|
EOF bool `json:"eof,omitempty"`
|
|
|
|
Path string `json:"path,omitempty"`
|
|
Offset int64 `json:"offset,omitempty"`
|
|
FileSize int64 `json:"file_size,omitempty"`
|
|
}
|
|
|
|
func serveCall(ctx context.Context, l *link, local node.Node, f frame) {
|
|
var p callParams
|
|
_ = json.Unmarshal(f.Params, &p)
|
|
|
|
switch f.Method {
|
|
case "start":
|
|
var spec node.StartSpec
|
|
if err := json.Unmarshal(f.Params, &spec); err != nil {
|
|
l.reply(f.ID, nil, err)
|
|
return
|
|
}
|
|
l.reply(f.ID, nil, local.Start(ctx, spec))
|
|
case "poweroff":
|
|
l.reply(f.ID, nil, local.Poweroff(ctx, p.Machine))
|
|
case "set_limits":
|
|
l.reply(f.ID, nil, local.SetLimits(ctx, p.Machine, p.Size))
|
|
case "set_quota":
|
|
l.reply(f.ID, nil, local.SetQuota(ctx, p.Drive, p.Machine, p.MB))
|
|
case "wait_stopped":
|
|
l.reply(f.ID, nil, local.WaitStopped(ctx, p.Machine))
|
|
case "export_home":
|
|
id, err := local.ExportHome(ctx, p.Drive, p.Machine, p.Parent)
|
|
l.reply(f.ID, callParams{ID: id}, err)
|
|
case "read_export":
|
|
data, eof, err := local.ReadExport(ctx, p.ID, min(p.Max, transferChunk))
|
|
l.reply(f.ID, callParams{Data: data, EOF: eof}, err)
|
|
case "close_export":
|
|
l.reply(f.ID, nil, local.CloseExport(ctx, p.ID))
|
|
case "import_home":
|
|
id, err := local.ImportHome(ctx, p.Drive, p.Machine)
|
|
l.reply(f.ID, callParams{ID: id}, err)
|
|
case "write_import":
|
|
l.reply(f.ID, nil, local.WriteImport(ctx, p.ID, p.Data))
|
|
case "end_receive":
|
|
l.reply(f.ID, nil, local.EndReceive(ctx, p.ID))
|
|
case "finish_import":
|
|
l.reply(f.ID, nil, local.FinishImport(ctx, p.ID, p.MB))
|
|
case "abort_import":
|
|
l.reply(f.ID, nil, local.AbortImport(ctx, p.ID))
|
|
case "delete_home":
|
|
l.reply(f.ID, nil, local.DeleteHome(ctx, p.Drive, p.Machine))
|
|
case "read_file":
|
|
data, size, err := local.ReadFile(ctx, p.Drive, p.Machine, p.Path, p.Offset, min(p.Max, transferChunk))
|
|
l.reply(f.ID, callParams{Data: data, FileSize: size}, err)
|
|
case "write_file":
|
|
l.reply(f.ID, nil, local.WriteFile(ctx, p.Drive, p.Machine, p.Path, p.Offset, p.Data))
|
|
case "open_agent":
|
|
agent, err := local.DialAgent(ctx, p.Machine, p.Secret)
|
|
if err != nil {
|
|
l.reply(f.ID, nil, err)
|
|
return
|
|
}
|
|
s := l.addStream(f.ID)
|
|
l.reply(f.ID, nil, nil)
|
|
pipe(ctx, agent, s)
|
|
default:
|
|
l.reply(f.ID, nil, fmt.Errorf("unknown call %q", f.Method))
|
|
}
|
|
}
|
|
|
|
func pipe(ctx context.Context, a, b node.MsgConn) {
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
copyTo := func(dst, src node.MsgConn) {
|
|
defer cancel()
|
|
for {
|
|
msg, err := src.Read(ctx)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if err := dst.Write(ctx, msg); err != nil {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
go copyTo(a, b)
|
|
copyTo(b, a)
|
|
<-ctx.Done()
|
|
a.Close()
|
|
b.Close()
|
|
}
|