package main import ( "context" "crypto/rand" "encoding/hex" "errors" "fmt" "math" "sort" "sync" "time" ) var ( ErrMaxedOut = errors.New("no node has room for this container") ErrTooBig = errors.New("bigger than max_container allows") ErrTooSmall = errors.New("smaller than the minimum size") MinContainer = Size{MemoryMB: 256, CPUs: 0.25} ) type State string const ( StateRunning State = "running" StateStopped State = "stopped" StateNone State = "not created" StateOffline State = "on an offline node" ) type Manager struct { cfg *Config store *Store local Node mu sync.RWMutex remotes map[string]*Remote onNodeBack func(node string) startMu sync.Mutex ownerMu keyedMutex migrating sync.Map } func NewManager(cfg *Config, store *Store, local Node) *Manager { return &Manager{cfg: cfg, store: store, local: local, remotes: map[string]*Remote{}} } func (m *Manager) node(name string) (Node, error) { if name == m.local.Name() { return m.local, nil } m.mu.RLock() r := m.remotes[name] m.mu.RUnlock() if r != nil && r.Online() { return r, nil } return nil, &NodeOfflineError{Node: name} } func (m *Manager) onlineNodes() []Node { nodes := []Node{m.local} m.mu.RLock() defer m.mu.RUnlock() for _, r := range m.remotes { if r.Online() { nodes = append(nodes, r) } } return nodes } func (m *Manager) OfflineNodes() []string { m.mu.RLock() defer m.mu.RUnlock() var out []string for name, r := range m.remotes { if !r.Online() { out = append(out, name) } } sort.Strings(out) return out } func (m *Manager) Views(ctx context.Context) ([]NodeView, error) { owners := m.store.Owners() var views []NodeView for _, n := range m.onlineNodes() { info, err := n.Info(ctx) if err != nil { if n == m.local { return nil, err } continue } views = append(views, buildView(info, owners, m.cfg.Container)) } sort.Slice(views, func(i, j int) bool { return views[i].Name < views[j].Name }) return views, nil } func (m *Manager) view(ctx context.Context, n Node) (NodeView, NodeInfo, error) { info, err := n.Info(ctx) if err != nil { return NodeView{}, info, err } return buildView(info, m.store.Owners(), m.cfg.Container), info, nil } func (m *Manager) Totals(ctx context.Context) (used, usable Size) { views, _ := m.Views(ctx) for _, v := range views { used, usable = used.Add(v.Used), usable.Add(v.Usable) } return } func (m *Manager) ownerRecord(id string, kind OwnerKind) (*Owner, error) { if u := m.store.Owner(id); u != nil { return u, nil } u := &Owner{ ID: id, Kind: kind, Machine: MachineName(id, kind), Secret: randomHex(32), Size: m.cfg.Container, StorageMinMB: m.cfg.Storage.MinMB, StorageMaxMB: m.cfg.Storage.MaxMB, LastActive: time.Now(), } return u, m.store.PutOwner(u) } func (m *Manager) Owner(id string, kind OwnerKind) (*Owner, error) { unlock := m.ownerMu.Lock(id) defer unlock() return m.ownerRecord(id, kind) } func (m *Manager) spec(u *Owner) StartSpec { return StartSpec{ Machine: u.Machine, Description: "slackspawn container for " + string(u.Kind) + " " + u.ID, Secret: u.Secret, Size: u.Size, Drive: u.Drive, StorageMaxMB: u.StorageMaxMB, Env: m.cfg.AgentEnv, } } func (m *Manager) Ensure(ctx context.Context, id string, kind OwnerKind) (*Owner, error) { if m.isMigrating(id) { return nil, ErrMigrating } unlock := m.ownerMu.Lock(id) defer unlock() u, err := m.ownerRecord(id, kind) if err != nil { return nil, err } if u.Node != "" { n, err := m.node(u.Node) if err != nil { return u, err } if n.IsRunning(u.Machine) { return u, nil } } m.startMu.Lock() defer m.startMu.Unlock() if u.Node == "" { views, err := m.Views(ctx) if err != nil { return u, err } node, drive, err := choosePlacement(views, u.Size, u.StorageMinMB) if err != nil { return u, err } u.Node, u.Drive = node, drive if err := m.store.SetPlacement(u.ID, node, drive); err != nil { return u, err } } n, err := m.node(u.Node) if err != nil { return u, err } v, _, err := m.view(ctx, n) if err != nil { return u, err } if !v.Used.Add(u.Size).Fits(v.Usable) { return u, &NodeFullError{Node: u.Node} } return u, n.Start(ctx, m.spec(u)) } // The old home folder stays on its node, untouched. func (m *Manager) Recreate(id string) error { if m.isMigrating(id) { return ErrMigrating } unlock := m.ownerMu.Lock(id) defer unlock() if err := m.store.SetPlacement(id, "", ""); err != nil { return err } return m.store.DeleteSessions(id) } func (m *Manager) DialAgent(ctx context.Context, u *Owner) (msgConn, error) { n, err := m.node(u.Node) if err != nil { return nil, err } return n.DialAgent(ctx, u.Machine, u.Secret) } func (m *Manager) State(u *Owner) State { if u == nil || u.Node == "" { return StateNone } n, err := m.node(u.Node) if err != nil { return StateOffline } if n.IsRunning(u.Machine) { return StateRunning } return StateStopped } func (m *Manager) Poweroff(ctx context.Context, u *Owner) error { n, err := m.node(u.Node) if err != nil { return err } return n.Poweroff(ctx, u.Machine) } func (m *Manager) RunningOwners(ctx context.Context) []Owner { byNodeMachine := map[string]Owner{} for _, u := range m.store.Owners() { byNodeMachine[u.Node+"/"+u.Machine] = u } var out []Owner for _, n := range m.onlineNodes() { running, err := n.Running(ctx) if err != nil { continue } for _, r := range running { if u, ok := byNodeMachine[n.Name()+"/"+r.Name]; ok { out = append(out, u) } } } return out } func (m *Manager) Resize(ctx context.Context, id string, kind OwnerKind, change func(Size) (Size, error)) (old, next Size, err error) { if m.isMigrating(id) { return old, old, ErrMigrating } unlock := m.ownerMu.Lock(id) defer unlock() m.startMu.Lock() defer m.startMu.Unlock() u, err := m.ownerRecord(id, kind) if err != nil { return Size{}, Size{}, err } old = u.Size if next, err = change(old); err != nil { return old, old, err } next.CPUs = math.Round(next.CPUs*100) / 100 switch { case !MinContainer.Fits(next): return old, old, ErrTooSmall case !next.Fits(m.cfg.MaxContainer): return old, old, ErrTooBig case next == old: return old, old, nil } if u.Node != "" { n, err := m.node(u.Node) if err != nil { return old, old, err } if n.IsRunning(u.Machine) { v, _, err := m.view(ctx, n) if err != nil { return old, old, err } growth := Size{max(0, next.MemoryMB-old.MemoryMB), math.Max(0, next.CPUs-old.CPUs)} if !v.Used.Add(growth).Fits(v.Usable) { return old, old, &NodeFullError{Node: u.Node} } if err := n.SetLimits(ctx, u.Machine, next); err != nil { return old, old, err } } } return old, next, m.store.SetSize(id, next) } type StorageChange struct { OldMin, OldMax, NewMin, NewMax int64 UsedMB int64 Drive *DriveReport } func (m *Manager) SetStorage(ctx context.Context, id string, kind OwnerKind, change func(min, max int64) (int64, int64, error)) (*StorageChange, error) { if m.isMigrating(id) { return &StorageChange{}, ErrMigrating } unlock := m.ownerMu.Lock(id) defer unlock() m.startMu.Lock() defer m.startMu.Unlock() u, err := m.ownerRecord(id, kind) if err != nil { return nil, err } c := &StorageChange{OldMin: u.StorageMinMB, OldMax: u.StorageMaxMB} if c.NewMin, c.NewMax, err = change(c.OldMin, c.OldMax); err != nil { return c, err } switch { case c.NewMin < MinStorageMB || c.NewMax < MinStorageMB: return c, ErrStorageTooSmall case c.NewMin > c.NewMax: return c, ErrMinOverMax case c.NewMax > m.cfg.Storage.LimitMB: return c, ErrStorageTooBig } if u.Node == "" { views, err := m.Views(ctx) if err != nil { return c, err } if _, _, err := choosePlacement(views, u.Size, c.NewMin); err != nil { return c, err } return c, m.store.SetStorage(id, c.NewMin, c.NewMax) } n, err := m.node(u.Node) if err != nil { return c, err } v, info, err := m.view(ctx, n) if err != nil { return c, err } for _, d := range info.Drives { if d.Path == u.Drive { c.UsedMB = d.HomesMB[u.Machine] } } if c.NewMax < c.UsedMB { return c, ErrBelowUsage } var r *DriveReport for i := range v.Drives { if v.Drives[i].Path == u.Drive { r = &v.Drives[i] } } if r == nil { return c, fmt.Errorf("drive %s is no longer configured on %s", u.Drive, u.Node) } extra := committed(c.NewMin, c.UsedMB) - committed(c.OldMin, c.UsedMB) if extra > 0 && extra > r.HeadroomMB() { return c, ErrStorageFull } if err := n.SetQuota(ctx, u.Drive, u.Machine, c.NewMax); err != nil { return c, err } r.CommittedMB += extra r.BookedMB += c.NewMax - c.OldMax c.Drive = r return c, m.store.SetStorage(id, c.NewMin, c.NewMax) } func (m *Manager) StorageUsed(ctx context.Context, u *Owner) (int64, bool) { if u == nil || u.Node == "" { return 0, false } n, err := m.node(u.Node) if err != nil { return 0, false } info, err := n.Info(ctx) if err != nil { return 0, false } for _, d := range info.Drives { if used, ok := d.HomesMB[u.Machine]; ok && d.Path == u.Drive { return used, true } } return 0, false } type keyedMutex struct{ m sync.Map } func (k *keyedMutex) Lock(key string) (unlock func()) { v, _ := k.m.LoadOrStore(key, &sync.Mutex{}) mu := v.(*sync.Mutex) mu.Lock() return mu.Unlock } func randomHex(n int) string { b := make([]byte, n) if _, err := rand.Read(b); err != nil { panic(err) } return hex.EncodeToString(b) }