package manager import ( "context" "errors" "fmt" "log" "math" "sort" "sync" "time" "slackerbot/internal/balance" "slackerbot/internal/config" "slackerbot/internal/model" "slackerbot/internal/node" "slackerbot/internal/state" "slackerbot/internal/storage" "slackerbot/internal/util" ) var ( ErrTooBig = errors.New("bigger than max_container allows") ErrTooSmall = errors.New("smaller than the minimum size") MinContainer = model.Size{MemoryMB: 256, CPUs: 0.25} ) type State string const ( StateRunning State = "running" StateStopped State = "stopped" StateNone State = "not created" StateOffline State = "on an offline node" ) type Manager struct { cfg *config.Config store *state.Store local node.Node mu sync.RWMutex remotes map[string]*Remote onNodeBack func(node string) startMu sync.Mutex ownerMu util.KeyedMutex migrating sync.Map } func New(cfg *config.Config, store *state.Store, local node.Node) *Manager { return &Manager{cfg: cfg, store: store, local: local, remotes: map[string]*Remote{}} } func (m *Manager) SetNodeBackHandler(fn func(node string)) { m.onNodeBack = fn } func (m *Manager) nodeByName(name string) (node.Node, error) { if name == m.local.Name() { return m.local, nil } m.mu.RLock() r := m.remotes[name] m.mu.RUnlock() if r != nil && r.Online() { return r, nil } return nil, &node.NodeOfflineError{Node: name} } func (m *Manager) onlineNodes() []node.Node { nodes := []node.Node{m.local} m.mu.RLock() defer m.mu.RUnlock() for _, r := range m.remotes { if r.Online() { nodes = append(nodes, r) } } return nodes } func (m *Manager) OfflineNodes() []string { m.mu.RLock() defer m.mu.RUnlock() var out []string for name, r := range m.remotes { if !r.Online() { out = append(out, name) } } sort.Strings(out) return out } func (m *Manager) Views(ctx context.Context) ([]balance.NodeView, error) { owners := m.store.Owners() var views []balance.NodeView for _, n := range m.onlineNodes() { info, err := n.Info(ctx) if err != nil { if n == m.local { return nil, err } continue } views = append(views, balance.BuildView(info, owners, m.cfg.Container)) } sort.Slice(views, func(i, j int) bool { return views[i].Name < views[j].Name }) return views, nil } func (m *Manager) view(ctx context.Context, n node.Node) (balance.NodeView, node.NodeInfo, error) { info, err := n.Info(ctx) if err != nil { return balance.NodeView{}, info, err } return balance.BuildView(info, m.store.Owners(), m.cfg.Container), info, nil } func (m *Manager) Totals(ctx context.Context) (used, usable model.Size) { views, _ := m.Views(ctx) for _, v := range views { used, usable = used.Add(v.Used), usable.Add(v.Usable) } return } func (m *Manager) ownerRecord(id string, kind model.OwnerKind) (*model.Owner, error) { if u := m.store.Owner(id); u != nil { return u, nil } u := &model.Owner{ ID: id, Kind: kind, Machine: model.MachineName(id, kind), Secret: util.RandomHex(32), Size: m.cfg.Container, StorageMinMB: m.cfg.Storage.MinMB, StorageMaxMB: m.cfg.Storage.MaxMB, LastActive: time.Now(), } return u, m.store.PutOwner(u) } func (m *Manager) Owner(id string, kind model.OwnerKind) (*model.Owner, error) { unlock := m.ownerMu.Lock(id) defer unlock() return m.ownerRecord(id, kind) } func (m *Manager) spec(u *model.Owner) node.StartSpec { return node.StartSpec{ Machine: u.Machine, Description: "slackerbot container for " + string(u.Kind) + " " + u.ID, Secret: u.Secret, Size: u.Size, Drive: u.Drive, StorageMaxMB: u.StorageMaxMB, Env: m.cfg.AgentEnv, } } func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (*model.Owner, error) { if m.IsMigrating(id) { return nil, ErrMigrating } unlock := m.ownerMu.Lock(id) defer unlock() u, err := m.ownerRecord(id, kind) if err != nil { return nil, err } if u.Node != "" { n, err := m.nodeByName(u.Node) if err != nil { return u, err } if n.IsRunning(u.Machine) { return u, nil } } m.startMu.Lock() defer m.startMu.Unlock() if u.Node == "" { views, err := m.Views(ctx) if err != nil { return u, err } target, drive, err := balance.ChoosePlacement(views, u.Size, u.StorageMinMB) if err != nil { return u, err } u.Node, u.Drive = target, drive if err := m.store.SetPlacement(u.ID, target, drive); err != nil { return u, err } } n, err := m.nodeByName(u.Node) if err != nil { return u, err } v, _, err := m.view(ctx, n) if err != nil { return u, err } if !v.Used.Add(u.Size).Fits(v.Usable) { return u, &node.NodeFullError{Node: u.Node} } log.Printf("container: starting %s for %s %s on %s (drive %s, %s)", u.Machine, u.Kind, u.ID, u.Node, u.Drive, u.Size) if err := n.Start(ctx, m.spec(u)); err != nil { log.Printf("container: starting %s on %s failed: %v", u.Machine, u.Node, err) return u, err } log.Printf("container: %s is running on %s", u.Machine, u.Node) return u, nil } // The old home folder stays on its node, untouched. func (m *Manager) Recreate(id string) error { if m.IsMigrating(id) { return ErrMigrating } unlock := m.ownerMu.Lock(id) defer unlock() if err := m.store.SetPlacement(id, "", ""); err != nil { return err } return m.store.DeleteSessions(id) } func (m *Manager) DialAgent(ctx context.Context, u *model.Owner) (node.MsgConn, error) { n, err := m.nodeByName(u.Node) if err != nil { return nil, err } return n.DialAgent(ctx, u.Machine, u.Secret) } func (m *Manager) State(u *model.Owner) State { if u == nil || u.Node == "" { return StateNone } n, err := m.nodeByName(u.Node) if err != nil { return StateOffline } if n.IsRunning(u.Machine) { return StateRunning } return StateStopped } func (m *Manager) Poweroff(ctx context.Context, u *model.Owner) error { n, err := m.nodeByName(u.Node) if err != nil { return err } log.Printf("container: shutting down %s for %s %s on %s", u.Machine, u.Kind, u.ID, u.Node) if err := n.Poweroff(ctx, u.Machine); err != nil { log.Printf("container: shutting down %s on %s failed: %v", u.Machine, u.Node, err) return err } return nil } func (m *Manager) RunningOwners(ctx context.Context) []model.Owner { byNodeMachine := map[string]model.Owner{} for _, u := range m.store.Owners() { byNodeMachine[u.Node+"/"+u.Machine] = u } var out []model.Owner for _, n := range m.onlineNodes() { running, err := n.Running(ctx) if err != nil { continue } for _, r := range running { if u, ok := byNodeMachine[n.Name()+"/"+r.Name]; ok { out = append(out, u) } } } return out } func (m *Manager) Resize(ctx context.Context, id string, kind model.OwnerKind, change func(model.Size) (model.Size, error)) (old, next model.Size, err error) { if m.IsMigrating(id) { return old, old, ErrMigrating } unlock := m.ownerMu.Lock(id) defer unlock() m.startMu.Lock() defer m.startMu.Unlock() u, err := m.ownerRecord(id, kind) if err != nil { return model.Size{}, model.Size{}, err } old = u.Size if next, err = change(old); err != nil { return old, old, err } next.CPUs = math.Round(next.CPUs*100) / 100 switch { case !MinContainer.Fits(next): return old, old, ErrTooSmall case !next.Fits(m.cfg.MaxContainer): return old, old, ErrTooBig case next == old: return old, old, nil } if u.Node != "" { n, err := m.nodeByName(u.Node) if err != nil { return old, old, err } if n.IsRunning(u.Machine) { v, _, err := m.view(ctx, n) if err != nil { return old, old, err } growth := model.Size{MemoryMB: max(0, next.MemoryMB-old.MemoryMB), CPUs: math.Max(0, next.CPUs-old.CPUs)} if !v.Used.Add(growth).Fits(v.Usable) { return old, old, &node.NodeFullError{Node: u.Node} } if err := n.SetLimits(ctx, u.Machine, next); err != nil { return old, old, err } } } return old, next, m.store.SetSize(id, next) } type StorageChange struct { OldMin, OldMax, NewMin, NewMax int64 UsedMB int64 Drive *storage.DriveReport } func (m *Manager) SetStorage(ctx context.Context, id string, kind model.OwnerKind, change func(min, max int64) (int64, int64, error)) (*StorageChange, error) { if m.IsMigrating(id) { return &StorageChange{}, ErrMigrating } unlock := m.ownerMu.Lock(id) defer unlock() m.startMu.Lock() defer m.startMu.Unlock() u, err := m.ownerRecord(id, kind) if err != nil { return nil, err } c := &StorageChange{OldMin: u.StorageMinMB, OldMax: u.StorageMaxMB} if c.NewMin, c.NewMax, err = change(c.OldMin, c.OldMax); err != nil { return c, err } switch { case c.NewMin < config.MinStorageMB || c.NewMax < config.MinStorageMB: return c, storage.ErrStorageTooSmall case c.NewMin > c.NewMax: return c, storage.ErrMinOverMax case c.NewMax > m.cfg.Storage.LimitMB: return c, storage.ErrStorageTooBig } if u.Node == "" { views, err := m.Views(ctx) if err != nil { return c, err } if _, _, err := balance.ChoosePlacement(views, u.Size, c.NewMin); err != nil { return c, err } return c, m.store.SetStorage(id, c.NewMin, c.NewMax) } n, err := m.nodeByName(u.Node) if err != nil { return c, err } v, info, err := m.view(ctx, n) if err != nil { return c, err } for _, d := range info.Drives { if d.Path == u.Drive { c.UsedMB = d.HomesMB[u.Machine] } } if c.NewMax < c.UsedMB { return c, storage.ErrBelowUsage } var r *storage.DriveReport for i := range v.Drives { if v.Drives[i].Path == u.Drive { r = &v.Drives[i] } } if r == nil { return c, fmt.Errorf("drive %s is no longer configured on %s", u.Drive, u.Node) } extra := storage.Committed(c.NewMin, c.UsedMB) - storage.Committed(c.OldMin, c.UsedMB) if extra > 0 && extra > r.HeadroomMB() { return c, storage.ErrStorageFull } if err := n.SetQuota(ctx, u.Drive, u.Machine, c.NewMax); err != nil { return c, err } r.CommittedMB += extra r.BookedMB += c.NewMax - c.OldMax c.Drive = r return c, m.store.SetStorage(id, c.NewMin, c.NewMax) } func (m *Manager) StorageUsed(ctx context.Context, u *model.Owner) (int64, bool) { if u == nil || u.Node == "" { return 0, false } n, err := m.nodeByName(u.Node) if err != nil { return 0, false } info, err := n.Info(ctx) if err != nil { return 0, false } for _, d := range info.Drives { if used, ok := d.HomesMB[u.Machine]; ok && d.Path == u.Drive { return used, true } } return 0, false }