Files
slacker-bot/slackerbot/internal/manager/manager.go
T
2026-11-01 15:10:39 -05:00

520 lines
12 KiB
Go

package manager
import (
"context"
"errors"
"fmt"
"log"
"math"
"sort"
"sync"
"time"
"slackerbot/internal/balance"
"slackerbot/internal/config"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/state"
"slackerbot/internal/storage"
"slackerbot/internal/util"
)
var (
ErrTooBig = errors.New("bigger than max_container allows")
ErrTooSmall = errors.New("smaller than the minimum size")
MinContainer = model.Size{MemoryMB: 256, CPUs: 0.25}
)
type State string
const (
StateRunning State = "running"
StateStopped State = "stopped"
StateNone State = "not created"
StateOffline State = "on an offline node"
)
type StartPhase string
const (
StartPlacing StartPhase = "placing"
StartBooting StartPhase = "booting"
StartRegistered StartPhase = "registered"
)
// StartEvent is only reported when Ensure really has to start the container.
type StartEvent struct {
Phase StartPhase
Node string
Drive string
Size model.Size
}
type Manager struct {
cfg *config.Config
store *state.Store
local node.Node
mu sync.RWMutex
remotes map[string]*Remote
onNodeBack func(node string)
startMu sync.Mutex
ownerMu util.KeyedMutex
migrating sync.Map
}
func New(cfg *config.Config, store *state.Store, local node.Node) *Manager {
return &Manager{cfg: cfg, store: store, local: local, remotes: map[string]*Remote{}}
}
func (m *Manager) SetNodeBackHandler(fn func(node string)) { m.onNodeBack = fn }
func (m *Manager) LocalName() string { return m.local.Name() }
func (m *Manager) nodeByName(name string) (node.Node, error) {
if name == m.local.Name() {
return m.local, nil
}
m.mu.RLock()
r := m.remotes[name]
m.mu.RUnlock()
if r != nil && r.Online() {
return r, nil
}
return nil, &node.NodeOfflineError{Node: name}
}
func (m *Manager) onlineNodes() []node.Node {
nodes := []node.Node{m.local}
m.mu.RLock()
defer m.mu.RUnlock()
for _, r := range m.remotes {
if r.Online() {
nodes = append(nodes, r)
}
}
return nodes
}
func (m *Manager) OfflineNodes() []string {
m.mu.RLock()
defer m.mu.RUnlock()
var out []string
for name, r := range m.remotes {
if !r.Online() {
out = append(out, name)
}
}
sort.Strings(out)
return out
}
func (m *Manager) Views(ctx context.Context) ([]balance.NodeView, error) {
owners := m.store.Owners()
var views []balance.NodeView
for _, n := range m.onlineNodes() {
info, err := n.Info(ctx)
if err != nil {
if n == m.local {
return nil, err
}
continue
}
views = append(views, balance.BuildView(info, owners, m.cfg.Container))
}
sort.Slice(views, func(i, j int) bool { return views[i].Name < views[j].Name })
return views, nil
}
func (m *Manager) view(ctx context.Context, n node.Node) (balance.NodeView, node.NodeInfo, error) {
info, err := n.Info(ctx)
if err != nil {
return balance.NodeView{}, info, err
}
return balance.BuildView(info, m.store.Owners(), m.cfg.Container), info, nil
}
func (m *Manager) Totals(ctx context.Context) (used, usable model.Size) {
views, _ := m.Views(ctx)
for _, v := range views {
used, usable = used.Add(v.Used), usable.Add(v.Usable)
}
return
}
func (m *Manager) ownerRecord(id string, kind model.OwnerKind) (*model.Owner, error) {
if u := m.store.Owner(id); u != nil {
return u, nil
}
u := &model.Owner{
ID: id,
Kind: kind,
Machine: model.MachineName(id, kind),
Secret: util.RandomHex(32),
Size: m.cfg.Container,
StorageMinMB: m.cfg.Storage.MinMB,
StorageMaxMB: m.cfg.Storage.MaxMB,
LastActive: time.Now(),
}
return u, m.store.PutOwner(u)
}
func (m *Manager) Owner(id string, kind model.OwnerKind) (*model.Owner, error) {
unlock := m.ownerMu.Lock(id)
defer unlock()
return m.ownerRecord(id, kind)
}
func (m *Manager) spec(u *model.Owner) node.StartSpec {
return node.StartSpec{
Machine: u.Machine,
Description: "slackerbot container for " + string(u.Kind) + " " + u.ID,
Secret: u.Secret,
Size: u.Size,
Drive: u.Drive,
StorageMaxMB: u.StorageMaxMB,
Env: m.cfg.AgentEnv,
}
}
func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (*model.Owner, error) {
return m.EnsureProgress(ctx, id, kind, nil)
}
func (m *Manager) EnsureProgress(ctx context.Context, id string, kind model.OwnerKind, progress func(StartEvent)) (*model.Owner, error) {
if progress == nil {
progress = func(StartEvent) {}
}
if m.IsMigrating(id) {
return nil, ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
u, err := m.ownerRecord(id, kind)
if err != nil {
return nil, err
}
if u.Node != "" {
n, err := m.nodeByName(u.Node)
if err != nil {
return u, err
}
if n.IsRunning(u.Machine) {
return u, nil
}
}
m.startMu.Lock()
defer m.startMu.Unlock()
if u.Node == "" {
progress(StartEvent{Phase: StartPlacing, Size: u.Size})
views, err := m.Views(ctx)
if err != nil {
return u, err
}
target, drive, err := balance.ChoosePlacement(views, u.Size, u.StorageMinMB)
if err != nil {
return u, err
}
u.Node, u.Drive = target, drive
if err := m.store.SetPlacement(u.ID, target, drive); err != nil {
return u, err
}
}
n, err := m.nodeByName(u.Node)
if err != nil {
return u, err
}
v, _, err := m.view(ctx, n)
if err != nil {
return u, err
}
if !v.Used.Add(u.Size).Fits(v.Usable) {
return u, &node.NodeFullError{Node: u.Node}
}
log.Printf("container: starting %s for %s %s on %s (drive %s, %s)", u.Machine, u.Kind, u.ID, u.Node, u.Drive, u.Size)
progress(StartEvent{Phase: StartBooting, Node: u.Node, Drive: u.Drive, Size: u.Size})
if err := n.Start(ctx, m.spec(u)); err != nil {
log.Printf("container: starting %s on %s failed: %v", u.Machine, u.Node, err)
return u, err
}
log.Printf("container: %s is running on %s", u.Machine, u.Node)
progress(StartEvent{Phase: StartRegistered, Node: u.Node, Drive: u.Drive, Size: u.Size})
return u, nil
}
// Restart stops a running container and starts it again, so it boots the
// node's current system.
func (m *Manager) Restart(ctx context.Context, id string, kind model.OwnerKind, progress func(StartEvent)) (*model.Owner, error) {
if m.IsMigrating(id) {
return nil, ErrMigrating
}
unlock := m.ownerMu.Lock(id)
u, err := m.ownerRecord(id, kind)
if err == nil && u.Node != "" {
var n node.Node
if n, err = m.nodeByName(u.Node); err == nil && n.IsRunning(u.Machine) {
log.Printf("container: restarting %s for %s %s on %s", u.Machine, u.Kind, u.ID, u.Node)
if err = n.Poweroff(ctx, u.Machine); err == nil {
err = n.WaitStopped(ctx, u.Machine)
}
}
}
unlock()
if err != nil {
return u, err
}
return m.EnsureProgress(ctx, id, kind, progress)
}
func (m *Manager) Outdated(ctx context.Context, u *model.Owner) bool {
if u == nil || u.Node == "" {
return false
}
n, err := m.nodeByName(u.Node)
if err != nil {
return false
}
running, err := n.Running(ctx)
if err != nil {
return false
}
for _, r := range running {
if r.Name == u.Machine {
return r.Outdated
}
}
return false
}
// The old home folder stays on its node, untouched.
func (m *Manager) Recreate(id string) error {
if m.IsMigrating(id) {
return ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
if err := m.store.SetPlacement(id, "", ""); err != nil {
return err
}
return m.store.DeleteSessions(id)
}
func (m *Manager) DialAgent(ctx context.Context, u *model.Owner) (node.MsgConn, error) {
n, err := m.nodeByName(u.Node)
if err != nil {
return nil, err
}
return n.DialAgent(ctx, u.Machine, u.Secret)
}
func (m *Manager) State(u *model.Owner) State {
if u == nil || u.Node == "" {
return StateNone
}
n, err := m.nodeByName(u.Node)
if err != nil {
return StateOffline
}
if n.IsRunning(u.Machine) {
return StateRunning
}
return StateStopped
}
func (m *Manager) Poweroff(ctx context.Context, u *model.Owner) error {
n, err := m.nodeByName(u.Node)
if err != nil {
return err
}
log.Printf("container: shutting down %s for %s %s on %s", u.Machine, u.Kind, u.ID, u.Node)
if err := n.Poweroff(ctx, u.Machine); err != nil {
log.Printf("container: shutting down %s on %s failed: %v", u.Machine, u.Node, err)
return err
}
return nil
}
func (m *Manager) WaitStopped(ctx context.Context, u *model.Owner) error {
n, err := m.nodeByName(u.Node)
if err != nil {
return err
}
return n.WaitStopped(ctx, u.Machine)
}
func (m *Manager) RunningOwners(ctx context.Context) []model.Owner {
byNodeMachine := map[string]model.Owner{}
for _, u := range m.store.Owners() {
byNodeMachine[u.Node+"/"+u.Machine] = u
}
var out []model.Owner
for _, n := range m.onlineNodes() {
running, err := n.Running(ctx)
if err != nil {
continue
}
for _, r := range running {
if u, ok := byNodeMachine[n.Name()+"/"+r.Name]; ok {
out = append(out, u)
}
}
}
return out
}
func (m *Manager) Resize(ctx context.Context, id string, kind model.OwnerKind, change func(model.Size) (model.Size, error)) (old, next model.Size, err error) {
if m.IsMigrating(id) {
return old, old, ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
m.startMu.Lock()
defer m.startMu.Unlock()
u, err := m.ownerRecord(id, kind)
if err != nil {
return model.Size{}, model.Size{}, err
}
old = u.Size
if next, err = change(old); err != nil {
return old, old, err
}
next.CPUs = math.Round(next.CPUs*100) / 100
switch {
case !MinContainer.Fits(next):
return old, old, ErrTooSmall
case !next.Fits(m.cfg.MaxContainer):
return old, old, ErrTooBig
case next == old:
return old, old, nil
}
if u.Node != "" {
n, err := m.nodeByName(u.Node)
if err != nil {
return old, old, err
}
if n.IsRunning(u.Machine) {
v, _, err := m.view(ctx, n)
if err != nil {
return old, old, err
}
growth := model.Size{MemoryMB: max(0, next.MemoryMB-old.MemoryMB), CPUs: math.Max(0, next.CPUs-old.CPUs)}
if !v.Used.Add(growth).Fits(v.Usable) {
return old, old, &node.NodeFullError{Node: u.Node}
}
if err := n.SetLimits(ctx, u.Machine, next); err != nil {
return old, old, err
}
}
}
return old, next, m.store.SetSize(id, next)
}
type StorageChange struct {
OldMin, OldMax, NewMin, NewMax int64
UsedMB int64
Drive *storage.DriveReport
}
func (m *Manager) SetStorage(ctx context.Context, id string, kind model.OwnerKind, change func(min, max int64) (int64, int64, error)) (*StorageChange, error) {
if m.IsMigrating(id) {
return &StorageChange{}, ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
m.startMu.Lock()
defer m.startMu.Unlock()
u, err := m.ownerRecord(id, kind)
if err != nil {
return nil, err
}
c := &StorageChange{OldMin: u.StorageMinMB, OldMax: u.StorageMaxMB}
if c.NewMin, c.NewMax, err = change(c.OldMin, c.OldMax); err != nil {
return c, err
}
switch {
case c.NewMin < config.MinStorageMB || c.NewMax < config.MinStorageMB:
return c, storage.ErrStorageTooSmall
case c.NewMin > c.NewMax:
return c, storage.ErrMinOverMax
case c.NewMax > m.cfg.Storage.LimitMB:
return c, storage.ErrStorageTooBig
}
if u.Node == "" {
views, err := m.Views(ctx)
if err != nil {
return c, err
}
if _, _, err := balance.ChoosePlacement(views, u.Size, c.NewMin); err != nil {
return c, err
}
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
}
n, err := m.nodeByName(u.Node)
if err != nil {
return c, err
}
v, info, err := m.view(ctx, n)
if err != nil {
return c, err
}
for _, d := range info.Drives {
if d.Path == u.Drive {
c.UsedMB = d.HomesMB[u.Machine]
}
}
if c.NewMax < c.UsedMB {
return c, storage.ErrBelowUsage
}
var r *storage.DriveReport
for i := range v.Drives {
if v.Drives[i].Path == u.Drive {
r = &v.Drives[i]
}
}
if r == nil {
return c, fmt.Errorf("drive %s is no longer configured on %s", u.Drive, u.Node)
}
extra := storage.Committed(c.NewMin, c.UsedMB) - storage.Committed(c.OldMin, c.UsedMB)
if extra > 0 && extra > r.HeadroomMB() {
return c, storage.ErrStorageFull
}
if err := n.SetQuota(ctx, u.Drive, u.Machine, c.NewMax); err != nil {
return c, err
}
r.CommittedMB += extra
r.BookedMB += c.NewMax - c.OldMax
c.Drive = r
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
}
func (m *Manager) StorageUsed(ctx context.Context, u *model.Owner) (int64, bool) {
if u == nil || u.Node == "" {
return 0, false
}
n, err := m.nodeByName(u.Node)
if err != nil {
return 0, false
}
info, err := n.Info(ctx)
if err != nil {
return 0, false
}
for _, d := range info.Drives {
if used, ok := d.HomesMB[u.Machine]; ok && d.Path == u.Drive {
return used, true
}
}
return 0, false
}