476 lines
11 KiB
Go
476 lines
11 KiB
Go
package manager
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"math"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
|
|
"slackerbot/internal/balance"
|
|
"slackerbot/internal/config"
|
|
"slackerbot/internal/model"
|
|
"slackerbot/internal/node"
|
|
"slackerbot/internal/state"
|
|
"slackerbot/internal/storage"
|
|
"slackerbot/internal/util"
|
|
)
|
|
|
|
var (
|
|
ErrTooBig = errors.New("bigger than max_container allows")
|
|
ErrTooSmall = errors.New("smaller than the minimum size")
|
|
|
|
MinContainer = model.Size{MemoryMB: 256, CPUs: 0.25}
|
|
)
|
|
|
|
type State string
|
|
|
|
const (
|
|
StateRunning State = "running"
|
|
StateStopped State = "stopped"
|
|
StateNone State = "not created"
|
|
StateOffline State = "on an offline node"
|
|
)
|
|
|
|
type StartPhase string
|
|
|
|
const (
|
|
StartPlacing StartPhase = "placing"
|
|
StartBooting StartPhase = "booting"
|
|
StartRegistered StartPhase = "registered"
|
|
)
|
|
|
|
// StartEvent is only reported when Ensure really has to start the container.
|
|
type StartEvent struct {
|
|
Phase StartPhase
|
|
Node string
|
|
Drive string
|
|
Size model.Size
|
|
}
|
|
|
|
type Manager struct {
|
|
cfg *config.Config
|
|
store *state.Store
|
|
local node.Node
|
|
|
|
mu sync.RWMutex
|
|
remotes map[string]*Remote
|
|
onNodeBack func(node string)
|
|
|
|
startMu sync.Mutex
|
|
ownerMu util.KeyedMutex
|
|
migrating sync.Map
|
|
}
|
|
|
|
func New(cfg *config.Config, store *state.Store, local node.Node) *Manager {
|
|
return &Manager{cfg: cfg, store: store, local: local, remotes: map[string]*Remote{}}
|
|
}
|
|
|
|
func (m *Manager) SetNodeBackHandler(fn func(node string)) { m.onNodeBack = fn }
|
|
|
|
func (m *Manager) LocalName() string { return m.local.Name() }
|
|
|
|
func (m *Manager) nodeByName(name string) (node.Node, error) {
|
|
if name == m.local.Name() {
|
|
return m.local, nil
|
|
}
|
|
m.mu.RLock()
|
|
r := m.remotes[name]
|
|
m.mu.RUnlock()
|
|
if r != nil && r.Online() {
|
|
return r, nil
|
|
}
|
|
return nil, &node.NodeOfflineError{Node: name}
|
|
}
|
|
|
|
func (m *Manager) onlineNodes() []node.Node {
|
|
nodes := []node.Node{m.local}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
for _, r := range m.remotes {
|
|
if r.Online() {
|
|
nodes = append(nodes, r)
|
|
}
|
|
}
|
|
return nodes
|
|
}
|
|
|
|
func (m *Manager) OfflineNodes() []string {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
var out []string
|
|
for name, r := range m.remotes {
|
|
if !r.Online() {
|
|
out = append(out, name)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
func (m *Manager) Views(ctx context.Context) ([]balance.NodeView, error) {
|
|
owners := m.store.Owners()
|
|
var views []balance.NodeView
|
|
for _, n := range m.onlineNodes() {
|
|
info, err := n.Info(ctx)
|
|
if err != nil {
|
|
if n == m.local {
|
|
return nil, err
|
|
}
|
|
continue
|
|
}
|
|
views = append(views, balance.BuildView(info, owners, m.cfg.Container))
|
|
}
|
|
sort.Slice(views, func(i, j int) bool { return views[i].Name < views[j].Name })
|
|
return views, nil
|
|
}
|
|
|
|
func (m *Manager) view(ctx context.Context, n node.Node) (balance.NodeView, node.NodeInfo, error) {
|
|
info, err := n.Info(ctx)
|
|
if err != nil {
|
|
return balance.NodeView{}, info, err
|
|
}
|
|
return balance.BuildView(info, m.store.Owners(), m.cfg.Container), info, nil
|
|
}
|
|
|
|
func (m *Manager) Totals(ctx context.Context) (used, usable model.Size) {
|
|
views, _ := m.Views(ctx)
|
|
for _, v := range views {
|
|
used, usable = used.Add(v.Used), usable.Add(v.Usable)
|
|
}
|
|
return
|
|
}
|
|
|
|
func (m *Manager) ownerRecord(id string, kind model.OwnerKind) (*model.Owner, error) {
|
|
if u := m.store.Owner(id); u != nil {
|
|
return u, nil
|
|
}
|
|
u := &model.Owner{
|
|
ID: id,
|
|
Kind: kind,
|
|
Machine: model.MachineName(id, kind),
|
|
Secret: util.RandomHex(32),
|
|
Size: m.cfg.Container,
|
|
StorageMinMB: m.cfg.Storage.MinMB,
|
|
StorageMaxMB: m.cfg.Storage.MaxMB,
|
|
LastActive: time.Now(),
|
|
}
|
|
return u, m.store.PutOwner(u)
|
|
}
|
|
|
|
func (m *Manager) Owner(id string, kind model.OwnerKind) (*model.Owner, error) {
|
|
unlock := m.ownerMu.Lock(id)
|
|
defer unlock()
|
|
return m.ownerRecord(id, kind)
|
|
}
|
|
|
|
func (m *Manager) spec(u *model.Owner) node.StartSpec {
|
|
return node.StartSpec{
|
|
Machine: u.Machine,
|
|
Description: "slackerbot container for " + string(u.Kind) + " " + u.ID,
|
|
Secret: u.Secret,
|
|
Size: u.Size,
|
|
Drive: u.Drive,
|
|
StorageMaxMB: u.StorageMaxMB,
|
|
Env: m.cfg.AgentEnv,
|
|
}
|
|
}
|
|
|
|
func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (*model.Owner, error) {
|
|
return m.EnsureProgress(ctx, id, kind, nil)
|
|
}
|
|
|
|
func (m *Manager) EnsureProgress(ctx context.Context, id string, kind model.OwnerKind, progress func(StartEvent)) (*model.Owner, error) {
|
|
if progress == nil {
|
|
progress = func(StartEvent) {}
|
|
}
|
|
if m.IsMigrating(id) {
|
|
return nil, ErrMigrating
|
|
}
|
|
unlock := m.ownerMu.Lock(id)
|
|
defer unlock()
|
|
|
|
u, err := m.ownerRecord(id, kind)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if u.Node != "" {
|
|
n, err := m.nodeByName(u.Node)
|
|
if err != nil {
|
|
return u, err
|
|
}
|
|
if n.IsRunning(u.Machine) {
|
|
return u, nil
|
|
}
|
|
}
|
|
|
|
m.startMu.Lock()
|
|
defer m.startMu.Unlock()
|
|
|
|
if u.Node == "" {
|
|
progress(StartEvent{Phase: StartPlacing, Size: u.Size})
|
|
views, err := m.Views(ctx)
|
|
if err != nil {
|
|
return u, err
|
|
}
|
|
target, drive, err := balance.ChoosePlacement(views, u.Size, u.StorageMinMB)
|
|
if err != nil {
|
|
return u, err
|
|
}
|
|
u.Node, u.Drive = target, drive
|
|
if err := m.store.SetPlacement(u.ID, target, drive); err != nil {
|
|
return u, err
|
|
}
|
|
}
|
|
|
|
n, err := m.nodeByName(u.Node)
|
|
if err != nil {
|
|
return u, err
|
|
}
|
|
v, _, err := m.view(ctx, n)
|
|
if err != nil {
|
|
return u, err
|
|
}
|
|
if !v.Used.Add(u.Size).Fits(v.Usable) {
|
|
return u, &node.NodeFullError{Node: u.Node}
|
|
}
|
|
log.Printf("container: starting %s for %s %s on %s (drive %s, %s)", u.Machine, u.Kind, u.ID, u.Node, u.Drive, u.Size)
|
|
progress(StartEvent{Phase: StartBooting, Node: u.Node, Drive: u.Drive, Size: u.Size})
|
|
if err := n.Start(ctx, m.spec(u)); err != nil {
|
|
log.Printf("container: starting %s on %s failed: %v", u.Machine, u.Node, err)
|
|
return u, err
|
|
}
|
|
log.Printf("container: %s is running on %s", u.Machine, u.Node)
|
|
progress(StartEvent{Phase: StartRegistered, Node: u.Node, Drive: u.Drive, Size: u.Size})
|
|
return u, nil
|
|
}
|
|
|
|
// The old home folder stays on its node, untouched.
|
|
func (m *Manager) Recreate(id string) error {
|
|
if m.IsMigrating(id) {
|
|
return ErrMigrating
|
|
}
|
|
unlock := m.ownerMu.Lock(id)
|
|
defer unlock()
|
|
if err := m.store.SetPlacement(id, "", ""); err != nil {
|
|
return err
|
|
}
|
|
return m.store.DeleteSessions(id)
|
|
}
|
|
|
|
func (m *Manager) DialAgent(ctx context.Context, u *model.Owner) (node.MsgConn, error) {
|
|
n, err := m.nodeByName(u.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return n.DialAgent(ctx, u.Machine, u.Secret)
|
|
}
|
|
|
|
func (m *Manager) State(u *model.Owner) State {
|
|
if u == nil || u.Node == "" {
|
|
return StateNone
|
|
}
|
|
n, err := m.nodeByName(u.Node)
|
|
if err != nil {
|
|
return StateOffline
|
|
}
|
|
if n.IsRunning(u.Machine) {
|
|
return StateRunning
|
|
}
|
|
return StateStopped
|
|
}
|
|
|
|
func (m *Manager) Poweroff(ctx context.Context, u *model.Owner) error {
|
|
n, err := m.nodeByName(u.Node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Printf("container: shutting down %s for %s %s on %s", u.Machine, u.Kind, u.ID, u.Node)
|
|
if err := n.Poweroff(ctx, u.Machine); err != nil {
|
|
log.Printf("container: shutting down %s on %s failed: %v", u.Machine, u.Node, err)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *Manager) WaitStopped(ctx context.Context, u *model.Owner) error {
|
|
n, err := m.nodeByName(u.Node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return n.WaitStopped(ctx, u.Machine)
|
|
}
|
|
|
|
func (m *Manager) RunningOwners(ctx context.Context) []model.Owner {
|
|
byNodeMachine := map[string]model.Owner{}
|
|
for _, u := range m.store.Owners() {
|
|
byNodeMachine[u.Node+"/"+u.Machine] = u
|
|
}
|
|
var out []model.Owner
|
|
for _, n := range m.onlineNodes() {
|
|
running, err := n.Running(ctx)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, r := range running {
|
|
if u, ok := byNodeMachine[n.Name()+"/"+r.Name]; ok {
|
|
out = append(out, u)
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (m *Manager) Resize(ctx context.Context, id string, kind model.OwnerKind, change func(model.Size) (model.Size, error)) (old, next model.Size, err error) {
|
|
if m.IsMigrating(id) {
|
|
return old, old, ErrMigrating
|
|
}
|
|
unlock := m.ownerMu.Lock(id)
|
|
defer unlock()
|
|
m.startMu.Lock()
|
|
defer m.startMu.Unlock()
|
|
|
|
u, err := m.ownerRecord(id, kind)
|
|
if err != nil {
|
|
return model.Size{}, model.Size{}, err
|
|
}
|
|
old = u.Size
|
|
if next, err = change(old); err != nil {
|
|
return old, old, err
|
|
}
|
|
next.CPUs = math.Round(next.CPUs*100) / 100
|
|
switch {
|
|
case !MinContainer.Fits(next):
|
|
return old, old, ErrTooSmall
|
|
case !next.Fits(m.cfg.MaxContainer):
|
|
return old, old, ErrTooBig
|
|
case next == old:
|
|
return old, old, nil
|
|
}
|
|
|
|
if u.Node != "" {
|
|
n, err := m.nodeByName(u.Node)
|
|
if err != nil {
|
|
return old, old, err
|
|
}
|
|
if n.IsRunning(u.Machine) {
|
|
v, _, err := m.view(ctx, n)
|
|
if err != nil {
|
|
return old, old, err
|
|
}
|
|
growth := model.Size{MemoryMB: max(0, next.MemoryMB-old.MemoryMB), CPUs: math.Max(0, next.CPUs-old.CPUs)}
|
|
if !v.Used.Add(growth).Fits(v.Usable) {
|
|
return old, old, &node.NodeFullError{Node: u.Node}
|
|
}
|
|
if err := n.SetLimits(ctx, u.Machine, next); err != nil {
|
|
return old, old, err
|
|
}
|
|
}
|
|
}
|
|
return old, next, m.store.SetSize(id, next)
|
|
}
|
|
|
|
type StorageChange struct {
|
|
OldMin, OldMax, NewMin, NewMax int64
|
|
UsedMB int64
|
|
Drive *storage.DriveReport
|
|
}
|
|
|
|
func (m *Manager) SetStorage(ctx context.Context, id string, kind model.OwnerKind, change func(min, max int64) (int64, int64, error)) (*StorageChange, error) {
|
|
if m.IsMigrating(id) {
|
|
return &StorageChange{}, ErrMigrating
|
|
}
|
|
unlock := m.ownerMu.Lock(id)
|
|
defer unlock()
|
|
m.startMu.Lock()
|
|
defer m.startMu.Unlock()
|
|
|
|
u, err := m.ownerRecord(id, kind)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
c := &StorageChange{OldMin: u.StorageMinMB, OldMax: u.StorageMaxMB}
|
|
if c.NewMin, c.NewMax, err = change(c.OldMin, c.OldMax); err != nil {
|
|
return c, err
|
|
}
|
|
switch {
|
|
case c.NewMin < config.MinStorageMB || c.NewMax < config.MinStorageMB:
|
|
return c, storage.ErrStorageTooSmall
|
|
case c.NewMin > c.NewMax:
|
|
return c, storage.ErrMinOverMax
|
|
case c.NewMax > m.cfg.Storage.LimitMB:
|
|
return c, storage.ErrStorageTooBig
|
|
}
|
|
|
|
if u.Node == "" {
|
|
views, err := m.Views(ctx)
|
|
if err != nil {
|
|
return c, err
|
|
}
|
|
if _, _, err := balance.ChoosePlacement(views, u.Size, c.NewMin); err != nil {
|
|
return c, err
|
|
}
|
|
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
|
|
}
|
|
|
|
n, err := m.nodeByName(u.Node)
|
|
if err != nil {
|
|
return c, err
|
|
}
|
|
v, info, err := m.view(ctx, n)
|
|
if err != nil {
|
|
return c, err
|
|
}
|
|
for _, d := range info.Drives {
|
|
if d.Path == u.Drive {
|
|
c.UsedMB = d.HomesMB[u.Machine]
|
|
}
|
|
}
|
|
if c.NewMax < c.UsedMB {
|
|
return c, storage.ErrBelowUsage
|
|
}
|
|
var r *storage.DriveReport
|
|
for i := range v.Drives {
|
|
if v.Drives[i].Path == u.Drive {
|
|
r = &v.Drives[i]
|
|
}
|
|
}
|
|
if r == nil {
|
|
return c, fmt.Errorf("drive %s is no longer configured on %s", u.Drive, u.Node)
|
|
}
|
|
extra := storage.Committed(c.NewMin, c.UsedMB) - storage.Committed(c.OldMin, c.UsedMB)
|
|
if extra > 0 && extra > r.HeadroomMB() {
|
|
return c, storage.ErrStorageFull
|
|
}
|
|
if err := n.SetQuota(ctx, u.Drive, u.Machine, c.NewMax); err != nil {
|
|
return c, err
|
|
}
|
|
r.CommittedMB += extra
|
|
r.BookedMB += c.NewMax - c.OldMax
|
|
c.Drive = r
|
|
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
|
|
}
|
|
|
|
func (m *Manager) StorageUsed(ctx context.Context, u *model.Owner) (int64, bool) {
|
|
if u == nil || u.Node == "" {
|
|
return 0, false
|
|
}
|
|
n, err := m.nodeByName(u.Node)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
info, err := n.Info(ctx)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
for _, d := range info.Drives {
|
|
if used, ok := d.HomesMB[u.Machine]; ok && d.Path == u.Drive {
|
|
return used, true
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|