241 lines
5.9 KiB
Go
241 lines
5.9 KiB
Go
package manager
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
|
|
"slackerbot/internal/balance"
|
|
"slackerbot/internal/model"
|
|
"slackerbot/internal/node"
|
|
"slackerbot/internal/storage"
|
|
)
|
|
|
|
const transferChunk = 4 << 20
|
|
|
|
var (
|
|
ErrMigrating = errors.New("this container is being moved")
|
|
ErrNoContainer = errors.New("there's no container here yet")
|
|
ErrSameNode = errors.New("the container is already on that node")
|
|
)
|
|
|
|
type UnknownNodeError struct{ Node string }
|
|
|
|
func (e *UnknownNodeError) Error() string { return fmt.Sprintf("there's no node called %s", e.Node) }
|
|
|
|
type MovePhase string
|
|
|
|
const (
|
|
PhaseCopy MovePhase = "copy"
|
|
PhaseFinal MovePhase = "final"
|
|
PhaseStart MovePhase = "start"
|
|
PhaseCleanup MovePhase = "cleanup"
|
|
)
|
|
|
|
type MoveProgress struct {
|
|
To string
|
|
Phase MovePhase
|
|
CopiedMB int64
|
|
TotalMB int64
|
|
}
|
|
|
|
type MoveOptions struct {
|
|
Target string
|
|
StartAfter bool
|
|
WaitIdle func(ctx context.Context)
|
|
Progress func(MoveProgress)
|
|
}
|
|
|
|
type MoveResult struct {
|
|
From, To string
|
|
WasRunning bool
|
|
Started bool
|
|
CopiedMB int64
|
|
LeftOld error
|
|
}
|
|
|
|
func (m *Manager) IsMigrating(id string) bool {
|
|
_, ok := m.migrating.Load(id)
|
|
return ok
|
|
}
|
|
|
|
// Migrate copies the owner's home folder to another node and switches the
|
|
// container over. A running container keeps running through the bulk copy
|
|
// and is only stopped for a final incremental sync.
|
|
func (m *Manager) Migrate(ctx context.Context, id string, opts MoveOptions) (*MoveResult, error) {
|
|
if _, busy := m.migrating.LoadOrStore(id, true); busy {
|
|
return nil, ErrMigrating
|
|
}
|
|
defer m.migrating.Delete(id)
|
|
unlock := m.ownerMu.Lock(id)
|
|
defer unlock()
|
|
|
|
u := m.store.Owner(id)
|
|
if u == nil || u.Node == "" {
|
|
return nil, ErrNoContainer
|
|
}
|
|
src, err := m.nodeByName(u.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
res := &MoveResult{From: u.Node, WasRunning: src.IsRunning(u.Machine)}
|
|
needStart := res.WasRunning || opts.StartAfter
|
|
|
|
usedMB, _ := m.StorageUsed(ctx, u)
|
|
target, drive, err := m.pickTarget(ctx, u, opts.Target, needStart, storage.Committed(u.StorageMinMB, usedMB))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
res.To = target
|
|
dst, err := m.nodeByName(target)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
progress := func(p MovePhase, copied int64) {
|
|
if opts.Progress != nil {
|
|
opts.Progress(MoveProgress{To: target, Phase: p, CopiedMB: copied >> 20, TotalMB: usedMB})
|
|
}
|
|
}
|
|
|
|
imp, err := dst.ImportHome(ctx, drive, u.Machine)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var exports []string
|
|
stopped := false
|
|
rollback := func(cause error) (*MoveResult, error) {
|
|
_ = dst.AbortImport(context.Background(), imp)
|
|
for _, e := range exports {
|
|
_ = src.CloseExport(context.Background(), e)
|
|
}
|
|
if stopped {
|
|
log.Printf("container: restarting %s on %s after a failed move", u.Machine, u.Node)
|
|
if err := src.Start(context.Background(), m.spec(u)); err != nil {
|
|
return res, fmt.Errorf("%w (and restarting it on %s failed: %v)", cause, u.Node, err)
|
|
}
|
|
}
|
|
return res, cause
|
|
}
|
|
|
|
var copied int64
|
|
send := func(parent string, phase MovePhase) error {
|
|
exp, err := src.ExportHome(ctx, u.Drive, u.Machine, parent)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
exports = append(exports, exp)
|
|
for {
|
|
data, eof, err := src.ReadExport(ctx, exp, transferChunk)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(data) > 0 {
|
|
if err := dst.WriteImport(ctx, imp, data); err != nil {
|
|
return err
|
|
}
|
|
copied += int64(len(data))
|
|
progress(phase, copied)
|
|
}
|
|
if eof {
|
|
return dst.EndReceive(ctx, imp)
|
|
}
|
|
}
|
|
}
|
|
|
|
progress(PhaseCopy, 0)
|
|
if err := send("", PhaseCopy); err != nil {
|
|
return rollback(err)
|
|
}
|
|
if res.WasRunning {
|
|
progress(PhaseFinal, copied)
|
|
if opts.WaitIdle != nil {
|
|
opts.WaitIdle(ctx)
|
|
}
|
|
log.Printf("container: shutting down %s on %s to move it to %s", u.Machine, u.Node, target)
|
|
if err := src.Poweroff(ctx, u.Machine); err != nil {
|
|
return rollback(err)
|
|
}
|
|
stopped = true
|
|
if err := src.WaitStopped(ctx, u.Machine); err != nil {
|
|
return rollback(err)
|
|
}
|
|
if err := send(exports[0], PhaseFinal); err != nil {
|
|
return rollback(err)
|
|
}
|
|
}
|
|
if err := dst.FinishImport(ctx, imp, u.StorageMaxMB); err != nil {
|
|
return rollback(err)
|
|
}
|
|
res.CopiedMB = copied >> 20
|
|
|
|
oldNode, oldDrive := u.Node, u.Drive
|
|
if err := m.store.SetPlacement(id, target, drive); err != nil {
|
|
return rollback(err)
|
|
}
|
|
u.Node, u.Drive = target, drive
|
|
|
|
progress(PhaseCleanup, copied)
|
|
for _, e := range exports {
|
|
_ = src.CloseExport(ctx, e)
|
|
}
|
|
if res.LeftOld = src.DeleteHome(ctx, oldDrive, u.Machine); res.LeftOld != nil {
|
|
log.Printf("move: old home of %s left on %s: %v", u.Machine, oldNode, res.LeftOld)
|
|
}
|
|
|
|
if !needStart {
|
|
return res, nil
|
|
}
|
|
progress(PhaseStart, copied)
|
|
log.Printf("container: starting %s on %s after the move from %s", u.Machine, target, oldNode)
|
|
if err := dst.Start(ctx, m.spec(u)); err != nil {
|
|
return res, fmt.Errorf("moved to %s, but starting it there failed: %w", target, err)
|
|
}
|
|
res.Started = true
|
|
return res, nil
|
|
}
|
|
|
|
func (m *Manager) pickTarget(ctx context.Context, u *model.Owner, want string, needStart bool, storageMB int64) (target, drive string, err error) {
|
|
if want == u.Node {
|
|
return "", "", ErrSameNode
|
|
}
|
|
m.startMu.Lock()
|
|
defer m.startMu.Unlock()
|
|
|
|
views, err := m.Views(ctx)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
compute := model.Size{}
|
|
if needStart {
|
|
compute = u.Size
|
|
}
|
|
var candidates []balance.NodeView
|
|
for _, v := range views {
|
|
if v.Name != u.Node && (want == "" || v.Name == want) {
|
|
candidates = append(candidates, v)
|
|
}
|
|
}
|
|
if want != "" && len(candidates) == 0 {
|
|
if m.KnownNode(want) {
|
|
return "", "", &node.NodeOfflineError{Node: want}
|
|
}
|
|
return "", "", &UnknownNodeError{Node: want}
|
|
}
|
|
target, drive, err = balance.ChoosePlacement(candidates, compute, storageMB)
|
|
if err != nil && want != "" {
|
|
return "", "", &node.NodeFullError{Node: want}
|
|
}
|
|
return target, drive, err
|
|
}
|
|
|
|
func (m *Manager) KnownNode(name string) bool {
|
|
if name == m.local.Name() {
|
|
return true
|
|
}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
_, ok := m.remotes[name]
|
|
return ok
|
|
}
|