package manager import ( "context" "errors" "fmt" "log" "slackerbot/internal/balance" "slackerbot/internal/model" "slackerbot/internal/node" "slackerbot/internal/storage" ) const transferChunk = 4 << 20 var ( ErrMigrating = errors.New("this container is being moved") ErrNoContainer = errors.New("there's no container here yet") ErrSameNode = errors.New("the container is already on that node") ) type UnknownNodeError struct{ Node string } func (e *UnknownNodeError) Error() string { return fmt.Sprintf("there's no node called %s", e.Node) } type MovePhase string const ( PhaseCopy MovePhase = "copy" PhaseFinal MovePhase = "final" PhaseStart MovePhase = "start" PhaseCleanup MovePhase = "cleanup" ) type MoveProgress struct { To string Phase MovePhase CopiedMB int64 TotalMB int64 } type MoveOptions struct { Target string StartAfter bool WaitIdle func(ctx context.Context) Progress func(MoveProgress) } type MoveResult struct { From, To string WasRunning bool Started bool CopiedMB int64 LeftOld error } func (m *Manager) IsMigrating(id string) bool { _, ok := m.migrating.Load(id) return ok } // Migrate copies the owner's home folder to another node and switches the // container over. A running container keeps running through the bulk copy // and is only stopped for a final incremental sync. func (m *Manager) Migrate(ctx context.Context, id string, opts MoveOptions) (*MoveResult, error) { if _, busy := m.migrating.LoadOrStore(id, true); busy { return nil, ErrMigrating } defer m.migrating.Delete(id) unlock := m.ownerMu.Lock(id) defer unlock() u := m.store.Owner(id) if u == nil || u.Node == "" { return nil, ErrNoContainer } src, err := m.nodeByName(u.Node) if err != nil { return nil, err } res := &MoveResult{From: u.Node, WasRunning: src.IsRunning(u.Machine)} needStart := res.WasRunning || opts.StartAfter usedMB, _ := m.StorageUsed(ctx, u) target, drive, err := m.pickTarget(ctx, u, opts.Target, needStart, storage.Committed(u.StorageMinMB, usedMB)) if err != nil { return nil, err } res.To = target dst, err := m.nodeByName(target) if err != nil { return nil, err } progress := func(p MovePhase, copied int64) { if opts.Progress != nil { opts.Progress(MoveProgress{To: target, Phase: p, CopiedMB: copied >> 20, TotalMB: usedMB}) } } imp, err := dst.ImportHome(ctx, drive, u.Machine) if err != nil { return nil, err } var exports []string stopped := false rollback := func(cause error) (*MoveResult, error) { _ = dst.AbortImport(context.Background(), imp) for _, e := range exports { _ = src.CloseExport(context.Background(), e) } if stopped { if err := src.Start(context.Background(), m.spec(u)); err != nil { return res, fmt.Errorf("%w (and restarting it on %s failed: %v)", cause, u.Node, err) } } return res, cause } var copied int64 send := func(parent string, phase MovePhase) error { exp, err := src.ExportHome(ctx, u.Drive, u.Machine, parent) if err != nil { return err } exports = append(exports, exp) for { data, eof, err := src.ReadExport(ctx, exp, transferChunk) if err != nil { return err } if len(data) > 0 { if err := dst.WriteImport(ctx, imp, data); err != nil { return err } copied += int64(len(data)) progress(phase, copied) } if eof { return dst.EndReceive(ctx, imp) } } } progress(PhaseCopy, 0) if err := send("", PhaseCopy); err != nil { return rollback(err) } if res.WasRunning { progress(PhaseFinal, copied) if opts.WaitIdle != nil { opts.WaitIdle(ctx) } if err := src.Poweroff(ctx, u.Machine); err != nil { return rollback(err) } stopped = true if err := src.WaitStopped(ctx, u.Machine); err != nil { return rollback(err) } if err := send(exports[0], PhaseFinal); err != nil { return rollback(err) } } if err := dst.FinishImport(ctx, imp, u.StorageMaxMB); err != nil { return rollback(err) } res.CopiedMB = copied >> 20 oldNode, oldDrive := u.Node, u.Drive if err := m.store.SetPlacement(id, target, drive); err != nil { return rollback(err) } u.Node, u.Drive = target, drive progress(PhaseCleanup, copied) for _, e := range exports { _ = src.CloseExport(ctx, e) } if res.LeftOld = src.DeleteHome(ctx, oldDrive, u.Machine); res.LeftOld != nil { log.Printf("move: old home of %s left on %s: %v", u.Machine, oldNode, res.LeftOld) } if !needStart { return res, nil } progress(PhaseStart, copied) if err := dst.Start(ctx, m.spec(u)); err != nil { return res, fmt.Errorf("moved to %s, but starting it there failed: %w", target, err) } res.Started = true return res, nil } func (m *Manager) pickTarget(ctx context.Context, u *model.Owner, want string, needStart bool, storageMB int64) (target, drive string, err error) { if want == u.Node { return "", "", ErrSameNode } m.startMu.Lock() defer m.startMu.Unlock() views, err := m.Views(ctx) if err != nil { return "", "", err } compute := model.Size{} if needStart { compute = u.Size } var candidates []balance.NodeView for _, v := range views { if v.Name != u.Node && (want == "" || v.Name == want) { candidates = append(candidates, v) } } if want != "" && len(candidates) == 0 { if m.KnownNode(want) { return "", "", &node.NodeOfflineError{Node: want} } return "", "", &UnknownNodeError{Node: want} } target, drive, err = balance.ChoosePlacement(candidates, compute, storageMB) if err != nil && want != "" { return "", "", &node.NodeFullError{Node: want} } return target, drive, err } func (m *Manager) KnownNode(name string) bool { if name == m.local.Name() { return true } m.mu.RLock() defer m.mu.RUnlock() _, ok := m.remotes[name] return ok }