first pass
This commit is contained in:
commit
7822529a38
31 files changed
+7050
No files matched your search
+230
@@ -0,0 +1,230 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrMigrating = errors.New("this container is being moved")
|
||||
ErrNoContainer = errors.New("there's no container here yet")
|
||||
ErrSameNode = errors.New("the container is already on that node")
|
||||
)
|
||||
|
||||
type UnknownNodeError struct{ Node string }
|
||||
|
||||
func (e *UnknownNodeError) Error() string { return fmt.Sprintf("there's no node called %s", e.Node) }
|
||||
|
||||
type MovePhase string
|
||||
|
||||
const (
|
||||
PhaseCopy MovePhase = "copy"
|
||||
PhaseFinal MovePhase = "final"
|
||||
PhaseStart MovePhase = "start"
|
||||
PhaseCleanup MovePhase = "cleanup"
|
||||
)
|
||||
|
||||
type MoveProgress struct {
|
||||
To string
|
||||
Phase MovePhase
|
||||
CopiedMB int64
|
||||
TotalMB int64
|
||||
}
|
||||
|
||||
type MoveOptions struct {
|
||||
Target string
|
||||
StartAfter bool
|
||||
WaitIdle func(ctx context.Context)
|
||||
Progress func(MoveProgress)
|
||||
}
|
||||
|
||||
type MoveResult struct {
|
||||
From, To string
|
||||
WasRunning bool
|
||||
Started bool
|
||||
CopiedMB int64
|
||||
LeftOld error
|
||||
}
|
||||
|
||||
func (m *Manager) isMigrating(id string) bool {
|
||||
_, ok := m.migrating.Load(id)
|
||||
return ok
|
||||
}
|
||||
|
||||
// Migrate copies the owner's home folder to another node and switches the
|
||||
// container over. A running container keeps running through the bulk copy
|
||||
// and is only stopped for a final incremental sync.
|
||||
func (m *Manager) Migrate(ctx context.Context, id string, opts MoveOptions) (*MoveResult, error) {
|
||||
if _, busy := m.migrating.LoadOrStore(id, true); busy {
|
||||
return nil, ErrMigrating
|
||||
}
|
||||
defer m.migrating.Delete(id)
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
|
||||
u := m.store.Owner(id)
|
||||
if u == nil || u.Node == "" {
|
||||
return nil, ErrNoContainer
|
||||
}
|
||||
src, err := m.node(u.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &MoveResult{From: u.Node, WasRunning: src.IsRunning(u.Machine)}
|
||||
needStart := res.WasRunning || opts.StartAfter
|
||||
|
||||
usedMB, _ := m.StorageUsed(ctx, u)
|
||||
target, drive, err := m.pickTarget(ctx, u, opts.Target, needStart, committed(u.StorageMinMB, usedMB))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.To = target
|
||||
dst, err := m.node(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
progress := func(p MovePhase, copied int64) {
|
||||
if opts.Progress != nil {
|
||||
opts.Progress(MoveProgress{To: target, Phase: p, CopiedMB: copied >> 20, TotalMB: usedMB})
|
||||
}
|
||||
}
|
||||
|
||||
imp, err := dst.ImportHome(ctx, drive, u.Machine)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var exports []string
|
||||
stopped := false
|
||||
rollback := func(cause error) (*MoveResult, error) {
|
||||
_ = dst.AbortImport(context.Background(), imp)
|
||||
for _, e := range exports {
|
||||
_ = src.CloseExport(context.Background(), e)
|
||||
}
|
||||
if stopped {
|
||||
if err := src.Start(context.Background(), m.spec(u)); err != nil {
|
||||
return res, fmt.Errorf("%w (and restarting it on %s failed: %v)", cause, u.Node, err)
|
||||
}
|
||||
}
|
||||
return res, cause
|
||||
}
|
||||
|
||||
var copied int64
|
||||
send := func(parent string, phase MovePhase) error {
|
||||
exp, err := src.ExportHome(ctx, u.Drive, u.Machine, parent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
exports = append(exports, exp)
|
||||
for {
|
||||
data, eof, err := src.ReadExport(ctx, exp, transferChunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(data) > 0 {
|
||||
if err := dst.WriteImport(ctx, imp, data); err != nil {
|
||||
return err
|
||||
}
|
||||
copied += int64(len(data))
|
||||
progress(phase, copied)
|
||||
}
|
||||
if eof {
|
||||
return dst.EndReceive(ctx, imp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
progress(PhaseCopy, 0)
|
||||
if err := send("", PhaseCopy); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
if res.WasRunning {
|
||||
progress(PhaseFinal, copied)
|
||||
if opts.WaitIdle != nil {
|
||||
opts.WaitIdle(ctx)
|
||||
}
|
||||
if err := src.Poweroff(ctx, u.Machine); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
stopped = true
|
||||
if err := src.WaitStopped(ctx, u.Machine); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
if err := send(exports[0], PhaseFinal); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
}
|
||||
if err := dst.FinishImport(ctx, imp, u.StorageMaxMB); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
res.CopiedMB = copied >> 20
|
||||
|
||||
oldNode, oldDrive := u.Node, u.Drive
|
||||
if err := m.store.SetPlacement(id, target, drive); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
u.Node, u.Drive = target, drive
|
||||
|
||||
progress(PhaseCleanup, copied)
|
||||
for _, e := range exports {
|
||||
_ = src.CloseExport(ctx, e)
|
||||
}
|
||||
if res.LeftOld = src.DeleteHome(ctx, oldDrive, u.Machine); res.LeftOld != nil {
|
||||
log.Printf("move: old home of %s left on %s: %v", u.Machine, oldNode, res.LeftOld)
|
||||
}
|
||||
|
||||
if !needStart {
|
||||
return res, nil
|
||||
}
|
||||
progress(PhaseStart, copied)
|
||||
if err := dst.Start(ctx, m.spec(u)); err != nil {
|
||||
return res, fmt.Errorf("moved to %s, but starting it there failed: %w", target, err)
|
||||
}
|
||||
res.Started = true
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (m *Manager) pickTarget(ctx context.Context, u *Owner, want string, needStart bool, storageMB int64) (node, drive string, err error) {
|
||||
if want == u.Node {
|
||||
return "", "", ErrSameNode
|
||||
}
|
||||
m.startMu.Lock()
|
||||
defer m.startMu.Unlock()
|
||||
|
||||
views, err := m.Views(ctx)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
compute := Size{}
|
||||
if needStart {
|
||||
compute = u.Size
|
||||
}
|
||||
var candidates []NodeView
|
||||
for _, v := range views {
|
||||
if v.Name != u.Node && (want == "" || v.Name == want) {
|
||||
candidates = append(candidates, v)
|
||||
}
|
||||
}
|
||||
if want != "" && len(candidates) == 0 {
|
||||
if m.KnownNode(want) {
|
||||
return "", "", &NodeOfflineError{Node: want}
|
||||
}
|
||||
return "", "", &UnknownNodeError{Node: want}
|
||||
}
|
||||
node, drive, err = choosePlacement(candidates, compute, storageMB)
|
||||
if err != nil && want != "" {
|
||||
return "", "", &NodeFullError{Node: want}
|
||||
}
|
||||
return node, drive, err
|
||||
}
|
||||
|
||||
func (m *Manager) KnownNode(name string) bool {
|
||||
if name == m.local.Name() {
|
||||
return true
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
_, ok := m.remotes[name]
|
||||
return ok
|
||||
}
|
||||
Reference in new issue
Block a user