better startup flow
This commit is contained in:
1 parent
d7f5dba677
commit
650bcff83a
8 files changed
+345
-68
No files matched your search
@@ -218,8 +218,11 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
|
||||
ownerID, kind := m.owner()
|
||||
var off *node.NodeOfflineError
|
||||
var full *node.NodeFullError
|
||||
u, err := b.mgr.Ensure(ctx, ownerID, kind)
|
||||
sl := &startLog{b: b, m: m, root: root, ownerID: ownerID}
|
||||
started := func(ev manager.StartEvent) { sl.event(ctx, ev) }
|
||||
u, err := b.mgr.EnsureProgress(ctx, ownerID, kind, started)
|
||||
if errors.As(err, &full) {
|
||||
sl.failed(ctx, fmt.Sprintf("`%s` is full, so the container is being moved to a node with room first.", full.Node))
|
||||
edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room…", full.Node))
|
||||
trigger := fmt.Sprintf("auto process (%s was full), for <@%s>'s message", full.Node, m.user)
|
||||
_, merr := b.runMove(ctx, m.channel, ownerID, manager.MoveOptions{StartAfter: true}, trigger, func(link string) {
|
||||
@@ -230,13 +233,15 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
|
||||
return
|
||||
}
|
||||
edit(":hourglass_flowing_sand: Moved. Getting the container ready…")
|
||||
u, err = b.mgr.Ensure(ctx, ownerID, kind)
|
||||
u, err = b.mgr.EnsureProgress(ctx, ownerID, kind, started)
|
||||
}
|
||||
if errors.As(err, &off) {
|
||||
sl.failed(ctx, fmt.Sprintf("Node `%s` is offline.", off.Node))
|
||||
b.deadSlave(ctx, m.channel, ph, ownerID, off.Node)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
sl.failed(ctx, b.explain(err))
|
||||
edit(b.explain(err))
|
||||
return
|
||||
}
|
||||
@@ -246,13 +251,16 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
|
||||
|
||||
acp, err := b.client(ctx, u)
|
||||
if errors.As(err, &off) {
|
||||
sl.failed(ctx, fmt.Sprintf("Node `%s` went offline.", off.Node))
|
||||
b.deadSlave(ctx, m.channel, ph, ownerID, off.Node)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
sl.failed(ctx, "The agent never became ready: "+err.Error())
|
||||
edit(":warning: Couldn't reach the agent: " + err.Error())
|
||||
return
|
||||
}
|
||||
sl.up(ctx)
|
||||
|
||||
var id string
|
||||
text := m.text
|
||||
@@ -510,7 +518,11 @@ func (b *Bot) stopFor(ctx context.Context, ownerID string) string {
|
||||
return ":warning: Couldn't stop it: " + err.Error()
|
||||
}
|
||||
b.dropClient(u)
|
||||
return "Shutting down the container. Its files are kept. Message me anytime to start it again."
|
||||
reply := "Shutting down the container. Its files are kept. Message me anytime to start it again."
|
||||
if link := b.announceStop(ctx, u, "Stopped on request."); link != "" {
|
||||
reply += fmt.Sprintf(" <%s|Follow the shutdown here>.", link)
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) {
|
||||
@@ -1074,6 +1086,9 @@ func (b *Bot) reapOnce(ctx context.Context) {
|
||||
b.dropClient(&u)
|
||||
msg := fmt.Sprintf("I stopped the container after %s without use. Its files are kept. "+
|
||||
"Message me anytime to start it again, or use `keep` to change how long it waits.", commands.Human(limit))
|
||||
if link := b.announceStop(ctx, &u, fmt.Sprintf("Unused for %s.", commands.Human(limit))); link != "" {
|
||||
msg += fmt.Sprintf(" <%s|Shutdown details>.", link)
|
||||
}
|
||||
b.say(ctx, u.ID, "", msg) // a user ID posts to their DM, a channel ID to the channel
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package bot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/slack-go/slack"
|
||||
|
||||
"slackerbot/internal/manager"
|
||||
"slackerbot/internal/model"
|
||||
)
|
||||
|
||||
type lifecycle struct {
|
||||
b *Bot
|
||||
channel string
|
||||
root string
|
||||
link string
|
||||
}
|
||||
|
||||
func (l *lifecycle) post(ctx context.Context, text string) {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
l.b.say(ctx, l.channel, l.root, text)
|
||||
}
|
||||
|
||||
func (l *lifecycle) url() string {
|
||||
if l == nil {
|
||||
return ""
|
||||
}
|
||||
return l.link
|
||||
}
|
||||
|
||||
func (b *Bot) permalink(ctx context.Context, channel, ts string) string {
|
||||
link, err := b.api.GetPermalinkContext(ctx, &slack.PermalinkParameters{Channel: channel, Ts: ts})
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return link
|
||||
}
|
||||
|
||||
// openLifecycle posts a new top-level message for the container, pins it and
|
||||
// unpins the one from its previous run.
|
||||
func (b *Bot) openLifecycle(ctx context.Context, u *model.Owner, headline string) *lifecycle {
|
||||
channel, ts, err := b.api.PostMessageContext(ctx, u.ID, slack.MsgOptionText(headline, false))
|
||||
if err != nil {
|
||||
log.Printf("slack post: %v", err)
|
||||
return nil
|
||||
}
|
||||
if u.LifecycleTS != "" {
|
||||
old := slack.NewRefToMessage(u.LifecycleChannel, u.LifecycleTS)
|
||||
if err := b.api.RemovePinContext(ctx, u.LifecycleChannel, old); err != nil {
|
||||
log.Printf("slack unpin: %v", err)
|
||||
}
|
||||
}
|
||||
if err := b.api.AddPinContext(ctx, channel, slack.NewRefToMessage(channel, ts)); err != nil {
|
||||
log.Printf("slack pin: %v", err)
|
||||
}
|
||||
if err := b.store.SetLifecycle(u.ID, channel, ts); err != nil {
|
||||
log.Printf("saving lifecycle message for %s: %v", u.ID, err)
|
||||
}
|
||||
return &lifecycle{b: b, channel: channel, root: ts, link: b.permalink(ctx, channel, ts)}
|
||||
}
|
||||
|
||||
func (b *Bot) currentLifecycle(ctx context.Context, u *model.Owner) *lifecycle {
|
||||
if u.LifecycleTS == "" {
|
||||
return b.openLifecycle(ctx, u, fmt.Sprintf(":package: Container `%s` for %s", u.Machine, ownerMention(u)))
|
||||
}
|
||||
return &lifecycle{b: b, channel: u.LifecycleChannel, root: u.LifecycleTS,
|
||||
link: b.permalink(ctx, u.LifecycleChannel, u.LifecycleTS)}
|
||||
}
|
||||
|
||||
func ownerMention(u *model.Owner) string {
|
||||
if u.Kind == model.OwnerUser {
|
||||
return "<@" + u.ID + ">"
|
||||
}
|
||||
return "<#" + u.ID + ">"
|
||||
}
|
||||
|
||||
// startLog reports a container start in its lifecycle thread. It stays silent
|
||||
// when the container is already running.
|
||||
type startLog struct {
|
||||
b *Bot
|
||||
m incoming
|
||||
root string
|
||||
ownerID string
|
||||
lc *lifecycle
|
||||
began time.Time
|
||||
}
|
||||
|
||||
func (s *startLog) open(ctx context.Context) {
|
||||
if s.lc != nil {
|
||||
return
|
||||
}
|
||||
u := s.b.store.Owner(s.ownerID)
|
||||
if u == nil {
|
||||
return
|
||||
}
|
||||
s.began = time.Now()
|
||||
head := fmt.Sprintf(":rocket: Starting container `%s` for %s. Started by <@%s> in <%s|this thread>.",
|
||||
u.Machine, ownerMention(u), s.m.user, s.b.permalink(ctx, s.m.channel, s.root))
|
||||
if s.lc = s.b.openLifecycle(ctx, u, head); s.lc != nil {
|
||||
s.b.say(ctx, s.m.channel, s.root, fmt.Sprintf(":rocket: Starting the container. <%s|Follow the startup here>.", s.lc.url()))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *startLog) event(ctx context.Context, ev manager.StartEvent) {
|
||||
s.open(ctx)
|
||||
switch ev.Phase {
|
||||
case manager.StartPlacing:
|
||||
s.lc.post(ctx, fmt.Sprintf(":mag: New container, so it needs a home. Waiting for the balancer to pick the node and drive with the most room for %s.", ev.Size))
|
||||
case manager.StartBooting:
|
||||
s.lc.post(ctx, fmt.Sprintf(":gear: Placed on `%s` (drive `%s`). Waiting for the node to create the home folder, apply the storage quota and launch the container with systemd-nspawn.", ev.Node, ev.Drive))
|
||||
case manager.StartRegistered:
|
||||
text := ":hourglass_flowing_sand: The container is launched and the node can see it. Waiting for the agent (goose) to answer on port 1984"
|
||||
if t := s.b.cfg.BootTimeout; t > 0 {
|
||||
text += fmt.Sprintf(", for up to %s", t)
|
||||
}
|
||||
s.lc.post(ctx, text+".")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *startLog) up(ctx context.Context) {
|
||||
if s.lc == nil {
|
||||
return
|
||||
}
|
||||
s.lc.post(ctx, fmt.Sprintf(":white_check_mark: <@%s> the container is up and the agent is ready (took %s).",
|
||||
s.m.user, time.Since(s.began).Round(time.Second)))
|
||||
}
|
||||
|
||||
func (s *startLog) failed(ctx context.Context, reason string) {
|
||||
if s.lc == nil {
|
||||
return
|
||||
}
|
||||
s.lc.post(ctx, ":x: The container didn't come up. "+reason)
|
||||
}
|
||||
|
||||
// announceStop posts the shutdown in the container's lifecycle thread and
|
||||
// returns a link to it. The thread gets a last message once the machine is gone.
|
||||
func (b *Bot) announceStop(ctx context.Context, u *model.Owner, reason string) string {
|
||||
lc := b.currentLifecycle(ctx, u)
|
||||
lc.post(ctx, ":octagonal_sign: Shutting down. "+reason+" Waiting for the container to power off.")
|
||||
go func() {
|
||||
bg := context.Background()
|
||||
if err := b.mgr.WaitStopped(bg, u); err != nil {
|
||||
lc.post(bg, ":warning: The container was asked to shut down but hasn't stopped: "+err.Error())
|
||||
return
|
||||
}
|
||||
lc.post(bg, ":white_check_mark: The container is down. Its files are kept, and a new message starts it again.")
|
||||
}()
|
||||
return lc.url()
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package bot
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/slack-go/slack"
|
||||
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/manager"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/state"
|
||||
)
|
||||
|
||||
func TestStartIsReportedInTheLifecycleThread(t *testing.T) {
|
||||
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
|
||||
srv := httptest.NewServer(log)
|
||||
defer srv.Close()
|
||||
store, _ := state.Open(filepath.Join(t.TempDir(), "state.json"))
|
||||
_ = store.PutOwner(&model.Owner{ID: "C1", Kind: model.OwnerChannel, Machine: "c-c1"})
|
||||
b := &Bot{store: store, cfg: &config.Config{BootTimeout: time.Minute}, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/"))}
|
||||
|
||||
sl := &startLog{b: b, m: incoming{user: "U1", channel: "C1", ts: "5.0"}, root: "5.0", ownerID: "C1"}
|
||||
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartPlacing, Size: model.Size{MemoryMB: 1024, CPUs: 1}})
|
||||
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartBooting, Node: "master", Drive: "/d"})
|
||||
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartRegistered, Node: "master", Drive: "/d"})
|
||||
sl.up(t.Context())
|
||||
|
||||
u := store.Owner("C1")
|
||||
if u.LifecycleTS == "" || u.LifecycleChannel == "" {
|
||||
t.Fatalf("the lifecycle message wasn't saved: %+v", u)
|
||||
}
|
||||
|
||||
log.mu.Lock()
|
||||
defer log.mu.Unlock()
|
||||
var inThread, topLevel, linked int
|
||||
for _, ts := range log.order {
|
||||
switch log.under[ts] {
|
||||
case u.LifecycleTS:
|
||||
inThread++
|
||||
case "5.0":
|
||||
if strings.Contains(log.text[ts], "https://slack.example/") {
|
||||
linked++
|
||||
}
|
||||
case "":
|
||||
topLevel++
|
||||
}
|
||||
}
|
||||
if topLevel != 1 || inThread != 4 || linked != 1 {
|
||||
t.Fatalf("top-level %d, in lifecycle thread %d, links in the user's thread %d (order %v, text %v)", topLevel, inThread, linked, log.order, log.text)
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,22 @@ const (
|
||||
StateOffline State = "on an offline node"
|
||||
)
|
||||
|
||||
type StartPhase string
|
||||
|
||||
const (
|
||||
StartPlacing StartPhase = "placing"
|
||||
StartBooting StartPhase = "booting"
|
||||
StartRegistered StartPhase = "registered"
|
||||
)
|
||||
|
||||
// StartEvent is only reported when Ensure really has to start the container.
|
||||
type StartEvent struct {
|
||||
Phase StartPhase
|
||||
Node string
|
||||
Drive string
|
||||
Size model.Size
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
cfg *config.Config
|
||||
store *state.Store
|
||||
@@ -162,6 +178,13 @@ func (m *Manager) spec(u *model.Owner) node.StartSpec {
|
||||
}
|
||||
|
||||
func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (*model.Owner, error) {
|
||||
return m.EnsureProgress(ctx, id, kind, nil)
|
||||
}
|
||||
|
||||
func (m *Manager) EnsureProgress(ctx context.Context, id string, kind model.OwnerKind, progress func(StartEvent)) (*model.Owner, error) {
|
||||
if progress == nil {
|
||||
progress = func(StartEvent) {}
|
||||
}
|
||||
if m.IsMigrating(id) {
|
||||
return nil, ErrMigrating
|
||||
}
|
||||
@@ -186,6 +209,7 @@ func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (
|
||||
defer m.startMu.Unlock()
|
||||
|
||||
if u.Node == "" {
|
||||
progress(StartEvent{Phase: StartPlacing, Size: u.Size})
|
||||
views, err := m.Views(ctx)
|
||||
if err != nil {
|
||||
return u, err
|
||||
@@ -212,11 +236,13 @@ func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (
|
||||
return u, &node.NodeFullError{Node: u.Node}
|
||||
}
|
||||
log.Printf("container: starting %s for %s %s on %s (drive %s, %s)", u.Machine, u.Kind, u.ID, u.Node, u.Drive, u.Size)
|
||||
progress(StartEvent{Phase: StartBooting, Node: u.Node, Drive: u.Drive, Size: u.Size})
|
||||
if err := n.Start(ctx, m.spec(u)); err != nil {
|
||||
log.Printf("container: starting %s on %s failed: %v", u.Machine, u.Node, err)
|
||||
return u, err
|
||||
}
|
||||
log.Printf("container: %s is running on %s", u.Machine, u.Node)
|
||||
progress(StartEvent{Phase: StartRegistered, Node: u.Node, Drive: u.Drive, Size: u.Size})
|
||||
return u, nil
|
||||
}
|
||||
|
||||
@@ -268,6 +294,14 @@ func (m *Manager) Poweroff(ctx context.Context, u *model.Owner) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) WaitStopped(ctx context.Context, u *model.Owner) error {
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return n.WaitStopped(ctx, u.Machine)
|
||||
}
|
||||
|
||||
func (m *Manager) RunningOwners(ctx context.Context) []model.Owner {
|
||||
byNodeMachine := map[string]model.Owner{}
|
||||
for _, u := range m.store.Owners() {
|
||||
|
||||
@@ -51,6 +51,10 @@ type Owner struct {
|
||||
|
||||
// 0 means the config default.
|
||||
IdleTimeout time.Duration `json:"idle_timeout,omitempty"`
|
||||
|
||||
// The pinned message whose thread holds the container's start and stop updates.
|
||||
LifecycleChannel string `json:"lifecycle_channel,omitempty"`
|
||||
LifecycleTS string `json:"lifecycle_ts,omitempty"`
|
||||
}
|
||||
|
||||
const Indefinite time.Duration = -1
|
||||
|
||||
@@ -154,6 +154,16 @@ func (s *Store) DeleteSessions(ownerID string) error {
|
||||
return s.save()
|
||||
}
|
||||
|
||||
func (s *Store) SetLifecycle(id, channel, ts string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u, ok := s.data.Owners[id]; ok {
|
||||
u.LifecycleChannel, u.LifecycleTS = channel, ts
|
||||
return s.save()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) SetIdleTimeout(id string, d time.Duration) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
Reference in new issue
Block a user