better startup flow
This commit is contained in:
1 parent
d7f5dba677
commit
650bcff83a
8 files changed
+345
-68
No files matched your search
@@ -218,8 +218,11 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
|
||||
ownerID, kind := m.owner()
|
||||
var off *node.NodeOfflineError
|
||||
var full *node.NodeFullError
|
||||
u, err := b.mgr.Ensure(ctx, ownerID, kind)
|
||||
sl := &startLog{b: b, m: m, root: root, ownerID: ownerID}
|
||||
started := func(ev manager.StartEvent) { sl.event(ctx, ev) }
|
||||
u, err := b.mgr.EnsureProgress(ctx, ownerID, kind, started)
|
||||
if errors.As(err, &full) {
|
||||
sl.failed(ctx, fmt.Sprintf("`%s` is full, so the container is being moved to a node with room first.", full.Node))
|
||||
edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room…", full.Node))
|
||||
trigger := fmt.Sprintf("auto process (%s was full), for <@%s>'s message", full.Node, m.user)
|
||||
_, merr := b.runMove(ctx, m.channel, ownerID, manager.MoveOptions{StartAfter: true}, trigger, func(link string) {
|
||||
@@ -230,13 +233,15 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
|
||||
return
|
||||
}
|
||||
edit(":hourglass_flowing_sand: Moved. Getting the container ready…")
|
||||
u, err = b.mgr.Ensure(ctx, ownerID, kind)
|
||||
u, err = b.mgr.EnsureProgress(ctx, ownerID, kind, started)
|
||||
}
|
||||
if errors.As(err, &off) {
|
||||
sl.failed(ctx, fmt.Sprintf("Node `%s` is offline.", off.Node))
|
||||
b.deadSlave(ctx, m.channel, ph, ownerID, off.Node)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
sl.failed(ctx, b.explain(err))
|
||||
edit(b.explain(err))
|
||||
return
|
||||
}
|
||||
@@ -246,13 +251,16 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
|
||||
|
||||
acp, err := b.client(ctx, u)
|
||||
if errors.As(err, &off) {
|
||||
sl.failed(ctx, fmt.Sprintf("Node `%s` went offline.", off.Node))
|
||||
b.deadSlave(ctx, m.channel, ph, ownerID, off.Node)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
sl.failed(ctx, "The agent never became ready: "+err.Error())
|
||||
edit(":warning: Couldn't reach the agent: " + err.Error())
|
||||
return
|
||||
}
|
||||
sl.up(ctx)
|
||||
|
||||
var id string
|
||||
text := m.text
|
||||
@@ -510,7 +518,11 @@ func (b *Bot) stopFor(ctx context.Context, ownerID string) string {
|
||||
return ":warning: Couldn't stop it: " + err.Error()
|
||||
}
|
||||
b.dropClient(u)
|
||||
return "Shutting down the container. Its files are kept. Message me anytime to start it again."
|
||||
reply := "Shutting down the container. Its files are kept. Message me anytime to start it again."
|
||||
if link := b.announceStop(ctx, u, "Stopped on request."); link != "" {
|
||||
reply += fmt.Sprintf(" <%s|Follow the shutdown here>.", link)
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) {
|
||||
@@ -1074,6 +1086,9 @@ func (b *Bot) reapOnce(ctx context.Context) {
|
||||
b.dropClient(&u)
|
||||
msg := fmt.Sprintf("I stopped the container after %s without use. Its files are kept. "+
|
||||
"Message me anytime to start it again, or use `keep` to change how long it waits.", commands.Human(limit))
|
||||
if link := b.announceStop(ctx, &u, fmt.Sprintf("Unused for %s.", commands.Human(limit))); link != "" {
|
||||
msg += fmt.Sprintf(" <%s|Shutdown details>.", link)
|
||||
}
|
||||
b.say(ctx, u.ID, "", msg) // a user ID posts to their DM, a channel ID to the channel
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package bot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/slack-go/slack"
|
||||
|
||||
"slackerbot/internal/manager"
|
||||
"slackerbot/internal/model"
|
||||
)
|
||||
|
||||
type lifecycle struct {
|
||||
b *Bot
|
||||
channel string
|
||||
root string
|
||||
link string
|
||||
}
|
||||
|
||||
func (l *lifecycle) post(ctx context.Context, text string) {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
l.b.say(ctx, l.channel, l.root, text)
|
||||
}
|
||||
|
||||
func (l *lifecycle) url() string {
|
||||
if l == nil {
|
||||
return ""
|
||||
}
|
||||
return l.link
|
||||
}
|
||||
|
||||
func (b *Bot) permalink(ctx context.Context, channel, ts string) string {
|
||||
link, err := b.api.GetPermalinkContext(ctx, &slack.PermalinkParameters{Channel: channel, Ts: ts})
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return link
|
||||
}
|
||||
|
||||
// openLifecycle posts a new top-level message for the container, pins it and
|
||||
// unpins the one from its previous run.
|
||||
func (b *Bot) openLifecycle(ctx context.Context, u *model.Owner, headline string) *lifecycle {
|
||||
channel, ts, err := b.api.PostMessageContext(ctx, u.ID, slack.MsgOptionText(headline, false))
|
||||
if err != nil {
|
||||
log.Printf("slack post: %v", err)
|
||||
return nil
|
||||
}
|
||||
if u.LifecycleTS != "" {
|
||||
old := slack.NewRefToMessage(u.LifecycleChannel, u.LifecycleTS)
|
||||
if err := b.api.RemovePinContext(ctx, u.LifecycleChannel, old); err != nil {
|
||||
log.Printf("slack unpin: %v", err)
|
||||
}
|
||||
}
|
||||
if err := b.api.AddPinContext(ctx, channel, slack.NewRefToMessage(channel, ts)); err != nil {
|
||||
log.Printf("slack pin: %v", err)
|
||||
}
|
||||
if err := b.store.SetLifecycle(u.ID, channel, ts); err != nil {
|
||||
log.Printf("saving lifecycle message for %s: %v", u.ID, err)
|
||||
}
|
||||
return &lifecycle{b: b, channel: channel, root: ts, link: b.permalink(ctx, channel, ts)}
|
||||
}
|
||||
|
||||
func (b *Bot) currentLifecycle(ctx context.Context, u *model.Owner) *lifecycle {
|
||||
if u.LifecycleTS == "" {
|
||||
return b.openLifecycle(ctx, u, fmt.Sprintf(":package: Container `%s` for %s", u.Machine, ownerMention(u)))
|
||||
}
|
||||
return &lifecycle{b: b, channel: u.LifecycleChannel, root: u.LifecycleTS,
|
||||
link: b.permalink(ctx, u.LifecycleChannel, u.LifecycleTS)}
|
||||
}
|
||||
|
||||
func ownerMention(u *model.Owner) string {
|
||||
if u.Kind == model.OwnerUser {
|
||||
return "<@" + u.ID + ">"
|
||||
}
|
||||
return "<#" + u.ID + ">"
|
||||
}
|
||||
|
||||
// startLog reports a container start in its lifecycle thread. It stays silent
|
||||
// when the container is already running.
|
||||
type startLog struct {
|
||||
b *Bot
|
||||
m incoming
|
||||
root string
|
||||
ownerID string
|
||||
lc *lifecycle
|
||||
began time.Time
|
||||
}
|
||||
|
||||
func (s *startLog) open(ctx context.Context) {
|
||||
if s.lc != nil {
|
||||
return
|
||||
}
|
||||
u := s.b.store.Owner(s.ownerID)
|
||||
if u == nil {
|
||||
return
|
||||
}
|
||||
s.began = time.Now()
|
||||
head := fmt.Sprintf(":rocket: Starting container `%s` for %s. Started by <@%s> in <%s|this thread>.",
|
||||
u.Machine, ownerMention(u), s.m.user, s.b.permalink(ctx, s.m.channel, s.root))
|
||||
if s.lc = s.b.openLifecycle(ctx, u, head); s.lc != nil {
|
||||
s.b.say(ctx, s.m.channel, s.root, fmt.Sprintf(":rocket: Starting the container. <%s|Follow the startup here>.", s.lc.url()))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *startLog) event(ctx context.Context, ev manager.StartEvent) {
|
||||
s.open(ctx)
|
||||
switch ev.Phase {
|
||||
case manager.StartPlacing:
|
||||
s.lc.post(ctx, fmt.Sprintf(":mag: New container, so it needs a home. Waiting for the balancer to pick the node and drive with the most room for %s.", ev.Size))
|
||||
case manager.StartBooting:
|
||||
s.lc.post(ctx, fmt.Sprintf(":gear: Placed on `%s` (drive `%s`). Waiting for the node to create the home folder, apply the storage quota and launch the container with systemd-nspawn.", ev.Node, ev.Drive))
|
||||
case manager.StartRegistered:
|
||||
text := ":hourglass_flowing_sand: The container is launched and the node can see it. Waiting for the agent (goose) to answer on port 1984"
|
||||
if t := s.b.cfg.BootTimeout; t > 0 {
|
||||
text += fmt.Sprintf(", for up to %s", t)
|
||||
}
|
||||
s.lc.post(ctx, text+".")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *startLog) up(ctx context.Context) {
|
||||
if s.lc == nil {
|
||||
return
|
||||
}
|
||||
s.lc.post(ctx, fmt.Sprintf(":white_check_mark: <@%s> the container is up and the agent is ready (took %s).",
|
||||
s.m.user, time.Since(s.began).Round(time.Second)))
|
||||
}
|
||||
|
||||
func (s *startLog) failed(ctx context.Context, reason string) {
|
||||
if s.lc == nil {
|
||||
return
|
||||
}
|
||||
s.lc.post(ctx, ":x: The container didn't come up. "+reason)
|
||||
}
|
||||
|
||||
// announceStop posts the shutdown in the container's lifecycle thread and
|
||||
// returns a link to it. The thread gets a last message once the machine is gone.
|
||||
func (b *Bot) announceStop(ctx context.Context, u *model.Owner, reason string) string {
|
||||
lc := b.currentLifecycle(ctx, u)
|
||||
lc.post(ctx, ":octagonal_sign: Shutting down. "+reason+" Waiting for the container to power off.")
|
||||
go func() {
|
||||
bg := context.Background()
|
||||
if err := b.mgr.WaitStopped(bg, u); err != nil {
|
||||
lc.post(bg, ":warning: The container was asked to shut down but hasn't stopped: "+err.Error())
|
||||
return
|
||||
}
|
||||
lc.post(bg, ":white_check_mark: The container is down. Its files are kept, and a new message starts it again.")
|
||||
}()
|
||||
return lc.url()
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package bot
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/slack-go/slack"
|
||||
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/manager"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/state"
|
||||
)
|
||||
|
||||
func TestStartIsReportedInTheLifecycleThread(t *testing.T) {
|
||||
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
|
||||
srv := httptest.NewServer(log)
|
||||
defer srv.Close()
|
||||
store, _ := state.Open(filepath.Join(t.TempDir(), "state.json"))
|
||||
_ = store.PutOwner(&model.Owner{ID: "C1", Kind: model.OwnerChannel, Machine: "c-c1"})
|
||||
b := &Bot{store: store, cfg: &config.Config{BootTimeout: time.Minute}, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/"))}
|
||||
|
||||
sl := &startLog{b: b, m: incoming{user: "U1", channel: "C1", ts: "5.0"}, root: "5.0", ownerID: "C1"}
|
||||
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartPlacing, Size: model.Size{MemoryMB: 1024, CPUs: 1}})
|
||||
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartBooting, Node: "master", Drive: "/d"})
|
||||
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartRegistered, Node: "master", Drive: "/d"})
|
||||
sl.up(t.Context())
|
||||
|
||||
u := store.Owner("C1")
|
||||
if u.LifecycleTS == "" || u.LifecycleChannel == "" {
|
||||
t.Fatalf("the lifecycle message wasn't saved: %+v", u)
|
||||
}
|
||||
|
||||
log.mu.Lock()
|
||||
defer log.mu.Unlock()
|
||||
var inThread, topLevel, linked int
|
||||
for _, ts := range log.order {
|
||||
switch log.under[ts] {
|
||||
case u.LifecycleTS:
|
||||
inThread++
|
||||
case "5.0":
|
||||
if strings.Contains(log.text[ts], "https://slack.example/") {
|
||||
linked++
|
||||
}
|
||||
case "":
|
||||
topLevel++
|
||||
}
|
||||
}
|
||||
if topLevel != 1 || inThread != 4 || linked != 1 {
|
||||
t.Fatalf("top-level %d, in lifecycle thread %d, links in the user's thread %d (order %v, text %v)", topLevel, inThread, linked, log.order, log.text)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user