better startup flow

This commit is contained in:
ari committed 2026-11-01 14:20:13 -05:00
1 parent d7f5dba677
commit 650bcff83a
8 files changed
+345 -68

No files matched your search

+18 -3
View File
@@ -218,8 +218,11 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
ownerID, kind := m.owner()
var off *node.NodeOfflineError
var full *node.NodeFullError
u, err := b.mgr.Ensure(ctx, ownerID, kind)
sl := &startLog{b: b, m: m, root: root, ownerID: ownerID}
started := func(ev manager.StartEvent) { sl.event(ctx, ev) }
u, err := b.mgr.EnsureProgress(ctx, ownerID, kind, started)
if errors.As(err, &full) {
sl.failed(ctx, fmt.Sprintf("`%s` is full, so the container is being moved to a node with room first.", full.Node))
edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room…", full.Node))
trigger := fmt.Sprintf("auto process (%s was full), for <@%s>'s message", full.Node, m.user)
_, merr := b.runMove(ctx, m.channel, ownerID, manager.MoveOptions{StartAfter: true}, trigger, func(link string) {
@@ -230,13 +233,15 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
return
}
edit(":hourglass_flowing_sand: Moved. Getting the container ready…")
u, err = b.mgr.Ensure(ctx, ownerID, kind)
u, err = b.mgr.EnsureProgress(ctx, ownerID, kind, started)
}
if errors.As(err, &off) {
sl.failed(ctx, fmt.Sprintf("Node `%s` is offline.", off.Node))
b.deadSlave(ctx, m.channel, ph, ownerID, off.Node)
return
}
if err != nil {
sl.failed(ctx, b.explain(err))
edit(b.explain(err))
return
}
@@ -246,13 +251,16 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
acp, err := b.client(ctx, u)
if errors.As(err, &off) {
sl.failed(ctx, fmt.Sprintf("Node `%s` went offline.", off.Node))
b.deadSlave(ctx, m.channel, ph, ownerID, off.Node)
return
}
if err != nil {
sl.failed(ctx, "The agent never became ready: "+err.Error())
edit(":warning: Couldn't reach the agent: " + err.Error())
return
}
sl.up(ctx)
var id string
text := m.text
@@ -510,7 +518,11 @@ func (b *Bot) stopFor(ctx context.Context, ownerID string) string {
return ":warning: Couldn't stop it: " + err.Error()
}
b.dropClient(u)
return "Shutting down the container. Its files are kept. Message me anytime to start it again."
reply := "Shutting down the container. Its files are kept. Message me anytime to start it again."
if link := b.announceStop(ctx, u, "Stopped on request."); link != "" {
reply += fmt.Sprintf(" <%s|Follow the shutdown here>.", link)
}
return reply
}
func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) {
@@ -1074,6 +1086,9 @@ func (b *Bot) reapOnce(ctx context.Context) {
b.dropClient(&u)
msg := fmt.Sprintf("I stopped the container after %s without use. Its files are kept. "+
"Message me anytime to start it again, or use `keep` to change how long it waits.", commands.Human(limit))
if link := b.announceStop(ctx, &u, fmt.Sprintf("Unused for %s.", commands.Human(limit))); link != "" {
msg += fmt.Sprintf(" <%s|Shutdown details>.", link)
}
b.say(ctx, u.ID, "", msg) // a user ID posts to their DM, a channel ID to the channel
}
}
+154
View File
@@ -0,0 +1,154 @@
package bot
import (
"context"
"fmt"
"log"
"time"
"github.com/slack-go/slack"
"slackerbot/internal/manager"
"slackerbot/internal/model"
)
type lifecycle struct {
b *Bot
channel string
root string
link string
}
func (l *lifecycle) post(ctx context.Context, text string) {
if l == nil {
return
}
l.b.say(ctx, l.channel, l.root, text)
}
func (l *lifecycle) url() string {
if l == nil {
return ""
}
return l.link
}
func (b *Bot) permalink(ctx context.Context, channel, ts string) string {
link, err := b.api.GetPermalinkContext(ctx, &slack.PermalinkParameters{Channel: channel, Ts: ts})
if err != nil {
return ""
}
return link
}
// openLifecycle posts a new top-level message for the container, pins it and
// unpins the one from its previous run.
func (b *Bot) openLifecycle(ctx context.Context, u *model.Owner, headline string) *lifecycle {
channel, ts, err := b.api.PostMessageContext(ctx, u.ID, slack.MsgOptionText(headline, false))
if err != nil {
log.Printf("slack post: %v", err)
return nil
}
if u.LifecycleTS != "" {
old := slack.NewRefToMessage(u.LifecycleChannel, u.LifecycleTS)
if err := b.api.RemovePinContext(ctx, u.LifecycleChannel, old); err != nil {
log.Printf("slack unpin: %v", err)
}
}
if err := b.api.AddPinContext(ctx, channel, slack.NewRefToMessage(channel, ts)); err != nil {
log.Printf("slack pin: %v", err)
}
if err := b.store.SetLifecycle(u.ID, channel, ts); err != nil {
log.Printf("saving lifecycle message for %s: %v", u.ID, err)
}
return &lifecycle{b: b, channel: channel, root: ts, link: b.permalink(ctx, channel, ts)}
}
func (b *Bot) currentLifecycle(ctx context.Context, u *model.Owner) *lifecycle {
if u.LifecycleTS == "" {
return b.openLifecycle(ctx, u, fmt.Sprintf(":package: Container `%s` for %s", u.Machine, ownerMention(u)))
}
return &lifecycle{b: b, channel: u.LifecycleChannel, root: u.LifecycleTS,
link: b.permalink(ctx, u.LifecycleChannel, u.LifecycleTS)}
}
func ownerMention(u *model.Owner) string {
if u.Kind == model.OwnerUser {
return "<@" + u.ID + ">"
}
return "<#" + u.ID + ">"
}
// startLog reports a container start in its lifecycle thread. It stays silent
// when the container is already running.
type startLog struct {
b *Bot
m incoming
root string
ownerID string
lc *lifecycle
began time.Time
}
func (s *startLog) open(ctx context.Context) {
if s.lc != nil {
return
}
u := s.b.store.Owner(s.ownerID)
if u == nil {
return
}
s.began = time.Now()
head := fmt.Sprintf(":rocket: Starting container `%s` for %s. Started by <@%s> in <%s|this thread>.",
u.Machine, ownerMention(u), s.m.user, s.b.permalink(ctx, s.m.channel, s.root))
if s.lc = s.b.openLifecycle(ctx, u, head); s.lc != nil {
s.b.say(ctx, s.m.channel, s.root, fmt.Sprintf(":rocket: Starting the container. <%s|Follow the startup here>.", s.lc.url()))
}
}
func (s *startLog) event(ctx context.Context, ev manager.StartEvent) {
s.open(ctx)
switch ev.Phase {
case manager.StartPlacing:
s.lc.post(ctx, fmt.Sprintf(":mag: New container, so it needs a home. Waiting for the balancer to pick the node and drive with the most room for %s.", ev.Size))
case manager.StartBooting:
s.lc.post(ctx, fmt.Sprintf(":gear: Placed on `%s` (drive `%s`). Waiting for the node to create the home folder, apply the storage quota and launch the container with systemd-nspawn.", ev.Node, ev.Drive))
case manager.StartRegistered:
text := ":hourglass_flowing_sand: The container is launched and the node can see it. Waiting for the agent (goose) to answer on port 1984"
if t := s.b.cfg.BootTimeout; t > 0 {
text += fmt.Sprintf(", for up to %s", t)
}
s.lc.post(ctx, text+".")
}
}
func (s *startLog) up(ctx context.Context) {
if s.lc == nil {
return
}
s.lc.post(ctx, fmt.Sprintf(":white_check_mark: <@%s> the container is up and the agent is ready (took %s).",
s.m.user, time.Since(s.began).Round(time.Second)))
}
func (s *startLog) failed(ctx context.Context, reason string) {
if s.lc == nil {
return
}
s.lc.post(ctx, ":x: The container didn't come up. "+reason)
}
// announceStop posts the shutdown in the container's lifecycle thread and
// returns a link to it. The thread gets a last message once the machine is gone.
func (b *Bot) announceStop(ctx context.Context, u *model.Owner, reason string) string {
lc := b.currentLifecycle(ctx, u)
lc.post(ctx, ":octagonal_sign: Shutting down. "+reason+" Waiting for the container to power off.")
go func() {
bg := context.Background()
if err := b.mgr.WaitStopped(bg, u); err != nil {
lc.post(bg, ":warning: The container was asked to shut down but hasn't stopped: "+err.Error())
return
}
lc.post(bg, ":white_check_mark: The container is down. Its files are kept, and a new message starts it again.")
}()
return lc.url()
}
+55
View File
@@ -0,0 +1,55 @@
package bot
import (
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slack-go/slack"
"slackerbot/internal/config"
"slackerbot/internal/manager"
"slackerbot/internal/model"
"slackerbot/internal/state"
)
func TestStartIsReportedInTheLifecycleThread(t *testing.T) {
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
srv := httptest.NewServer(log)
defer srv.Close()
store, _ := state.Open(filepath.Join(t.TempDir(), "state.json"))
_ = store.PutOwner(&model.Owner{ID: "C1", Kind: model.OwnerChannel, Machine: "c-c1"})
b := &Bot{store: store, cfg: &config.Config{BootTimeout: time.Minute}, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/"))}
sl := &startLog{b: b, m: incoming{user: "U1", channel: "C1", ts: "5.0"}, root: "5.0", ownerID: "C1"}
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartPlacing, Size: model.Size{MemoryMB: 1024, CPUs: 1}})
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartBooting, Node: "master", Drive: "/d"})
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartRegistered, Node: "master", Drive: "/d"})
sl.up(t.Context())
u := store.Owner("C1")
if u.LifecycleTS == "" || u.LifecycleChannel == "" {
t.Fatalf("the lifecycle message wasn't saved: %+v", u)
}
log.mu.Lock()
defer log.mu.Unlock()
var inThread, topLevel, linked int
for _, ts := range log.order {
switch log.under[ts] {
case u.LifecycleTS:
inThread++
case "5.0":
if strings.Contains(log.text[ts], "https://slack.example/") {
linked++
}
case "":
topLevel++
}
}
if topLevel != 1 || inThread != 4 || linked != 1 {
t.Fatalf("top-level %d, in lifecycle thread %d, links in the user's thread %d (order %v, text %v)", topLevel, inThread, linked, log.order, log.text)
}
}