better startup flow

This commit is contained in:
ari committed 2026-11-01 14:20:13 -05:00
1 parent d7f5dba677
commit 650bcff83a
8 files changed
+345 -68

No files matched your search

+18 -3
View File
@@ -218,8 +218,11 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
ownerID, kind := m.owner()
var off *node.NodeOfflineError
var full *node.NodeFullError
u, err := b.mgr.Ensure(ctx, ownerID, kind)
sl := &startLog{b: b, m: m, root: root, ownerID: ownerID}
started := func(ev manager.StartEvent) { sl.event(ctx, ev) }
u, err := b.mgr.EnsureProgress(ctx, ownerID, kind, started)
if errors.As(err, &full) {
sl.failed(ctx, fmt.Sprintf("`%s` is full, so the container is being moved to a node with room first.", full.Node))
edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room…", full.Node))
trigger := fmt.Sprintf("auto process (%s was full), for <@%s>'s message", full.Node, m.user)
_, merr := b.runMove(ctx, m.channel, ownerID, manager.MoveOptions{StartAfter: true}, trigger, func(link string) {
@@ -230,13 +233,15 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
return
}
edit(":hourglass_flowing_sand: Moved. Getting the container ready…")
u, err = b.mgr.Ensure(ctx, ownerID, kind)
u, err = b.mgr.EnsureProgress(ctx, ownerID, kind, started)
}
if errors.As(err, &off) {
sl.failed(ctx, fmt.Sprintf("Node `%s` is offline.", off.Node))
b.deadSlave(ctx, m.channel, ph, ownerID, off.Node)
return
}
if err != nil {
sl.failed(ctx, b.explain(err))
edit(b.explain(err))
return
}
@@ -246,13 +251,16 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
acp, err := b.client(ctx, u)
if errors.As(err, &off) {
sl.failed(ctx, fmt.Sprintf("Node `%s` went offline.", off.Node))
b.deadSlave(ctx, m.channel, ph, ownerID, off.Node)
return
}
if err != nil {
sl.failed(ctx, "The agent never became ready: "+err.Error())
edit(":warning: Couldn't reach the agent: " + err.Error())
return
}
sl.up(ctx)
var id string
text := m.text
@@ -510,7 +518,11 @@ func (b *Bot) stopFor(ctx context.Context, ownerID string) string {
return ":warning: Couldn't stop it: " + err.Error()
}
b.dropClient(u)
return "Shutting down the container. Its files are kept. Message me anytime to start it again."
reply := "Shutting down the container. Its files are kept. Message me anytime to start it again."
if link := b.announceStop(ctx, u, "Stopped on request."); link != "" {
reply += fmt.Sprintf(" <%s|Follow the shutdown here>.", link)
}
return reply
}
func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) {
@@ -1074,6 +1086,9 @@ func (b *Bot) reapOnce(ctx context.Context) {
b.dropClient(&u)
msg := fmt.Sprintf("I stopped the container after %s without use. Its files are kept. "+
"Message me anytime to start it again, or use `keep` to change how long it waits.", commands.Human(limit))
if link := b.announceStop(ctx, &u, fmt.Sprintf("Unused for %s.", commands.Human(limit))); link != "" {
msg += fmt.Sprintf(" <%s|Shutdown details>.", link)
}
b.say(ctx, u.ID, "", msg) // a user ID posts to their DM, a channel ID to the channel
}
}
+154
View File
@@ -0,0 +1,154 @@
package bot
import (
"context"
"fmt"
"log"
"time"
"github.com/slack-go/slack"
"slackerbot/internal/manager"
"slackerbot/internal/model"
)
type lifecycle struct {
b *Bot
channel string
root string
link string
}
func (l *lifecycle) post(ctx context.Context, text string) {
if l == nil {
return
}
l.b.say(ctx, l.channel, l.root, text)
}
func (l *lifecycle) url() string {
if l == nil {
return ""
}
return l.link
}
func (b *Bot) permalink(ctx context.Context, channel, ts string) string {
link, err := b.api.GetPermalinkContext(ctx, &slack.PermalinkParameters{Channel: channel, Ts: ts})
if err != nil {
return ""
}
return link
}
// openLifecycle posts a new top-level message for the container, pins it and
// unpins the one from its previous run.
func (b *Bot) openLifecycle(ctx context.Context, u *model.Owner, headline string) *lifecycle {
channel, ts, err := b.api.PostMessageContext(ctx, u.ID, slack.MsgOptionText(headline, false))
if err != nil {
log.Printf("slack post: %v", err)
return nil
}
if u.LifecycleTS != "" {
old := slack.NewRefToMessage(u.LifecycleChannel, u.LifecycleTS)
if err := b.api.RemovePinContext(ctx, u.LifecycleChannel, old); err != nil {
log.Printf("slack unpin: %v", err)
}
}
if err := b.api.AddPinContext(ctx, channel, slack.NewRefToMessage(channel, ts)); err != nil {
log.Printf("slack pin: %v", err)
}
if err := b.store.SetLifecycle(u.ID, channel, ts); err != nil {
log.Printf("saving lifecycle message for %s: %v", u.ID, err)
}
return &lifecycle{b: b, channel: channel, root: ts, link: b.permalink(ctx, channel, ts)}
}
func (b *Bot) currentLifecycle(ctx context.Context, u *model.Owner) *lifecycle {
if u.LifecycleTS == "" {
return b.openLifecycle(ctx, u, fmt.Sprintf(":package: Container `%s` for %s", u.Machine, ownerMention(u)))
}
return &lifecycle{b: b, channel: u.LifecycleChannel, root: u.LifecycleTS,
link: b.permalink(ctx, u.LifecycleChannel, u.LifecycleTS)}
}
func ownerMention(u *model.Owner) string {
if u.Kind == model.OwnerUser {
return "<@" + u.ID + ">"
}
return "<#" + u.ID + ">"
}
// startLog reports a container start in its lifecycle thread. It stays silent
// when the container is already running.
type startLog struct {
b *Bot
m incoming
root string
ownerID string
lc *lifecycle
began time.Time
}
func (s *startLog) open(ctx context.Context) {
if s.lc != nil {
return
}
u := s.b.store.Owner(s.ownerID)
if u == nil {
return
}
s.began = time.Now()
head := fmt.Sprintf(":rocket: Starting container `%s` for %s. Started by <@%s> in <%s|this thread>.",
u.Machine, ownerMention(u), s.m.user, s.b.permalink(ctx, s.m.channel, s.root))
if s.lc = s.b.openLifecycle(ctx, u, head); s.lc != nil {
s.b.say(ctx, s.m.channel, s.root, fmt.Sprintf(":rocket: Starting the container. <%s|Follow the startup here>.", s.lc.url()))
}
}
func (s *startLog) event(ctx context.Context, ev manager.StartEvent) {
s.open(ctx)
switch ev.Phase {
case manager.StartPlacing:
s.lc.post(ctx, fmt.Sprintf(":mag: New container, so it needs a home. Waiting for the balancer to pick the node and drive with the most room for %s.", ev.Size))
case manager.StartBooting:
s.lc.post(ctx, fmt.Sprintf(":gear: Placed on `%s` (drive `%s`). Waiting for the node to create the home folder, apply the storage quota and launch the container with systemd-nspawn.", ev.Node, ev.Drive))
case manager.StartRegistered:
text := ":hourglass_flowing_sand: The container is launched and the node can see it. Waiting for the agent (goose) to answer on port 1984"
if t := s.b.cfg.BootTimeout; t > 0 {
text += fmt.Sprintf(", for up to %s", t)
}
s.lc.post(ctx, text+".")
}
}
func (s *startLog) up(ctx context.Context) {
if s.lc == nil {
return
}
s.lc.post(ctx, fmt.Sprintf(":white_check_mark: <@%s> the container is up and the agent is ready (took %s).",
s.m.user, time.Since(s.began).Round(time.Second)))
}
func (s *startLog) failed(ctx context.Context, reason string) {
if s.lc == nil {
return
}
s.lc.post(ctx, ":x: The container didn't come up. "+reason)
}
// announceStop posts the shutdown in the container's lifecycle thread and
// returns a link to it. The thread gets a last message once the machine is gone.
func (b *Bot) announceStop(ctx context.Context, u *model.Owner, reason string) string {
lc := b.currentLifecycle(ctx, u)
lc.post(ctx, ":octagonal_sign: Shutting down. "+reason+" Waiting for the container to power off.")
go func() {
bg := context.Background()
if err := b.mgr.WaitStopped(bg, u); err != nil {
lc.post(bg, ":warning: The container was asked to shut down but hasn't stopped: "+err.Error())
return
}
lc.post(bg, ":white_check_mark: The container is down. Its files are kept, and a new message starts it again.")
}()
return lc.url()
}
+55
View File
@@ -0,0 +1,55 @@
package bot
import (
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slack-go/slack"
"slackerbot/internal/config"
"slackerbot/internal/manager"
"slackerbot/internal/model"
"slackerbot/internal/state"
)
func TestStartIsReportedInTheLifecycleThread(t *testing.T) {
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
srv := httptest.NewServer(log)
defer srv.Close()
store, _ := state.Open(filepath.Join(t.TempDir(), "state.json"))
_ = store.PutOwner(&model.Owner{ID: "C1", Kind: model.OwnerChannel, Machine: "c-c1"})
b := &Bot{store: store, cfg: &config.Config{BootTimeout: time.Minute}, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/"))}
sl := &startLog{b: b, m: incoming{user: "U1", channel: "C1", ts: "5.0"}, root: "5.0", ownerID: "C1"}
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartPlacing, Size: model.Size{MemoryMB: 1024, CPUs: 1}})
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartBooting, Node: "master", Drive: "/d"})
sl.event(t.Context(), manager.StartEvent{Phase: manager.StartRegistered, Node: "master", Drive: "/d"})
sl.up(t.Context())
u := store.Owner("C1")
if u.LifecycleTS == "" || u.LifecycleChannel == "" {
t.Fatalf("the lifecycle message wasn't saved: %+v", u)
}
log.mu.Lock()
defer log.mu.Unlock()
var inThread, topLevel, linked int
for _, ts := range log.order {
switch log.under[ts] {
case u.LifecycleTS:
inThread++
case "5.0":
if strings.Contains(log.text[ts], "https://slack.example/") {
linked++
}
case "":
topLevel++
}
}
if topLevel != 1 || inThread != 4 || linked != 1 {
t.Fatalf("top-level %d, in lifecycle thread %d, links in the user's thread %d (order %v, text %v)", topLevel, inThread, linked, log.order, log.text)
}
}
+34
View File
@@ -35,6 +35,22 @@ const (
StateOffline State = "on an offline node"
)
type StartPhase string
const (
StartPlacing StartPhase = "placing"
StartBooting StartPhase = "booting"
StartRegistered StartPhase = "registered"
)
// StartEvent is only reported when Ensure really has to start the container.
type StartEvent struct {
Phase StartPhase
Node string
Drive string
Size model.Size
}
type Manager struct {
cfg *config.Config
store *state.Store
@@ -162,6 +178,13 @@ func (m *Manager) spec(u *model.Owner) node.StartSpec {
}
func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (*model.Owner, error) {
return m.EnsureProgress(ctx, id, kind, nil)
}
func (m *Manager) EnsureProgress(ctx context.Context, id string, kind model.OwnerKind, progress func(StartEvent)) (*model.Owner, error) {
if progress == nil {
progress = func(StartEvent) {}
}
if m.IsMigrating(id) {
return nil, ErrMigrating
}
@@ -186,6 +209,7 @@ func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (
defer m.startMu.Unlock()
if u.Node == "" {
progress(StartEvent{Phase: StartPlacing, Size: u.Size})
views, err := m.Views(ctx)
if err != nil {
return u, err
@@ -212,11 +236,13 @@ func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (
return u, &node.NodeFullError{Node: u.Node}
}
log.Printf("container: starting %s for %s %s on %s (drive %s, %s)", u.Machine, u.Kind, u.ID, u.Node, u.Drive, u.Size)
progress(StartEvent{Phase: StartBooting, Node: u.Node, Drive: u.Drive, Size: u.Size})
if err := n.Start(ctx, m.spec(u)); err != nil {
log.Printf("container: starting %s on %s failed: %v", u.Machine, u.Node, err)
return u, err
}
log.Printf("container: %s is running on %s", u.Machine, u.Node)
progress(StartEvent{Phase: StartRegistered, Node: u.Node, Drive: u.Drive, Size: u.Size})
return u, nil
}
@@ -268,6 +294,14 @@ func (m *Manager) Poweroff(ctx context.Context, u *model.Owner) error {
return nil
}
func (m *Manager) WaitStopped(ctx context.Context, u *model.Owner) error {
n, err := m.nodeByName(u.Node)
if err != nil {
return err
}
return n.WaitStopped(ctx, u.Machine)
}
func (m *Manager) RunningOwners(ctx context.Context) []model.Owner {
byNodeMachine := map[string]model.Owner{}
for _, u := range m.store.Owners() {
+4
View File
@@ -51,6 +51,10 @@ type Owner struct {
// 0 means the config default.
IdleTimeout time.Duration `json:"idle_timeout,omitempty"`
// The pinned message whose thread holds the container's start and stop updates.
LifecycleChannel string `json:"lifecycle_channel,omitempty"`
LifecycleTS string `json:"lifecycle_ts,omitempty"`
}
const Indefinite time.Duration = -1
+10
View File
@@ -154,6 +154,16 @@ func (s *Store) DeleteSessions(ownerID string) error {
return s.save()
}
func (s *Store) SetLifecycle(id, channel, ts string) error {
s.mu.Lock()
defer s.mu.Unlock()
if u, ok := s.data.Owners[id]; ok {
u.LifecycleChannel, u.LifecycleTS = channel, ts
return s.save()
}
return nil
}
func (s *Store) SetIdleTimeout(id string, d time.Duration) error {
s.mu.Lock()
defer s.mu.Unlock()