resume active sessions

This commit is contained in:
ari committed 2026-11-01 15:52:46 -05:00
1 parent 9e94c05fb6
commit 917a361370
19 files changed
+871 -42

No files matched your search

+2
View File
@@ -7,6 +7,7 @@ go_library(
"assistant.go",
"bot.go",
"files.go",
"inflight.go",
"lifecycle.go",
"listen.go",
"manage.go",
@@ -40,6 +41,7 @@ go_test(
srcs = [
"agent_test.go",
"bot_test.go",
"inflight_test.go",
"lifecycle_test.go",
"update_test.go",
"version_test.go",
+39 -5
View File
@@ -55,6 +55,7 @@ const helpText = "I pass your messages to an agent running in its own container.
"• `@slacker /listen channel all-messages` / `@slacker /listen channel only-mentions`: set the channel's default for threads " +
"(a top-level `/listen all-messages` does the same). `@slacker /listen channel`: show it. `listen` without the slash works too\n" +
"• `permissions`: who can scale, change storage, move or set timeouts here\n" +
"• `in-flight` / `in-flight all`: super users only. Shows the replies I'd pick back up after a restart, for debugging\n" +
"• `managers`: who the user managers are\n" +
"• Managers: `grant @alice move scale`, `revoke @alice move` / `revoke @alice`, `permissions reset`, " +
"`grant default @alice all` / `revoke default @alice`, `dm storage on|off`, `managers add|remove @alice`\n" +
@@ -85,6 +86,8 @@ type Bot struct {
waitMu sync.Mutex
waiting map[string]map[string]bool
turnsWG sync.WaitGroup
}
type incoming struct {
@@ -122,7 +125,10 @@ func New(cfg *config.Config, store *state.Store, mgr *manager.Manager) (*Bot, er
func (b *Bot) Run(ctx context.Context) error {
go b.events(ctx)
go b.reaper(ctx)
return b.sm.RunContext(ctx)
go b.resumeTurns(ctx)
err := b.sm.RunContext(ctx)
b.waitForTurns(shutdownGrace)
return err
}
func (b *Bot) events(ctx context.Context) {
@@ -199,6 +205,13 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
if commandable && m.threadTS != "" && !m.heard && b.moveCommand(ctx, m, m.threadTS) {
return
}
b.converse(ctx, m, nil)
}
// converse runs one turn with the agent. prior is the turn a restart cut off, if this picks it back up.
func (b *Bot) converse(ctx context.Context, m incoming, prior *model.Turn) {
b.turnsWG.Add(1)
defer b.turnsWG.Done()
root := m.threadTS
if root == "" {
root = m.ts
@@ -209,6 +222,9 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
defer unlock()
sess := b.store.Session(m.channel, root)
if prior != nil {
b.say(ctx, m.channel, root, ":arrows_counterclockwise: slackerbot restarted part-way through this reply, so I'm picking it back up.")
}
_, ph, err := b.api.PostMessageContext(ctx, m.channel,
slack.MsgOptionText(":hourglass_flowing_sand: Getting the container ready…", false),
slack.MsgOptionTS(root))
@@ -216,6 +232,8 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
log.Printf("slack post: %v", err)
return
}
turn := b.trackTurn(m, root, ph, prior)
defer b.untrackTurn(ctx, turn)
edit := func(text string) {
_, _, _, _ = b.api.UpdateMessageContext(ctx, m.channel, ph, slack.MsgOptionText(text, false))
}
@@ -276,10 +294,15 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
sl.up(ctx)
var id string
// A new session (the old one was dropped) never saw the cut-off turn, so it gets the message again.
carryOn := turn.Prompted && sess != nil
text := m.text
if !m.dm {
text = b.displayName(ctx, m.user) + ": " + text
}
if carryOn {
text = resumeNote
}
if sess == nil {
if id, err = acp.NewSession(ctx, b.cfg.AgentCwd); err != nil {
edit(":warning: Couldn't start a session: " + err.Error())
@@ -304,10 +327,10 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
}
}
if m.dm {
if m.dm && !carryOn {
text = b.viewingNote(ctx, m.channel, root) + text
}
if len(m.files) > 0 {
if len(m.files) > 0 && !carryOn {
edit(":paperclip: Copying the attachments into the container…")
note, problems := b.attach(ctx, u, m)
if note != "" {
@@ -329,6 +352,8 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
b.setAssistantStatus(ctx, m.channel, root, "is working…")
defer b.setAssistantStatus(ctx, m.channel, root, "")
}
turn.Prompted = true
b.saveTurn(turn)
st := NewStreamer(b.api, m.channel, root, ph)
var sends tools.Calls
cmds := b.agentCommands(u, m, root)
@@ -337,7 +362,10 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
cmds.Update(ctx, up)
st.Update(up)
})
if pctx.Err() != nil && ctx.Err() == nil {
switch {
case ctx.Err() != nil:
stop, err = stopInterrupted, nil
case pctx.Err() != nil:
stop, err = "cancelled", nil
}
st.Finish(stop, err)
@@ -346,7 +374,7 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) {
_ = b.store.Touch(u.ID)
b.offerUpdate(ctx, u, m.channel, root)
switch {
case stop == "cancelled":
case stop == "cancelled", stop == stopInterrupted:
outcome = ""
case err == nil:
outcome = "white_check_mark"
@@ -424,6 +452,12 @@ func (b *Bot) runCommand(ctx context.Context, m incoming, verb, args string, age
reply = b.permissionsList(ctx, m.channel, kind)
case verb == "storage" || verb == "disk":
return b.storageFor(ctx, ownerID, kind, args)
case isInFlight(verb) && (args == "" || strings.EqualFold(args, "all")):
reply = b.inFlightFor(m, args != "", agent)
if !agent {
b.whisper(ctx, m, reply)
return "", true
}
default:
return "", false
}
+179
View File
@@ -0,0 +1,179 @@
package bot
import (
"context"
"fmt"
"log"
"sort"
"strings"
"time"
"unicode/utf8"
"github.com/slack-go/slack"
"slackerbot/internal/model"
)
const (
stopInterrupted = "interrupted"
shutdownGrace = 5 * time.Second
resumeWithin = time.Hour
maxResumes = 3
resumeNote = "(slackerbot restarted while you were working on the last message, so your turn was cut off. " +
"Carry on from where you stopped, without redoing work that's already done.)"
inFlightSlackLimit = 35000
)
func isInFlight(verb string) bool { return verb == "in-flight" || verb == "inflight" }
func (b *Bot) trackTurn(m incoming, root, replyTS string, prior *model.Turn) *model.Turn {
ownerID, _ := m.owner()
t := &model.Turn{
Channel: m.channel, ThreadTS: root, MessageTS: m.ts, ReplyTS: replyTS,
User: m.user, OwnerID: ownerID, DM: m.dm, Heard: m.heard, Text: m.text,
StartedAt: time.Now(),
}
for _, f := range m.files {
t.Files = append(t.Files, model.TurnFile{Name: f.Name, Mode: f.Mode, URL: f.URLPrivateDownload, Size: f.Size})
}
if prior != nil {
t.Prompted, t.Resumes, t.StartedAt = prior.Prompted, prior.Resumes+1, prior.StartedAt
}
b.saveTurn(t)
return t
}
func (b *Bot) saveTurn(t *model.Turn) {
if err := b.store.PutTurn(t); err != nil {
log.Printf("in-flight: saving %s/%s: %v", t.Channel, t.ThreadTS, err)
}
}
// A turn cut off by shutdown stays recorded, so the next start picks it back up.
func (b *Bot) untrackTurn(ctx context.Context, t *model.Turn) {
if ctx.Err() != nil {
return
}
if err := b.store.DeleteTurn(t.Channel, t.ThreadTS); err != nil {
log.Printf("in-flight: clearing %s/%s: %v", t.Channel, t.ThreadTS, err)
}
}
// waitForTurns gives running turns a moment to post their last edit before the process exits.
func (b *Bot) waitForTurns(grace time.Duration) {
done := make(chan struct{})
go func() {
b.turnsWG.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(grace):
}
}
func turnMessage(t model.Turn) incoming {
m := incoming{user: t.User, channel: t.Channel, ts: t.MessageTS, text: t.Text, dm: t.DM, heard: t.Heard}
if t.ThreadTS != t.MessageTS {
m.threadTS = t.ThreadTS
}
for _, f := range t.Files {
m.files = append(m.files, slack.File{Name: f.Name, Mode: f.Mode, URLPrivateDownload: f.URL, Size: f.Size})
}
return m
}
func (b *Bot) resumeTurns(ctx context.Context) {
for _, t := range b.store.Turns() {
go b.resume(ctx, t)
}
}
func (b *Bot) resume(ctx context.Context, t model.Turn) {
if time.Since(t.StartedAt) > resumeWithin || t.Resumes >= maxResumes {
log.Printf("in-flight: giving up on %s/%s (started %s, resumed %d times)", t.Channel, t.ThreadTS, t.StartedAt.Format(time.RFC3339), t.Resumes)
b.say(ctx, t.Channel, t.ThreadTS, ":warning: slackerbot restarted while it was answering this, and it's too late to pick it back up. "+
"Send your message again if you still need it.")
b.unreact(ctx, t.Channel, t.MessageTS, "eyes")
b.react(ctx, t.Channel, t.MessageTS, "warning")
if err := b.store.DeleteTurn(t.Channel, t.ThreadTS); err != nil {
log.Printf("in-flight: clearing %s/%s: %v", t.Channel, t.ThreadTS, err)
}
return
}
log.Printf("in-flight: resuming %s/%s for %s after a restart (prompted %v, resume %d)", t.Channel, t.ThreadTS, t.User, t.Prompted, t.Resumes+1)
b.converse(ctx, turnMessage(t), &t)
}
// inFlightFor shows the in-flight files as they are on disk. They hold message
// text, so only super users and user managers may read them.
func (b *Bot) inFlightFor(m incoming, all, agent bool) string {
p := b.store.Permissions()
channel := m.channel
if all {
channel = ""
if !p.IsManager(m.user) && len(p.Default[m.user]) == 0 {
return ":lock: Only user managers and super users on the default list can see every channel's in-flight replies."
}
} else if !p.IsSuperUser(m.channel, m.user) {
return ":lock: Only super users here and user managers can see the in-flight replies."
}
files, err := b.store.InFlightFiles(channel)
if err != nil {
return ":warning: Couldn't read the in-flight files: " + err.Error()
}
if len(files) == 0 {
if all {
return "No replies are in flight anywhere."
}
return "No replies are in flight here."
}
limit := inFlightSlackLimit
if agent {
limit = maxReplyBytes / 2
}
names := make([]string, 0, len(files))
for name := range files {
names = append(names, name)
}
sort.Strings(names)
var sb strings.Builder
sb.WriteString("*In-flight replies* (what I'd pick back up after a restart):\n")
for i, name := range names {
data := strings.TrimSpace(string(files[name]))
room := limit - sb.Len() - len(name) - 20
if room <= 0 {
fmt.Fprintf(&sb, "_(%d more files not shown)_", len(names)-i)
break
}
if len(data) > room {
data = cutBytes(data, room) + fmt.Sprintf("\n… (cut off, %d bytes in total)", len(files[name]))
}
fmt.Fprintf(&sb, "`%s`\n```\n%s\n```\n", name, data)
}
return strings.TrimSpace(sb.String())
}
// The files hold other people's messages, so only the asker sees them.
func (b *Bot) whisper(ctx context.Context, m incoming, text string) {
opts := []slack.MsgOption{slack.MsgOptionText(text, false)}
if m.threadTS != "" {
opts = append(opts, slack.MsgOptionTS(m.threadTS))
}
if _, err := b.api.PostEphemeralContext(ctx, m.channel, m.user, opts...); err != nil {
log.Printf("slack ephemeral: %v", err)
}
}
func cutBytes(s string, n int) string {
if len(s) <= n {
return s
}
for n > 0 && !utf8.RuneStart(s[n]) {
n--
}
return s[:n]
}
+177
View File
@@ -0,0 +1,177 @@
package bot
import (
"context"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/slack-go/slack"
"slackerbot/internal/acp"
"slackerbot/internal/model"
"slackerbot/internal/testutil"
"slackerbot/internal/testutil/clustertest"
)
type turnHarness struct {
c *clustertest.Cluster
log *slackLog
url string
goose *testutil.FakeGoose
}
func newTurnHarness(t *testing.T) *turnHarness {
c := clustertest.New(t)
c.Cfg.AgentCwd = "/root"
c.AddOwner(t, testutil.HomeData(), true)
u := c.Store.Owner("C1")
u.Secret = "agent-secret"
_ = c.Store.PutOwner(u)
c.Store.SeedPermissions(model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UBOSS": {"all"}, "USUPER": {"keep"}}})
goose := testutil.NewFakeGoose()
c.Master.Goose = goose
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
srv := httptest.NewServer(log)
t.Cleanup(srv.Close)
return &turnHarness{c: c, log: log, url: srv.URL, goose: goose}
}
// Each bot is a fresh process: nothing in memory carries over, only the store.
func (h *turnHarness) bot() *Bot {
return &Bot{cfg: h.c.Cfg, store: h.c.Store, mgr: h.c.Mgr, api: slack.New("xoxb", slack.OptionAPIURL(h.url+"/")),
acps: map[string]*acp.ACP{}, busy: map[string]int{}}
}
func (s *slackLog) anyText(want string) bool {
s.mu.Lock()
defer s.mu.Unlock()
for _, text := range s.text {
if strings.Contains(text, want) {
return true
}
}
return false
}
func TestRestartResumesInFlightTurn(t *testing.T) {
h := newTurnHarness(t)
c := h.c
h.goose.Hold(true)
ctx, shutdown := context.WithCancel(c.Ctx)
done := make(chan struct{})
go func() {
defer close(done)
h.bot().onMessage(ctx, incoming{user: "UANA", channel: "C1", ts: "5.0", text: "build the thing"})
}()
select {
case <-h.goose.Holding:
case <-time.After(10 * time.Second):
t.Fatal("the prompt never reached the agent")
}
turns := c.Store.Turns()
if len(turns) != 1 || !turns[0].Prompted || turns[0].ThreadTS != "5.0" || turns[0].User != "UANA" {
t.Fatalf("the running turn should be recorded: %+v", turns)
}
if _, err := os.Stat(filepath.Join(c.Cfg.StateDir, "in-flight-C1.json")); err != nil {
t.Fatal(err)
}
shutdown()
<-done
if len(c.Store.Turns()) != 1 {
t.Fatal("a turn cut off by shutdown must stay recorded")
}
if !h.log.anyText("slackerbot is restarting") {
t.Fatal("the cut-off reply should say it'll be picked back up")
}
h.goose.Hold(false)
h.bot().resumeTurns(c.Ctx)
testutil.WaitFor(t, "the turn to be resumed and finish", func() bool { return len(c.Store.Turns()) == 0 })
prompts := h.goose.Prompts()
if len(prompts) != 2 || !strings.Contains(prompts[0], "build the thing") || prompts[1] != resumeNote {
t.Fatalf("the agent should be told to carry on, not get the message twice: %q", prompts)
}
if !h.log.anyText("picking it back up") {
t.Fatal("the thread should say the reply is being picked back up")
}
if _, err := os.Stat(filepath.Join(c.Cfg.StateDir, "in-flight-C1.json")); !os.IsNotExist(err) {
t.Fatal("the in-flight file should go once the turn is done")
}
}
func TestResumeSendsMessageTheAgentNeverGot(t *testing.T) {
h := newTurnHarness(t)
_ = h.c.Store.PutTurn(&model.Turn{Channel: "C1", ThreadTS: "7.0", MessageTS: "7.0", User: "UANA", OwnerID: "C1",
Text: "check the logs", StartedAt: time.Now()})
h.bot().resumeTurns(h.c.Ctx)
testutil.WaitFor(t, "the turn to finish", func() bool { return len(h.c.Store.Turns()) == 0 })
if prompts := h.goose.Prompts(); len(prompts) != 1 || !strings.Contains(prompts[0], "check the logs") {
t.Fatalf("a turn cut off before the prompt should send the message: %q", prompts)
}
}
func TestStaleTurnIsDroppedNotResumed(t *testing.T) {
h := newTurnHarness(t)
_ = h.c.Store.PutTurn(&model.Turn{Channel: "C1", ThreadTS: "8.0", MessageTS: "8.0", User: "UANA", OwnerID: "C1",
Text: "old", Prompted: true, StartedAt: time.Now().Add(-2 * resumeWithin)})
_ = h.c.Store.PutTurn(&model.Turn{Channel: "C1", ThreadTS: "9.0", MessageTS: "9.0", User: "UANA", OwnerID: "C1",
Text: "looping", Prompted: true, Resumes: maxResumes, StartedAt: time.Now()})
h.bot().resumeTurns(h.c.Ctx)
testutil.WaitFor(t, "the turns to be dropped", func() bool { return len(h.c.Store.Turns()) == 0 })
if len(h.goose.Prompts()) != 0 {
t.Fatal("stale turns must not reach the agent")
}
if !h.log.anyText("too late to pick it back up") {
t.Fatal("the thread should say to send the message again")
}
}
func TestInFlightIsForSuperUsers(t *testing.T) {
h := newTurnHarness(t)
_ = h.c.Store.PutTurn(&model.Turn{Channel: "C1", ThreadTS: "1.0", MessageTS: "1.0", Text: "secret plans", StartedAt: time.Now()})
_ = h.c.Store.PutTurn(&model.Turn{Channel: "D2", ThreadTS: "2.0", MessageTS: "2.0", Text: "dm plans", DM: true, StartedAt: time.Now()})
b := h.bot()
ask := func(user, channel string, all bool) string {
return b.inFlightFor(incoming{user: user, channel: channel}, all, true)
}
if got := ask("UANA", "C1", false); !strings.Contains(got, ":lock:") || strings.Contains(got, "secret plans") {
t.Fatalf("ordinary users can't read it: %q", got)
}
if got := ask("USUPER", "C1", false); !strings.Contains(got, "in-flight-C1.json") || !strings.Contains(got, "secret plans") || strings.Contains(got, "dm plans") {
t.Fatalf("a super user sees this channel's file: %q", got)
}
if got := ask("USUPER", "C1", true); !strings.Contains(got, "in-flight-D2.json") || !strings.Contains(got, "secret plans") {
t.Fatalf("a default-list super user sees every file: %q", got)
}
if got := ask("UBOSS", "C3", false); !strings.Contains(got, "No replies are in flight here") {
t.Fatalf("empty channel: %q", got)
}
_ = h.c.Store.UpdatePermissions(func(p *model.PermissionsConfig) error {
p.Grant("C1", []string{"UANA"}, []model.Action{model.ActMove})
return nil
})
if got := ask("UANA", "C1", false); !strings.Contains(got, "secret plans") {
t.Fatalf("a channel's own super users see its file: %q", got)
}
if got := ask("UANA", "C1", true); !strings.Contains(got, ":lock:") {
t.Fatalf("every channel needs the default list or a user manager: %q", got)
}
}
func TestInFlightCommandIsPrivate(t *testing.T) {
h := newTurnHarness(t)
b := h.bot()
reply, ok := b.runCommand(h.c.Ctx, incoming{user: "UBOSS", channel: "C1", ts: "1.0"}, "in-flight", "", false)
if !ok || reply != "" {
t.Fatalf("in-flight should be answered ephemerally: %q %v", reply, ok)
}
if _, ok := b.runCommand(h.c.Ctx, incoming{user: "UBOSS", channel: "C1", ts: "1.0"}, "in-flight", "entertainment is great", false); ok {
t.Fatal("a sentence that starts with in-flight goes to the agent")
}
}
+2
View File
@@ -91,6 +91,8 @@ func (s *Streamer) Finish(stopReason string, err error) {
note = "_(the agent declined this one)_"
case stopReason == "cancelled":
note = "_(cancelled)_"
case stopReason == stopInterrupted:
note = "_(slackerbot is restarting. I'll pick this reply back up once it's back.)_"
}
if note != "" {
if s.full.Len() > 0 {