package bot import ( "cmp" "context" "errors" "fmt" "log" "math" "sort" "strconv" "strings" "sync" "time" "unicode/utf8" "github.com/slack-go/slack" "github.com/slack-go/slack/slackevents" "github.com/slack-go/slack/socketmode" "slackerbot/internal/acp" "slackerbot/internal/balance" "slackerbot/internal/commands" "slackerbot/internal/config" "slackerbot/internal/manager" "slackerbot/internal/model" "slackerbot/internal/node" "slackerbot/internal/state" "slackerbot/internal/storage" "slackerbot/internal/tools" "slackerbot/internal/units" "slackerbot/internal/util" ) const helpText = "I pass your messages to an agent running in its own container.\n" + "• In a DM, the container is yours alone. In a channel, it belongs to the channel " + "and anyone here can use it by @mentioning me.\n" + "• Every new message starts a new session. Reply in its thread to keep going.\n" + "• In a channel thread I only read replies that @mention me, unless the thread or channel is switched with `listen` (see below).\n" + "• React with :x: to stop a reply part-way, or send `cancel all` to stop every reply here.\n" + "*Commands* (send as a new message, not in a thread. `move` and `listen` work in threads too):\n" + "• `status`: state, size and idle limit, with buttons\n" + "• `sessions`: past sessions, with links\n" + "• `running`: the replies I'm working on here right now\n" + "• `cancel all`: cancel every reply I'm working on here (sessions are kept)\n" + "• `stop`: shut the container down (files are kept)\n" + "• `scale up` / `scale down`: double or halve RAM and CPU\n" + "• `scale 8G 4cpu`: set an exact size. `scale +2G` or `scale -1cpu`: change by that much\n" + "• `keep 24h` (or `3d`, `1d12h`, `2w`): stop after that long unused\n" + "• `keep indefinite`: never stop for being unused. `keep default`: back to normal. `keep`: show it\n" + "• `storage 20G` / `storage +5G` / `storage -2G`: set or change the home folder's max size (`disk` works too)\n" + "• `storage min 8G`: set the guaranteed space. `storage min 8G max 20G`: set both. `storage`: show usage\n" + "• `nodes`: every node and what's free on it\n" + "• `move ` (or `move to `, `migrate `): move the container to that node. `move auto`: let me pick the best one\n" + "• `@slacker /listen all-messages` / `@slacker /listen only-mentions`: in a session's thread, read every reply or only @mentions. " + "`@slacker /listen`: show this thread's setting\n" + "• `@slacker /listen channel all-messages` / `@slacker /listen channel only-mentions`: set the channel's default for threads " + "(a top-level `/listen all-messages` does the same). `@slacker /listen channel`: show it. `listen` without the slash works too\n" + "• `permissions`: who can scale, change storage, move or set timeouts here\n" + "• `in-flight` / `in-flight all`: super users only. Shows the replies I'd pick back up after a restart, for debugging\n" + "• `managers`: who the user managers are\n" + "• Managers: `grant @alice move scale`, `revoke @alice move` / `revoke @alice`, `permissions reset`, " + "`grant default @alice all` / `revoke default @alice`, `dm storage on|off`, `managers add|remove @alice`\n" + "• `help`: this message" type Bot struct { cfg *config.Config store *state.Store mgr *manager.Manager api *slack.Client sm *socketmode.Client botID string acpMu sync.Mutex acps map[string]*acp.ACP dialMu util.KeyedMutex sessMu util.KeyedMutex busyMu sync.Mutex busy map[string]int names sync.Map updateAsked sync.Map // machine → asked to restart onto a newer system channels sync.Map // channel ID → *slack.Channel running sync.Map // "channel/ts" of a prompt's message or reply → its cancel func viewing sync.Map // assistant "channel/thread" → channel the user has open told sync.Map // assistant "channel/thread" → last channel the agent was told about waitMu sync.Mutex waiting map[string]map[string]bool turnsWG sync.WaitGroup } type incoming struct { user, channel, ts, threadTS, text string dm bool heard bool files []slack.File } func (m incoming) owner() (string, model.OwnerKind) { if m.dm { return m.user, model.OwnerUser } return m.channel, model.OwnerChannel } func New(cfg *config.Config, store *state.Store, mgr *manager.Manager) (*Bot, error) { api := slack.New(cfg.Slack.BotToken, slack.OptionAppLevelToken(cfg.Slack.AppToken)) auth, err := api.AuthTest() if err != nil { return nil, fmt.Errorf("slack auth: %w", err) } b := &Bot{ cfg: cfg, store: store, mgr: mgr, api: api, sm: socketmode.New(api), botID: auth.UserID, acps: map[string]*acp.ACP{}, busy: map[string]int{}, waiting: map[string]map[string]bool{}, } mgr.SetNodeBackHandler(b.nodeBack) return b, nil } func (b *Bot) Run(ctx context.Context) error { go b.events(ctx) go b.reaper(ctx) go b.resumeTurns(ctx) err := b.sm.RunContext(ctx) b.waitForTurns(shutdownGrace) return err } func (b *Bot) events(ctx context.Context) { for { select { case <-ctx.Done(): return case evt, ok := <-b.sm.Events: if !ok { return } switch evt.Type { case socketmode.EventTypeEventsAPI: if evt.Request != nil { b.sm.Ack(*evt.Request) } if ev, ok := evt.Data.(slackevents.EventsAPIEvent); ok { go b.onEvent(ctx, ev) } case socketmode.EventTypeInteractive: if evt.Request != nil { b.sm.Ack(*evt.Request) } if cb, ok := evt.Data.(slack.InteractionCallback); ok { go b.onAction(ctx, cb) } } } } } func (b *Bot) onEvent(ctx context.Context, ev slackevents.EventsAPIEvent) { switch e := ev.InnerEvent.Data.(type) { case *slackevents.AppMentionEvent: if e.BotID != "" { return } b.onMessage(ctx, incoming{e.User, e.Channel, e.TimeStamp, e.ThreadTimeStamp, b.stripMention(e.Text), false, false, eventFiles(ev)}) case *slackevents.MessageEvent: if e.BotID != "" || (e.SubType != "" && e.SubType != "file_share") || e.User == "" || e.User == b.botID { return } text := b.stripMention(e.Text) switch { case e.ChannelType == "im": b.onMessage(ctx, incoming{e.User, e.Channel, e.TimeStamp, e.ThreadTimeStamp, text, true, false, eventFiles(ev)}) case b.listening(e.Channel, e.ThreadTimeStamp, e.Text): b.onMessage(ctx, incoming{e.User, e.Channel, e.TimeStamp, e.ThreadTimeStamp, text, false, true, eventFiles(ev)}) } case *slackevents.ReactionAddedEvent: b.onReaction(e) case *slackevents.AssistantThreadStartedEvent: b.assistantStarted(ctx, e.AssistantThread) case *slackevents.AssistantThreadContextChangedEvent: b.setViewing(e.AssistantThread) } } func (b *Bot) stripMention(text string) string { return strings.TrimSpace(strings.ReplaceAll(text, "<@"+b.botID+">", "")) } func (b *Bot) onMessage(ctx context.Context, m incoming) { // A message with files always goes to the agent, so a file with no text isn't read as `help`. commandable := len(m.files) == 0 if commandable && b.listenCommand(ctx, m) { return } // Every assistant-pane chat is a DM thread, so its first message counts as new. fresh := m.threadTS == "" || m.dm && b.store.Session(m.channel, m.threadTS) == nil if commandable && fresh && b.command(ctx, m) { return } if commandable && m.threadTS != "" && !m.heard && b.moveCommand(ctx, m, m.threadTS) { return } b.converse(ctx, m, nil) } // converse runs one turn with the agent. prior is the turn a restart cut off, if this picks it back up. func (b *Bot) converse(ctx context.Context, m incoming, prior *model.Turn) { b.turnsWG.Add(1) defer b.turnsWG.Done() root := m.threadTS if root == "" { root = m.ts } // Look the session up only after locking, so two first messages can't both create one. unlock := b.sessMu.Lock(m.channel + "/" + root) defer unlock() sess := b.store.Session(m.channel, root) if prior != nil { b.say(ctx, m.channel, root, ":arrows_counterclockwise: slackerbot restarted part-way through this reply, so I'm picking it back up.") } _, ph, err := b.api.PostMessageContext(ctx, m.channel, slack.MsgOptionText(":hourglass_flowing_sand: Getting the container ready…", false), slack.MsgOptionTS(root)) if err != nil { log.Printf("slack post: %v", err) return } turn := b.trackTurn(m, root, ph, prior) defer b.untrackTurn(ctx, turn) edit := func(text string) { _, _, _, _ = b.api.UpdateMessageContext(ctx, m.channel, ph, slack.MsgOptionText(text, false)) } b.react(ctx, m.channel, m.ts, "eyes") outcome := "warning" defer func() { b.unreact(ctx, m.channel, m.ts, "eyes") if outcome != "" { b.react(ctx, m.channel, m.ts, outcome) } }() ownerID, kind := m.owner() var off *node.NodeOfflineError var full *node.NodeFullError sl := &startLog{b: b, m: m, root: root, ownerID: ownerID} started := func(ev manager.StartEvent) { sl.event(ctx, ev) } u, err := b.mgr.EnsureProgress(ctx, ownerID, kind, started) if errors.As(err, &full) { sl.failed(ctx, fmt.Sprintf("`%s` is full, so the container is being moved to a node with room first.", full.Node)) edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room…", full.Node)) trigger := fmt.Sprintf("auto process (%s was full), for <@%s>'s message", full.Node, m.user) _, merr := b.runMove(ctx, m.channel, ownerID, manager.MoveOptions{StartAfter: true}, trigger, func(link string) { edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room. <%s|Follow the move here>.", full.Node, link)) }) if merr != nil { edit(b.explain(err) + "\nMoving it elsewhere didn't work either, see the move thread.") return } edit(":hourglass_flowing_sand: Moved. Getting the container ready…") u, err = b.mgr.EnsureProgress(ctx, ownerID, kind, started) } if errors.As(err, &off) { sl.failed(ctx, fmt.Sprintf("Node `%s` is offline.", off.Node)) b.deadSlave(ctx, m.channel, ph, ownerID, off.Node) return } if err != nil { sl.failed(ctx, b.explain(err)) edit(b.explain(err)) return } b.setBusy(u.Machine, +1) defer b.setBusy(u.Machine, -1) _ = b.store.Touch(u.ID) acp, err := b.client(ctx, u) if errors.As(err, &off) { sl.failed(ctx, fmt.Sprintf("Node `%s` went offline.", off.Node)) b.deadSlave(ctx, m.channel, ph, ownerID, off.Node) return } if err != nil { sl.failed(ctx, "The agent never became ready: "+err.Error()) edit(":warning: Couldn't reach the agent: " + err.Error()) return } sl.up(ctx) var id string // A new session (the old one was dropped) never saw the cut-off turn, so it gets the message again. carryOn := turn.Prompted && sess != nil text := m.text if !m.dm { text = b.displayName(ctx, m.user) + ": " + text } if carryOn { text = resumeNote } if sess == nil { if id, err = acp.NewSession(ctx, b.cfg.AgentCwd); err != nil { edit(":warning: Couldn't start a session: " + err.Error()) return } _ = b.store.AddSession(&model.Session{ Channel: m.channel, ThreadTS: root, OwnerID: u.ID, ACPID: id, Title: title(m.text), CreatedAt: time.Now(), }) log.Printf("thread: new thread %s/%s for %s %s (started by %s, session %s)", m.channel, root, u.Kind, u.ID, m.user, id) if w := b.where(ctx, m.channel); w != "" { text = "(You're in " + w + ". Each message starts with the sender's name.)\n" + text } if m.dm && m.threadTS != "" { b.setAssistantTitle(ctx, m.channel, root, title(m.text)) } } else { id = sess.ACPID if err := acp.LoadSession(ctx, id, b.cfg.AgentCwd); err != nil { edit(":warning: Couldn't reopen this session (" + err.Error() + "). Send me a new message to start a fresh one.") return } } if m.dm && !carryOn { text = b.viewingNote(ctx, m.channel, root) + text } if len(m.files) > 0 && !carryOn { edit(":paperclip: Copying the attachments into the container…") note, problems := b.attach(ctx, u, m) if note != "" { text += "\n" + note } if len(problems) > 0 { b.say(ctx, m.channel, root, ":warning: "+strings.Join(problems, "\n")) } } pctx, cancel := context.WithCancel(ctx) defer cancel() for _, ts := range []string{m.ts, ph} { key := m.channel + "/" + ts b.running.Store(key, cancel) defer b.running.Delete(key) } if m.dm && m.threadTS != "" { b.setAssistantStatus(ctx, m.channel, root, "is working…") defer b.setAssistantStatus(ctx, m.channel, root, "") } turn.Prompted = true b.saveTurn(turn) st := NewStreamer(b.api, m.channel, root, ph) var sends tools.Calls cmds := b.agentCommands(u, m, root) stop, err := acp.Prompt(pctx, id, text, func(up acpUpdate) { sends.Update(up) cmds.Update(ctx, up) st.Update(up) }) switch { case ctx.Err() != nil: stop, err = stopInterrupted, nil case pctx.Err() != nil: stop, err = "cancelled", nil } st.Finish(stop, err) b.upload(ctx, u, m.channel, root, sends.Files()) cmds.Finish(ctx) _ = b.store.Touch(u.ID) b.offerUpdate(ctx, u, m.channel, root) switch { case stop == "cancelled", stop == stopInterrupted: outcome = "" case err == nil: outcome = "white_check_mark" } } func splitCommand(text string) (verb, args string) { verb, args, _ = strings.Cut(strings.TrimSpace(text), " ") verb = strings.ToLower(strings.TrimRight(verb, ".!?")) args = strings.TrimRight(strings.TrimSpace(args), " .!?") return verb, args } func (b *Bot) command(ctx context.Context, m incoming) bool { verb, args := splitCommand(m.text) if verb == "move" || verb == "migrate" { return b.moveCommand(ctx, m, cmp.Or(m.threadTS, m.ts)) } reply, ok := b.runCommand(ctx, m, verb, args, false) if reply != "" { b.say(ctx, m.channel, m.threadTS, reply) } return ok } // runCommand returns ok=false when the text isn't a command, so it belongs to // the agent. An empty reply means it was posted already, with buttons. The // agent can't press buttons, so it gets plain text instead. func (b *Bot) runCommand(ctx context.Context, m incoming, verb, args string, agent bool) (reply string, ok bool) { ownerID, kind := m.owner() if verb == "grant" || verb == "revoke" || verb == "dm" || verb == "managers" || (verb == "permissions" && args != "") { return b.manageCommand(ctx, m, verb, args) } if commands.IsListen(verb) { c, err := commands.ParseListen(args) if err != nil { return "", false } return b.listenReply(m, c), true } if need := commands.NeededPermission(verb, args); need != "" { if denied := b.denial(ctx, m.channel, m.user, kind, need); denied != "" { return denied, true } } if verb == "stop" || verb == "scale" || verb == "storage" || verb == "disk" { if reply, ok := b.blocked(ctx, m.channel, ownerID, agent); ok { return reply, true } } switch { case verb == "" || verb == "help" && args == "": reply = helpText if warning := b.versionWarning(ctx); warning != "" { reply = warning + "\n\n" + reply } case verb == "sessions" && args == "": reply = b.sessionList(ctx, ownerID) case verb == "running" && args == "": reply = b.runningList(ctx, m.channel) case isCancelAll(verb, args): reply = b.cancelAll(m, agent) case verb == "status" && args == "": if agent { return b.statusText(ctx, ownerID, kind), true } b.postStatus(ctx, m.channel, m.threadTS, ownerID, kind) return "", true case verb == "stop" && args == "": reply = b.stopFor(ctx, ownerID) case verb == "scale": return b.scaleFor(ctx, ownerID, kind, args) case verb == "keep": return b.keepFor(ownerID, kind, args) case verb == "nodes" && args == "": reply = b.nodesList(ctx, ownerID) case verb == "permissions" && args == "": reply = b.permissionsList(ctx, m.channel, kind) case verb == "storage" || verb == "disk": return b.storageFor(ctx, ownerID, kind, args) case isInFlight(verb) && (args == "" || strings.EqualFold(args, "all")): reply = b.inFlightFor(m, args != "", agent) if !agent { b.whisper(ctx, m, reply) return "", true } default: return "", false } return reply, true } func (b *Bot) onAction(ctx context.Context, cb slack.InteractionCallback) { if cb.Type != slack.InteractionTypeBlockActions { return } thread := cb.Message.ThreadTimestamp for _, a := range cb.ActionCallback.BlockActions { if a.ActionID == "cancel" { if cancel, ok := b.running.Load(cb.Channel.ID + "/" + a.Value); ok { cancel.(context.CancelFunc)() } continue } ownerID := a.Value if ownerID != cb.User.ID && ownerID != cb.Channel.ID { continue } kind := model.OwnerChannel if ownerID == cb.User.ID { kind = model.OwnerUser } u := b.store.Owner(ownerID) if need := map[string]model.Action{"recreate": model.ActMove, "scale": model.ActScale}[a.ActionID]; need != "" && !b.permitted(ctx, cb.Channel.ID, thread, cb.User.ID, kind, need) { continue } switch a.ActionID { case "recreate": if u != nil { b.dropClient(u) } if err := b.mgr.Recreate(ownerID); err != nil { b.say(ctx, cb.Channel.ID, thread, ":warning: "+err.Error()) continue } b.say(ctx, cb.Channel.ID, thread, "OK. Your next message starts a fresh container on a healthy node. "+ "Old threads start new sessions when you reply in them.") case "wait": if u == nil || u.Node == "" { continue } b.waitMu.Lock() if b.waiting[u.Node] == nil { b.waiting[u.Node] = map[string]bool{} } b.waiting[u.Node][cb.Channel.ID] = true b.waitMu.Unlock() b.say(ctx, cb.Channel.ID, thread, fmt.Sprintf("OK, waiting for %s. I'll post here when it's back up.", u.Node)) case "restart": b.restartFor(ctx, cb, ownerID, kind) case "later": b.settlePrompt(ctx, cb, "OK, not now. The update applies the next time the container starts, "+ "for example after `stop` and a new message.") case "stop", "scale": if b.mgr.State(u) == manager.StateOffline { b.deadSlave(ctx, cb.Channel.ID, "", ownerID, u.Node) continue } if a.ActionID == "stop" { b.say(ctx, cb.Channel.ID, thread, b.stopFor(ctx, ownerID)) } else { reply, _ := b.scaleFor(ctx, ownerID, kind, "up") b.say(ctx, cb.Channel.ID, thread, reply) } } } } func (b *Bot) sessionList(ctx context.Context, ownerID string) string { list := b.store.Sessions(ownerID, 15) if len(list) == 0 { return "No sessions here yet. Send me a message to start one." } var sb strings.Builder sb.WriteString("*Sessions* (open one and reply in its thread to pick it back up):\n") for _, s := range list { link, err := b.api.GetPermalinkContext(ctx, &slack.PermalinkParameters{Channel: s.Channel, Ts: s.ThreadTS}) name := s.Title if err == nil { name = fmt.Sprintf("<%s|%s>", link, s.Title) } fmt.Fprintf(&sb, "• %s · %s\n", name, s.CreatedAt.Format("Jan 2, 15:04")) } return sb.String() } func (b *Bot) statusText(ctx context.Context, ownerID string, kind model.OwnerKind) string { u := b.store.Owner(ownerID) state := b.mgr.State(u) size := b.cfg.Container if u != nil { size = u.Size } var sb strings.Builder whose := "This channel's container" if kind == model.OwnerUser { whose = "Your container" } fmt.Fprintf(&sb, "*%s:* %s\n*model.Size:* %s\n", whose, state, size) if u != nil && u.Node != "" { fmt.Fprintf(&sb, "*Runs on:* %s\n", u.Node) } if u != nil && state == manager.StateRunning { fmt.Fprintf(&sb, "*Unused for:* %s\n", commands.Human(time.Since(u.LastActive))) } fmt.Fprintf(&sb, "*Stops when unused for:* %s\n", b.describeIdle(u)) fmt.Fprintf(&sb, "*Home folder:* %s\n", b.describeStorage(ctx, u)) views, _ := b.mgr.Views(ctx) if warning := versionWarning(b.mgr.LocalName(), views); warning != "" { sb.WriteString(warning + "\n") } sb.WriteString("*Nodes:*") masterVersion, _ := versionMismatches(b.mgr.LocalName(), views) for _, v := range views { sb.WriteString("\n• " + describeNode(v) + versionNote(b.mgr.LocalName(), masterVersion, v)) } for _, name := range b.mgr.OfflineNodes() { fmt.Fprintf(&sb, "\n• %s: offline", name) } return sb.String() } func (b *Bot) postStatus(ctx context.Context, channel, thread, ownerID string, kind model.OwnerKind) { u := b.store.Owner(ownerID) state := b.mgr.State(u) size := b.cfg.Container if u != nil { size = u.Size } text := b.statusText(ctx, ownerID, kind) blocks := []slack.Block{slack.NewSectionBlock(slack.NewTextBlockObject(slack.MarkdownType, text, false, false), nil, nil)} var buttons []slack.BlockElement if state == manager.StateRunning { stop := slack.NewButtonBlockElement("stop", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Stop container", false, false)) stop.Style = slack.StyleDanger buttons = append(buttons, stop) } if u != nil && size != b.cfg.MaxContainer { buttons = append(buttons, slack.NewButtonBlockElement("scale", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Scale up", false, false))) } if len(buttons) > 0 { blocks = append(blocks, slack.NewActionBlock("container", buttons...)) } opts := []slack.MsgOption{slack.MsgOptionText(text, false), slack.MsgOptionBlocks(blocks...)} if thread != "" { opts = append(opts, slack.MsgOptionTS(thread)) } if _, _, err := b.api.PostMessageContext(ctx, channel, opts...); err != nil { log.Printf("slack post: %v", err) } } func (b *Bot) stopFor(ctx context.Context, ownerID string) string { u := b.store.Owner(ownerID) if b.mgr.State(u) != manager.StateRunning { return "The container isn't running." } if err := b.mgr.Poweroff(ctx, u); err != nil { return ":warning: Couldn't stop it: " + err.Error() } b.dropClient(u) reply := "Shutting down the container. Its files are kept. Message me anytime to start it again." if link := b.announceStop(ctx, u, "Stopped on request."); link != "" { reply += fmt.Sprintf(" <%s|Follow the shutdown here>.", link) } return reply } func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) { old, next, err := b.mgr.Resize(ctx, ownerID, kind, func(cur model.Size) (model.Size, error) { return commands.ParseScale(args, cur) }) switch { case errors.Is(err, commands.ErrNotCommand): return "", false case errors.Is(err, manager.ErrTooBig): return fmt.Sprintf("That's bigger than allowed. The max is %s.", b.cfg.MaxContainer), true case errors.Is(err, manager.ErrTooSmall): return fmt.Sprintf("That's too small. The minimum is %s.", manager.MinContainer), true case errors.Is(err, balance.ErrMaxedOut) || errors.As(err, new(*node.NodeFullError)): return b.explain(err), true case err != nil: return fmt.Sprintf(":warning: %s. Try `scale 8G 4cpu`, `scale +2G`, `scale -1cpu` or `scale up`.", err), true case next == old: return fmt.Sprintf("No change. The container already has %s.", old), true } if b.mgr.State(b.store.Owner(ownerID)) != manager.StateRunning { return fmt.Sprintf("Done! The container will start with %s next time (was %s).", next, old), true } reply = fmt.Sprintf("Done! The container now has %s (was %s).", next, old) if next.MemoryMB < old.MemoryMB { reply += " If it was using more memory than the new limit, some programs in it may be stopped." } return reply, true } func (b *Bot) keepFor(ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) { k, err := commands.ParseKeep(args) switch { case errors.Is(err, commands.ErrNotCommand): return "", false case err != nil: return ":warning: " + err.Error(), true } u, err := b.mgr.Owner(ownerID, kind) if err != nil { return ":warning: " + err.Error(), true } var d time.Duration switch { case k.Show: return "This container stops when unused for " + b.describeIdle(u) + ".", true case k.Reset: d = 0 case k.Indefinite: if !b.cfg.AllowIndefinite { return "Keeping containers indefinitely is turned off on this server.", true } d = model.Indefinite default: d = k.Limit if d < 10*time.Minute { return "That's too short. The minimum is 10m.", true } if max := b.cfg.MaxIdleTimeout; max > 0 && d > max { return fmt.Sprintf("That's longer than allowed. The max is %s.", commands.Human(max)), true } } if err := b.store.SetIdleTimeout(ownerID, d); err != nil { return ":warning: " + err.Error(), true } _ = b.store.Touch(ownerID) u.IdleTimeout = d if d == model.Indefinite { return "Done! This container won't be stopped for being unused. Send `stop` when you're finished, " + "or `keep default` to go back to normal.", true } return "Done! This container stops when unused for " + b.describeIdle(u) + ".", true } func (b *Bot) storageFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) { if strings.TrimSpace(args) == "" { u, err := b.mgr.Owner(ownerID, kind) if err != nil { return ":warning: " + err.Error(), true } return "Home folder storage: " + b.describeStorage(ctx, u) + ".", true } c, err := b.mgr.SetStorage(ctx, ownerID, kind, func(min, max int64) (int64, int64, error) { return commands.ParseStorage(args, min, max) }) switch { case errors.Is(err, commands.ErrNotCommand): return "", false case errors.Is(err, storage.ErrStorageTooSmall): return fmt.Sprintf("That's too small. The minimum is %s.", units.FormatMB(config.MinStorageMB)), true case errors.Is(err, storage.ErrMinOverMax): return fmt.Sprintf("The guaranteed space (%s) can't be more than the max (%s).", units.FormatMB(c.NewMin), units.FormatMB(c.NewMax)), true case errors.Is(err, storage.ErrStorageTooBig): return fmt.Sprintf("The max can't go above %s.", units.FormatMB(b.cfg.Storage.LimitMB)), true case errors.Is(err, storage.ErrBelowUsage): return fmt.Sprintf("The home folder already holds %s, so the max can't go below that.", units.FormatMB(c.UsedMB)), true case errors.Is(err, storage.ErrStorageFull): return fmt.Sprintf(":no_entry: There isn't enough free space on the drive to guarantee %s.", units.FormatMB(c.NewMin)), true case err != nil: return fmt.Sprintf(":warning: %s. Try `storage 20G`, `storage +5G` or `storage min 8G max 20G`.", err), true } reply = fmt.Sprintf("Done! Home folder: %s guaranteed, up to %s (was %s guaranteed, up to %s).", units.FormatMB(c.NewMin), units.FormatMB(c.NewMax), units.FormatMB(c.OldMin), units.FormatMB(c.OldMax)) if c.Drive != nil && c.Drive.Overbooked() { reply += fmt.Sprintf("\n:warning: Heads up: drive %s is overbooked. Its containers may use up to %s in total, "+ "but only %s is available. Guaranteed space is safe, but if they all fill up, writes will start failing.", c.Drive.Path, units.FormatMB(c.Drive.BookedMB), units.FormatMB(c.Drive.UsableMB)) } return reply, true } func (b *Bot) describeStorage(ctx context.Context, u *model.Owner) string { if u == nil { return fmt.Sprintf("%s guaranteed, up to %s", units.FormatMB(b.cfg.Storage.MinMB), units.FormatMB(b.cfg.Storage.MaxMB)) } text := fmt.Sprintf("%s guaranteed, up to %s", units.FormatMB(u.StorageMinMB), units.FormatMB(u.StorageMaxMB)) if used, ok := b.mgr.StorageUsed(ctx, u); ok { text = units.FormatMB(used) + " used, " + text } if u.Node != "" { text += fmt.Sprintf(" (on %s:%s)", u.Node, u.Drive) } return text } func describeNode(v balance.NodeView) string { us, cm, _ := v.Storage() free := v.Usable.Sub(v.Used) text := fmt.Sprintf("*%s*: %s of %s CPUs free · %s of %s RAM free · %s of %s storage free · %d running, %d total", v.Name, strconv.FormatFloat(math.Max(0, free.CPUs), 'f', -1, 64), strconv.FormatFloat(v.Usable.CPUs, 'f', -1, 64), units.FormatMB(int64(max(0, free.MemoryMB))), units.FormatMB(int64(v.Usable.MemoryMB)), units.FormatMB(max(0, us-cm)), units.FormatMB(us), v.Running, v.Homes()) for _, d := range v.Drives { if d.Overbooked() { text += fmt.Sprintf(" · :warning: %s overbooked (caps add up to %s of %s)", d.Path, units.FormatMB(d.BookedMB), units.FormatMB(d.UsableMB)) } } return text } func (b *Bot) moveTarget(args string) (target string, ok bool) { words := strings.Fields(strings.ToLower(args)) switch { case len(words) == 0, len(words) == 1 && words[0] == "auto": return "", true case len(words) == 1: return words[0], true case len(words) == 2 && words[0] == "to": return words[1], true } return "", false } func (b *Bot) permitted(ctx context.Context, channel, thread, user string, kind model.OwnerKind, a model.Action) bool { text := b.denial(ctx, channel, user, kind, a) if text == "" { return true } b.say(ctx, channel, thread, text) return false } // denial explains why the user may not do a, or is empty when they may. func (b *Bot) denial(ctx context.Context, channel, user string, kind model.OwnerKind, a model.Action) string { p := b.store.Permissions() if p.Allowed(kind, channel, user, a) { return "" } if kind == model.OwnerUser { return fmt.Sprintf(":lock: Changing *%s* from DMs is turned off on this server.", a) } holders := p.Holders(channel, a) if len(holders) == 0 { return fmt.Sprintf(":lock: Nobody here has *%s* permission. An admin can grant it in the config.", a) } names := make([]string, len(holders)) for i, h := range holders { names[i] = b.displayName(ctx, h) } return fmt.Sprintf(":lock: Only super users with *%s* permission can do that here: %s.", a, strings.Join(names, ", ")) } func (b *Bot) permissionsList(ctx context.Context, channel string, kind model.OwnerKind) string { p := b.store.Permissions() if kind == model.OwnerUser { var lines []string for _, a := range model.AllActions { state := "allowed" if !p.Allowed(model.OwnerUser, channel, "", a) { state = "turned off" } lines = append(lines, fmt.Sprintf("• *%s*: %s", a, state)) } return "*In DMs, you can change your own container:*\n" + strings.Join(lines, "\n") + "\nUser managers can switch these with `dm on|off`." } list, own := p.SuperUsers(channel) source := "the server-wide default list (this channel has no list of its own)" if own { source = "this channel's own list" } users := make([]string, 0, len(list)) for user := range list { users = append(users, user) } sort.Strings(users) lines := []string{fmt.Sprintf("*Super users here*, from %s:", source)} for _, user := range users { actions := p.ActionsFor(channel, user) names := make([]string, len(actions)) for i, a := range actions { names[i] = string(a) } lines = append(lines, fmt.Sprintf("• %s: %s", b.displayName(ctx, user), strings.Join(names, ", "))) } managerNames := make([]string, len(p.Managers)) for i, u := range p.Managers { managerNames[i] = b.displayName(ctx, u) } lines = append(lines, "*User managers* (can change any channel's list): "+strings.Join(managerNames, ", "), "Everyone can talk to the agent and use `status`, `nodes`, `sessions` and `stop`.") return strings.Join(lines, "\n") } // blocked explains why the container can't be changed right now. In Slack a // dead node gets its recreate/wait buttons instead, and the reply is empty. func (b *Bot) blocked(ctx context.Context, channel, ownerID string, agent bool) (reply string, ok bool) { if b.mgr.IsMigrating(ownerID) { return b.explain(manager.ErrMigrating), true } if u := b.store.Owner(ownerID); b.mgr.State(u) == manager.StateOffline { if agent { return fmt.Sprintf("Node %s is offline right now.", u.Node), true } b.deadSlave(ctx, channel, "", ownerID, u.Node) return "", true } return "", false } func (b *Bot) moveCommand(ctx context.Context, m incoming, replyThread string) bool { verb, args := splitCommand(m.text) if verb != "move" && verb != "migrate" { return false } reply, ok := b.move(ctx, m, args, replyThread, false) if reply != "" { b.say(ctx, m.channel, replyThread, reply) } return ok } func (b *Bot) move(ctx context.Context, m incoming, args, replyThread string, agent bool) (reply string, ok bool) { target, ok := b.moveTarget(args) if !ok { return "", false } ownerID, kind := m.owner() if denied := b.denial(ctx, m.channel, m.user, kind, model.ActMove); denied != "" { return denied, true } if reply, ok := b.blocked(ctx, m.channel, ownerID, agent); ok { return reply, true } trigger := "<@" + m.user + ">" if !agent { go b.moveFor(ctx, m.channel, ownerID, target, trigger, replyThread) return "", true } go b.moveFor(ctx, m.channel, ownerID, target, "the agent, for "+trigger, replyThread) return "The move has started, and its progress is posted in Slack. The container is stopped briefly near the end, " + "so finish your reply now. The user can carry on in this thread once it's done.", true } func (b *Bot) moveFor(ctx context.Context, channel, ownerID, target, trigger, replyThread string) { b.runMove(ctx, channel, ownerID, manager.MoveOptions{Target: target}, trigger, func(link string) { b.say(ctx, channel, replyThread, fmt.Sprintf(":truck: Moving this container. <%s|Follow the move here>.", link)) }) } // Every move gets its own top-level message saying who started it, with // progress in that message's thread. func (b *Bot) runMove(ctx context.Context, channel, ownerID string, opts manager.MoveOptions, trigger string, announce func(link string)) (*manager.MoveResult, error) { before := b.store.Owner(ownerID) from := "its node" if before != nil && before.Node != "" { from = before.Node } where := opts.Target if where == "" { where = "best node" } head := fmt.Sprintf(":truck: Container move started by %s: %s → %s", trigger, from, where) _, root, err := b.api.PostMessageContext(ctx, channel, slack.MsgOptionText(head, false)) if err != nil { log.Printf("slack post: %v", err) } if announce != nil && root != "" { link, _ := b.api.GetPermalinkContext(ctx, &slack.PermalinkParameters{Channel: channel, Ts: root}) announce(link) } _, progressTS, err := b.api.PostMessageContext(ctx, channel, slack.MsgOptionText(":hourglass_flowing_sand: Checking where it can go…", false), slack.MsgOptionTS(root)) if err != nil { log.Printf("slack post: %v", err) } opts.WaitIdle = func(ctx context.Context) { if before != nil { b.waitIdle(ctx, before.Machine) } } opts.Progress = b.progressEditor(ctx, channel, progressTS) res, err := b.mgr.Migrate(ctx, ownerID, opts) if before != nil { b.dropClient(before) } text, outcome := b.moveOutcome(ownerID, opts.Target, res, err) _, _, _, _ = b.api.UpdateMessageContext(ctx, channel, progressTS, slack.MsgOptionText(text, false)) _, _, _, _ = b.api.UpdateMessageContext(ctx, channel, root, slack.MsgOptionText(head+" · "+outcome, false)) return res, err } func (b *Bot) moveOutcome(ownerID, target string, res *manager.MoveResult, err error) (text, outcome string) { failed := ":x: didn't move" var unknown *manager.UnknownNodeError var full *node.NodeFullError var off *node.NodeOfflineError switch { case errors.Is(err, manager.ErrNoContainer): return "There's no container here yet. Send me a message to create one.", failed case errors.Is(err, manager.ErrSameNode): return "The container is already on " + target + ".", failed case errors.As(err, &unknown): return fmt.Sprintf("There's no node called %s. Send `nodes` to see them.", unknown.Node), failed case errors.As(err, &off) && res == nil: return fmt.Sprintf("%s is offline right now, so nothing can move there.", off.Node), failed case errors.As(err, &full) && res == nil: return fmt.Sprintf(":no_entry: %s doesn't have room for this container. Send `nodes` to see what's free.", full.Node), failed case errors.Is(err, balance.ErrMaxedOut) || errors.Is(err, storage.ErrStorageFull): return ":no_entry: No other node has room for this container right now. Send `nodes` to see what's free.", failed case errors.Is(err, manager.ErrMigrating): return b.explain(err), failed } if err != nil { if after := b.store.Owner(ownerID); res != nil && after != nil && after.Node == res.To { return fmt.Sprintf(":warning: Moved to %s, but: %s", res.To, err), ":warning: moved to " + res.To + " with problems" } return ":warning: The move didn't work, so the container stays on its old node: " + err.Error(), failed } text = fmt.Sprintf(":white_check_mark: Moved from %s to %s (%s copied).", res.From, res.To, units.FormatMB(res.CopiedMB)) if res.Started { text += " It's running there now, and its sessions came along." } if res.LeftOld != nil { text += fmt.Sprintf("\n:warning: The old copy on %s couldn't be deleted: %s", res.From, res.LeftOld) } return text, ":white_check_mark: done, now on " + res.To } func (b *Bot) progressEditor(ctx context.Context, channel, ts string) func(manager.MoveProgress) { var last time.Time var lastPhase manager.MovePhase return func(p manager.MoveProgress) { if p.Phase == lastPhase && time.Since(last) < 3*time.Second { return } last, lastPhase = time.Now(), p.Phase copied := units.FormatMB(p.CopiedMB) if p.TotalMB > 0 { copied += " of ~" + units.FormatMB(p.TotalMB) } var text string switch p.Phase { case manager.PhaseCopy: text = fmt.Sprintf(":truck: Moving to %s… copying the home folder (%s). The container keeps running for now.", p.To, copied) case manager.PhaseFinal: text = fmt.Sprintf(":truck: Moving to %s… final sync of recent changes. The container is briefly stopped.", p.To) case manager.PhaseCleanup: text = fmt.Sprintf(":truck: Moving to %s… copied %s, switching over.", p.To, copied) case manager.PhaseStart: text = fmt.Sprintf(":truck: Moving to %s… starting it there.", p.To) } _, _, _, _ = b.api.UpdateMessageContext(ctx, channel, ts, slack.MsgOptionText(text, false)) } } func (b *Bot) waitIdle(ctx context.Context, machine string) { deadline := time.Now().Add(2 * time.Minute) for b.isBusy(machine) && time.Now().Before(deadline) { select { case <-ctx.Done(): return case <-time.After(time.Second): } } } func (b *Bot) nodesList(ctx context.Context, ownerID string) string { views, err := b.mgr.Views(ctx) if err != nil { return ":warning: " + err.Error() } here := "" if u := b.store.Owner(ownerID); u != nil { here = u.Node } var sb strings.Builder master := b.mgr.LocalName() if warning := versionWarning(master, views); warning != "" { sb.WriteString(warning + "\n\n") } sb.WriteString("*Nodes* (what's free on each):") masterVersion, _ := versionMismatches(master, views) for _, v := range views { sb.WriteString("\n• " + describeNode(v) + versionNote(master, masterVersion, v)) if v.Name == here { sb.WriteString(" ← this container") } } for _, name := range b.mgr.OfflineNodes() { fmt.Fprintf(&sb, "\n• *%s*: offline", name) if name == here { sb.WriteString(" ← this container") } } sb.WriteString("\nSend `move ` to move this container, or `move auto` to let me pick.") return sb.String() } func (b *Bot) deadSlave(ctx context.Context, channel, replaceTS, ownerID, node string) { text := fmt.Sprintf("Channel bot is on a dead slave %s, create new container or wait for the slave to be back up?", node) blocks := []slack.Block{ slack.NewSectionBlock(slack.NewTextBlockObject(slack.MarkdownType, text, false, false), nil, nil), slack.NewActionBlock("dead_slave", slack.NewButtonBlockElement("recreate", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Create new container", false, false)), slack.NewButtonBlockElement("wait", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Wait", false, false)), ), } opts := []slack.MsgOption{slack.MsgOptionText(text, false), slack.MsgOptionBlocks(blocks...)} var err error if replaceTS != "" { _, _, _, err = b.api.UpdateMessageContext(ctx, channel, replaceTS, opts...) } else { _, _, err = b.api.PostMessageContext(ctx, channel, opts...) } if err != nil { log.Printf("slack post: %v", err) } } func (b *Bot) nodeBack(node string) { b.waitMu.Lock() channels := b.waiting[node] delete(b.waiting, node) b.waitMu.Unlock() for channel := range channels { b.say(context.Background(), channel, "", fmt.Sprintf("%s is back up. Message me to pick up where you left off.", node)) } } func (b *Bot) idleLimit(u *model.Owner) (limit time.Duration, indefinite bool) { switch { case u == nil || u.IdleTimeout == 0: return b.cfg.IdleTimeout, false case u.IdleTimeout == model.Indefinite: return 0, true } return u.IdleTimeout, false } func (b *Bot) describeIdle(u *model.Owner) string { limit, indefinite := b.idleLimit(u) switch { case indefinite: return "never (kept indefinitely)" case u == nil || u.IdleTimeout == 0: return commands.Human(limit) + " (default)" } return commands.Human(limit) } func (b *Bot) explain(err error) string { var full *node.NodeFullError switch { case errors.Is(err, manager.ErrMigrating): return ":truck: This container is being moved right now. Try again in a moment." case errors.As(err, &full): return fmt.Sprintf(":no_entry: Sorry, %s (where this container's files live) is maxed out right now. "+ "Try again a bit later.", full.Node) case errors.Is(err, balance.ErrMaxedOut): used, usable := b.mgr.Totals(context.Background()) return fmt.Sprintf(":no_entry: Sorry, every node is maxed out right now (%s of %s in use). "+ "Try again a bit later.", used, usable) case errors.Is(err, storage.ErrStorageFull): return ":no_entry: Sorry, there isn't enough storage left on any node to set up a new container. Try again later." } return ":warning: " + err.Error() } func clientKey(u *model.Owner) string { return u.Node + "/" + u.Machine } func (b *Bot) client(ctx context.Context, u *model.Owner) (*acp.ACP, error) { key := clientKey(u) unlock := b.dialMu.Lock(key) defer unlock() b.acpMu.Lock() c := b.acps[key] b.acpMu.Unlock() if c != nil { select { case <-c.Done(): default: return c, nil } } conn, err := b.mgr.DialAgent(ctx, u) if err != nil { return nil, err } initCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() if c, err = acp.New(initCtx, conn); err != nil { return nil, err } c.Servers = b.mcpServers() b.acpMu.Lock() b.acps[key] = c b.acpMu.Unlock() return c, nil } func (b *Bot) dropClient(u *model.Owner) { b.acpMu.Lock() defer b.acpMu.Unlock() if c := b.acps[clientKey(u)]; c != nil { c.Close() delete(b.acps, clientKey(u)) } } func (b *Bot) setBusy(machine string, delta int) { b.busyMu.Lock() defer b.busyMu.Unlock() b.busy[machine] += delta } func (b *Bot) isBusy(machine string) bool { b.busyMu.Lock() defer b.busyMu.Unlock() return b.busy[machine] > 0 } func (b *Bot) reaper(ctx context.Context) { t := time.NewTicker(10 * time.Minute) defer t.Stop() for { b.reapOnce(ctx) select { case <-ctx.Done(): return case <-t.C: } } } func (b *Bot) reapOnce(ctx context.Context) { for _, owner := range b.mgr.RunningOwners(ctx) { u := owner if b.isBusy(u.Machine) { continue } limit, indefinite := b.idleLimit(&u) if indefinite || time.Since(u.LastActive) < limit { continue } log.Printf("cleanup: stopping %s on %s (idle since %s)", u.Machine, u.Node, u.LastActive.Format(time.RFC3339)) if err := b.mgr.Poweroff(ctx, &u); err != nil { log.Printf("cleanup: stopping %s: %v", u.Machine, err) continue } b.dropClient(&u) msg := fmt.Sprintf("I stopped the container after %s without use. Its files are kept. "+ "Message me anytime to start it again, or use `keep` to change how long it waits.", commands.Human(limit)) if link := b.announceStop(ctx, &u, fmt.Sprintf("Unused for %s.", commands.Human(limit))); link != "" { msg += fmt.Sprintf(" <%s|Shutdown details>.", link) } b.say(ctx, u.ID, "", msg) // a user ID posts to their DM, a channel ID to the channel } } func (b *Bot) say(ctx context.Context, channel, thread, text string) { opts := []slack.MsgOption{slack.MsgOptionText(text, false)} if thread != "" { opts = append(opts, slack.MsgOptionTS(thread)) } if _, _, err := b.api.PostMessageContext(ctx, channel, opts...); err != nil { log.Printf("slack post: %v", err) } } func (b *Bot) displayName(ctx context.Context, userID string) string { if v, ok := b.names.Load(userID); ok { return v.(string) } name := userID if u, err := b.api.GetUserInfoContext(ctx, userID); err == nil { for _, n := range []string{u.Profile.DisplayName, u.RealName, u.Name} { if n != "" { name = n break } } b.names.Store(userID, name) } return name } func title(text string) string { line, _, _ := strings.Cut(strings.TrimSpace(text), "\n") if utf8.RuneCountInString(line) > 60 { r := []rune(line) line = string(r[:60]) + "…" } if line == "" { line = "untitled" } return line }