package bot import ( "cmp" "context" "errors" "fmt" "log" "math" "sort" "strconv" "strings" "sync" "time" "unicode/utf8" "github.com/slack-go/slack" "github.com/slack-go/slack/slackevents" "github.com/slack-go/slack/socketmode" "slackerbot/internal/acp" "slackerbot/internal/balance" "slackerbot/internal/commands" "slackerbot/internal/config" "slackerbot/internal/manager" "slackerbot/internal/model" "slackerbot/internal/node" "slackerbot/internal/state" "slackerbot/internal/storage" "slackerbot/internal/units" "slackerbot/internal/util" ) const helpText = "I pass your messages to an agent running in its own container.\n" + "• In a DM, the container is yours alone. In a channel, it belongs to the channel " + "and anyone here can use it by @mentioning me.\n" + "• Every new message starts a new session. Reply in its thread to keep going.\n" + "• In a channel thread I only read replies that @mention me. `/listen all-messages` in the thread makes me read them all, " + "`/listen channel all-messages` sets the channel's default.\n" + "• React with :x: to stop a reply part-way.\n" + "*Commands* (send as a new message, not in a thread):\n" + "• `status`: state, size and idle limit, with buttons\n" + "• `sessions`: past sessions, with links\n" + "• `stop`: shut the container down (files are kept)\n" + "• `scale up` / `scale down`: double or halve RAM and CPU\n" + "• `scale 8G 4cpu`: set an exact size. `scale +2G` or `scale -1cpu`: change by that much\n" + "• `keep 24h` (or `3d`, `1d12h`, `2w`): stop after that long unused\n" + "• `keep indefinite`: never stop for being unused. `keep default`: back to normal. `keep`: show it\n" + "• `storage 20G` / `storage +5G`: set or change the home folder's max size\n" + "• `storage min 8G`: set the guaranteed space. `storage`: show usage\n" + "• `nodes`: every node and what's free on it\n" + "• `move `: move the container to that node. `move auto`: let me pick the best one. Works in threads too\n" + "• `permissions`: who can scale, change storage, move or set timeouts here\n" + "• Managers: `grant @alice move scale`, `revoke @alice`, `permissions reset`, " + "`grant default @alice all`, `dm storage on|off`, `managers add|remove @alice`\n" + "• `help`: this message" type Bot struct { cfg *config.Config store *state.Store mgr *manager.Manager api *slack.Client sm *socketmode.Client botID string acpMu sync.Mutex acps map[string]*acp.ACP dialMu util.KeyedMutex sessMu util.KeyedMutex busyMu sync.Mutex busy map[string]int names sync.Map channels sync.Map // channel ID → *slack.Channel running sync.Map // "channel/ts" of a prompt's message or reply → its cancel func viewing sync.Map // assistant "channel/thread" → channel the user has open told sync.Map // assistant "channel/thread" → last channel the agent was told about waitMu sync.Mutex waiting map[string]map[string]bool } type incoming struct { user, channel, ts, threadTS, text string dm bool heard bool } func (m incoming) owner() (string, model.OwnerKind) { if m.dm { return m.user, model.OwnerUser } return m.channel, model.OwnerChannel } func New(cfg *config.Config, store *state.Store, mgr *manager.Manager) (*Bot, error) { api := slack.New(cfg.Slack.BotToken, slack.OptionAppLevelToken(cfg.Slack.AppToken)) auth, err := api.AuthTest() if err != nil { return nil, fmt.Errorf("slack auth: %w", err) } b := &Bot{ cfg: cfg, store: store, mgr: mgr, api: api, sm: socketmode.New(api), botID: auth.UserID, acps: map[string]*acp.ACP{}, busy: map[string]int{}, waiting: map[string]map[string]bool{}, } mgr.SetNodeBackHandler(b.nodeBack) return b, nil } func (b *Bot) Run(ctx context.Context) error { go b.events(ctx) go b.reaper(ctx) return b.sm.RunContext(ctx) } func (b *Bot) events(ctx context.Context) { for { select { case <-ctx.Done(): return case evt, ok := <-b.sm.Events: if !ok { return } switch evt.Type { case socketmode.EventTypeEventsAPI: if evt.Request != nil { b.sm.Ack(*evt.Request) } if ev, ok := evt.Data.(slackevents.EventsAPIEvent); ok { go b.onEvent(ctx, ev) } case socketmode.EventTypeInteractive: if evt.Request != nil { b.sm.Ack(*evt.Request) } if cb, ok := evt.Data.(slack.InteractionCallback); ok { go b.onAction(ctx, cb) } } } } } func (b *Bot) onEvent(ctx context.Context, ev slackevents.EventsAPIEvent) { switch e := ev.InnerEvent.Data.(type) { case *slackevents.AppMentionEvent: if e.BotID != "" { return } b.onMessage(ctx, incoming{e.User, e.Channel, e.TimeStamp, e.ThreadTimeStamp, b.stripMention(e.Text), false, false}) case *slackevents.MessageEvent: if e.BotID != "" || e.SubType != "" || e.User == "" || e.User == b.botID { return } text := b.stripMention(e.Text) switch { case e.ChannelType == "im": b.onMessage(ctx, incoming{e.User, e.Channel, e.TimeStamp, e.ThreadTimeStamp, text, true, false}) case b.listening(e.Channel, e.ThreadTimeStamp, e.Text): b.onMessage(ctx, incoming{e.User, e.Channel, e.TimeStamp, e.ThreadTimeStamp, text, false, true}) } case *slackevents.ReactionAddedEvent: b.onReaction(e) case *slackevents.AssistantThreadStartedEvent: b.assistantStarted(ctx, e.AssistantThread) case *slackevents.AssistantThreadContextChangedEvent: b.setViewing(e.AssistantThread) } } func (b *Bot) stripMention(text string) string { return strings.TrimSpace(strings.ReplaceAll(text, "<@"+b.botID+">", "")) } func (b *Bot) onMessage(ctx context.Context, m incoming) { if b.listenCommand(ctx, m) { return } // Every assistant-pane chat is a DM thread, so its first message counts as new. fresh := m.threadTS == "" || m.dm && b.store.Session(m.channel, m.threadTS) == nil if fresh && b.command(ctx, m) { return } if m.threadTS != "" && !m.heard && b.moveCommand(ctx, m, m.threadTS) { return } root := m.threadTS if root == "" { root = m.ts } // Look the session up only after locking, so two first messages can't both create one. unlock := b.sessMu.Lock(m.channel + "/" + root) defer unlock() sess := b.store.Session(m.channel, root) _, ph, err := b.api.PostMessageContext(ctx, m.channel, slack.MsgOptionText(":hourglass_flowing_sand: Getting the container ready…", false), slack.MsgOptionTS(root)) if err != nil { log.Printf("slack post: %v", err) return } edit := func(text string) { _, _, _, _ = b.api.UpdateMessageContext(ctx, m.channel, ph, slack.MsgOptionText(text, false)) } b.react(ctx, m.channel, m.ts, "eyes") outcome := "warning" defer func() { b.unreact(ctx, m.channel, m.ts, "eyes") if outcome != "" { b.react(ctx, m.channel, m.ts, outcome) } }() ownerID, kind := m.owner() var off *node.NodeOfflineError var full *node.NodeFullError sl := &startLog{b: b, m: m, root: root, ownerID: ownerID} started := func(ev manager.StartEvent) { sl.event(ctx, ev) } u, err := b.mgr.EnsureProgress(ctx, ownerID, kind, started) if errors.As(err, &full) { sl.failed(ctx, fmt.Sprintf("`%s` is full, so the container is being moved to a node with room first.", full.Node)) edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room…", full.Node)) trigger := fmt.Sprintf("auto process (%s was full), for <@%s>'s message", full.Node, m.user) _, merr := b.runMove(ctx, m.channel, ownerID, manager.MoveOptions{StartAfter: true}, trigger, func(link string) { edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room. <%s|Follow the move here>.", full.Node, link)) }) if merr != nil { edit(b.explain(err) + "\nMoving it elsewhere didn't work either, see the move thread.") return } edit(":hourglass_flowing_sand: Moved. Getting the container ready…") u, err = b.mgr.EnsureProgress(ctx, ownerID, kind, started) } if errors.As(err, &off) { sl.failed(ctx, fmt.Sprintf("Node `%s` is offline.", off.Node)) b.deadSlave(ctx, m.channel, ph, ownerID, off.Node) return } if err != nil { sl.failed(ctx, b.explain(err)) edit(b.explain(err)) return } b.setBusy(u.Machine, +1) defer b.setBusy(u.Machine, -1) _ = b.store.Touch(u.ID) acp, err := b.client(ctx, u) if errors.As(err, &off) { sl.failed(ctx, fmt.Sprintf("Node `%s` went offline.", off.Node)) b.deadSlave(ctx, m.channel, ph, ownerID, off.Node) return } if err != nil { sl.failed(ctx, "The agent never became ready: "+err.Error()) edit(":warning: Couldn't reach the agent: " + err.Error()) return } sl.up(ctx) var id string text := m.text if !m.dm { text = b.displayName(ctx, m.user) + ": " + text } if sess == nil { if id, err = acp.NewSession(ctx, b.cfg.AgentCwd); err != nil { edit(":warning: Couldn't start a session: " + err.Error()) return } _ = b.store.AddSession(&model.Session{ Channel: m.channel, ThreadTS: root, OwnerID: u.ID, ACPID: id, Title: title(m.text), CreatedAt: time.Now(), }) log.Printf("thread: new thread %s/%s for %s %s (started by %s, session %s)", m.channel, root, u.Kind, u.ID, m.user, id) if w := b.where(ctx, m.channel); w != "" { text = "(You're in " + w + ". Each message starts with the sender's name.)\n" + text } if m.dm && m.threadTS != "" { b.setAssistantTitle(ctx, m.channel, root, title(m.text)) } } else { id = sess.ACPID if err := acp.LoadSession(ctx, id, b.cfg.AgentCwd); err != nil { edit(":warning: Couldn't reopen this session (" + err.Error() + "). Send me a new message to start a fresh one.") return } } if m.dm { text = b.viewingNote(ctx, m.channel, root) + text } edit("_working…_") pctx, cancel := context.WithCancel(ctx) defer cancel() for _, ts := range []string{m.ts, ph} { key := m.channel + "/" + ts b.running.Store(key, cancel) defer b.running.Delete(key) } if m.dm && m.threadTS != "" { b.setAssistantStatus(ctx, m.channel, root, "is working…") defer b.setAssistantStatus(ctx, m.channel, root, "") } st := NewStreamer(b.api, m.channel, root, ph) stop, err := acp.Prompt(pctx, id, text, st.Update) if pctx.Err() != nil && ctx.Err() == nil { stop, err = "cancelled", nil } st.Finish(stop, err) _ = b.store.Touch(u.ID) switch { case stop == "cancelled": outcome = "" case err == nil: outcome = "white_check_mark" } } func (b *Bot) command(ctx context.Context, m incoming) bool { ownerID, kind := m.owner() verb, args, _ := strings.Cut(strings.TrimSpace(m.text), " ") verb = strings.ToLower(strings.TrimRight(verb, ".!?")) args = strings.TrimRight(strings.TrimSpace(args), " .!?") if verb == "move" || verb == "migrate" { return b.moveCommand(ctx, m, cmp.Or(m.threadTS, m.ts)) } if verb == "grant" || verb == "revoke" || verb == "dm" || verb == "managers" || (verb == "permissions" && args != "") { return b.manageCommand(ctx, m, verb, args) } if need := commands.NeededPermission(verb, args); need != "" && !b.permitted(ctx, m.channel, m.threadTS, m.user, kind, need) { return true } if verb == "stop" || verb == "scale" || verb == "storage" || verb == "disk" { if b.blocked(ctx, m.channel, m.threadTS, ownerID) { return true } } reply := "" switch { case verb == "" || verb == "help" && args == "": reply = helpText case verb == "sessions" && args == "": reply = b.sessionList(ctx, ownerID) case verb == "status" && args == "": b.postStatus(ctx, m.channel, m.threadTS, ownerID, kind) return true case verb == "stop" && args == "": reply = b.stopFor(ctx, ownerID) case verb == "scale": var ok bool if reply, ok = b.scaleFor(ctx, ownerID, kind, args); !ok { return false } case verb == "keep": var ok bool if reply, ok = b.keepFor(ownerID, kind, args); !ok { return false } case verb == "nodes" && args == "": reply = b.nodesList(ctx, ownerID) case verb == "permissions" && args == "": reply = b.permissionsList(ctx, m.channel, kind) case verb == "storage" || verb == "disk": var ok bool if reply, ok = b.storageFor(ctx, ownerID, kind, args); !ok { return false } default: return false } b.say(ctx, m.channel, m.threadTS, reply) return true } func (b *Bot) onAction(ctx context.Context, cb slack.InteractionCallback) { if cb.Type != slack.InteractionTypeBlockActions { return } thread := cb.Message.ThreadTimestamp for _, a := range cb.ActionCallback.BlockActions { ownerID := a.Value if ownerID != cb.User.ID && ownerID != cb.Channel.ID { continue } kind := model.OwnerChannel if ownerID == cb.User.ID { kind = model.OwnerUser } u := b.store.Owner(ownerID) if need := map[string]model.Action{"recreate": model.ActMove, "scale": model.ActScale}[a.ActionID]; need != "" && !b.permitted(ctx, cb.Channel.ID, thread, cb.User.ID, kind, need) { continue } switch a.ActionID { case "recreate": if u != nil { b.dropClient(u) } if err := b.mgr.Recreate(ownerID); err != nil { b.say(ctx, cb.Channel.ID, thread, ":warning: "+err.Error()) continue } b.say(ctx, cb.Channel.ID, thread, "OK. Your next message starts a fresh container on a healthy node. "+ "Old threads start new sessions when you reply in them.") case "wait": if u == nil || u.Node == "" { continue } b.waitMu.Lock() if b.waiting[u.Node] == nil { b.waiting[u.Node] = map[string]bool{} } b.waiting[u.Node][cb.Channel.ID] = true b.waitMu.Unlock() b.say(ctx, cb.Channel.ID, thread, fmt.Sprintf("OK, waiting for %s. I'll post here when it's back up.", u.Node)) case "stop", "scale": if b.mgr.State(u) == manager.StateOffline { b.deadSlave(ctx, cb.Channel.ID, "", ownerID, u.Node) continue } if a.ActionID == "stop" { b.say(ctx, cb.Channel.ID, thread, b.stopFor(ctx, ownerID)) } else { reply, _ := b.scaleFor(ctx, ownerID, kind, "up") b.say(ctx, cb.Channel.ID, thread, reply) } } } } func (b *Bot) sessionList(ctx context.Context, ownerID string) string { list := b.store.Sessions(ownerID, 15) if len(list) == 0 { return "No sessions here yet. Send me a message to start one." } var sb strings.Builder sb.WriteString("*Sessions* (open one and reply in its thread to pick it back up):\n") for _, s := range list { link, err := b.api.GetPermalinkContext(ctx, &slack.PermalinkParameters{Channel: s.Channel, Ts: s.ThreadTS}) name := s.Title if err == nil { name = fmt.Sprintf("<%s|%s>", link, s.Title) } fmt.Fprintf(&sb, "• %s · %s\n", name, s.CreatedAt.Format("Jan 2, 15:04")) } return sb.String() } func (b *Bot) postStatus(ctx context.Context, channel, thread, ownerID string, kind model.OwnerKind) { u := b.store.Owner(ownerID) state := b.mgr.State(u) size := b.cfg.Container if u != nil { size = u.Size } var sb strings.Builder whose := "This channel's container" if kind == model.OwnerUser { whose = "Your container" } fmt.Fprintf(&sb, "*%s:* %s\n*model.Size:* %s\n", whose, state, size) if u != nil && u.Node != "" { fmt.Fprintf(&sb, "*Runs on:* %s\n", u.Node) } if u != nil && state == manager.StateRunning { fmt.Fprintf(&sb, "*Unused for:* %s\n", commands.Human(time.Since(u.LastActive))) } fmt.Fprintf(&sb, "*Stops when unused for:* %s\n", b.describeIdle(u)) fmt.Fprintf(&sb, "*Home folder:* %s\n", b.describeStorage(ctx, u)) sb.WriteString("*Nodes:*") views, _ := b.mgr.Views(ctx) for _, v := range views { sb.WriteString("\n• " + describeNode(v)) } for _, name := range b.mgr.OfflineNodes() { fmt.Fprintf(&sb, "\n• %s: offline", name) } text := sb.String() blocks := []slack.Block{slack.NewSectionBlock(slack.NewTextBlockObject(slack.MarkdownType, text, false, false), nil, nil)} var buttons []slack.BlockElement if state == manager.StateRunning { stop := slack.NewButtonBlockElement("stop", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Stop container", false, false)) stop.Style = slack.StyleDanger buttons = append(buttons, stop) } if u != nil && size != b.cfg.MaxContainer { buttons = append(buttons, slack.NewButtonBlockElement("scale", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Scale up", false, false))) } if len(buttons) > 0 { blocks = append(blocks, slack.NewActionBlock("container", buttons...)) } opts := []slack.MsgOption{slack.MsgOptionText(text, false), slack.MsgOptionBlocks(blocks...)} if thread != "" { opts = append(opts, slack.MsgOptionTS(thread)) } if _, _, err := b.api.PostMessageContext(ctx, channel, opts...); err != nil { log.Printf("slack post: %v", err) } } func (b *Bot) stopFor(ctx context.Context, ownerID string) string { u := b.store.Owner(ownerID) if b.mgr.State(u) != manager.StateRunning { return "The container isn't running." } if err := b.mgr.Poweroff(ctx, u); err != nil { return ":warning: Couldn't stop it: " + err.Error() } b.dropClient(u) reply := "Shutting down the container. Its files are kept. Message me anytime to start it again." if link := b.announceStop(ctx, u, "Stopped on request."); link != "" { reply += fmt.Sprintf(" <%s|Follow the shutdown here>.", link) } return reply } func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) { old, next, err := b.mgr.Resize(ctx, ownerID, kind, func(cur model.Size) (model.Size, error) { return commands.ParseScale(args, cur) }) switch { case errors.Is(err, commands.ErrNotCommand): return "", false case errors.Is(err, manager.ErrTooBig): return fmt.Sprintf("That's bigger than allowed. The max is %s.", b.cfg.MaxContainer), true case errors.Is(err, manager.ErrTooSmall): return fmt.Sprintf("That's too small. The minimum is %s.", manager.MinContainer), true case errors.Is(err, balance.ErrMaxedOut) || errors.As(err, new(*node.NodeFullError)): return b.explain(err), true case err != nil: return fmt.Sprintf(":warning: %s. Try `scale 8G 4cpu`, `scale +2G`, `scale -1cpu` or `scale up`.", err), true case next == old: return fmt.Sprintf("No change. The container already has %s.", old), true } if b.mgr.State(b.store.Owner(ownerID)) != manager.StateRunning { return fmt.Sprintf("Done! The container will start with %s next time (was %s).", next, old), true } reply = fmt.Sprintf("Done! The container now has %s (was %s).", next, old) if next.MemoryMB < old.MemoryMB { reply += " If it was using more memory than the new limit, some programs in it may be stopped." } return reply, true } func (b *Bot) keepFor(ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) { k, err := commands.ParseKeep(args) switch { case errors.Is(err, commands.ErrNotCommand): return "", false case err != nil: return ":warning: " + err.Error(), true } u, err := b.mgr.Owner(ownerID, kind) if err != nil { return ":warning: " + err.Error(), true } var d time.Duration switch { case k.Show: return "This container stops when unused for " + b.describeIdle(u) + ".", true case k.Reset: d = 0 case k.Indefinite: if !b.cfg.AllowIndefinite { return "Keeping containers indefinitely is turned off on this server.", true } d = model.Indefinite default: d = k.Limit if d < 10*time.Minute { return "That's too short. The minimum is 10m.", true } if max := b.cfg.MaxIdleTimeout; max > 0 && d > max { return fmt.Sprintf("That's longer than allowed. The max is %s.", commands.Human(max)), true } } if err := b.store.SetIdleTimeout(ownerID, d); err != nil { return ":warning: " + err.Error(), true } _ = b.store.Touch(ownerID) u.IdleTimeout = d if d == model.Indefinite { return "Done! This container won't be stopped for being unused. Send `stop` when you're finished, " + "or `keep default` to go back to normal.", true } return "Done! This container stops when unused for " + b.describeIdle(u) + ".", true } func (b *Bot) storageFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) { if strings.TrimSpace(args) == "" { u, err := b.mgr.Owner(ownerID, kind) if err != nil { return ":warning: " + err.Error(), true } return "Home folder storage: " + b.describeStorage(ctx, u) + ".", true } c, err := b.mgr.SetStorage(ctx, ownerID, kind, func(min, max int64) (int64, int64, error) { return commands.ParseStorage(args, min, max) }) switch { case errors.Is(err, commands.ErrNotCommand): return "", false case errors.Is(err, storage.ErrStorageTooSmall): return fmt.Sprintf("That's too small. The minimum is %s.", units.FormatMB(config.MinStorageMB)), true case errors.Is(err, storage.ErrMinOverMax): return fmt.Sprintf("The guaranteed space (%s) can't be more than the max (%s).", units.FormatMB(c.NewMin), units.FormatMB(c.NewMax)), true case errors.Is(err, storage.ErrStorageTooBig): return fmt.Sprintf("The max can't go above %s.", units.FormatMB(b.cfg.Storage.LimitMB)), true case errors.Is(err, storage.ErrBelowUsage): return fmt.Sprintf("The home folder already holds %s, so the max can't go below that.", units.FormatMB(c.UsedMB)), true case errors.Is(err, storage.ErrStorageFull): return fmt.Sprintf(":no_entry: There isn't enough free space on the drive to guarantee %s.", units.FormatMB(c.NewMin)), true case err != nil: return fmt.Sprintf(":warning: %s. Try `storage 20G`, `storage +5G` or `storage min 8G max 20G`.", err), true } reply = fmt.Sprintf("Done! Home folder: %s guaranteed, up to %s (was %s guaranteed, up to %s).", units.FormatMB(c.NewMin), units.FormatMB(c.NewMax), units.FormatMB(c.OldMin), units.FormatMB(c.OldMax)) if c.Drive != nil && c.Drive.Overbooked() { reply += fmt.Sprintf("\n:warning: Heads up: drive %s is overbooked. Its containers may use up to %s in total, "+ "but only %s is available. Guaranteed space is safe, but if they all fill up, writes will start failing.", c.Drive.Path, units.FormatMB(c.Drive.BookedMB), units.FormatMB(c.Drive.UsableMB)) } return reply, true } func (b *Bot) describeStorage(ctx context.Context, u *model.Owner) string { if u == nil { return fmt.Sprintf("%s guaranteed, up to %s", units.FormatMB(b.cfg.Storage.MinMB), units.FormatMB(b.cfg.Storage.MaxMB)) } text := fmt.Sprintf("%s guaranteed, up to %s", units.FormatMB(u.StorageMinMB), units.FormatMB(u.StorageMaxMB)) if used, ok := b.mgr.StorageUsed(ctx, u); ok { text = units.FormatMB(used) + " used, " + text } if u.Node != "" { text += fmt.Sprintf(" (on %s:%s)", u.Node, u.Drive) } return text } func describeNode(v balance.NodeView) string { us, cm, _ := v.Storage() free := v.Usable.Sub(v.Used) text := fmt.Sprintf("*%s*: %s of %s CPUs free · %s of %s RAM free · %s of %s storage free · %d running, %d total", v.Name, strconv.FormatFloat(math.Max(0, free.CPUs), 'f', -1, 64), strconv.FormatFloat(v.Usable.CPUs, 'f', -1, 64), units.FormatMB(int64(max(0, free.MemoryMB))), units.FormatMB(int64(v.Usable.MemoryMB)), units.FormatMB(max(0, us-cm)), units.FormatMB(us), v.Running, v.Homes()) for _, d := range v.Drives { if d.Overbooked() { text += fmt.Sprintf(" · :warning: %s overbooked (caps add up to %s of %s)", d.Path, units.FormatMB(d.BookedMB), units.FormatMB(d.UsableMB)) } } return text } func (b *Bot) moveTarget(args string) (target string, ok bool) { words := strings.Fields(strings.ToLower(args)) switch { case len(words) == 0, len(words) == 1 && words[0] == "auto": return "", true case len(words) == 1: return words[0], true case len(words) == 2 && words[0] == "to": return words[1], true } return "", false } func (b *Bot) permitted(ctx context.Context, channel, thread, user string, kind model.OwnerKind, a model.Action) bool { p := b.store.Permissions() if p.Allowed(kind, channel, user, a) { return true } var text string if kind == model.OwnerUser { text = fmt.Sprintf(":lock: Changing *%s* from DMs is turned off on this server.", a) } else if holders := p.Holders(channel, a); len(holders) == 0 { text = fmt.Sprintf(":lock: Nobody here has *%s* permission. An admin can grant it in the config.", a) } else { names := make([]string, len(holders)) for i, h := range holders { names[i] = b.displayName(ctx, h) } text = fmt.Sprintf(":lock: Only super users with *%s* permission can do that here: %s.", a, strings.Join(names, ", ")) } b.say(ctx, channel, thread, text) return false } func (b *Bot) permissionsList(ctx context.Context, channel string, kind model.OwnerKind) string { p := b.store.Permissions() if kind == model.OwnerUser { var lines []string for _, a := range model.AllActions { state := "allowed" if !p.Allowed(model.OwnerUser, channel, "", a) { state = "turned off" } lines = append(lines, fmt.Sprintf("• *%s*: %s", a, state)) } return "*In DMs, you can change your own container:*\n" + strings.Join(lines, "\n") + "\nUser managers can switch these with `dm on|off`." } list, own := p.SuperUsers(channel) source := "the server-wide default list (this channel has no list of its own)" if own { source = "this channel's own list" } users := make([]string, 0, len(list)) for user := range list { users = append(users, user) } sort.Strings(users) lines := []string{fmt.Sprintf("*Super users here*, from %s:", source)} for _, user := range users { actions := p.ActionsFor(channel, user) names := make([]string, len(actions)) for i, a := range actions { names[i] = string(a) } lines = append(lines, fmt.Sprintf("• %s: %s", b.displayName(ctx, user), strings.Join(names, ", "))) } managerNames := make([]string, len(p.Managers)) for i, u := range p.Managers { managerNames[i] = b.displayName(ctx, u) } lines = append(lines, "*User managers* (can change any channel's list): "+strings.Join(managerNames, ", "), "Everyone can talk to the agent and use `status`, `nodes`, `sessions` and `stop`.") return strings.Join(lines, "\n") } func (b *Bot) blocked(ctx context.Context, channel, thread, ownerID string) bool { if b.mgr.IsMigrating(ownerID) { b.say(ctx, channel, thread, b.explain(manager.ErrMigrating)) return true } if u := b.store.Owner(ownerID); b.mgr.State(u) == manager.StateOffline { b.deadSlave(ctx, channel, "", ownerID, u.Node) return true } return false } func (b *Bot) moveCommand(ctx context.Context, m incoming, replyThread string) bool { verb, args, _ := strings.Cut(strings.TrimSpace(m.text), " ") verb = strings.ToLower(strings.TrimRight(verb, ".!?")) if verb != "move" && verb != "migrate" { return false } target, ok := b.moveTarget(strings.TrimRight(strings.TrimSpace(args), " .!?")) if !ok { return false } ownerID, kind := m.owner() if !b.permitted(ctx, m.channel, replyThread, m.user, kind, model.ActMove) { return true } if b.blocked(ctx, m.channel, replyThread, ownerID) { return true } go b.moveFor(ctx, m.channel, ownerID, target, "<@"+m.user+">", replyThread) return true } func (b *Bot) moveFor(ctx context.Context, channel, ownerID, target, trigger, replyThread string) { b.runMove(ctx, channel, ownerID, manager.MoveOptions{Target: target}, trigger, func(link string) { b.say(ctx, channel, replyThread, fmt.Sprintf(":truck: Moving this container. <%s|Follow the move here>.", link)) }) } // Every move gets its own top-level message saying who started it, with // progress in that message's thread. func (b *Bot) runMove(ctx context.Context, channel, ownerID string, opts manager.MoveOptions, trigger string, announce func(link string)) (*manager.MoveResult, error) { before := b.store.Owner(ownerID) from := "its node" if before != nil && before.Node != "" { from = before.Node } where := opts.Target if where == "" { where = "best node" } head := fmt.Sprintf(":truck: Container move started by %s: %s → %s", trigger, from, where) _, root, err := b.api.PostMessageContext(ctx, channel, slack.MsgOptionText(head, false)) if err != nil { log.Printf("slack post: %v", err) } if announce != nil && root != "" { link, _ := b.api.GetPermalinkContext(ctx, &slack.PermalinkParameters{Channel: channel, Ts: root}) announce(link) } _, progressTS, err := b.api.PostMessageContext(ctx, channel, slack.MsgOptionText(":hourglass_flowing_sand: Checking where it can go…", false), slack.MsgOptionTS(root)) if err != nil { log.Printf("slack post: %v", err) } opts.WaitIdle = func(ctx context.Context) { if before != nil { b.waitIdle(ctx, before.Machine) } } opts.Progress = b.progressEditor(ctx, channel, progressTS) res, err := b.mgr.Migrate(ctx, ownerID, opts) if before != nil { b.dropClient(before) } text, outcome := b.moveOutcome(ownerID, opts.Target, res, err) _, _, _, _ = b.api.UpdateMessageContext(ctx, channel, progressTS, slack.MsgOptionText(text, false)) _, _, _, _ = b.api.UpdateMessageContext(ctx, channel, root, slack.MsgOptionText(head+" · "+outcome, false)) return res, err } func (b *Bot) moveOutcome(ownerID, target string, res *manager.MoveResult, err error) (text, outcome string) { failed := ":x: didn't move" var unknown *manager.UnknownNodeError var full *node.NodeFullError var off *node.NodeOfflineError switch { case errors.Is(err, manager.ErrNoContainer): return "There's no container here yet. Send me a message to create one.", failed case errors.Is(err, manager.ErrSameNode): return "The container is already on " + target + ".", failed case errors.As(err, &unknown): return fmt.Sprintf("There's no node called %s. Send `nodes` to see them.", unknown.Node), failed case errors.As(err, &off) && res == nil: return fmt.Sprintf("%s is offline right now, so nothing can move there.", off.Node), failed case errors.As(err, &full) && res == nil: return fmt.Sprintf(":no_entry: %s doesn't have room for this container. Send `nodes` to see what's free.", full.Node), failed case errors.Is(err, balance.ErrMaxedOut) || errors.Is(err, storage.ErrStorageFull): return ":no_entry: No other node has room for this container right now. Send `nodes` to see what's free.", failed case errors.Is(err, manager.ErrMigrating): return b.explain(err), failed } if err != nil { if after := b.store.Owner(ownerID); res != nil && after != nil && after.Node == res.To { return fmt.Sprintf(":warning: Moved to %s, but: %s", res.To, err), ":warning: moved to " + res.To + " with problems" } return ":warning: The move didn't work, so the container stays on its old node: " + err.Error(), failed } text = fmt.Sprintf(":white_check_mark: Moved from %s to %s (%s copied).", res.From, res.To, units.FormatMB(res.CopiedMB)) if res.Started { text += " It's running there now, and its sessions came along." } if res.LeftOld != nil { text += fmt.Sprintf("\n:warning: The old copy on %s couldn't be deleted: %s", res.From, res.LeftOld) } return text, ":white_check_mark: done, now on " + res.To } func (b *Bot) progressEditor(ctx context.Context, channel, ts string) func(manager.MoveProgress) { var last time.Time var lastPhase manager.MovePhase return func(p manager.MoveProgress) { if p.Phase == lastPhase && time.Since(last) < 3*time.Second { return } last, lastPhase = time.Now(), p.Phase copied := units.FormatMB(p.CopiedMB) if p.TotalMB > 0 { copied += " of ~" + units.FormatMB(p.TotalMB) } var text string switch p.Phase { case manager.PhaseCopy: text = fmt.Sprintf(":truck: Moving to %s… copying the home folder (%s). The container keeps running for now.", p.To, copied) case manager.PhaseFinal: text = fmt.Sprintf(":truck: Moving to %s… final sync of recent changes. The container is briefly stopped.", p.To) case manager.PhaseCleanup: text = fmt.Sprintf(":truck: Moving to %s… copied %s, switching over.", p.To, copied) case manager.PhaseStart: text = fmt.Sprintf(":truck: Moving to %s… starting it there.", p.To) } _, _, _, _ = b.api.UpdateMessageContext(ctx, channel, ts, slack.MsgOptionText(text, false)) } } func (b *Bot) waitIdle(ctx context.Context, machine string) { deadline := time.Now().Add(2 * time.Minute) for b.isBusy(machine) && time.Now().Before(deadline) { select { case <-ctx.Done(): return case <-time.After(time.Second): } } } func (b *Bot) nodesList(ctx context.Context, ownerID string) string { views, err := b.mgr.Views(ctx) if err != nil { return ":warning: " + err.Error() } here := "" if u := b.store.Owner(ownerID); u != nil { here = u.Node } var sb strings.Builder sb.WriteString("*Nodes* (what's free on each):") for _, v := range views { sb.WriteString("\n• " + describeNode(v)) if v.Name == here { sb.WriteString(" ← this container") } } for _, name := range b.mgr.OfflineNodes() { fmt.Fprintf(&sb, "\n• *%s*: offline", name) if name == here { sb.WriteString(" ← this container") } } sb.WriteString("\nSend `move ` to move this container, or `move auto` to let me pick.") return sb.String() } func (b *Bot) deadSlave(ctx context.Context, channel, replaceTS, ownerID, node string) { text := fmt.Sprintf("Channel bot is on a dead slave %s, create new container or wait for the slave to be back up?", node) blocks := []slack.Block{ slack.NewSectionBlock(slack.NewTextBlockObject(slack.MarkdownType, text, false, false), nil, nil), slack.NewActionBlock("dead_slave", slack.NewButtonBlockElement("recreate", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Create new container", false, false)), slack.NewButtonBlockElement("wait", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Wait", false, false)), ), } opts := []slack.MsgOption{slack.MsgOptionText(text, false), slack.MsgOptionBlocks(blocks...)} var err error if replaceTS != "" { _, _, _, err = b.api.UpdateMessageContext(ctx, channel, replaceTS, opts...) } else { _, _, err = b.api.PostMessageContext(ctx, channel, opts...) } if err != nil { log.Printf("slack post: %v", err) } } func (b *Bot) nodeBack(node string) { b.waitMu.Lock() channels := b.waiting[node] delete(b.waiting, node) b.waitMu.Unlock() for channel := range channels { b.say(context.Background(), channel, "", fmt.Sprintf("%s is back up. Message me to pick up where you left off.", node)) } } func (b *Bot) idleLimit(u *model.Owner) (limit time.Duration, indefinite bool) { switch { case u == nil || u.IdleTimeout == 0: return b.cfg.IdleTimeout, false case u.IdleTimeout == model.Indefinite: return 0, true } return u.IdleTimeout, false } func (b *Bot) describeIdle(u *model.Owner) string { limit, indefinite := b.idleLimit(u) switch { case indefinite: return "never (kept indefinitely)" case u == nil || u.IdleTimeout == 0: return commands.Human(limit) + " (default)" } return commands.Human(limit) } func (b *Bot) explain(err error) string { var full *node.NodeFullError switch { case errors.Is(err, manager.ErrMigrating): return ":truck: This container is being moved right now. Try again in a moment." case errors.As(err, &full): return fmt.Sprintf(":no_entry: Sorry, %s (where this container's files live) is maxed out right now. "+ "Try again a bit later.", full.Node) case errors.Is(err, balance.ErrMaxedOut): used, usable := b.mgr.Totals(context.Background()) return fmt.Sprintf(":no_entry: Sorry, every node is maxed out right now (%s of %s in use). "+ "Try again a bit later.", used, usable) case errors.Is(err, storage.ErrStorageFull): return ":no_entry: Sorry, there isn't enough storage left on any node to set up a new container. Try again later." } return ":warning: " + err.Error() } func clientKey(u *model.Owner) string { return u.Node + "/" + u.Machine } func (b *Bot) client(ctx context.Context, u *model.Owner) (*acp.ACP, error) { key := clientKey(u) unlock := b.dialMu.Lock(key) defer unlock() b.acpMu.Lock() c := b.acps[key] b.acpMu.Unlock() if c != nil { select { case <-c.Done(): default: return c, nil } } conn, err := b.mgr.DialAgent(ctx, u) if err != nil { return nil, err } initCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() if c, err = acp.New(initCtx, conn); err != nil { return nil, err } b.acpMu.Lock() b.acps[key] = c b.acpMu.Unlock() return c, nil } func (b *Bot) dropClient(u *model.Owner) { b.acpMu.Lock() defer b.acpMu.Unlock() if c := b.acps[clientKey(u)]; c != nil { c.Close() delete(b.acps, clientKey(u)) } } func (b *Bot) setBusy(machine string, delta int) { b.busyMu.Lock() defer b.busyMu.Unlock() b.busy[machine] += delta } func (b *Bot) isBusy(machine string) bool { b.busyMu.Lock() defer b.busyMu.Unlock() return b.busy[machine] > 0 } func (b *Bot) reaper(ctx context.Context) { t := time.NewTicker(10 * time.Minute) defer t.Stop() for { b.reapOnce(ctx) select { case <-ctx.Done(): return case <-t.C: } } } func (b *Bot) reapOnce(ctx context.Context) { for _, owner := range b.mgr.RunningOwners(ctx) { u := owner if b.isBusy(u.Machine) { continue } limit, indefinite := b.idleLimit(&u) if indefinite || time.Since(u.LastActive) < limit { continue } log.Printf("cleanup: stopping %s on %s (idle since %s)", u.Machine, u.Node, u.LastActive.Format(time.RFC3339)) if err := b.mgr.Poweroff(ctx, &u); err != nil { log.Printf("cleanup: stopping %s: %v", u.Machine, err) continue } b.dropClient(&u) msg := fmt.Sprintf("I stopped the container after %s without use. Its files are kept. "+ "Message me anytime to start it again, or use `keep` to change how long it waits.", commands.Human(limit)) if link := b.announceStop(ctx, &u, fmt.Sprintf("Unused for %s.", commands.Human(limit))); link != "" { msg += fmt.Sprintf(" <%s|Shutdown details>.", link) } b.say(ctx, u.ID, "", msg) // a user ID posts to their DM, a channel ID to the channel } } func (b *Bot) say(ctx context.Context, channel, thread, text string) { opts := []slack.MsgOption{slack.MsgOptionText(text, false)} if thread != "" { opts = append(opts, slack.MsgOptionTS(thread)) } if _, _, err := b.api.PostMessageContext(ctx, channel, opts...); err != nil { log.Printf("slack post: %v", err) } } func (b *Bot) displayName(ctx context.Context, userID string) string { if v, ok := b.names.Load(userID); ok { return v.(string) } name := userID if u, err := b.api.GetUserInfoContext(ctx, userID); err == nil { for _, n := range []string{u.Profile.DisplayName, u.RealName, u.Name} { if n != "" { name = n break } } b.names.Store(userID, name) } return name } func title(text string) string { line, _, _ := strings.Cut(strings.TrimSpace(text), "\n") if utf8.RuneCountInString(line) > 60 { r := []rune(line) line = string(r[:60]) + "…" } if line == "" { line = "untitled" } return line }