1273 lines
43 KiB
Go
1273 lines
43 KiB
Go
package bot
|
|
|
|
import (
|
|
"cmp"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"math"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"github.com/slack-go/slack"
|
|
"github.com/slack-go/slack/slackevents"
|
|
"github.com/slack-go/slack/socketmode"
|
|
|
|
"slackerbot/internal/acp"
|
|
"slackerbot/internal/balance"
|
|
"slackerbot/internal/commands"
|
|
"slackerbot/internal/config"
|
|
"slackerbot/internal/manager"
|
|
"slackerbot/internal/model"
|
|
"slackerbot/internal/node"
|
|
"slackerbot/internal/state"
|
|
"slackerbot/internal/storage"
|
|
"slackerbot/internal/tools"
|
|
"slackerbot/internal/units"
|
|
"slackerbot/internal/util"
|
|
)
|
|
|
|
const helpText = "I pass your messages to an agent running in its own container.\n" +
|
|
"• In a DM, the container is yours alone. In a channel, it belongs to the channel " +
|
|
"and anyone here can use it by @mentioning me.\n" +
|
|
"• Every new message starts a new session. Reply in its thread to keep going.\n" +
|
|
"• In a channel thread I only read replies that @mention me, unless the thread or channel is switched with `listen` (see below).\n" +
|
|
"• React with :x: to stop a reply part-way.\n" +
|
|
"*Commands* (send as a new message, not in a thread. `move` and `listen` work in threads too):\n" +
|
|
"• `status`: state, size and idle limit, with buttons\n" +
|
|
"• `sessions`: past sessions, with links\n" +
|
|
"• `stop`: shut the container down (files are kept)\n" +
|
|
"• `scale up` / `scale down`: double or halve RAM and CPU\n" +
|
|
"• `scale 8G 4cpu`: set an exact size. `scale +2G` or `scale -1cpu`: change by that much\n" +
|
|
"• `keep 24h` (or `3d`, `1d12h`, `2w`): stop after that long unused\n" +
|
|
"• `keep indefinite`: never stop for being unused. `keep default`: back to normal. `keep`: show it\n" +
|
|
"• `storage 20G` / `storage +5G` / `storage -2G`: set or change the home folder's max size (`disk` works too)\n" +
|
|
"• `storage min 8G`: set the guaranteed space. `storage min 8G max 20G`: set both. `storage`: show usage\n" +
|
|
"• `nodes`: every node and what's free on it\n" +
|
|
"• `move <node>` (or `move to <node>`, `migrate <node>`): move the container to that node. `move auto`: let me pick the best one\n" +
|
|
"• `@slacker /listen all-messages` / `@slacker /listen only-mentions`: in a session's thread, read every reply or only @mentions. " +
|
|
"`@slacker /listen`: show this thread's setting\n" +
|
|
"• `@slacker /listen channel all-messages` / `@slacker /listen channel only-mentions`: set the channel's default for threads " +
|
|
"(a top-level `/listen all-messages` does the same). `@slacker /listen channel`: show it. `listen` without the slash works too\n" +
|
|
"• `permissions`: who can scale, change storage, move or set timeouts here\n" +
|
|
"• `in-flight` / `in-flight all`: super users only. Shows the replies I'd pick back up after a restart, for debugging\n" +
|
|
"• `managers`: who the user managers are\n" +
|
|
"• Managers: `grant @alice move scale`, `revoke @alice move` / `revoke @alice`, `permissions reset`, " +
|
|
"`grant default @alice all` / `revoke default @alice`, `dm storage on|off`, `managers add|remove @alice`\n" +
|
|
"• `help`: this message"
|
|
|
|
type Bot struct {
|
|
cfg *config.Config
|
|
store *state.Store
|
|
mgr *manager.Manager
|
|
api *slack.Client
|
|
sm *socketmode.Client
|
|
botID string
|
|
|
|
acpMu sync.Mutex
|
|
acps map[string]*acp.ACP
|
|
dialMu util.KeyedMutex
|
|
sessMu util.KeyedMutex
|
|
busyMu sync.Mutex
|
|
busy map[string]int
|
|
names sync.Map
|
|
|
|
updateAsked sync.Map // machine → asked to restart onto a newer system
|
|
|
|
channels sync.Map // channel ID → *slack.Channel
|
|
running sync.Map // "channel/ts" of a prompt's message or reply → its cancel func
|
|
viewing sync.Map // assistant "channel/thread" → channel the user has open
|
|
told sync.Map // assistant "channel/thread" → last channel the agent was told about
|
|
|
|
waitMu sync.Mutex
|
|
waiting map[string]map[string]bool
|
|
|
|
turnsWG sync.WaitGroup
|
|
}
|
|
|
|
type incoming struct {
|
|
user, channel, ts, threadTS, text string
|
|
dm bool
|
|
heard bool
|
|
files []slack.File
|
|
}
|
|
|
|
func (m incoming) owner() (string, model.OwnerKind) {
|
|
if m.dm {
|
|
return m.user, model.OwnerUser
|
|
}
|
|
return m.channel, model.OwnerChannel
|
|
}
|
|
|
|
func New(cfg *config.Config, store *state.Store, mgr *manager.Manager) (*Bot, error) {
|
|
api := slack.New(cfg.Slack.BotToken, slack.OptionAppLevelToken(cfg.Slack.AppToken))
|
|
auth, err := api.AuthTest()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("slack auth: %w", err)
|
|
}
|
|
b := &Bot{
|
|
cfg: cfg, store: store, mgr: mgr, api: api,
|
|
sm: socketmode.New(api),
|
|
botID: auth.UserID,
|
|
acps: map[string]*acp.ACP{},
|
|
busy: map[string]int{},
|
|
waiting: map[string]map[string]bool{},
|
|
}
|
|
mgr.SetNodeBackHandler(b.nodeBack)
|
|
return b, nil
|
|
}
|
|
|
|
func (b *Bot) Run(ctx context.Context) error {
|
|
go b.events(ctx)
|
|
go b.reaper(ctx)
|
|
go b.resumeTurns(ctx)
|
|
err := b.sm.RunContext(ctx)
|
|
b.waitForTurns(shutdownGrace)
|
|
return err
|
|
}
|
|
|
|
func (b *Bot) events(ctx context.Context) {
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case evt, ok := <-b.sm.Events:
|
|
if !ok {
|
|
return
|
|
}
|
|
switch evt.Type {
|
|
case socketmode.EventTypeEventsAPI:
|
|
if evt.Request != nil {
|
|
b.sm.Ack(*evt.Request)
|
|
}
|
|
if ev, ok := evt.Data.(slackevents.EventsAPIEvent); ok {
|
|
go b.onEvent(ctx, ev)
|
|
}
|
|
case socketmode.EventTypeInteractive:
|
|
if evt.Request != nil {
|
|
b.sm.Ack(*evt.Request)
|
|
}
|
|
if cb, ok := evt.Data.(slack.InteractionCallback); ok {
|
|
go b.onAction(ctx, cb)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (b *Bot) onEvent(ctx context.Context, ev slackevents.EventsAPIEvent) {
|
|
switch e := ev.InnerEvent.Data.(type) {
|
|
case *slackevents.AppMentionEvent:
|
|
if e.BotID != "" {
|
|
return
|
|
}
|
|
b.onMessage(ctx, incoming{e.User, e.Channel, e.TimeStamp, e.ThreadTimeStamp, b.stripMention(e.Text), false, false, eventFiles(ev)})
|
|
case *slackevents.MessageEvent:
|
|
if e.BotID != "" || (e.SubType != "" && e.SubType != "file_share") || e.User == "" || e.User == b.botID {
|
|
return
|
|
}
|
|
text := b.stripMention(e.Text)
|
|
switch {
|
|
case e.ChannelType == "im":
|
|
b.onMessage(ctx, incoming{e.User, e.Channel, e.TimeStamp, e.ThreadTimeStamp, text, true, false, eventFiles(ev)})
|
|
case b.listening(e.Channel, e.ThreadTimeStamp, e.Text):
|
|
b.onMessage(ctx, incoming{e.User, e.Channel, e.TimeStamp, e.ThreadTimeStamp, text, false, true, eventFiles(ev)})
|
|
}
|
|
case *slackevents.ReactionAddedEvent:
|
|
b.onReaction(e)
|
|
case *slackevents.AssistantThreadStartedEvent:
|
|
b.assistantStarted(ctx, e.AssistantThread)
|
|
case *slackevents.AssistantThreadContextChangedEvent:
|
|
b.setViewing(e.AssistantThread)
|
|
}
|
|
}
|
|
|
|
func (b *Bot) stripMention(text string) string {
|
|
return strings.TrimSpace(strings.ReplaceAll(text, "<@"+b.botID+">", ""))
|
|
}
|
|
|
|
func (b *Bot) onMessage(ctx context.Context, m incoming) {
|
|
// A message with files always goes to the agent, so a file with no text isn't read as `help`.
|
|
commandable := len(m.files) == 0
|
|
if commandable && b.listenCommand(ctx, m) {
|
|
return
|
|
}
|
|
// Every assistant-pane chat is a DM thread, so its first message counts as new.
|
|
fresh := m.threadTS == "" || m.dm && b.store.Session(m.channel, m.threadTS) == nil
|
|
if commandable && fresh && b.command(ctx, m) {
|
|
return
|
|
}
|
|
if commandable && m.threadTS != "" && !m.heard && b.moveCommand(ctx, m, m.threadTS) {
|
|
return
|
|
}
|
|
b.converse(ctx, m, nil)
|
|
}
|
|
|
|
// converse runs one turn with the agent. prior is the turn a restart cut off, if this picks it back up.
|
|
func (b *Bot) converse(ctx context.Context, m incoming, prior *model.Turn) {
|
|
b.turnsWG.Add(1)
|
|
defer b.turnsWG.Done()
|
|
root := m.threadTS
|
|
if root == "" {
|
|
root = m.ts
|
|
}
|
|
|
|
// Look the session up only after locking, so two first messages can't both create one.
|
|
unlock := b.sessMu.Lock(m.channel + "/" + root)
|
|
defer unlock()
|
|
sess := b.store.Session(m.channel, root)
|
|
|
|
if prior != nil {
|
|
b.say(ctx, m.channel, root, ":arrows_counterclockwise: slackerbot restarted part-way through this reply, so I'm picking it back up.")
|
|
}
|
|
_, ph, err := b.api.PostMessageContext(ctx, m.channel,
|
|
slack.MsgOptionText(":hourglass_flowing_sand: Getting the container ready…", false),
|
|
slack.MsgOptionTS(root))
|
|
if err != nil {
|
|
log.Printf("slack post: %v", err)
|
|
return
|
|
}
|
|
turn := b.trackTurn(m, root, ph, prior)
|
|
defer b.untrackTurn(ctx, turn)
|
|
edit := func(text string) {
|
|
_, _, _, _ = b.api.UpdateMessageContext(ctx, m.channel, ph, slack.MsgOptionText(text, false))
|
|
}
|
|
b.react(ctx, m.channel, m.ts, "eyes")
|
|
outcome := "warning"
|
|
defer func() {
|
|
b.unreact(ctx, m.channel, m.ts, "eyes")
|
|
if outcome != "" {
|
|
b.react(ctx, m.channel, m.ts, outcome)
|
|
}
|
|
}()
|
|
|
|
ownerID, kind := m.owner()
|
|
var off *node.NodeOfflineError
|
|
var full *node.NodeFullError
|
|
sl := &startLog{b: b, m: m, root: root, ownerID: ownerID}
|
|
started := func(ev manager.StartEvent) { sl.event(ctx, ev) }
|
|
u, err := b.mgr.EnsureProgress(ctx, ownerID, kind, started)
|
|
if errors.As(err, &full) {
|
|
sl.failed(ctx, fmt.Sprintf("`%s` is full, so the container is being moved to a node with room first.", full.Node))
|
|
edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room…", full.Node))
|
|
trigger := fmt.Sprintf("auto process (%s was full), for <@%s>'s message", full.Node, m.user)
|
|
_, merr := b.runMove(ctx, m.channel, ownerID, manager.MoveOptions{StartAfter: true}, trigger, func(link string) {
|
|
edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room. <%s|Follow the move here>.", full.Node, link))
|
|
})
|
|
if merr != nil {
|
|
edit(b.explain(err) + "\nMoving it elsewhere didn't work either, see the move thread.")
|
|
return
|
|
}
|
|
edit(":hourglass_flowing_sand: Moved. Getting the container ready…")
|
|
u, err = b.mgr.EnsureProgress(ctx, ownerID, kind, started)
|
|
}
|
|
if errors.As(err, &off) {
|
|
sl.failed(ctx, fmt.Sprintf("Node `%s` is offline.", off.Node))
|
|
b.deadSlave(ctx, m.channel, ph, ownerID, off.Node)
|
|
return
|
|
}
|
|
if err != nil {
|
|
sl.failed(ctx, b.explain(err))
|
|
edit(b.explain(err))
|
|
return
|
|
}
|
|
b.setBusy(u.Machine, +1)
|
|
defer b.setBusy(u.Machine, -1)
|
|
_ = b.store.Touch(u.ID)
|
|
|
|
acp, err := b.client(ctx, u)
|
|
if errors.As(err, &off) {
|
|
sl.failed(ctx, fmt.Sprintf("Node `%s` went offline.", off.Node))
|
|
b.deadSlave(ctx, m.channel, ph, ownerID, off.Node)
|
|
return
|
|
}
|
|
if err != nil {
|
|
sl.failed(ctx, "The agent never became ready: "+err.Error())
|
|
edit(":warning: Couldn't reach the agent: " + err.Error())
|
|
return
|
|
}
|
|
sl.up(ctx)
|
|
|
|
var id string
|
|
// A new session (the old one was dropped) never saw the cut-off turn, so it gets the message again.
|
|
carryOn := turn.Prompted && sess != nil
|
|
text := m.text
|
|
if !m.dm {
|
|
text = b.displayName(ctx, m.user) + ": " + text
|
|
}
|
|
if carryOn {
|
|
text = resumeNote
|
|
}
|
|
if sess == nil {
|
|
if id, err = acp.NewSession(ctx, b.cfg.AgentCwd); err != nil {
|
|
edit(":warning: Couldn't start a session: " + err.Error())
|
|
return
|
|
}
|
|
_ = b.store.AddSession(&model.Session{
|
|
Channel: m.channel, ThreadTS: root, OwnerID: u.ID,
|
|
ACPID: id, Title: title(m.text), CreatedAt: time.Now(),
|
|
})
|
|
log.Printf("thread: new thread %s/%s for %s %s (started by %s, session %s)", m.channel, root, u.Kind, u.ID, m.user, id)
|
|
if w := b.where(ctx, m.channel); w != "" {
|
|
text = "(You're in " + w + ". Each message starts with the sender's name.)\n" + text
|
|
}
|
|
if m.dm && m.threadTS != "" {
|
|
b.setAssistantTitle(ctx, m.channel, root, title(m.text))
|
|
}
|
|
} else {
|
|
id = sess.ACPID
|
|
if err := acp.LoadSession(ctx, id, b.cfg.AgentCwd); err != nil {
|
|
edit(":warning: Couldn't reopen this session (" + err.Error() + "). Send me a new message to start a fresh one.")
|
|
return
|
|
}
|
|
}
|
|
|
|
if m.dm && !carryOn {
|
|
text = b.viewingNote(ctx, m.channel, root) + text
|
|
}
|
|
if len(m.files) > 0 && !carryOn {
|
|
edit(":paperclip: Copying the attachments into the container…")
|
|
note, problems := b.attach(ctx, u, m)
|
|
if note != "" {
|
|
text += "\n" + note
|
|
}
|
|
if len(problems) > 0 {
|
|
b.say(ctx, m.channel, root, ":warning: "+strings.Join(problems, "\n"))
|
|
}
|
|
}
|
|
|
|
pctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
for _, ts := range []string{m.ts, ph} {
|
|
key := m.channel + "/" + ts
|
|
b.running.Store(key, cancel)
|
|
defer b.running.Delete(key)
|
|
}
|
|
if m.dm && m.threadTS != "" {
|
|
b.setAssistantStatus(ctx, m.channel, root, "is working…")
|
|
defer b.setAssistantStatus(ctx, m.channel, root, "")
|
|
}
|
|
turn.Prompted = true
|
|
b.saveTurn(turn)
|
|
st := NewStreamer(b.api, m.channel, root, ph)
|
|
var sends tools.Calls
|
|
cmds := b.agentCommands(u, m, root)
|
|
stop, err := acp.Prompt(pctx, id, text, func(up acpUpdate) {
|
|
sends.Update(up)
|
|
cmds.Update(ctx, up)
|
|
st.Update(up)
|
|
})
|
|
switch {
|
|
case ctx.Err() != nil:
|
|
stop, err = stopInterrupted, nil
|
|
case pctx.Err() != nil:
|
|
stop, err = "cancelled", nil
|
|
}
|
|
st.Finish(stop, err)
|
|
b.upload(ctx, u, m.channel, root, sends.Files())
|
|
cmds.Finish(ctx)
|
|
_ = b.store.Touch(u.ID)
|
|
b.offerUpdate(ctx, u, m.channel, root)
|
|
switch {
|
|
case stop == "cancelled", stop == stopInterrupted:
|
|
outcome = ""
|
|
case err == nil:
|
|
outcome = "white_check_mark"
|
|
}
|
|
}
|
|
|
|
func splitCommand(text string) (verb, args string) {
|
|
verb, args, _ = strings.Cut(strings.TrimSpace(text), " ")
|
|
verb = strings.ToLower(strings.TrimRight(verb, ".!?"))
|
|
args = strings.TrimRight(strings.TrimSpace(args), " .!?")
|
|
return verb, args
|
|
}
|
|
|
|
func (b *Bot) command(ctx context.Context, m incoming) bool {
|
|
verb, args := splitCommand(m.text)
|
|
if verb == "move" || verb == "migrate" {
|
|
return b.moveCommand(ctx, m, cmp.Or(m.threadTS, m.ts))
|
|
}
|
|
reply, ok := b.runCommand(ctx, m, verb, args, false)
|
|
if reply != "" {
|
|
b.say(ctx, m.channel, m.threadTS, reply)
|
|
}
|
|
return ok
|
|
}
|
|
|
|
// runCommand returns ok=false when the text isn't a command, so it belongs to
|
|
// the agent. An empty reply means it was posted already, with buttons. The
|
|
// agent can't press buttons, so it gets plain text instead.
|
|
func (b *Bot) runCommand(ctx context.Context, m incoming, verb, args string, agent bool) (reply string, ok bool) {
|
|
ownerID, kind := m.owner()
|
|
if verb == "grant" || verb == "revoke" || verb == "dm" || verb == "managers" || (verb == "permissions" && args != "") {
|
|
return b.manageCommand(ctx, m, verb, args)
|
|
}
|
|
if commands.IsListen(verb) {
|
|
c, err := commands.ParseListen(args)
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
return b.listenReply(m, c), true
|
|
}
|
|
if need := commands.NeededPermission(verb, args); need != "" {
|
|
if denied := b.denial(ctx, m.channel, m.user, kind, need); denied != "" {
|
|
return denied, true
|
|
}
|
|
}
|
|
if verb == "stop" || verb == "scale" || verb == "storage" || verb == "disk" {
|
|
if reply, ok := b.blocked(ctx, m.channel, ownerID, agent); ok {
|
|
return reply, true
|
|
}
|
|
}
|
|
|
|
switch {
|
|
case verb == "" || verb == "help" && args == "":
|
|
reply = helpText
|
|
if warning := b.versionWarning(ctx); warning != "" {
|
|
reply = warning + "\n\n" + reply
|
|
}
|
|
case verb == "sessions" && args == "":
|
|
reply = b.sessionList(ctx, ownerID)
|
|
case verb == "status" && args == "":
|
|
if agent {
|
|
return b.statusText(ctx, ownerID, kind), true
|
|
}
|
|
b.postStatus(ctx, m.channel, m.threadTS, ownerID, kind)
|
|
return "", true
|
|
case verb == "stop" && args == "":
|
|
reply = b.stopFor(ctx, ownerID)
|
|
case verb == "scale":
|
|
return b.scaleFor(ctx, ownerID, kind, args)
|
|
case verb == "keep":
|
|
return b.keepFor(ownerID, kind, args)
|
|
case verb == "nodes" && args == "":
|
|
reply = b.nodesList(ctx, ownerID)
|
|
case verb == "permissions" && args == "":
|
|
reply = b.permissionsList(ctx, m.channel, kind)
|
|
case verb == "storage" || verb == "disk":
|
|
return b.storageFor(ctx, ownerID, kind, args)
|
|
case isInFlight(verb) && (args == "" || strings.EqualFold(args, "all")):
|
|
reply = b.inFlightFor(m, args != "", agent)
|
|
if !agent {
|
|
b.whisper(ctx, m, reply)
|
|
return "", true
|
|
}
|
|
default:
|
|
return "", false
|
|
}
|
|
return reply, true
|
|
}
|
|
|
|
func (b *Bot) onAction(ctx context.Context, cb slack.InteractionCallback) {
|
|
if cb.Type != slack.InteractionTypeBlockActions {
|
|
return
|
|
}
|
|
thread := cb.Message.ThreadTimestamp
|
|
for _, a := range cb.ActionCallback.BlockActions {
|
|
if a.ActionID == "cancel" {
|
|
if cancel, ok := b.running.Load(cb.Channel.ID + "/" + a.Value); ok {
|
|
cancel.(context.CancelFunc)()
|
|
}
|
|
continue
|
|
}
|
|
ownerID := a.Value
|
|
if ownerID != cb.User.ID && ownerID != cb.Channel.ID {
|
|
continue
|
|
}
|
|
kind := model.OwnerChannel
|
|
if ownerID == cb.User.ID {
|
|
kind = model.OwnerUser
|
|
}
|
|
u := b.store.Owner(ownerID)
|
|
if need := map[string]model.Action{"recreate": model.ActMove, "scale": model.ActScale}[a.ActionID]; need != "" &&
|
|
!b.permitted(ctx, cb.Channel.ID, thread, cb.User.ID, kind, need) {
|
|
continue
|
|
}
|
|
switch a.ActionID {
|
|
case "recreate":
|
|
if u != nil {
|
|
b.dropClient(u)
|
|
}
|
|
if err := b.mgr.Recreate(ownerID); err != nil {
|
|
b.say(ctx, cb.Channel.ID, thread, ":warning: "+err.Error())
|
|
continue
|
|
}
|
|
b.say(ctx, cb.Channel.ID, thread, "OK. Your next message starts a fresh container on a healthy node. "+
|
|
"Old threads start new sessions when you reply in them.")
|
|
case "wait":
|
|
if u == nil || u.Node == "" {
|
|
continue
|
|
}
|
|
b.waitMu.Lock()
|
|
if b.waiting[u.Node] == nil {
|
|
b.waiting[u.Node] = map[string]bool{}
|
|
}
|
|
b.waiting[u.Node][cb.Channel.ID] = true
|
|
b.waitMu.Unlock()
|
|
b.say(ctx, cb.Channel.ID, thread, fmt.Sprintf("OK, waiting for %s. I'll post here when it's back up.", u.Node))
|
|
case "restart":
|
|
b.restartFor(ctx, cb, ownerID, kind)
|
|
case "later":
|
|
b.settlePrompt(ctx, cb, "OK, not now. The update applies the next time the container starts, "+
|
|
"for example after `stop` and a new message.")
|
|
case "stop", "scale":
|
|
if b.mgr.State(u) == manager.StateOffline {
|
|
b.deadSlave(ctx, cb.Channel.ID, "", ownerID, u.Node)
|
|
continue
|
|
}
|
|
if a.ActionID == "stop" {
|
|
b.say(ctx, cb.Channel.ID, thread, b.stopFor(ctx, ownerID))
|
|
} else {
|
|
reply, _ := b.scaleFor(ctx, ownerID, kind, "up")
|
|
b.say(ctx, cb.Channel.ID, thread, reply)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (b *Bot) sessionList(ctx context.Context, ownerID string) string {
|
|
list := b.store.Sessions(ownerID, 15)
|
|
if len(list) == 0 {
|
|
return "No sessions here yet. Send me a message to start one."
|
|
}
|
|
var sb strings.Builder
|
|
sb.WriteString("*Sessions* (open one and reply in its thread to pick it back up):\n")
|
|
for _, s := range list {
|
|
link, err := b.api.GetPermalinkContext(ctx, &slack.PermalinkParameters{Channel: s.Channel, Ts: s.ThreadTS})
|
|
name := s.Title
|
|
if err == nil {
|
|
name = fmt.Sprintf("<%s|%s>", link, s.Title)
|
|
}
|
|
fmt.Fprintf(&sb, "• %s · %s\n", name, s.CreatedAt.Format("Jan 2, 15:04"))
|
|
}
|
|
return sb.String()
|
|
}
|
|
|
|
func (b *Bot) statusText(ctx context.Context, ownerID string, kind model.OwnerKind) string {
|
|
u := b.store.Owner(ownerID)
|
|
state := b.mgr.State(u)
|
|
size := b.cfg.Container
|
|
if u != nil {
|
|
size = u.Size
|
|
}
|
|
var sb strings.Builder
|
|
whose := "This channel's container"
|
|
if kind == model.OwnerUser {
|
|
whose = "Your container"
|
|
}
|
|
fmt.Fprintf(&sb, "*%s:* %s\n*model.Size:* %s\n", whose, state, size)
|
|
if u != nil && u.Node != "" {
|
|
fmt.Fprintf(&sb, "*Runs on:* %s\n", u.Node)
|
|
}
|
|
if u != nil && state == manager.StateRunning {
|
|
fmt.Fprintf(&sb, "*Unused for:* %s\n", commands.Human(time.Since(u.LastActive)))
|
|
}
|
|
fmt.Fprintf(&sb, "*Stops when unused for:* %s\n", b.describeIdle(u))
|
|
fmt.Fprintf(&sb, "*Home folder:* %s\n", b.describeStorage(ctx, u))
|
|
views, _ := b.mgr.Views(ctx)
|
|
if warning := versionWarning(b.mgr.LocalName(), views); warning != "" {
|
|
sb.WriteString(warning + "\n")
|
|
}
|
|
sb.WriteString("*Nodes:*")
|
|
masterVersion, _ := versionMismatches(b.mgr.LocalName(), views)
|
|
for _, v := range views {
|
|
sb.WriteString("\n• " + describeNode(v) + versionNote(b.mgr.LocalName(), masterVersion, v))
|
|
}
|
|
for _, name := range b.mgr.OfflineNodes() {
|
|
fmt.Fprintf(&sb, "\n• %s: offline", name)
|
|
}
|
|
return sb.String()
|
|
}
|
|
|
|
func (b *Bot) postStatus(ctx context.Context, channel, thread, ownerID string, kind model.OwnerKind) {
|
|
u := b.store.Owner(ownerID)
|
|
state := b.mgr.State(u)
|
|
size := b.cfg.Container
|
|
if u != nil {
|
|
size = u.Size
|
|
}
|
|
text := b.statusText(ctx, ownerID, kind)
|
|
blocks := []slack.Block{slack.NewSectionBlock(slack.NewTextBlockObject(slack.MarkdownType, text, false, false), nil, nil)}
|
|
|
|
var buttons []slack.BlockElement
|
|
if state == manager.StateRunning {
|
|
stop := slack.NewButtonBlockElement("stop", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Stop container", false, false))
|
|
stop.Style = slack.StyleDanger
|
|
buttons = append(buttons, stop)
|
|
}
|
|
if u != nil && size != b.cfg.MaxContainer {
|
|
buttons = append(buttons, slack.NewButtonBlockElement("scale", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Scale up", false, false)))
|
|
}
|
|
if len(buttons) > 0 {
|
|
blocks = append(blocks, slack.NewActionBlock("container", buttons...))
|
|
}
|
|
|
|
opts := []slack.MsgOption{slack.MsgOptionText(text, false), slack.MsgOptionBlocks(blocks...)}
|
|
if thread != "" {
|
|
opts = append(opts, slack.MsgOptionTS(thread))
|
|
}
|
|
if _, _, err := b.api.PostMessageContext(ctx, channel, opts...); err != nil {
|
|
log.Printf("slack post: %v", err)
|
|
}
|
|
}
|
|
|
|
func (b *Bot) stopFor(ctx context.Context, ownerID string) string {
|
|
u := b.store.Owner(ownerID)
|
|
if b.mgr.State(u) != manager.StateRunning {
|
|
return "The container isn't running."
|
|
}
|
|
if err := b.mgr.Poweroff(ctx, u); err != nil {
|
|
return ":warning: Couldn't stop it: " + err.Error()
|
|
}
|
|
b.dropClient(u)
|
|
reply := "Shutting down the container. Its files are kept. Message me anytime to start it again."
|
|
if link := b.announceStop(ctx, u, "Stopped on request."); link != "" {
|
|
reply += fmt.Sprintf(" <%s|Follow the shutdown here>.", link)
|
|
}
|
|
return reply
|
|
}
|
|
|
|
func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) {
|
|
old, next, err := b.mgr.Resize(ctx, ownerID, kind, func(cur model.Size) (model.Size, error) {
|
|
return commands.ParseScale(args, cur)
|
|
})
|
|
switch {
|
|
case errors.Is(err, commands.ErrNotCommand):
|
|
return "", false
|
|
case errors.Is(err, manager.ErrTooBig):
|
|
return fmt.Sprintf("That's bigger than allowed. The max is %s.", b.cfg.MaxContainer), true
|
|
case errors.Is(err, manager.ErrTooSmall):
|
|
return fmt.Sprintf("That's too small. The minimum is %s.", manager.MinContainer), true
|
|
case errors.Is(err, balance.ErrMaxedOut) || errors.As(err, new(*node.NodeFullError)):
|
|
return b.explain(err), true
|
|
case err != nil:
|
|
return fmt.Sprintf(":warning: %s. Try `scale 8G 4cpu`, `scale +2G`, `scale -1cpu` or `scale up`.", err), true
|
|
case next == old:
|
|
return fmt.Sprintf("No change. The container already has %s.", old), true
|
|
}
|
|
|
|
if b.mgr.State(b.store.Owner(ownerID)) != manager.StateRunning {
|
|
return fmt.Sprintf("Done! The container will start with %s next time (was %s).", next, old), true
|
|
}
|
|
reply = fmt.Sprintf("Done! The container now has %s (was %s).", next, old)
|
|
if next.MemoryMB < old.MemoryMB {
|
|
reply += " If it was using more memory than the new limit, some programs in it may be stopped."
|
|
}
|
|
return reply, true
|
|
}
|
|
|
|
func (b *Bot) keepFor(ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) {
|
|
k, err := commands.ParseKeep(args)
|
|
switch {
|
|
case errors.Is(err, commands.ErrNotCommand):
|
|
return "", false
|
|
case err != nil:
|
|
return ":warning: " + err.Error(), true
|
|
}
|
|
|
|
u, err := b.mgr.Owner(ownerID, kind)
|
|
if err != nil {
|
|
return ":warning: " + err.Error(), true
|
|
}
|
|
|
|
var d time.Duration
|
|
switch {
|
|
case k.Show:
|
|
return "This container stops when unused for " + b.describeIdle(u) + ".", true
|
|
case k.Reset:
|
|
d = 0
|
|
case k.Indefinite:
|
|
if !b.cfg.AllowIndefinite {
|
|
return "Keeping containers indefinitely is turned off on this server.", true
|
|
}
|
|
d = model.Indefinite
|
|
default:
|
|
d = k.Limit
|
|
if d < 10*time.Minute {
|
|
return "That's too short. The minimum is 10m.", true
|
|
}
|
|
if max := b.cfg.MaxIdleTimeout; max > 0 && d > max {
|
|
return fmt.Sprintf("That's longer than allowed. The max is %s.", commands.Human(max)), true
|
|
}
|
|
}
|
|
|
|
if err := b.store.SetIdleTimeout(ownerID, d); err != nil {
|
|
return ":warning: " + err.Error(), true
|
|
}
|
|
_ = b.store.Touch(ownerID)
|
|
u.IdleTimeout = d
|
|
if d == model.Indefinite {
|
|
return "Done! This container won't be stopped for being unused. Send `stop` when you're finished, " +
|
|
"or `keep default` to go back to normal.", true
|
|
}
|
|
return "Done! This container stops when unused for " + b.describeIdle(u) + ".", true
|
|
}
|
|
|
|
func (b *Bot) storageFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) {
|
|
if strings.TrimSpace(args) == "" {
|
|
u, err := b.mgr.Owner(ownerID, kind)
|
|
if err != nil {
|
|
return ":warning: " + err.Error(), true
|
|
}
|
|
return "Home folder storage: " + b.describeStorage(ctx, u) + ".", true
|
|
}
|
|
|
|
c, err := b.mgr.SetStorage(ctx, ownerID, kind, func(min, max int64) (int64, int64, error) {
|
|
return commands.ParseStorage(args, min, max)
|
|
})
|
|
switch {
|
|
case errors.Is(err, commands.ErrNotCommand):
|
|
return "", false
|
|
case errors.Is(err, storage.ErrStorageTooSmall):
|
|
return fmt.Sprintf("That's too small. The minimum is %s.", units.FormatMB(config.MinStorageMB)), true
|
|
case errors.Is(err, storage.ErrMinOverMax):
|
|
return fmt.Sprintf("The guaranteed space (%s) can't be more than the max (%s).", units.FormatMB(c.NewMin), units.FormatMB(c.NewMax)), true
|
|
case errors.Is(err, storage.ErrStorageTooBig):
|
|
return fmt.Sprintf("The max can't go above %s.", units.FormatMB(b.cfg.Storage.LimitMB)), true
|
|
case errors.Is(err, storage.ErrBelowUsage):
|
|
return fmt.Sprintf("The home folder already holds %s, so the max can't go below that.", units.FormatMB(c.UsedMB)), true
|
|
case errors.Is(err, storage.ErrStorageFull):
|
|
return fmt.Sprintf(":no_entry: There isn't enough free space on the drive to guarantee %s.", units.FormatMB(c.NewMin)), true
|
|
case err != nil:
|
|
return fmt.Sprintf(":warning: %s. Try `storage 20G`, `storage +5G` or `storage min 8G max 20G`.", err), true
|
|
}
|
|
|
|
reply = fmt.Sprintf("Done! Home folder: %s guaranteed, up to %s (was %s guaranteed, up to %s).",
|
|
units.FormatMB(c.NewMin), units.FormatMB(c.NewMax), units.FormatMB(c.OldMin), units.FormatMB(c.OldMax))
|
|
if c.Drive != nil && c.Drive.Overbooked() {
|
|
reply += fmt.Sprintf("\n:warning: Heads up: drive %s is overbooked. Its containers may use up to %s in total, "+
|
|
"but only %s is available. Guaranteed space is safe, but if they all fill up, writes will start failing.",
|
|
c.Drive.Path, units.FormatMB(c.Drive.BookedMB), units.FormatMB(c.Drive.UsableMB))
|
|
}
|
|
return reply, true
|
|
}
|
|
|
|
func (b *Bot) describeStorage(ctx context.Context, u *model.Owner) string {
|
|
if u == nil {
|
|
return fmt.Sprintf("%s guaranteed, up to %s", units.FormatMB(b.cfg.Storage.MinMB), units.FormatMB(b.cfg.Storage.MaxMB))
|
|
}
|
|
text := fmt.Sprintf("%s guaranteed, up to %s", units.FormatMB(u.StorageMinMB), units.FormatMB(u.StorageMaxMB))
|
|
if used, ok := b.mgr.StorageUsed(ctx, u); ok {
|
|
text = units.FormatMB(used) + " used, " + text
|
|
}
|
|
if u.Node != "" {
|
|
text += fmt.Sprintf(" (on %s:%s)", u.Node, u.Drive)
|
|
}
|
|
return text
|
|
}
|
|
|
|
func describeNode(v balance.NodeView) string {
|
|
us, cm, _ := v.Storage()
|
|
free := v.Usable.Sub(v.Used)
|
|
text := fmt.Sprintf("*%s*: %s of %s CPUs free · %s of %s RAM free · %s of %s storage free · %d running, %d total", v.Name,
|
|
strconv.FormatFloat(math.Max(0, free.CPUs), 'f', -1, 64), strconv.FormatFloat(v.Usable.CPUs, 'f', -1, 64),
|
|
units.FormatMB(int64(max(0, free.MemoryMB))), units.FormatMB(int64(v.Usable.MemoryMB)),
|
|
units.FormatMB(max(0, us-cm)), units.FormatMB(us), v.Running, v.Homes())
|
|
for _, d := range v.Drives {
|
|
if d.Overbooked() {
|
|
text += fmt.Sprintf(" · :warning: %s overbooked (caps add up to %s of %s)", d.Path, units.FormatMB(d.BookedMB), units.FormatMB(d.UsableMB))
|
|
}
|
|
}
|
|
return text
|
|
}
|
|
|
|
func (b *Bot) moveTarget(args string) (target string, ok bool) {
|
|
words := strings.Fields(strings.ToLower(args))
|
|
switch {
|
|
case len(words) == 0, len(words) == 1 && words[0] == "auto":
|
|
return "", true
|
|
case len(words) == 1:
|
|
return words[0], true
|
|
case len(words) == 2 && words[0] == "to":
|
|
return words[1], true
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func (b *Bot) permitted(ctx context.Context, channel, thread, user string, kind model.OwnerKind, a model.Action) bool {
|
|
text := b.denial(ctx, channel, user, kind, a)
|
|
if text == "" {
|
|
return true
|
|
}
|
|
b.say(ctx, channel, thread, text)
|
|
return false
|
|
}
|
|
|
|
// denial explains why the user may not do a, or is empty when they may.
|
|
func (b *Bot) denial(ctx context.Context, channel, user string, kind model.OwnerKind, a model.Action) string {
|
|
p := b.store.Permissions()
|
|
if p.Allowed(kind, channel, user, a) {
|
|
return ""
|
|
}
|
|
if kind == model.OwnerUser {
|
|
return fmt.Sprintf(":lock: Changing *%s* from DMs is turned off on this server.", a)
|
|
}
|
|
holders := p.Holders(channel, a)
|
|
if len(holders) == 0 {
|
|
return fmt.Sprintf(":lock: Nobody here has *%s* permission. An admin can grant it in the config.", a)
|
|
}
|
|
names := make([]string, len(holders))
|
|
for i, h := range holders {
|
|
names[i] = b.displayName(ctx, h)
|
|
}
|
|
return fmt.Sprintf(":lock: Only super users with *%s* permission can do that here: %s.", a, strings.Join(names, ", "))
|
|
}
|
|
|
|
func (b *Bot) permissionsList(ctx context.Context, channel string, kind model.OwnerKind) string {
|
|
p := b.store.Permissions()
|
|
if kind == model.OwnerUser {
|
|
var lines []string
|
|
for _, a := range model.AllActions {
|
|
state := "allowed"
|
|
if !p.Allowed(model.OwnerUser, channel, "", a) {
|
|
state = "turned off"
|
|
}
|
|
lines = append(lines, fmt.Sprintf("• *%s*: %s", a, state))
|
|
}
|
|
return "*In DMs, you can change your own container:*\n" + strings.Join(lines, "\n") +
|
|
"\nUser managers can switch these with `dm <permission> on|off`."
|
|
}
|
|
list, own := p.SuperUsers(channel)
|
|
source := "the server-wide default list (this channel has no list of its own)"
|
|
if own {
|
|
source = "this channel's own list"
|
|
}
|
|
users := make([]string, 0, len(list))
|
|
for user := range list {
|
|
users = append(users, user)
|
|
}
|
|
sort.Strings(users)
|
|
lines := []string{fmt.Sprintf("*Super users here*, from %s:", source)}
|
|
for _, user := range users {
|
|
actions := p.ActionsFor(channel, user)
|
|
names := make([]string, len(actions))
|
|
for i, a := range actions {
|
|
names[i] = string(a)
|
|
}
|
|
lines = append(lines, fmt.Sprintf("• %s: %s", b.displayName(ctx, user), strings.Join(names, ", ")))
|
|
}
|
|
managerNames := make([]string, len(p.Managers))
|
|
for i, u := range p.Managers {
|
|
managerNames[i] = b.displayName(ctx, u)
|
|
}
|
|
lines = append(lines,
|
|
"*User managers* (can change any channel's list): "+strings.Join(managerNames, ", "),
|
|
"Everyone can talk to the agent and use `status`, `nodes`, `sessions` and `stop`.")
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
// blocked explains why the container can't be changed right now. In Slack a
|
|
// dead node gets its recreate/wait buttons instead, and the reply is empty.
|
|
func (b *Bot) blocked(ctx context.Context, channel, ownerID string, agent bool) (reply string, ok bool) {
|
|
if b.mgr.IsMigrating(ownerID) {
|
|
return b.explain(manager.ErrMigrating), true
|
|
}
|
|
if u := b.store.Owner(ownerID); b.mgr.State(u) == manager.StateOffline {
|
|
if agent {
|
|
return fmt.Sprintf("Node %s is offline right now.", u.Node), true
|
|
}
|
|
b.deadSlave(ctx, channel, "", ownerID, u.Node)
|
|
return "", true
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func (b *Bot) moveCommand(ctx context.Context, m incoming, replyThread string) bool {
|
|
verb, args := splitCommand(m.text)
|
|
if verb != "move" && verb != "migrate" {
|
|
return false
|
|
}
|
|
reply, ok := b.move(ctx, m, args, replyThread, false)
|
|
if reply != "" {
|
|
b.say(ctx, m.channel, replyThread, reply)
|
|
}
|
|
return ok
|
|
}
|
|
|
|
func (b *Bot) move(ctx context.Context, m incoming, args, replyThread string, agent bool) (reply string, ok bool) {
|
|
target, ok := b.moveTarget(args)
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
ownerID, kind := m.owner()
|
|
if denied := b.denial(ctx, m.channel, m.user, kind, model.ActMove); denied != "" {
|
|
return denied, true
|
|
}
|
|
if reply, ok := b.blocked(ctx, m.channel, ownerID, agent); ok {
|
|
return reply, true
|
|
}
|
|
trigger := "<@" + m.user + ">"
|
|
if !agent {
|
|
go b.moveFor(ctx, m.channel, ownerID, target, trigger, replyThread)
|
|
return "", true
|
|
}
|
|
go b.moveFor(ctx, m.channel, ownerID, target, "the agent, for "+trigger, replyThread)
|
|
return "The move has started, and its progress is posted in Slack. The container is stopped briefly near the end, " +
|
|
"so finish your reply now. The user can carry on in this thread once it's done.", true
|
|
}
|
|
|
|
func (b *Bot) moveFor(ctx context.Context, channel, ownerID, target, trigger, replyThread string) {
|
|
b.runMove(ctx, channel, ownerID, manager.MoveOptions{Target: target}, trigger, func(link string) {
|
|
b.say(ctx, channel, replyThread, fmt.Sprintf(":truck: Moving this container. <%s|Follow the move here>.", link))
|
|
})
|
|
}
|
|
|
|
// Every move gets its own top-level message saying who started it, with
|
|
// progress in that message's thread.
|
|
func (b *Bot) runMove(ctx context.Context, channel, ownerID string, opts manager.MoveOptions, trigger string, announce func(link string)) (*manager.MoveResult, error) {
|
|
before := b.store.Owner(ownerID)
|
|
from := "its node"
|
|
if before != nil && before.Node != "" {
|
|
from = before.Node
|
|
}
|
|
where := opts.Target
|
|
if where == "" {
|
|
where = "best node"
|
|
}
|
|
head := fmt.Sprintf(":truck: Container move started by %s: %s → %s", trigger, from, where)
|
|
_, root, err := b.api.PostMessageContext(ctx, channel, slack.MsgOptionText(head, false))
|
|
if err != nil {
|
|
log.Printf("slack post: %v", err)
|
|
}
|
|
if announce != nil && root != "" {
|
|
link, _ := b.api.GetPermalinkContext(ctx, &slack.PermalinkParameters{Channel: channel, Ts: root})
|
|
announce(link)
|
|
}
|
|
_, progressTS, err := b.api.PostMessageContext(ctx, channel,
|
|
slack.MsgOptionText(":hourglass_flowing_sand: Checking where it can go…", false), slack.MsgOptionTS(root))
|
|
if err != nil {
|
|
log.Printf("slack post: %v", err)
|
|
}
|
|
|
|
opts.WaitIdle = func(ctx context.Context) {
|
|
if before != nil {
|
|
b.waitIdle(ctx, before.Machine)
|
|
}
|
|
}
|
|
opts.Progress = b.progressEditor(ctx, channel, progressTS)
|
|
res, err := b.mgr.Migrate(ctx, ownerID, opts)
|
|
if before != nil {
|
|
b.dropClient(before)
|
|
}
|
|
|
|
text, outcome := b.moveOutcome(ownerID, opts.Target, res, err)
|
|
_, _, _, _ = b.api.UpdateMessageContext(ctx, channel, progressTS, slack.MsgOptionText(text, false))
|
|
_, _, _, _ = b.api.UpdateMessageContext(ctx, channel, root, slack.MsgOptionText(head+" · "+outcome, false))
|
|
return res, err
|
|
}
|
|
|
|
func (b *Bot) moveOutcome(ownerID, target string, res *manager.MoveResult, err error) (text, outcome string) {
|
|
failed := ":x: didn't move"
|
|
var unknown *manager.UnknownNodeError
|
|
var full *node.NodeFullError
|
|
var off *node.NodeOfflineError
|
|
switch {
|
|
case errors.Is(err, manager.ErrNoContainer):
|
|
return "There's no container here yet. Send me a message to create one.", failed
|
|
case errors.Is(err, manager.ErrSameNode):
|
|
return "The container is already on " + target + ".", failed
|
|
case errors.As(err, &unknown):
|
|
return fmt.Sprintf("There's no node called %s. Send `nodes` to see them.", unknown.Node), failed
|
|
case errors.As(err, &off) && res == nil:
|
|
return fmt.Sprintf("%s is offline right now, so nothing can move there.", off.Node), failed
|
|
case errors.As(err, &full) && res == nil:
|
|
return fmt.Sprintf(":no_entry: %s doesn't have room for this container. Send `nodes` to see what's free.", full.Node), failed
|
|
case errors.Is(err, balance.ErrMaxedOut) || errors.Is(err, storage.ErrStorageFull):
|
|
return ":no_entry: No other node has room for this container right now. Send `nodes` to see what's free.", failed
|
|
case errors.Is(err, manager.ErrMigrating):
|
|
return b.explain(err), failed
|
|
}
|
|
if err != nil {
|
|
if after := b.store.Owner(ownerID); res != nil && after != nil && after.Node == res.To {
|
|
return fmt.Sprintf(":warning: Moved to %s, but: %s", res.To, err), ":warning: moved to " + res.To + " with problems"
|
|
}
|
|
return ":warning: The move didn't work, so the container stays on its old node: " + err.Error(), failed
|
|
}
|
|
text = fmt.Sprintf(":white_check_mark: Moved from %s to %s (%s copied).", res.From, res.To, units.FormatMB(res.CopiedMB))
|
|
if res.Started {
|
|
text += " It's running there now, and its sessions came along."
|
|
}
|
|
if res.LeftOld != nil {
|
|
text += fmt.Sprintf("\n:warning: The old copy on %s couldn't be deleted: %s", res.From, res.LeftOld)
|
|
}
|
|
return text, ":white_check_mark: done, now on " + res.To
|
|
}
|
|
|
|
func (b *Bot) progressEditor(ctx context.Context, channel, ts string) func(manager.MoveProgress) {
|
|
var last time.Time
|
|
var lastPhase manager.MovePhase
|
|
return func(p manager.MoveProgress) {
|
|
if p.Phase == lastPhase && time.Since(last) < 3*time.Second {
|
|
return
|
|
}
|
|
last, lastPhase = time.Now(), p.Phase
|
|
copied := units.FormatMB(p.CopiedMB)
|
|
if p.TotalMB > 0 {
|
|
copied += " of ~" + units.FormatMB(p.TotalMB)
|
|
}
|
|
var text string
|
|
switch p.Phase {
|
|
case manager.PhaseCopy:
|
|
text = fmt.Sprintf(":truck: Moving to %s… copying the home folder (%s). The container keeps running for now.", p.To, copied)
|
|
case manager.PhaseFinal:
|
|
text = fmt.Sprintf(":truck: Moving to %s… final sync of recent changes. The container is briefly stopped.", p.To)
|
|
case manager.PhaseCleanup:
|
|
text = fmt.Sprintf(":truck: Moving to %s… copied %s, switching over.", p.To, copied)
|
|
case manager.PhaseStart:
|
|
text = fmt.Sprintf(":truck: Moving to %s… starting it there.", p.To)
|
|
}
|
|
_, _, _, _ = b.api.UpdateMessageContext(ctx, channel, ts, slack.MsgOptionText(text, false))
|
|
}
|
|
}
|
|
|
|
func (b *Bot) waitIdle(ctx context.Context, machine string) {
|
|
deadline := time.Now().Add(2 * time.Minute)
|
|
for b.isBusy(machine) && time.Now().Before(deadline) {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
}
|
|
|
|
func (b *Bot) nodesList(ctx context.Context, ownerID string) string {
|
|
views, err := b.mgr.Views(ctx)
|
|
if err != nil {
|
|
return ":warning: " + err.Error()
|
|
}
|
|
here := ""
|
|
if u := b.store.Owner(ownerID); u != nil {
|
|
here = u.Node
|
|
}
|
|
var sb strings.Builder
|
|
master := b.mgr.LocalName()
|
|
if warning := versionWarning(master, views); warning != "" {
|
|
sb.WriteString(warning + "\n\n")
|
|
}
|
|
sb.WriteString("*Nodes* (what's free on each):")
|
|
masterVersion, _ := versionMismatches(master, views)
|
|
for _, v := range views {
|
|
sb.WriteString("\n• " + describeNode(v) + versionNote(master, masterVersion, v))
|
|
if v.Name == here {
|
|
sb.WriteString(" ← this container")
|
|
}
|
|
}
|
|
for _, name := range b.mgr.OfflineNodes() {
|
|
fmt.Fprintf(&sb, "\n• *%s*: offline", name)
|
|
if name == here {
|
|
sb.WriteString(" ← this container")
|
|
}
|
|
}
|
|
sb.WriteString("\nSend `move <node>` to move this container, or `move auto` to let me pick.")
|
|
return sb.String()
|
|
}
|
|
|
|
func (b *Bot) deadSlave(ctx context.Context, channel, replaceTS, ownerID, node string) {
|
|
text := fmt.Sprintf("Channel bot is on a dead slave %s, create new container or wait for the slave to be back up?", node)
|
|
blocks := []slack.Block{
|
|
slack.NewSectionBlock(slack.NewTextBlockObject(slack.MarkdownType, text, false, false), nil, nil),
|
|
slack.NewActionBlock("dead_slave",
|
|
slack.NewButtonBlockElement("recreate", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Create new container", false, false)),
|
|
slack.NewButtonBlockElement("wait", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Wait", false, false)),
|
|
),
|
|
}
|
|
opts := []slack.MsgOption{slack.MsgOptionText(text, false), slack.MsgOptionBlocks(blocks...)}
|
|
var err error
|
|
if replaceTS != "" {
|
|
_, _, _, err = b.api.UpdateMessageContext(ctx, channel, replaceTS, opts...)
|
|
} else {
|
|
_, _, err = b.api.PostMessageContext(ctx, channel, opts...)
|
|
}
|
|
if err != nil {
|
|
log.Printf("slack post: %v", err)
|
|
}
|
|
}
|
|
|
|
func (b *Bot) nodeBack(node string) {
|
|
b.waitMu.Lock()
|
|
channels := b.waiting[node]
|
|
delete(b.waiting, node)
|
|
b.waitMu.Unlock()
|
|
for channel := range channels {
|
|
b.say(context.Background(), channel, "", fmt.Sprintf("%s is back up. Message me to pick up where you left off.", node))
|
|
}
|
|
}
|
|
|
|
func (b *Bot) idleLimit(u *model.Owner) (limit time.Duration, indefinite bool) {
|
|
switch {
|
|
case u == nil || u.IdleTimeout == 0:
|
|
return b.cfg.IdleTimeout, false
|
|
case u.IdleTimeout == model.Indefinite:
|
|
return 0, true
|
|
}
|
|
return u.IdleTimeout, false
|
|
}
|
|
|
|
func (b *Bot) describeIdle(u *model.Owner) string {
|
|
limit, indefinite := b.idleLimit(u)
|
|
switch {
|
|
case indefinite:
|
|
return "never (kept indefinitely)"
|
|
case u == nil || u.IdleTimeout == 0:
|
|
return commands.Human(limit) + " (default)"
|
|
}
|
|
return commands.Human(limit)
|
|
}
|
|
|
|
func (b *Bot) explain(err error) string {
|
|
var full *node.NodeFullError
|
|
switch {
|
|
case errors.Is(err, manager.ErrMigrating):
|
|
return ":truck: This container is being moved right now. Try again in a moment."
|
|
case errors.As(err, &full):
|
|
return fmt.Sprintf(":no_entry: Sorry, %s (where this container's files live) is maxed out right now. "+
|
|
"Try again a bit later.", full.Node)
|
|
case errors.Is(err, balance.ErrMaxedOut):
|
|
used, usable := b.mgr.Totals(context.Background())
|
|
return fmt.Sprintf(":no_entry: Sorry, every node is maxed out right now (%s of %s in use). "+
|
|
"Try again a bit later.", used, usable)
|
|
case errors.Is(err, storage.ErrStorageFull):
|
|
return ":no_entry: Sorry, there isn't enough storage left on any node to set up a new container. Try again later."
|
|
}
|
|
return ":warning: " + err.Error()
|
|
}
|
|
|
|
func clientKey(u *model.Owner) string { return u.Node + "/" + u.Machine }
|
|
|
|
func (b *Bot) client(ctx context.Context, u *model.Owner) (*acp.ACP, error) {
|
|
key := clientKey(u)
|
|
unlock := b.dialMu.Lock(key)
|
|
defer unlock()
|
|
|
|
b.acpMu.Lock()
|
|
c := b.acps[key]
|
|
b.acpMu.Unlock()
|
|
if c != nil {
|
|
select {
|
|
case <-c.Done():
|
|
default:
|
|
return c, nil
|
|
}
|
|
}
|
|
|
|
conn, err := b.mgr.DialAgent(ctx, u)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
initCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
if c, err = acp.New(initCtx, conn); err != nil {
|
|
return nil, err
|
|
}
|
|
c.Servers = b.mcpServers()
|
|
b.acpMu.Lock()
|
|
b.acps[key] = c
|
|
b.acpMu.Unlock()
|
|
return c, nil
|
|
}
|
|
|
|
func (b *Bot) dropClient(u *model.Owner) {
|
|
b.acpMu.Lock()
|
|
defer b.acpMu.Unlock()
|
|
if c := b.acps[clientKey(u)]; c != nil {
|
|
c.Close()
|
|
delete(b.acps, clientKey(u))
|
|
}
|
|
}
|
|
|
|
func (b *Bot) setBusy(machine string, delta int) {
|
|
b.busyMu.Lock()
|
|
defer b.busyMu.Unlock()
|
|
b.busy[machine] += delta
|
|
}
|
|
|
|
func (b *Bot) isBusy(machine string) bool {
|
|
b.busyMu.Lock()
|
|
defer b.busyMu.Unlock()
|
|
return b.busy[machine] > 0
|
|
}
|
|
|
|
func (b *Bot) reaper(ctx context.Context) {
|
|
t := time.NewTicker(10 * time.Minute)
|
|
defer t.Stop()
|
|
for {
|
|
b.reapOnce(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func (b *Bot) reapOnce(ctx context.Context) {
|
|
for _, owner := range b.mgr.RunningOwners(ctx) {
|
|
u := owner
|
|
if b.isBusy(u.Machine) {
|
|
continue
|
|
}
|
|
limit, indefinite := b.idleLimit(&u)
|
|
if indefinite || time.Since(u.LastActive) < limit {
|
|
continue
|
|
}
|
|
log.Printf("cleanup: stopping %s on %s (idle since %s)", u.Machine, u.Node, u.LastActive.Format(time.RFC3339))
|
|
if err := b.mgr.Poweroff(ctx, &u); err != nil {
|
|
log.Printf("cleanup: stopping %s: %v", u.Machine, err)
|
|
continue
|
|
}
|
|
b.dropClient(&u)
|
|
msg := fmt.Sprintf("I stopped the container after %s without use. Its files are kept. "+
|
|
"Message me anytime to start it again, or use `keep` to change how long it waits.", commands.Human(limit))
|
|
if link := b.announceStop(ctx, &u, fmt.Sprintf("Unused for %s.", commands.Human(limit))); link != "" {
|
|
msg += fmt.Sprintf(" <%s|Shutdown details>.", link)
|
|
}
|
|
b.say(ctx, u.ID, "", msg) // a user ID posts to their DM, a channel ID to the channel
|
|
}
|
|
}
|
|
|
|
func (b *Bot) say(ctx context.Context, channel, thread, text string) {
|
|
opts := []slack.MsgOption{slack.MsgOptionText(text, false)}
|
|
if thread != "" {
|
|
opts = append(opts, slack.MsgOptionTS(thread))
|
|
}
|
|
if _, _, err := b.api.PostMessageContext(ctx, channel, opts...); err != nil {
|
|
log.Printf("slack post: %v", err)
|
|
}
|
|
}
|
|
|
|
func (b *Bot) displayName(ctx context.Context, userID string) string {
|
|
if v, ok := b.names.Load(userID); ok {
|
|
return v.(string)
|
|
}
|
|
name := userID
|
|
if u, err := b.api.GetUserInfoContext(ctx, userID); err == nil {
|
|
for _, n := range []string{u.Profile.DisplayName, u.RealName, u.Name} {
|
|
if n != "" {
|
|
name = n
|
|
break
|
|
}
|
|
}
|
|
b.names.Store(userID, name)
|
|
}
|
|
return name
|
|
}
|
|
|
|
func title(text string) string {
|
|
line, _, _ := strings.Cut(strings.TrimSpace(text), "\n")
|
|
if utf8.RuneCountInString(line) > 60 {
|
|
r := []rune(line)
|
|
line = string(r[:60]) + "…"
|
|
}
|
|
if line == "" {
|
|
line = "untitled"
|
|
}
|
|
return line
|
|
}
|