From 25fcf85855d26966aaeb195d2a83ee3241da2131 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Sun, 1 Nov 2026 15:10:39 -0500 Subject: [PATCH] restart fix + slack commands --- AGENTS.md | 2 + ARCHITECTURE.md | 47 +++--- USERGUIDE.md | 2 + config.example.toml | 3 +- slackerbot/internal/bot/BUILD.bazel | 5 + slackerbot/internal/bot/agent.go | 102 ++++++++++++ slackerbot/internal/bot/agent_test.go | 83 ++++++++++ slackerbot/internal/bot/bot.go | 171 ++++++++++++++------- slackerbot/internal/bot/manage.go | 21 ++- slackerbot/internal/bot/update.go | 80 ++++++++++ slackerbot/internal/bot/update_test.go | 92 +++++++++++ slackerbot/internal/local/local.go | 66 ++++++-- slackerbot/internal/manager/BUILD.bazel | 1 + slackerbot/internal/manager/manager.go | 44 ++++++ slackerbot/internal/manager/update_test.go | 55 +++++++ slackerbot/internal/node/node.go | 2 + slackerbot/internal/testutil/testutil.go | 6 +- slackerbot/internal/tools/BUILD.bazel | 7 +- slackerbot/internal/tools/commands.go | 159 +++++++++++++++++++ slackerbot/internal/tools/commands_test.go | 94 +++++++++++ slackerbot/internal/tools/server.go | 149 ++++++++++++++++-- slackerbot/internal/tools/session.go | 5 +- 22 files changed, 1077 insertions(+), 119 deletions(-) create mode 100644 slackerbot/internal/bot/agent.go create mode 100644 slackerbot/internal/bot/agent_test.go create mode 100644 slackerbot/internal/bot/update.go create mode 100644 slackerbot/internal/bot/update_test.go create mode 100644 slackerbot/internal/manager/update_test.go create mode 100644 slackerbot/internal/tools/commands.go create mode 100644 slackerbot/internal/tools/commands_test.go diff --git a/AGENTS.md b/AGENTS.md index a0fe197..dcd2fdf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,7 @@ nix flake check --no-build ## Docs Update the matching doc in the same change: + - Slack commands, replies or permissions → USERGUIDE.md - Packages, data flow, protocol, placement → ARCHITECTURE.md - Config, Nix module options, setup steps → INSTALLATION.md and `config.example.toml` @@ -71,5 +72,6 @@ Designs that aren't built yet live in `plan/`. When you build one, move what it ## Git +- Before you start changing anything, ask the human whether to work in a git worktree (`git worktree add`). Wait for an answer, and don't create a worktree or edit files until you have one. - Don't commit, push or rewrite history unless asked. - Never commit build output (`result`, the `slackerbot` binary, `.direnv/`). diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 15a1791..644b080 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -33,7 +33,11 @@ flowchart LR **Files:** attachments on a message (`file_share`, or `files` on an `app_mention`) are downloaded by the bot and written into the home with the node's `write_file` call, then their paths are added to the prompt. The other way, goose gets the `slackerbot` MCP server (`slackerbot tools`, run inside the container as `slackerbot-tools`) through `mcpServers` in `session/new` and `session/load`. Its `send_file` tool only checks the path and size and answers "queued". The bot watches the ACP `tool_call` / `tool_call_update` messages, and when a `send_file` call completes it reads the file with `read_file` and uploads it to the thread with `files.uploadV2`. Bytes go node → master → Slack, so slaves never need a Slack token. Both calls work in 4 MB chunks, open files with `os.Root` so symlinks and `..` can't leave the home, and are capped by `max_send_file_mb`. An older slave answers "unknown call", which surfaces as `node.ErrTooOld` and a polite reply. -**Containers** are `systemd-nspawn` machines started through `systemd-run`, each in its own `slackerbot-.service` unit, with CPU and memory limits. They boot the NixOS system built by `nix build .#agent` (`nix/container.nix`), which runs `goose serve` on port 1984. `/nix/store` is shared read-only. `/root` is bind-mounted from a btrfs subvolume on one of the node's drives, and is the only thing that persists. The agent's env file (API keys, the per-container goose secret) is written by the host and bind-mounted in read-only. +**Commands as tools:** the same MCP server also lists one tool per Slack command (`tools.Commands`: `status`, `scale`, `keep`, `storage`, `move`, `listen`, `grant`, …), each with an optional `args` string that's appended to the verb, exactly as a user would type it. The MCP server can't reach the bot, so the call goes round: the bot spots the `tool_call` over ACP (`tools.CommandCall`), runs the command through the same code as Slack (`bot.runCommand`, `bot.move`) as the user whose message started the turn, and writes the reply with `write_file` to `/root/.slackerbot/replies/-.json`. The server waits up to a minute for a file with that prefix and claims it by deleting it. Commands that change something are also posted in the thread. `stop` waits for the turn to end (`agentCommands.Finish`), and `move` reports as soon as it starts, since its final pass waits for the reply to finish anyway. + +**Containers** are `systemd-nspawn` machines started through `systemd-run`, each in its own `slackerbot-.service` unit, with CPU and memory limits. They boot the NixOS system built by `nix build .#agent` (`nix/container.nix`), which runs `goose serve` on port 1984. `/nix/store` is shared read-only. `/root` is bind-mounted from a btrfs subvolume on one of the node's drives, and is the only thing that persists. `ro_binds` and `rw_binds` add more mounts, and their missing source folders are created first. The agent's env file (API keys, the per-container goose secret) is written by the host and bind-mounted in read-only. + +**System updates:** each node reports running containers whose `slackerbot-.service` command ends with a different system's `init` than the node's current `system` (`outdated` in `RunningMachine`, read over D-Bus; unknown counts as current). After a reply, the bot asks once per container (`bot/update.go`, remembered in memory) whether to restart. **Restart now** calls `manager.Restart` (power off, wait, then the normal start), unless a reply is in progress. **State** lives only on the master, in `state_dir/state.json`: owners (container, size, node, drive, storage limits, idle limit, lifecycle message), sessions (Slack thread → ACP session id, plus the thread's listen mode), each channel's default listen mode, and permissions. Nodes report what's running and how much disk each home uses. Nothing else is stored. @@ -84,26 +88,26 @@ flowchart TD `util` (`KeyedMutex`, `RandomHex`) is used by `bot`, `manager` and `local`, and left out of the diagram to keep it readable. -| Package | Role | -| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `main.go` | Flags, wiring, master vs slave | -| `model` | Shared types: `Size`, `Owner`, `Session`, permission rules, machine names | -| `units` | Parses and prints sizes (`5G`, `1.5 TB`) | -| `commands` | Parses Slack command text: `scale`, `keep`, `storage`, `grant`, and which permission a command needs | -| `config` | Loads and validates `config.toml` | -| `storage` | Drive math (`DriveReport`, headroom, overbooking) and btrfs subvolume/quota commands | -| `state` | `state.json` persistence | -| `node` | `Node` interface, wire types, offline/full errors, the WebSocket connection used to reach goose | -| `local` | The `Node` on this machine: nspawn via `systemd-run`, machined and systemd over D-Bus, capacity detection, btrfs send/receive | -| `balance` | Builds a view of each node and picks where a container goes | -| `manager` | `Manager` (owners, placement, start/stop/resize/storage), the cluster link (`Remote`, `ServeCluster`, `RunSlave`) and migration. These three share a package because they're tightly coupled | -| `tools` | The in-container MCP server (`send_file`), how goose starts it, and spotting its calls in ACP updates | -| `acp` | JSON-RPC client for goose's ACP endpoint. Tool permission requests are auto-approved | -| `bot` | Slack events and buttons, replies, permission checks, `grant`/`revoke`, move announcements, idle reaper, streaming replies, thread listen modes, stop reactions, the assistant pane | -| `util` | `KeyedMutex`, `RandomHex` | -| `testutil`, `testutil/clustertest` | Fake nodes, and a real master + slave harness for tests | -| `nix/container.nix` | NixOS system for each container | -| `nix/host.nix` | NixOS module: the service, the `ve-*` network, firewall | +| Package | Role | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `main.go` | Flags, wiring, master vs slave | +| `model` | Shared types: `Size`, `Owner`, `Session`, permission rules, machine names | +| `units` | Parses and prints sizes (`5G`, `1.5 TB`) | +| `commands` | Parses Slack command text: `scale`, `keep`, `storage`, `grant`, and which permission a command needs | +| `config` | Loads and validates `config.toml` | +| `storage` | Drive math (`DriveReport`, headroom, overbooking) and btrfs subvolume/quota commands | +| `state` | `state.json` persistence | +| `node` | `Node` interface, wire types, offline/full errors, the WebSocket connection used to reach goose | +| `local` | The `Node` on this machine: nspawn via `systemd-run`, machined and systemd over D-Bus, capacity detection, btrfs send/receive | +| `balance` | Builds a view of each node and picks where a container goes | +| `manager` | `Manager` (owners, placement, start/stop/resize/storage), the cluster link (`Remote`, `ServeCluster`, `RunSlave`) and migration. These three share a package because they're tightly coupled | +| `tools` | The in-container MCP server (`send_file` and the Slack commands), how goose starts it, spotting its calls in ACP updates, and the reply files commands are answered with | +| `acp` | JSON-RPC client for goose's ACP endpoint. Tool permission requests are auto-approved | +| `bot` | Slack events and buttons, replies, permission checks, `grant`/`revoke`, move announcements, idle reaper, streaming replies, thread listen modes, stop reactions, the assistant pane, commands the agent calls as tools | +| `util` | `KeyedMutex`, `RandomHex` | +| `testutil`, `testutil/clustertest` | Fake nodes, and a real master + slave harness for tests | +| `nix/container.nix` | NixOS system for each container | +| `nix/host.nix` | NixOS module: the service, the `ve-*` network, firewall | There are two builds. Nix (`flake.nix`) builds the deployable package, the container system and the host module. Bazel (`MODULE.bazel`, with `BUILD.bazel` files generated by Gazelle) builds and tests the Go code and packages the static binary as a tarball. Both take their dependencies from `slackerbot/go.mod`. @@ -111,6 +115,7 @@ There are two builds. Nix (`flake.nix`) builds the deployable package, the conta - The agent's tool calls are auto-approved, since the container is the sandbox (`acp.handleRequest`). - Anyone in a channel controls that channel's container and everything the agent can reach from it. +- The agent runs commands with the sender's permissions, so whatever steers it (a file, a web page) can scale, move or re-grant as far as that person may. The thread shows every change it makes. - Containers run as root without user namespaces, and container root can get around btrfs quotas. That's fine for fair sharing among trusted users, but not safe for hostile ones. - Restarting the master makes slaves look offline for a few seconds. Restarting the bot doesn't stop containers. - `state.json` is fine for hundreds of users. Swap the `state` package for SQLite if you need more. diff --git a/USERGUIDE.md b/USERGUIDE.md index fdb93c7..ed0d92b 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -26,6 +26,8 @@ In DMs the bot reads every message. You can also chat with it in Slack's assista 9. **Attach files** to your message (images, logs, zips) and they're copied into the container at `/root/slack-uploads//`, and the agent is told where. A message with files always goes to the agent, never to a command. Files over the server's size limit (default 100 MB) are left out, and the home folder's max size still applies. 10. **The agent can send you files.** It has a `send_file` tool, and the files land in the thread when its reply ends, where Slack previews images, video and PDFs. At most 5 files per reply, each under the server's size limit. +11. **The agent can run the commands below for you**, so you can ask it things like "give yourself 8G" or "keep running over the weekend". It runs them as you, the sender of the message it's answering, with your permissions. Anything that changes the container is posted in the thread as ":robot_face: The agent ran `scale 8G` for Alice", with the bot's answer. If the agent stops the container, it waits until the reply ends. Its moves get their own thread like any other move. It can do everything a command can except `help`, and it can't press buttons. +12. **System updates ask first.** When the server gets a new container system (new tools, for example) while yours is running, the bot asks once, under the agent's next reply, whether to restart onto it. **Restart now** restarts the container right away (it waits if the agent is still replying), and **Later** leaves it alone. Either way, `/root` and your sessions are kept, and the update also applies the next time the container starts for any reason. Other replies you may get: diff --git a/config.example.toml b/config.example.toml index 1f93806..1afa29b 100644 --- a/config.example.toml +++ b/config.example.toml @@ -11,7 +11,8 @@ boot_timeout = "2m" # Largest file the agent may send to Slack, and the largest attachment copied into a container. max_send_file_mb = 100 -# "{machine}" becomes the container name, e.g. c-c07abc123. +# Extra mounts as "host:container". "{machine}" becomes the container name, e.g. c-c07abc123. +# Missing host folders are created. ro_binds = [] rw_binds = [] diff --git a/slackerbot/internal/bot/BUILD.bazel b/slackerbot/internal/bot/BUILD.bazel index 20d87fc..dec4bb0 100644 --- a/slackerbot/internal/bot/BUILD.bazel +++ b/slackerbot/internal/bot/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "bot", srcs = [ + "agent.go", "assistant.go", "bot.go", "files.go", @@ -10,6 +11,7 @@ go_library( "listen.go", "manage.go", "stream.go", + "update.go", "version.go", ], importpath = "slackerbot/internal/bot", @@ -36,8 +38,10 @@ go_library( go_test( name = "bot_test", srcs = [ + "agent_test.go", "bot_test.go", "lifecycle_test.go", + "update_test.go", "version_test.go", ], embed = [":bot"], @@ -50,6 +54,7 @@ go_test( "//slackerbot/internal/state", "//slackerbot/internal/testutil", "//slackerbot/internal/testutil/clustertest", + "//slackerbot/internal/tools", "@com_github_slack_go_slack//:slack", "@com_github_slack_go_slack//slackevents", ], diff --git a/slackerbot/internal/bot/agent.go b/slackerbot/internal/bot/agent.go new file mode 100644 index 0000000..b85dad7 --- /dev/null +++ b/slackerbot/internal/bot/agent.go @@ -0,0 +1,102 @@ +package bot + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log" + "sync" + + "slackerbot/internal/model" + "slackerbot/internal/tools" +) + +const maxReplyBytes = 1 << 20 + +// agentCommands runs the Slack commands the agent calls as tools during one +// turn, as the person whose message started it. Only stop waits for the turn +// to end, because stopping the container would cut the reply off. +type agentCommands struct { + b *Bot + u *model.Owner + m incoming + root string + + wg sync.WaitGroup + mu sync.Mutex + stopWhen bool +} + +func (b *Bot) agentCommands(u *model.Owner, m incoming, root string) *agentCommands { + m.threadTS = root + return &agentCommands{b: b, u: u, m: m, root: root} +} + +// Update must not block: it's called from the ACP read loop. +func (a *agentCommands) Update(ctx context.Context, up acpUpdate) { + c, args, ok := tools.CommandCall(up) + if !ok || c.Missing(args) { + return + } + a.wg.Add(1) + go func() { + defer a.wg.Done() + reply, isErr := a.run(ctx, c, args) + a.answer(ctx, c, args, reply, isErr) + }() +} + +func (a *agentCommands) run(ctx context.Context, c tools.SlackCommand, args string) (string, bool) { + text := c.Text(args) + log.Printf("agent: %s ran %q for %s in %s/%s", a.u.Machine, text, a.m.user, a.m.channel, a.root) + if c.Name == tools.Stop { + return a.queueStop(), false + } + verb, rest := splitCommand(text) + var reply string + var ok bool + if c.Name == tools.Move { + reply, ok = a.b.move(ctx, a.m, rest, a.root, true) + } else { + reply, ok = a.b.runCommand(ctx, a.m, verb, rest, true) + } + if !ok { + return fmt.Sprintf("`%s` isn't something slackerbot understands. Check the tool's description for the args it takes.", text), true + } + if c.Changes(args) && c.Name != tools.Move { + a.b.say(ctx, a.m.channel, a.root, fmt.Sprintf(":robot_face: The agent ran `%s` for %s:\n%s", text, a.b.displayName(ctx, a.m.user), reply)) + } + return reply, false +} + +func (a *agentCommands) queueStop() string { + if a.b.store.Owner(a.u.ID) == nil { + return "There's no container to stop." + } + a.mu.Lock() + a.stopWhen = true + a.mu.Unlock() + return "The container will shut down as soon as this reply ends. Its files are kept, and the next message starts it again." +} + +func (a *agentCommands) answer(ctx context.Context, c tools.SlackCommand, args, text string, isErr bool) { + data, _ := json.Marshal(tools.Reply{Text: text, Error: isErr}) + p := tools.ReplyPath(a.b.cfg.AgentCwd, c.Name, args) + if err := a.b.mgr.WriteFile(ctx, a.u, p, maxReplyBytes, bytes.NewReader(data)); err != nil { + log.Printf("agent: answering %s in %s: %v", c.Name, a.u.Machine, err) + } +} + +// Finish waits for the turn's commands, then does what had to wait for the turn to end. +func (a *agentCommands) Finish(ctx context.Context) { + a.wg.Wait() + a.mu.Lock() + stop := a.stopWhen + a.mu.Unlock() + if !stop { + return + } + reply := a.b.stopFor(ctx, a.u.ID) + a.b.say(ctx, a.m.channel, a.root, fmt.Sprintf(":robot_face: The agent ran `stop` for %s:\n%s", a.b.displayName(ctx, a.m.user), reply)) +} diff --git a/slackerbot/internal/bot/agent_test.go b/slackerbot/internal/bot/agent_test.go new file mode 100644 index 0000000..b6cf0db --- /dev/null +++ b/slackerbot/internal/bot/agent_test.go @@ -0,0 +1,83 @@ +package bot + +import ( + "encoding/json" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/slack-go/slack" + + "slackerbot/internal/acp" + "slackerbot/internal/manager" + "slackerbot/internal/model" + "slackerbot/internal/testutil" + "slackerbot/internal/testutil/clustertest" + "slackerbot/internal/tools" +) + +func TestAgentRunsCommandsAsTheSender(t *testing.T) { + c := clustertest.New(t) + c.Cfg.AgentCwd = "/root" + c.Cfg.IdleTimeout = 6 * time.Hour + c.Cfg.MaxIdleTimeout = 7 * 24 * time.Hour + c.AddOwner(t, testutil.HomeData(), true) + c.Store.SeedPermissions(model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UBOSS": {"all"}}}) + log := &slackLog{text: map[string]string{}, under: map[string]string{}} + srv := httptest.NewServer(log) + defer srv.Close() + b := &Bot{cfg: c.Cfg, store: c.Store, mgr: c.Mgr, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/")), + acps: map[string]*acp.ACP{}, busy: map[string]int{}} + + call := func(user, tool, args string) (tools.Reply, bool) { + u := c.Store.Owner("C1") + cmds := b.agentCommands(u, incoming{user: user, channel: "C1", ts: "5.0"}, "5.0") + raw, _ := json.Marshal(map[string]string{"args": args}) + cmds.Update(c.Ctx, acp.Update{Kind: "tool_call", ToolID: "1", ToolName: "slackerbot__" + tool, RawInput: raw}) + cmds.Finish(c.Ctx) + c.Master.Mu.Lock() + defer c.Master.Mu.Unlock() + for key, data := range c.Master.Files { + if strings.HasPrefix(key, "/d/c-c1/.slackerbot/replies/") { + delete(c.Master.Files, key) + var r tools.Reply + _ = json.Unmarshal(data, &r) + return r, true + } + } + return tools.Reply{}, false + } + + if r, ok := call("UBOSS", "keep", "3h"); !ok || !strings.Contains(r.Text, "3h") { + t.Fatalf("keep: %+v %v", r, ok) + } + if got := c.Store.Owner("C1").IdleTimeout; got != 3*time.Hour { + t.Fatalf("keep wasn't saved: %s", got) + } + if r, _ := call("UANA", "scale", "up"); !strings.Contains(r.Text, ":lock:") { + t.Fatalf("the sender's permissions apply: %+v", r) + } + if r, _ := call("UANA", "status", ""); !strings.Contains(r.Text, "This channel's container") { + t.Fatalf("status: %+v", r) + } + if r, _ := call("UANA", "scale", "the tests"); !r.Error { + t.Fatalf("nonsense args should be an error: %+v", r) + } + if r, _ := call("UANA", "stop", ""); !strings.Contains(r.Text, "as soon as this reply ends") { + t.Fatalf("stop: %+v", r) + } + testutil.WaitFor(t, "container to stop", func() bool { return c.Mgr.State(c.Store.Owner("C1")) != manager.StateRunning }) + + log.mu.Lock() + defer log.mu.Unlock() + var seen []string + for _, ts := range log.order { + if log.under[ts] == "5.0" && strings.Contains(log.text[ts], "The agent ran") { + seen = append(seen, log.text[ts]) + } + } + if len(seen) != 3 || !strings.Contains(seen[0], "`keep 3h`") || !strings.Contains(seen[1], ":lock:") || !strings.Contains(seen[2], "`stop`") { + t.Fatalf("changes should be posted in the thread, got %q", seen) + } +} diff --git a/slackerbot/internal/bot/bot.go b/slackerbot/internal/bot/bot.go index 4c171eb..9dd91b9 100644 --- a/slackerbot/internal/bot/bot.go +++ b/slackerbot/internal/bot/bot.go @@ -72,6 +72,8 @@ type Bot struct { busy map[string]int names sync.Map + updateAsked sync.Map // machine → asked to restart onto a newer system + channels sync.Map // channel ID → *slack.Channel running sync.Map // "channel/ts" of a prompt's message or reply → its cancel func viewing sync.Map // assistant "channel/thread" → channel the user has open @@ -326,8 +328,10 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) { } st := NewStreamer(b.api, m.channel, root, ph) var sends tools.Calls + cmds := b.agentCommands(u, m, root) stop, err := acp.Prompt(pctx, id, text, func(up acpUpdate) { sends.Update(up) + cmds.Update(ctx, up) st.Update(up) }) if pctx.Err() != nil && ctx.Err() == nil { @@ -335,7 +339,9 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) { } st.Finish(stop, err) b.upload(ctx, u, m.channel, root, sends.Files()) + cmds.Finish(ctx) _ = b.store.Touch(u.ID) + b.offerUpdate(ctx, u, m.channel, root) switch { case stop == "cancelled": outcome = "" @@ -344,28 +350,51 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) { } } -func (b *Bot) command(ctx context.Context, m incoming) bool { - ownerID, kind := m.owner() - verb, args, _ := strings.Cut(strings.TrimSpace(m.text), " ") +func splitCommand(text string) (verb, args string) { + verb, args, _ = strings.Cut(strings.TrimSpace(text), " ") verb = strings.ToLower(strings.TrimRight(verb, ".!?")) args = strings.TrimRight(strings.TrimSpace(args), " .!?") + return verb, args +} +func (b *Bot) command(ctx context.Context, m incoming) bool { + verb, args := splitCommand(m.text) if verb == "move" || verb == "migrate" { return b.moveCommand(ctx, m, cmp.Or(m.threadTS, m.ts)) } + reply, ok := b.runCommand(ctx, m, verb, args, false) + if reply != "" { + b.say(ctx, m.channel, m.threadTS, reply) + } + return ok +} + +// runCommand returns ok=false when the text isn't a command, so it belongs to +// the agent. An empty reply means it was posted already, with buttons. The +// agent can't press buttons, so it gets plain text instead. +func (b *Bot) runCommand(ctx context.Context, m incoming, verb, args string, agent bool) (reply string, ok bool) { + ownerID, kind := m.owner() if verb == "grant" || verb == "revoke" || verb == "dm" || verb == "managers" || (verb == "permissions" && args != "") { return b.manageCommand(ctx, m, verb, args) } - if need := commands.NeededPermission(verb, args); need != "" && !b.permitted(ctx, m.channel, m.threadTS, m.user, kind, need) { - return true + if commands.IsListen(verb) { + c, err := commands.ParseListen(args) + if err != nil { + return "", false + } + return b.listenReply(m, c), true + } + if need := commands.NeededPermission(verb, args); need != "" { + if denied := b.denial(ctx, m.channel, m.user, kind, need); denied != "" { + return denied, true + } } if verb == "stop" || verb == "scale" || verb == "storage" || verb == "disk" { - if b.blocked(ctx, m.channel, m.threadTS, ownerID) { - return true + if reply, ok := b.blocked(ctx, m.channel, ownerID, agent); ok { + return reply, true } } - reply := "" switch { case verb == "" || verb == "help" && args == "": reply = helpText @@ -375,34 +404,27 @@ func (b *Bot) command(ctx context.Context, m incoming) bool { case verb == "sessions" && args == "": reply = b.sessionList(ctx, ownerID) case verb == "status" && args == "": + if agent { + return b.statusText(ctx, ownerID, kind), true + } b.postStatus(ctx, m.channel, m.threadTS, ownerID, kind) - return true + return "", true case verb == "stop" && args == "": reply = b.stopFor(ctx, ownerID) case verb == "scale": - var ok bool - if reply, ok = b.scaleFor(ctx, ownerID, kind, args); !ok { - return false - } + return b.scaleFor(ctx, ownerID, kind, args) case verb == "keep": - var ok bool - if reply, ok = b.keepFor(ownerID, kind, args); !ok { - return false - } + return b.keepFor(ownerID, kind, args) case verb == "nodes" && args == "": reply = b.nodesList(ctx, ownerID) case verb == "permissions" && args == "": reply = b.permissionsList(ctx, m.channel, kind) case verb == "storage" || verb == "disk": - var ok bool - if reply, ok = b.storageFor(ctx, ownerID, kind, args); !ok { - return false - } + return b.storageFor(ctx, ownerID, kind, args) default: - return false + return "", false } - b.say(ctx, m.channel, m.threadTS, reply) - return true + return reply, true } func (b *Bot) onAction(ctx context.Context, cb slack.InteractionCallback) { @@ -446,6 +468,11 @@ func (b *Bot) onAction(ctx context.Context, cb slack.InteractionCallback) { b.waiting[u.Node][cb.Channel.ID] = true b.waitMu.Unlock() b.say(ctx, cb.Channel.ID, thread, fmt.Sprintf("OK, waiting for %s. I'll post here when it's back up.", u.Node)) + case "restart": + b.restartFor(ctx, cb, ownerID, kind) + case "later": + b.settlePrompt(ctx, cb, "OK, not now. The update applies the next time the container starts, "+ + "for example after `stop` and a new message.") case "stop", "scale": if b.mgr.State(u) == manager.StateOffline { b.deadSlave(ctx, cb.Channel.ID, "", ownerID, u.Node) @@ -479,7 +506,7 @@ func (b *Bot) sessionList(ctx context.Context, ownerID string) string { return sb.String() } -func (b *Bot) postStatus(ctx context.Context, channel, thread, ownerID string, kind model.OwnerKind) { +func (b *Bot) statusText(ctx context.Context, ownerID string, kind model.OwnerKind) string { u := b.store.Owner(ownerID) state := b.mgr.State(u) size := b.cfg.Container @@ -512,8 +539,17 @@ func (b *Bot) postStatus(ctx context.Context, channel, thread, ownerID string, k for _, name := range b.mgr.OfflineNodes() { fmt.Fprintf(&sb, "\n• %s: offline", name) } + return sb.String() +} - text := sb.String() +func (b *Bot) postStatus(ctx context.Context, channel, thread, ownerID string, kind model.OwnerKind) { + u := b.store.Owner(ownerID) + state := b.mgr.State(u) + size := b.cfg.Container + if u != nil { + size = u.Size + } + text := b.statusText(ctx, ownerID, kind) blocks := []slack.Block{slack.NewSectionBlock(slack.NewTextBlockObject(slack.MarkdownType, text, false, false), nil, nil)} var buttons []slack.BlockElement @@ -712,26 +748,34 @@ func (b *Bot) moveTarget(args string) (target string, ok bool) { } func (b *Bot) permitted(ctx context.Context, channel, thread, user string, kind model.OwnerKind, a model.Action) bool { - p := b.store.Permissions() - if p.Allowed(kind, channel, user, a) { + text := b.denial(ctx, channel, user, kind, a) + if text == "" { return true } - var text string - if kind == model.OwnerUser { - text = fmt.Sprintf(":lock: Changing *%s* from DMs is turned off on this server.", a) - } else if holders := p.Holders(channel, a); len(holders) == 0 { - text = fmt.Sprintf(":lock: Nobody here has *%s* permission. An admin can grant it in the config.", a) - } else { - names := make([]string, len(holders)) - for i, h := range holders { - names[i] = b.displayName(ctx, h) - } - text = fmt.Sprintf(":lock: Only super users with *%s* permission can do that here: %s.", a, strings.Join(names, ", ")) - } b.say(ctx, channel, thread, text) return false } +// denial explains why the user may not do a, or is empty when they may. +func (b *Bot) denial(ctx context.Context, channel, user string, kind model.OwnerKind, a model.Action) string { + p := b.store.Permissions() + if p.Allowed(kind, channel, user, a) { + return "" + } + if kind == model.OwnerUser { + return fmt.Sprintf(":lock: Changing *%s* from DMs is turned off on this server.", a) + } + holders := p.Holders(channel, a) + if len(holders) == 0 { + return fmt.Sprintf(":lock: Nobody here has *%s* permission. An admin can grant it in the config.", a) + } + names := make([]string, len(holders)) + for i, h := range holders { + names[i] = b.displayName(ctx, h) + } + return fmt.Sprintf(":lock: Only super users with *%s* permission can do that here: %s.", a, strings.Join(names, ", ")) +} + func (b *Bot) permissionsList(ctx context.Context, channel string, kind model.OwnerKind) string { p := b.store.Permissions() if kind == model.OwnerUser { @@ -775,37 +819,54 @@ func (b *Bot) permissionsList(ctx context.Context, channel string, kind model.Ow return strings.Join(lines, "\n") } -func (b *Bot) blocked(ctx context.Context, channel, thread, ownerID string) bool { +// blocked explains why the container can't be changed right now. In Slack a +// dead node gets its recreate/wait buttons instead, and the reply is empty. +func (b *Bot) blocked(ctx context.Context, channel, ownerID string, agent bool) (reply string, ok bool) { if b.mgr.IsMigrating(ownerID) { - b.say(ctx, channel, thread, b.explain(manager.ErrMigrating)) - return true + return b.explain(manager.ErrMigrating), true } if u := b.store.Owner(ownerID); b.mgr.State(u) == manager.StateOffline { + if agent { + return fmt.Sprintf("Node %s is offline right now.", u.Node), true + } b.deadSlave(ctx, channel, "", ownerID, u.Node) - return true + return "", true } - return false + return "", false } func (b *Bot) moveCommand(ctx context.Context, m incoming, replyThread string) bool { - verb, args, _ := strings.Cut(strings.TrimSpace(m.text), " ") - verb = strings.ToLower(strings.TrimRight(verb, ".!?")) + verb, args := splitCommand(m.text) if verb != "move" && verb != "migrate" { return false } - target, ok := b.moveTarget(strings.TrimRight(strings.TrimSpace(args), " .!?")) + reply, ok := b.move(ctx, m, args, replyThread, false) + if reply != "" { + b.say(ctx, m.channel, replyThread, reply) + } + return ok +} + +func (b *Bot) move(ctx context.Context, m incoming, args, replyThread string, agent bool) (reply string, ok bool) { + target, ok := b.moveTarget(args) if !ok { - return false + return "", false } ownerID, kind := m.owner() - if !b.permitted(ctx, m.channel, replyThread, m.user, kind, model.ActMove) { - return true + if denied := b.denial(ctx, m.channel, m.user, kind, model.ActMove); denied != "" { + return denied, true } - if b.blocked(ctx, m.channel, replyThread, ownerID) { - return true + if reply, ok := b.blocked(ctx, m.channel, ownerID, agent); ok { + return reply, true } - go b.moveFor(ctx, m.channel, ownerID, target, "<@"+m.user+">", replyThread) - return true + trigger := "<@" + m.user + ">" + if !agent { + go b.moveFor(ctx, m.channel, ownerID, target, trigger, replyThread) + return "", true + } + go b.moveFor(ctx, m.channel, ownerID, target, "the agent, for "+trigger, replyThread) + return "The move has started, and its progress is posted in Slack. The container is stopped briefly near the end, " + + "so finish your reply now. The user can carry on in this thread once it's done.", true } func (b *Bot) moveFor(ctx context.Context, channel, ownerID, target, trigger, replyThread string) { diff --git a/slackerbot/internal/bot/manage.go b/slackerbot/internal/bot/manage.go index bd33bff..8dbacce 100644 --- a/slackerbot/internal/bot/manage.go +++ b/slackerbot/internal/bot/manage.go @@ -32,14 +32,11 @@ func actionList(actions []model.Action) string { // manageCommand handles permission changes. It returns false if the message // isn't one of these commands, so it goes to the agent. -func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) bool { +func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) (string, bool) { _, kind := m.owner() p := b.store.Permissions() - reply := func(text string) bool { - b.say(ctx, m.channel, m.threadTS, text) - return true - } - denied := func(global bool) bool { + reply := func(text string) (string, bool) { return text, true } + denied := func(global bool) (string, bool) { who := "user managers" if !global { who = "user managers and people with *manage* permission here" @@ -58,7 +55,7 @@ func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) case "grant", "revoke": g, err := commands.ParseGrant(args) if errors.Is(err, commands.ErrNotCommand) { - return false + return "", false } if err != nil { return reply(":warning: " + err.Error() + ". Try `grant @alice move scale` or `revoke @alice`.") @@ -119,11 +116,11 @@ func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) case "dm": words := strings.Fields(strings.ToLower(args)) if len(words) != 2 || !slices.Contains(model.AllActions, model.Action(words[0])) { - return false + return "", false } on, ok := commands.ParseToggle(words[1]) if !ok { - return false + return "", false } if !p.IsManager(m.user) { return denied(true) @@ -145,7 +142,7 @@ func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) op := strings.ToLower(words[0]) g, err := commands.ParseGrant(strings.Join(words[1:], " ")) if (op != "add" && op != "remove") || err != nil || len(g.Actions) > 0 { - return false + return "", false } if !p.IsManager(m.user) { return denied(true) @@ -171,7 +168,7 @@ func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) case "permissions": if strings.ToLower(args) != "reset" { - return false + return "", false } if kind == model.OwnerUser { return reply("DMs don't have a super-user list to reset.") @@ -184,5 +181,5 @@ func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) } return reply("Done. This channel uses the default super-user list again.") } - return false + return "", false } diff --git a/slackerbot/internal/bot/update.go b/slackerbot/internal/bot/update.go new file mode 100644 index 0000000..11de640 --- /dev/null +++ b/slackerbot/internal/bot/update.go @@ -0,0 +1,80 @@ +package bot + +import ( + "context" + "log" + + "github.com/slack-go/slack" + + "slackerbot/internal/manager" + "slackerbot/internal/model" +) + +const updateText = ":arrows_counterclockwise: A system update is waiting for this container. " + + "Restarting applies it. Files in /root and your sessions are kept." + +// offerUpdate asks once per container whether to restart onto the node's +// current system. Asking again needs a newer system or a bot restart. +func (b *Bot) offerUpdate(ctx context.Context, u *model.Owner, channel, thread string) { + if !b.mgr.Outdated(ctx, u) { + b.updateAsked.Delete(u.Machine) + return + } + if _, asked := b.updateAsked.LoadOrStore(u.Machine, true); asked { + return + } + restart := slack.NewButtonBlockElement("restart", u.ID, slack.NewTextBlockObject(slack.PlainTextType, "Restart now", false, false)) + restart.Style = slack.StylePrimary + later := slack.NewButtonBlockElement("later", u.ID, slack.NewTextBlockObject(slack.PlainTextType, "Later", false, false)) + blocks := []slack.Block{ + slack.NewSectionBlock(slack.NewTextBlockObject(slack.MarkdownType, updateText, false, false), nil, nil), + slack.NewActionBlock("update", restart, later), + } + if _, _, err := b.api.PostMessageContext(ctx, channel, slack.MsgOptionText(updateText, false), + slack.MsgOptionBlocks(blocks...), slack.MsgOptionTS(thread)); err != nil { + log.Printf("slack post: %v", err) + } +} + +// settlePrompt replaces the update prompt, so its buttons can't be pressed twice. +func (b *Bot) settlePrompt(ctx context.Context, cb slack.InteractionCallback, text string) { + block := slack.NewSectionBlock(slack.NewTextBlockObject(slack.MarkdownType, text, false, false), nil, nil) + if _, _, _, err := b.api.UpdateMessageContext(ctx, cb.Channel.ID, cb.Message.Timestamp, + slack.MsgOptionText(text, false), slack.MsgOptionBlocks(block)); err != nil { + log.Printf("slack update: %v", err) + } +} + +func (b *Bot) restartFor(ctx context.Context, cb slack.InteractionCallback, ownerID string, kind model.OwnerKind) { + channel, thread := cb.Channel.ID, cb.Message.ThreadTimestamp + u := b.store.Owner(ownerID) + switch b.mgr.State(u) { + case manager.StateOffline: + b.deadSlave(ctx, channel, "", ownerID, u.Node) + return + case manager.StateRunning: + default: + b.settlePrompt(ctx, cb, ":white_check_mark: The container isn't running, so it gets the update when it next starts.") + return + } + if b.isBusy(u.Machine) { + b.say(ctx, channel, thread, "The agent is still replying in this container. Press *Restart now* again once it's done.") + return + } + b.settlePrompt(ctx, cb, ":arrows_counterclockwise: Restarting the container to apply the update…") + b.currentLifecycle(ctx, u).post(ctx, ":octagonal_sign: Shutting down to apply a system update, asked by <@"+cb.User.ID+">.") + b.dropClient(u) + + sl := &startLog{b: b, m: incoming{user: cb.User.ID, channel: channel}, root: thread, ownerID: ownerID} + u, err := b.mgr.Restart(ctx, ownerID, kind, func(ev manager.StartEvent) { sl.event(ctx, ev) }) + if err == nil { + _, err = b.client(ctx, u) + } + if err != nil { + sl.failed(ctx, b.explain(err)) + b.say(ctx, channel, thread, ":warning: The restart didn't finish: "+b.explain(err)) + return + } + sl.up(ctx) + b.say(ctx, channel, thread, ":white_check_mark: Restarted with the update. Carry on in any thread.") +} diff --git a/slackerbot/internal/bot/update_test.go b/slackerbot/internal/bot/update_test.go new file mode 100644 index 0000000..c33d2ac --- /dev/null +++ b/slackerbot/internal/bot/update_test.go @@ -0,0 +1,92 @@ +package bot + +import ( + "net/http/httptest" + "strings" + "testing" + + "github.com/slack-go/slack" + + "slackerbot/internal/acp" + "slackerbot/internal/model" + "slackerbot/internal/testutil" + "slackerbot/internal/testutil/clustertest" +) + +func updateBot(t *testing.T) (*Bot, *clustertest.Cluster, *slackLog) { + c := clustertest.New(t) + c.AddOwner(t, testutil.HomeData(), true) + log := &slackLog{text: map[string]string{}, under: map[string]string{}} + srv := httptest.NewServer(log) + t.Cleanup(srv.Close) + b := &Bot{cfg: c.Cfg, store: c.Store, mgr: c.Mgr, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/")), + acps: map[string]*acp.ACP{}, busy: map[string]int{}} + return b, c, log +} + +func setOutdated(c *clustertest.Cluster, outdated bool) { + c.Master.Mu.Lock() + defer c.Master.Mu.Unlock() + c.Master.Outdated["c-c1"] = outdated +} + +func (s *slackLog) prompts() int { + s.mu.Lock() + defer s.mu.Unlock() + n := 0 + for _, text := range s.text { + if strings.Contains(text, "A system update is waiting") { + n++ + } + } + return n +} + +func TestUpdatePromptOncePerSystem(t *testing.T) { + b, c, log := updateBot(t) + u := c.Store.Owner("C1") + + b.offerUpdate(c.Ctx, u, "C1", "100.0") + if log.prompts() != 0 { + t.Fatal("no prompt for a container on the current system") + } + setOutdated(c, true) + b.offerUpdate(c.Ctx, u, "C1", "100.0") + b.offerUpdate(c.Ctx, u, "C1", "200.0") + if log.prompts() != 1 { + t.Fatalf("want one prompt for one pending update, got %d", log.prompts()) + } + setOutdated(c, false) + b.offerUpdate(c.Ctx, u, "C1", "100.0") + setOutdated(c, true) + b.offerUpdate(c.Ctx, u, "C1", "100.0") + if log.prompts() != 2 { + t.Fatalf("a newer update should be offered again, got %d prompts", log.prompts()) + } +} + +func TestRestartWaitsForReply(t *testing.T) { + b, c, log := updateBot(t) + setOutdated(c, true) + b.setBusy("c-c1", +1) + + cb := slack.InteractionCallback{Channel: slack.Channel{GroupConversation: slack.GroupConversation{ + Conversation: slack.Conversation{ID: "C1"}}}} + cb.Message.Timestamp, cb.Message.ThreadTimestamp = "5.0", "100.0" + b.restartFor(c.Ctx, cb, "C1", model.OwnerChannel) + + c.Master.Mu.Lock() + still := c.Master.Outdated["c-c1"] + c.Master.Mu.Unlock() + if !still { + t.Fatal("a busy container must not be restarted") + } + log.mu.Lock() + defer log.mu.Unlock() + for _, text := range log.text { + if strings.Contains(text, "still replying") { + return + } + } + t.Fatal("the user should be told to try again once the reply is done") +} diff --git a/slackerbot/internal/local/local.go b/slackerbot/internal/local/local.go index 84bdf16..1855410 100644 --- a/slackerbot/internal/local/local.go +++ b/slackerbot/internal/local/local.go @@ -29,6 +29,9 @@ const ( machinedName = "org.freedesktop.machine1" machinedPath = "/org/freedesktop/machine1" machinedMgr = "org.freedesktop.machine1.Manager" + systemdName = "org.freedesktop.systemd1" + systemdPath = "/org/freedesktop/systemd1" + systemdMgr = "org.freedesktop.systemd1.Manager" sigPoweroff = 38 // SIGRTMIN+4, what `machinectl poweroff` sends agentEnvInside = "/var/lib/slackerbot/agent.env" @@ -114,7 +117,7 @@ func (l *Local) Running(ctx context.Context) ([]node.RunningMachine, error) { if !model.IsOurs(x.Name) { continue } - r := node.RunningMachine{Name: x.Name} + r := node.RunningMachine{Name: x.Name, Outdated: l.outdated(x.Name)} if v, err := l.bus.Object(machinedName, x.Path).GetProperty("org.freedesktop.machine1.Machine.Timestamp"); err == nil { if us, ok := v.Value().(uint64); ok { r.Since = time.UnixMicro(int64(us)) @@ -125,6 +128,40 @@ func (l *Local) Running(ctx context.Context) ([]node.RunningMachine, error) { return out, nil } +// The unit's command ends with the init of the system the container booted. +// When it can't be read, the container counts as up to date. +func (l *Local) outdated(machine string) bool { + args := l.execStart(unitName(machine)) + if len(args) == 0 { + return false + } + init := args[len(args)-1] + return filepath.Base(init) == "init" && filepath.Dir(init) != l.system +} + +func (l *Local) execStart(unit string) []string { + var path dbus.ObjectPath + if err := l.bus.Object(systemdName, systemdPath).Call(systemdMgr+".GetUnit", 0, unit).Store(&path); err != nil { + return nil + } + v, err := l.bus.Object(systemdName, path).GetProperty(systemdName + ".Service.ExecStart") + if err != nil { + return nil + } + var cmds []struct { + Path string + Args []string + IgnoreErrors bool + StartTime, StartTimeMono, ExitTime, ExitTimeMono uint64 + PID uint32 + Code, Status int32 + } + if err := v.Store(&cmds); err != nil || len(cmds) == 0 { + return nil + } + return cmds[0].Args +} + func (l *Local) IsRunning(machine string) bool { var p dbus.ObjectPath return l.machined().Call(machinedMgr+".GetMachine", 0, machine).Store(&p) == nil @@ -201,16 +238,25 @@ func (l *Local) Start(ctx context.Context, spec node.StartSpec) error { "--bind-ro=" + envFile + ":" + agentEnvInside, "--bind=" + home + ":/root", } - for _, b := range l.cfg.ROBinds { - args = append(args, "--bind-ro="+strings.ReplaceAll(b, "{machine}", spec.Machine)) - } - for _, b := range l.cfg.RWBinds { - b = strings.ReplaceAll(b, "{machine}", spec.Machine) - src, _, _ := strings.Cut(b, ":") - if err := os.MkdirAll(src, 0o755); err != nil { - return err + binds := func(list []string, flag string) error { + for _, b := range list { + b = strings.ReplaceAll(b, "{machine}", spec.Machine) + src, _, _ := strings.Cut(b, ":") + // Missing sources make systemd-nspawn fail; existing ones may be files. + if _, err := os.Stat(src); errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(src, 0o755); err != nil { + return err + } + } + args = append(args, flag+b) } - args = append(args, "--bind="+b) + return nil + } + if err := binds(l.cfg.ROBinds, "--bind-ro="); err != nil { + return err + } + if err := binds(l.cfg.RWBinds, "--bind="); err != nil { + return err } args = append(args, filepath.Join(l.system, "init")) diff --git a/slackerbot/internal/manager/BUILD.bazel b/slackerbot/internal/manager/BUILD.bazel index f8ff289..d6959e3 100644 --- a/slackerbot/internal/manager/BUILD.bazel +++ b/slackerbot/internal/manager/BUILD.bazel @@ -28,6 +28,7 @@ go_test( "cluster_test.go", "files_test.go", "migrate_test.go", + "update_test.go", ], embed = [":manager"], deps = [ diff --git a/slackerbot/internal/manager/manager.go b/slackerbot/internal/manager/manager.go index c595561..eab25b8 100644 --- a/slackerbot/internal/manager/manager.go +++ b/slackerbot/internal/manager/manager.go @@ -248,6 +248,50 @@ func (m *Manager) EnsureProgress(ctx context.Context, id string, kind model.Owne return u, nil } +// Restart stops a running container and starts it again, so it boots the +// node's current system. +func (m *Manager) Restart(ctx context.Context, id string, kind model.OwnerKind, progress func(StartEvent)) (*model.Owner, error) { + if m.IsMigrating(id) { + return nil, ErrMigrating + } + unlock := m.ownerMu.Lock(id) + u, err := m.ownerRecord(id, kind) + if err == nil && u.Node != "" { + var n node.Node + if n, err = m.nodeByName(u.Node); err == nil && n.IsRunning(u.Machine) { + log.Printf("container: restarting %s for %s %s on %s", u.Machine, u.Kind, u.ID, u.Node) + if err = n.Poweroff(ctx, u.Machine); err == nil { + err = n.WaitStopped(ctx, u.Machine) + } + } + } + unlock() + if err != nil { + return u, err + } + return m.EnsureProgress(ctx, id, kind, progress) +} + +func (m *Manager) Outdated(ctx context.Context, u *model.Owner) bool { + if u == nil || u.Node == "" { + return false + } + n, err := m.nodeByName(u.Node) + if err != nil { + return false + } + running, err := n.Running(ctx) + if err != nil { + return false + } + for _, r := range running { + if r.Name == u.Machine { + return r.Outdated + } + } + return false +} + // The old home folder stays on its node, untouched. func (m *Manager) Recreate(id string) error { if m.IsMigrating(id) { diff --git a/slackerbot/internal/manager/update_test.go b/slackerbot/internal/manager/update_test.go new file mode 100644 index 0000000..cb69e88 --- /dev/null +++ b/slackerbot/internal/manager/update_test.go @@ -0,0 +1,55 @@ +package manager_test + +import ( + "testing" + + "slackerbot/internal/model" + "slackerbot/internal/testutil" + "slackerbot/internal/testutil/clustertest" +) + +func TestRestartAppliesUpdate(t *testing.T) { + c := clustertest.New(t) + c.AddOwner(t, testutil.HomeData(), true) + u := c.Store.Owner("C1") + if c.Mgr.Outdated(c.Ctx, u) { + t.Fatal("a container on the current system isn't outdated") + } + + c.Master.Mu.Lock() + c.Master.Outdated["c-c1"] = true + c.Master.Mu.Unlock() + if !c.Mgr.Outdated(c.Ctx, u) { + t.Fatal("the node said the container is outdated") + } + + if _, err := c.Mgr.Restart(c.Ctx, "C1", model.OwnerChannel, nil); err != nil { + t.Fatal(err) + } + if !c.Master.IsRunning("c-c1") || c.Mgr.Outdated(c.Ctx, u) { + t.Fatal("after a restart the container should run the current system") + } + if c.Master.Last.Machine != "c-c1" { + t.Fatal("the container wasn't started again") + } +} + +func TestOutdatedThroughSlave(t *testing.T) { + c := clustertest.New(t) + u := &model.Owner{ID: "C2", Kind: model.OwnerChannel, Machine: "c-c2", Node: "slave-1", Drive: "/d"} + c.Slave.Mu.Lock() + c.Slave.Runs["c-c2"], c.Slave.Outdated["c-c2"] = true, true + c.Slave.Mu.Unlock() + testutil.WaitFor(t, "slave to report the outdated container", func() bool { return c.Mgr.Outdated(c.Ctx, u) }) +} + +func TestRestartStoppedContainerJustStarts(t *testing.T) { + c := clustertest.New(t) + c.AddOwner(t, testutil.HomeData(), false) + if _, err := c.Mgr.Restart(c.Ctx, "C1", model.OwnerChannel, nil); err != nil { + t.Fatal(err) + } + if !c.Master.IsRunning("c-c1") { + t.Fatal("a stopped container should just start") + } +} diff --git a/slackerbot/internal/node/node.go b/slackerbot/internal/node/node.go index 32ba0d3..7081810 100644 --- a/slackerbot/internal/node/node.go +++ b/slackerbot/internal/node/node.go @@ -61,6 +61,8 @@ type DriveInfo struct { type RunningMachine struct { Name string `json:"name"` Since time.Time `json:"since"` + // Booted an older system than the node has now. + Outdated bool `json:"outdated,omitempty"` } type StartSpec struct { diff --git a/slackerbot/internal/testutil/testutil.go b/slackerbot/internal/testutil/testutil.go index 27904a0..e5cd1e3 100644 --- a/slackerbot/internal/testutil/testutil.go +++ b/slackerbot/internal/testutil/testutil.go @@ -19,6 +19,7 @@ type FakeNode struct { name string Mu sync.Mutex Runs map[string]bool + Outdated map[string]bool // machines that booted an older system; cleared on Start Last node.StartSpec Homes map[string][]byte Files map[string][]byte @@ -31,7 +32,7 @@ type FakeNode struct { } func NewFakeNode(name string) *FakeNode { - return &FakeNode{name: name, Runs: map[string]bool{}, Homes: map[string][]byte{}, Files: map[string][]byte{}, + return &FakeNode{name: name, Runs: map[string]bool{}, Outdated: map[string]bool{}, Homes: map[string][]byte{}, Files: map[string][]byte{}, Exports: map[string]*fakeExport{}, Imports: map[string]*fakeImport{}, Usable: model.Size{MemoryMB: 8192, CPUs: 4}} } @@ -52,7 +53,7 @@ func (f *FakeNode) Running(context.Context) ([]node.RunningMachine, error) { defer f.Mu.Unlock() var out []node.RunningMachine for m := range f.Runs { - out = append(out, node.RunningMachine{Name: m}) + out = append(out, node.RunningMachine{Name: m, Outdated: f.Outdated[m]}) } return out, nil } @@ -65,6 +66,7 @@ func (f *FakeNode) Start(_ context.Context, spec node.StartSpec) error { f.Mu.Lock() defer f.Mu.Unlock() f.Last, f.Runs[spec.Machine] = spec, true + delete(f.Outdated, spec.Machine) return nil } func (f *FakeNode) Poweroff(_ context.Context, machine string) error { diff --git a/slackerbot/internal/tools/BUILD.bazel b/slackerbot/internal/tools/BUILD.bazel index f1b820b..a14adcf 100644 --- a/slackerbot/internal/tools/BUILD.bazel +++ b/slackerbot/internal/tools/BUILD.bazel @@ -3,17 +3,22 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "tools", srcs = [ + "commands.go", "server.go", "session.go", ], importpath = "slackerbot/internal/tools", visibility = ["//slackerbot:__subpackages__"], - deps = ["//slackerbot/internal/acp"], + deps = [ + "//slackerbot/internal/acp", + "//slackerbot/internal/util", + ], ) go_test( name = "tools_test", srcs = [ + "commands_test.go", "server_test.go", "session_test.go", ], diff --git a/slackerbot/internal/tools/commands.go b/slackerbot/internal/tools/commands.go new file mode 100644 index 0000000..916c19d --- /dev/null +++ b/slackerbot/internal/tools/commands.go @@ -0,0 +1,159 @@ +package tools + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "path/filepath" + "strings" + + "slackerbot/internal/acp" + "slackerbot/internal/util" +) + +// SlackCommand is a Slack command offered to the agent as a tool. Its text is Verb +// followed by the call's args, exactly as a user would type it. +type SlackCommand struct { + Name string + Verb string + Help string + // Describes the args parameter. Empty when the command takes none. + Args string + Required bool + // With no args it only shows a setting. + Shows bool +} + +const ( + Stop = "stop" + Move = "move" +) + +var Commands = []SlackCommand{ + {Name: "status", Verb: "status", Shows: true, + Help: "Show this container's state, size, node, time unused, idle limit, home folder usage and every node."}, + {Name: "sessions", Verb: "sessions", Shows: true, + Help: "List this container's recent sessions (Slack threads), with links."}, + {Name: "nodes", Verb: "nodes", Shows: true, + Help: "List every node with the CPU, RAM and storage free on it, and where this container runs."}, + {Name: "permissions", Verb: "permissions", Shows: true, + Help: "Show who may scale, change storage, move or set the idle limit here, and who the user managers are."}, + {Name: Stop, Verb: "stop", + Help: "Shut this container down once your current reply is finished. The home folder is kept, and the next message starts it again."}, + {Name: "scale", Verb: "scale", Required: true, + Help: "Change this container's RAM and CPU. A running container changes right away; shrinking memory may stop programs.", + Args: "`up` or `down` to double or halve, an exact size like `8G 4cpu`, or a change like `+2G` or `-1cpu`"}, + {Name: "keep", Verb: "keep", Shows: true, + Help: "Show or set how long this container may sit unused before it's stopped.", + Args: "A time like `24h`, `3d` or `2w`, `indefinite` to never stop it, or `default` to reset. Empty shows the current limit"}, + {Name: "storage", Verb: "storage", Shows: true, + Help: "Show the home folder's usage, or change its guaranteed space and max size.", + Args: "A new max like `20G`, a change like `+5G`, or `min 8G` / `min 8G max 20G`. Empty shows usage"}, + {Name: Move, Verb: "move", + Help: "Move this container to another node. Progress is posted in Slack. Near the end the container is briefly stopped, so finish your reply soon after calling this.", + Args: "A node name from `nodes`, or `auto` (or empty) to pick the best one"}, + {Name: "listen", Verb: "listen", Shows: true, + Help: "Show or change which replies in a channel thread reach you. DMs always read every message.", + Args: "`all-messages` to read every reply in this thread, `only-mentions` for replies that @mention the bot. Put `channel` first to set the channel's default. Empty shows the setting"}, + {Name: "grant", Verb: "grant", Required: true, + Help: "Give people permissions in this channel. People are Slack mentions like <@U012AB>, as they appear in messages.", + Args: "Mentions then permissions (scale, storage, move, keep, manage or all), like `<@U012AB> move scale`. Start with `default` to change the server-wide list"}, + {Name: "revoke", Verb: "revoke", Required: true, + Help: "Take permissions away in this channel. People are Slack mentions like <@U012AB>.", + Args: "Mentions, then optionally permissions, like `<@U012AB> move`. Only mentions removes everything. Start with `default` for the server-wide list"}, + {Name: "reset_permissions", Verb: "permissions reset", + Help: "Make this channel use the server-wide default list of super users again."}, + {Name: "dm_permission", Verb: "dm", Required: true, + Help: "Allow or forbid everyone changing a setting on their own DM container. User managers only.", + Args: "A permission and on or off, like `storage off`"}, + {Name: "managers", Verb: "managers", Shows: true, + Help: "Show, add or remove user managers.", + Args: "`add <@U012AB>` or `remove <@U012AB>`. Empty lists them"}, +} + +func commandNamed(name string) (SlackCommand, bool) { + for _, c := range Commands { + if c.Name == name { + return c, true + } + } + return SlackCommand{}, false +} + +func (c SlackCommand) Text(args string) string { + return strings.TrimSpace(c.Verb + " " + strings.TrimSpace(args)) +} + +// Missing calls are refused by the server, so the bot doesn't run them. +func (c SlackCommand) Missing(args string) bool { + return c.Required && strings.TrimSpace(args) == "" +} + +func (c SlackCommand) Changes(args string) bool { + return !c.Shows || strings.TrimSpace(args) != "" +} + +// Reply is what the bot answers a command call with. +type Reply struct { + Text string `json:"text"` + Error bool `json:"error,omitempty"` +} + +// The MCP server can't reach the bot, so the bot runs the command when it +// sees the call over ACP and drops the reply here for the server to pick up. +// The server only knows the tool and its args, so they name the file. +func replyDir(home string) string { return filepath.Join(home, ".slackerbot", "replies") } + +func replyPrefix(name, args string) string { + sum := sha256.Sum256([]byte(name + "\x00" + strings.TrimSpace(args))) + return hex.EncodeToString(sum[:8]) + "-" +} + +func ReplyPath(home, name, args string) string { + return filepath.Join(replyDir(home), replyPrefix(name, args)+util.RandomHex(6)+".json") +} + +func argsOf(raw json.RawMessage) string { + var in struct { + Args string `json:"args"` + } + _ = json.Unmarshal(raw, &in) + return in.Args +} + +// toolName is the slackerbot tool an ACP tool call is for. Goose names it in +// _meta, and older versions only in the title ("slackerbot: send file · …"). +func toolName(u acp.Update) (string, bool) { + if name, ok := strings.CutPrefix(u.ToolName, ServerName+"__"); ok { + return name, true + } + best := "" + for _, name := range append([]string{SendFile}, commandNames()...) { + t := ServerName + ": " + strings.ReplaceAll(name, "_", " ") + if (u.Title == t || strings.HasPrefix(u.Title, t+" ")) && len(name) > len(best) { + best = name + } + } + return best, best != "" +} + +func commandNames() []string { + names := make([]string, len(Commands)) + for i, c := range Commands { + names[i] = c.Name + } + return names +} + +// CommandCall reports whether an ACP update starts a command tool call, and with which args. +func CommandCall(u acp.Update) (c SlackCommand, args string, ok bool) { + if u.Kind != "tool_call" { + return SlackCommand{}, "", false + } + name, ok := toolName(u) + if !ok { + return SlackCommand{}, "", false + } + c, ok = commandNamed(name) + return c, argsOf(u.RawInput), ok +} diff --git a/slackerbot/internal/tools/commands_test.go b/slackerbot/internal/tools/commands_test.go new file mode 100644 index 0000000..969ad5f --- /dev/null +++ b/slackerbot/internal/tools/commands_test.go @@ -0,0 +1,94 @@ +package tools + +import ( + "bytes" + "encoding/json" + "io" + "os" + "strings" + "testing" + "time" + + "slackerbot/internal/acp" +) + +func TestEveryCommandIsListed(t *testing.T) { + s := &Server{Home: t.TempDir(), MaxMB: 1} + var names []string + for _, tool := range s.tools() { + names = append(names, tool.(map[string]any)["name"].(string)) + } + for _, want := range []string{"send_file", "status", "sessions", "nodes", "permissions", "stop", "scale", "keep", + "storage", "move", "listen", "grant", "revoke", "reset_permissions", "dm_permission", "managers"} { + if !strings.Contains(" "+strings.Join(names, " ")+" ", " "+want+" ") { + t.Fatalf("%s missing from %v", want, names) + } + } +} + +func TestCommandWaitsForTheBotsReply(t *testing.T) { + home := t.TempDir() + s := &Server{Home: home, MaxMB: 1, Wait: 300 * time.Millisecond} + os.MkdirAll(replyDir(home), 0o755) + old := ReplyPath(home, "keep", "24h") + os.WriteFile(old, []byte(`{"text":"stale"}`), 0o644) + os.Chtimes(old, time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + + inR, inW := io.Pipe() + var out bytes.Buffer + done := make(chan error) + go func() { done <- s.Serve(inR, &out) }() + inW.Write([]byte(`{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"keep","arguments":{"args":" 24h"}}}` + "\n")) + inW.Write([]byte(`{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"scale","arguments":{}}}` + "\n")) + inW.Write([]byte(`{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"nodes","arguments":{}}}` + "\n")) + time.Sleep(50 * time.Millisecond) + data, _ := json.Marshal(Reply{Text: "Done! 24h."}) + os.WriteFile(ReplyPath(home, "keep", "24h"), data, 0o644) + inW.Close() + if err := <-done; err != nil { + t.Fatal(err) + } + + got := map[int]string{} + errs := map[int]bool{} + for _, line := range strings.Split(strings.TrimSpace(out.String()), "\n") { + var r struct { + ID int + Result struct { + Content []struct{ Text string } + IsError bool + } + } + _ = json.Unmarshal([]byte(line), &r) + got[r.ID], errs[r.ID] = r.Result.Content[0].Text, r.Result.IsError + } + if got[1] != "Done! 24h." || errs[1] { + t.Fatalf("keep: %q", got[1]) + } + if !strings.Contains(got[2], "args is required") || !errs[2] { + t.Fatalf("scale without args: %q", got[2]) + } + if !strings.Contains(got[3], "didn't answer in time") || !errs[3] { + t.Fatalf("unanswered: %q", got[3]) + } + if left, _ := os.ReadDir(replyDir(home)); len(left) != 0 { + t.Fatalf("replies should be cleaned up, left %v", left) + } +} + +func TestCommandCall(t *testing.T) { + c, args, ok := CommandCall(acp.Update{Kind: "tool_call", ToolName: "slackerbot__reset_permissions"}) + if !ok || c.Text(args) != "permissions reset" { + t.Fatalf("by tool name: %+v %q %v", c, args, ok) + } + c, args, ok = CommandCall(acp.Update{Kind: "tool_call", Title: "slackerbot: scale · up", RawInput: json.RawMessage(`{"args":"up"}`)}) + if !ok || c.Text(args) != "scale up" { + t.Fatalf("by title: %+v %q %v", c, args, ok) + } + if _, _, ok := CommandCall(acp.Update{Kind: "tool_call", ToolName: "slackerbot__send_file"}); ok { + t.Fatal("send_file isn't a command") + } + if _, _, ok := CommandCall(acp.Update{Kind: "tool_call", ToolName: "developer__shell", Title: "status"}); ok { + t.Fatal("other servers' tools aren't commands") + } +} diff --git a/slackerbot/internal/tools/server.go b/slackerbot/internal/tools/server.go index ac97b9c..0316c22 100644 --- a/slackerbot/internal/tools/server.go +++ b/slackerbot/internal/tools/server.go @@ -10,6 +10,8 @@ import ( "os" "path/filepath" "strings" + "sync" + "time" ) const ( @@ -22,9 +24,20 @@ const sendFileHelp = "Send a file to the user in Slack: images, videos, PDFs, ar "It's uploaded into the conversation when your turn ends, so mention it in your reply. " + "At most a few files per turn." +const commandNote = " It's the same as the Slack command `%s`, and runs with the permissions of the person whose message you're answering." + +const ( + defaultWait = time.Minute + pollEvery = 200 * time.Millisecond + // A reply older than this when a call starts belongs to an earlier call that gave up. + staleAfter = 30 * time.Second +) + type Server struct { Home string MaxMB int64 + // How long a command waits for the bot. Zero means a minute. + Wait time.Duration } type msg struct { @@ -45,20 +58,105 @@ func (s *Server) Serve(in io.Reader, out io.Writer) error { sc := bufio.NewScanner(in) sc.Buffer(make([]byte, 1<<20), 16<<20) enc := json.NewEncoder(out) + var mu sync.Mutex + var wg sync.WaitGroup + defer wg.Wait() + write := func(reply msg) error { + mu.Lock() + defer mu.Unlock() + return enc.Encode(reply) + } for sc.Scan() { var m msg if json.Unmarshal(sc.Bytes(), &m) != nil || len(m.ID) == 0 { continue } reply := msg{JSONRPC: "2.0", ID: m.ID} + // Commands wait on the bot, so they mustn't hold up other requests. + if c, args, ok := s.commandCall(m); ok { + wg.Add(1) + go func() { + defer wg.Done() + reply.Result = toolResult(s.await(c, args)) + _ = write(reply) + }() + continue + } reply.Result, reply.Error = s.handle(m) - if err := enc.Encode(reply); err != nil { + if err := write(reply); err != nil { return err } } return sc.Err() } +func toolResult(text string, isErr bool) map[string]any { + return map[string]any{"content": []any{map[string]string{"type": "text", "text": text}}, "isError": isErr} +} + +func (s *Server) commandCall(m msg) (SlackCommand, string, bool) { + if m.Method != "tools/call" { + return SlackCommand{}, "", false + } + var p struct { + Name string `json:"name"` + Arguments json.RawMessage `json:"arguments"` + } + _ = json.Unmarshal(m.Params, &p) + c, ok := commandNamed(p.Name) + return c, argsOf(p.Arguments), ok +} + +func (s *Server) await(c SlackCommand, args string) (string, bool) { + if c.Missing(args) { + return "args is required: " + c.Args, true + } + wait := s.Wait + if wait == 0 { + wait = defaultWait + } + start := time.Now() + for { + if r, ok := claimReply(replyDir(s.Home), replyPrefix(c.Name, args), start); ok { + return r.Text, r.Error + } + if time.Since(start) > wait { + return "slackerbot didn't answer in time, so it may not have run. Its master may be too old for this tool, " + + "or the home folder may be full. Ask the user to send the command in Slack instead: " + c.Text(args), true + } + time.Sleep(pollEvery) + } +} + +// claimReply takes a reply by deleting it, so two waiting calls can't both get it. +// The bot writes a file in one go, so one that doesn't parse yet is still being written. +func claimReply(dir, prefix string, since time.Time) (Reply, bool) { + entries, _ := os.ReadDir(dir) + for _, e := range entries { + if !strings.HasPrefix(e.Name(), prefix) { + continue + } + p := filepath.Join(dir, e.Name()) + info, err := e.Info() + if err != nil { + continue + } + if info.ModTime().Before(since.Add(-staleAfter)) { + _ = os.Remove(p) + continue + } + data, err := os.ReadFile(p) + var r Reply + if err != nil || json.Unmarshal(data, &r) != nil { + continue + } + if os.Remove(p) == nil { + return r, true + } + } + return Reply{}, false +} + func (s *Server) handle(m msg) (any, *rpcError) { switch m.Method { case "initialize": @@ -74,18 +172,7 @@ func (s *Server) handle(m msg) (any, *rpcError) { case "ping": return map[string]any{}, nil case "tools/list": - return map[string]any{"tools": []any{map[string]any{ - "name": SendFile, - "description": fmt.Sprintf(sendFileHelp, s.Home, s.MaxMB), - "inputSchema": map[string]any{ - "type": "object", - "properties": map[string]any{ - "path": map[string]string{"type": "string", "description": "Path of the file, e.g. " + s.Home + "/out/chart.png"}, - "comment": map[string]string{"type": "string", "description": "Optional message posted with the file"}, - }, - "required": []string{"path"}, - }, - }}}, nil + return map[string]any{"tools": s.tools()}, nil case "tools/call": var p struct { Name string `json:"name"` @@ -101,11 +188,45 @@ func (s *Server) handle(m msg) (any, *rpcError) { if err := s.Check(p.Arguments.Path); err != nil { text, isErr = err.Error(), true } - return map[string]any{"content": []any{map[string]string{"type": "text", "text": text}}, "isError": isErr}, nil + return toolResult(text, isErr), nil } return nil, &rpcError{-32601, "method not found: " + m.Method} } +func (s *Server) tools() []any { + list := []any{map[string]any{ + "name": SendFile, + "description": fmt.Sprintf(sendFileHelp, s.Home, s.MaxMB), + "inputSchema": map[string]any{ + "type": "object", + "properties": map[string]any{ + "path": map[string]string{"type": "string", "description": "Path of the file, e.g. " + s.Home + "/out/chart.png"}, + "comment": map[string]string{"type": "string", "description": "Optional message posted with the file"}, + }, + "required": []string{"path"}, + }, + }} + for _, c := range Commands { + schema := map[string]any{"type": "object", "properties": map[string]any{}} + if c.Args != "" { + schema["properties"] = map[string]any{"args": map[string]string{"type": "string", "description": c.Args}} + } + if c.Required { + schema["required"] = []string{"args"} + } + usage := c.Verb + if c.Args != "" { + usage += " " + } + list = append(list, map[string]any{ + "name": c.Name, + "description": c.Help + fmt.Sprintf(commandNote, usage), + "inputSchema": schema, + }) + } + return list +} + func (s *Server) Check(path string) error { if path == "" { return fmt.Errorf("path is required") diff --git a/slackerbot/internal/tools/session.go b/slackerbot/internal/tools/session.go index 5f4cff9..c7b5669 100644 --- a/slackerbot/internal/tools/session.go +++ b/slackerbot/internal/tools/session.go @@ -5,7 +5,6 @@ import ( "flag" "os" "strconv" - "strings" "sync" "slackerbot/internal/acp" @@ -47,8 +46,8 @@ type Calls struct { } func isSendFile(u acp.Update) bool { - return u.ToolName == ServerName+"__"+SendFile || - strings.HasPrefix(u.Title, ServerName+": "+strings.ReplaceAll(SendFile, "_", " ")) + name, ok := toolName(u) + return ok && name == SendFile } func (c *Calls) Update(u acp.Update) {