diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ce0f77d --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# built binaries +/slackspawn +/slackerbot/slackerbot +result +result-* +.commandcode/ diff --git a/README.md b/README.md index 773ff28..863c459 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# slackspawn +# slackerbot A Slack bot that runs goose agents in NixOS containers and connects them to Slack. @@ -118,29 +118,34 @@ People named in a `grant`, `revoke` or `managers` change are tagged in the confi ## Files -| File | What's in it | +All Go code lives in `slackerbot/` (the Go module root). Each package under `slackerbot/internal/` does one job: + +| Package | What's in it | |---|---| | `main.go` | Startup: config, capacity check, Slack connection | -| `config.go` | Reads `config.toml` | -| `capacity.go` | Reads the node's RAM/CPU and decides whether another container fits | -| `node.go` | What every node (this machine or a slave) can do | -| `local.go` | Runs containers on this machine (`systemd-run` + machined over D-Bus, btrfs homes) | -| `manager.go` | Owners, placement, resizing and storage across all nodes | -| `balance.go` | Picks the node that keeps CPU, RAM and storage most even | -| `cluster.go` | The master/slave link: TLS WebSocket, calls, agent relay, certificate pinning | -| `migrate.go` | Moving a container between nodes | -| `transfer.go` | Copying a home folder with btrfs snapshots, send and receive | -| `acp.go` | Talks to goose (ACP over WebSocket at `/acp`) | -| `stream.go` | Streams agent replies into Slack by editing messages | -| `bot.go` | Slack events, commands, buttons, idle cleanup | -| `commands.go` | Understands `scale` sizes and `keep` times | -| `permissions.go` | Super users and per-action permissions | -| `manage.go` | `grant`, `revoke`, `dm`, `managers` and `permissions reset` | -| `storage.go` | Home folder sizes, btrfs quotas, and the drive math | -| `store.go` | Remembers owners (DMs and channels), sessions, last activity and permissions in `state.json` | +| `config` | Reads `config.toml` | +| `model` | Shared types: sizes, owners, sessions, super users and per-action permissions | +| `units` | Reads and prints sizes like `5G` | +| `commands` | Understands `scale`, `keep`, `storage` and `grant` messages | +| `storage` | Drive math, btrfs subvolumes and quotas | +| `state` | Remembers owners (DMs and channels), sessions, last activity and permissions in `state.json` | +| `node` | What every node (this machine or a slave) can do | +| `local` | Runs containers on this machine (`systemd-run` + machined over D-Bus, btrfs homes, send/receive), and reads its RAM/CPU | +| `balance` | Picks the node that keeps CPU, RAM and storage most even | +| `manager` | Owners, placement, resizing and storage across all nodes; the master/slave link (TLS WebSocket, agent relay, certificate pinning); moving containers | +| `acp` | Talks to goose (ACP over WebSocket at `/acp`) | +| `bot` | Slack events, commands, buttons, idle cleanup, streaming replies, `grant`/`revoke`/`dm`/`managers` | +| `util`, `testutil` | Small helpers, and fakes for tests | | `nix/container.nix` | The NixOS system each container boots, with goose on 1984 | | `nix/host.nix` | Host module: runs the bot and gives containers a network | +## Development + +`nix develop` gives a shell with Go, gopls and the other tools. Then, inside `slackerbot/`: +``` +go build ./... && go vet ./... && go test ./... +``` + ## Setup **1. Slack app.** Create an app from `slack-manifest.yaml`. Then: @@ -149,22 +154,22 @@ People named in a `grant`, `revoke` or `managers` change are tagged in the confi **2. Build the container system.** ``` -nix build .#nixosConfigurations.agent.config.system.build.toplevel -o /var/lib/slackspawn/agent-system +nix build .#agent -o /var/lib/slackerbot/agent-system ``` Rebuild the same way after changing `nix/container.nix`. Each container picks up the new build on its next start. **3. Host config.** Add the flake's `nixosModules.host` to your host: ```nix -imports = [ slackspawn.nixosModules.host ]; -services.slackspawn = { +imports = [ slackerbot.nixosModules.host ]; +services.slackerbot = { enable = true; - configFile = ./slackspawn.toml; # based on config.example.toml - environmentFile = "/etc/slackspawn/secrets"; # SLACK_APP_TOKEN=…, SLACK_BOT_TOKEN=…, ANTHROPIC_API_KEY=… + configFile = ./slackerbot.toml; # based on config.example.toml + environmentFile = "/etc/slackerbot/secrets"; # SLACK_APP_TOKEN=…, SLACK_BOT_TOKEN=…, ANTHROPIC_API_KEY=… }; ``` -On the first build, Nix will complain about `vendorHash`. Paste the hash it prints into `flake.nix`. +After changing Go dependencies, update `vendorHash` in `flake.nix` (Nix prints the new one on the next build). -**Not using NixOS for the host?** Run `go build`, then start `./slackspawn -config config.toml` as root. Make sure systemd-networkd manages `ve-*` links (systemd ships `80-container-ve.network` for this). +**Not using NixOS for the host?** Run `nix build .#slackerbot` (or `go build` inside `slackerbot/`), then start `slackerbot -config config.toml` as root. Make sure systemd-networkd manages `ve-*` links (systemd ships `80-container-ve.network` for this). ## How capacity works @@ -192,13 +197,13 @@ Every drive must be btrfs. The bot turns on quotas for each drive at startup. ## Running more than one node One binary, two modes: -- `slackspawn -mode master`: runs the Slack bot and also runs containers itself. -- `slackspawn -mode slave`: runs containers only, for the master it points at. +- `slackerbot -mode master`: runs the Slack bot and also runs containers itself. +- `slackerbot -mode slave`: runs containers only, for the master it points at. **Setup** 1. Every node needs its own config with `system`, `[[drives]]` and `host_reserve`, plus the same `cluster.secret` (16+ characters). Each also needs the NixOS container build. Slaves don't need Slack tokens or `agent_env`, since the master sends those. -2. On the master, set `cluster.listen` (e.g. `":7946"`) and open that port. On NixOS, set `services.slackspawn.clusterPort = 7946`. -3. On each slave, set `cluster.master = "wss://:7946"` and give it a unique `cluster.name` (default: hostname). On NixOS, set `services.slackspawn.mode = "slave"`. +2. On the master, set `cluster.listen` (e.g. `":7946"`) and open that port. On NixOS, set `services.slackerbot.clusterPort = 7946`. +3. On each slave, set `cluster.master = "wss://:7946"` and give it a unique `cluster.name` (default: hostname). On NixOS, set `services.slackerbot.mode = "slave"`. 4. The master logs its certificate fingerprint at startup. Put it in each slave's `cluster.master_fingerprint`. If you leave it empty, a slave trusts the first certificate it sees, saves it to `state_dir/master.fingerprint`, and refuses any other from then on. **How it works** @@ -229,13 +234,13 @@ You can't move off an offline node (the files are there), and a move fails if th ## Things to know -- **The agent's actions are auto-approved.** When goose asks permission to run a tool, the bot says yes, since the container is the sandbox. To ask the user instead, change `handleRequest` in `acp.go` to post Slack buttons. +- **The agent's actions are auto-approved.** When goose asks permission to run a tool, the bot says yes, since the container is the sandbox. To ask the user instead, change `handleRequest` in `slackerbot/internal/acp` to post Slack buttons. - **Anyone in a channel controls its container,** including everything the agent can reach from it. Only add the bot to channels whose members you'd trust with that. - **Containers run as root without user namespaces.** They're isolated, but not strongly enough for people you don't trust. For untrusted users, look at nspawn's `--private-users=pick` (not turned on here, since it needs testing with a shared `/nix/store`). - **goose version:** `nix/container.nix` uses `goose serve`, which needs a recent goose. If nixpkgs' `goose-cli` is too old, override it with a newer build. -- **Transport:** the bot uses the WebSocket side of goose's `/acp` endpoint. If your goose version only offers the HTTP+SSE side, `DialACP` in `acp.go` is the one place to change. +- **Transport:** the bot uses the WebSocket side of goose's `/acp` endpoint. If your goose version only offers the HTTP+SSE side, `DialACP` in `slackerbot/internal/acp` is the one place to change. - **btrfs quotas aren't a hard security boundary here.** Container root keeps enough privileges that a determined user could raise their own quota. They're good for fair sharing, not for hostile users. - **Restarting the master** briefly makes every slave look offline until they reconnect, which takes a few seconds. Messages in that window get the dead-slave reply. -- **Restarting the bot doesn't stop containers.** They run as their own systemd units (`spawn-u-….service`), and the bot reconnects on the next message. -- **State** is a small JSON file (`state_dir/state.json`). It's fine for hundreds of users. Swap `store.go` for SQLite if you need more. +- **Restarting the bot doesn't stop containers.** They run as their own systemd units (`slackerbot-u-….service`), and the bot reconnects on the next message. +- **State** is a small JSON file (`state_dir/state.json`). It's fine for hundreds of users. Swap the `state` package for SQLite if you need more. - Home folders live at `/u-` or `/c-`. To wipe one for good, stop the container, run `btrfs subvolume delete` on its home, and remove its entry from `state.json` with the bot stopped. Nothing deletes homes automatically, so their guaranteed space stays taken until you do. diff --git a/cluster_test.go b/cluster_test.go deleted file mode 100644 index 25c54bd..0000000 --- a/cluster_test.go +++ /dev/null @@ -1,437 +0,0 @@ -package main - -import ( - "context" - "errors" - "net" - "os" - "path/filepath" - "strings" - "sync" - "testing" - "time" -) - -func TestPlacementBalancesAllThree(t *testing.T) { - drive := func(usable, committed int64) []DriveReport { - return []DriveReport{{Path: "/d", UsableMB: usable, CommittedMB: committed}} - } - want := Size{MemoryMB: 2048, CPUs: 1} - views := []NodeView{ - {Name: "cpu-busy", Usable: Size{16384, 8}, Used: Size{2048, 6}, Drives: drive(100_000, 10_000)}, - {Name: "ram-busy", Usable: Size{16384, 8}, Used: Size{12288, 1}, Drives: drive(100_000, 10_000)}, - {Name: "disk-busy", Usable: Size{16384, 8}, Used: Size{2048, 1}, Drives: drive(100_000, 80_000)}, - {Name: "calm", Usable: Size{16384, 8}, Used: Size{4096, 2}, Drives: drive(100_000, 20_000)}, - } - node, d, err := choosePlacement(views, want, 5120) - if err != nil || node != "calm" || d != "/d" { - t.Fatalf("got %s %s %v, want calm", node, d, err) - } -} - -func TestPlacementSpreadsAcrossEqualNodes(t *testing.T) { - views := []NodeView{ - {Name: "a", Usable: Size{8192, 4}, Drives: []DriveReport{{Path: "/a", UsableMB: 50_000}}}, - {Name: "b", Usable: Size{8192, 4}, Drives: []DriveReport{{Path: "/b", UsableMB: 50_000}}}, - } - want := Size{2048, 1} - counts := map[string]int{} - for i := 0; i < 4; i++ { - node, _, err := choosePlacement(views, want, 5120) - if err != nil { - t.Fatal(err) - } - counts[node]++ - for j := range views { - if views[j].Name == node { - views[j].Used = views[j].Used.Add(want) - views[j].Drives[0].CommittedMB += 5120 - } - } - } - if counts["a"] != 2 || counts["b"] != 2 { - t.Fatalf("uneven spread: %v", counts) - } -} - -func TestPlacementWhenFull(t *testing.T) { - full := []NodeView{{Name: "a", Usable: Size{4096, 2}, Used: Size{4096, 2}, Drives: []DriveReport{{Path: "/a", UsableMB: 50_000}}}} - if _, _, err := choosePlacement(full, Size{1024, 1}, 1024); !errors.Is(err, ErrMaxedOut) { - t.Fatalf("want maxed out, got %v", err) - } - noDisk := []NodeView{{Name: "a", Usable: Size{4096, 2}, Drives: []DriveReport{{Path: "/a", UsableMB: 1000}}}} - if _, _, err := choosePlacement(noDisk, Size{1024, 1}, 5120); !errors.Is(err, ErrStorageFull) { - t.Fatalf("want storage full, got %v", err) - } -} - -func TestBuildView(t *testing.T) { - info := NodeInfo{ - Name: "n1", - Usable: Size{16384, 8}, - Running: []RunningMachine{{Name: "c-a"}, {Name: "u-stranger"}}, - Drives: []DriveInfo{{Path: "/d", FreeMB: 90_000, ReserveMB: 10_000, - HomesMB: map[string]int64{"c-a": 8000, "c-b": 100, "u-stranger": 3000}}}, - } - owners := []Owner{ - {ID: "A", Machine: "c-a", Node: "n1", Drive: "/d", Size: Size{4096, 2}, StorageMinMB: 5120, StorageMaxMB: 10240}, - {ID: "B", Machine: "c-b", Node: "n1", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240}, - {ID: "C", Machine: "c-c", Node: "n1", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240}, - {ID: "D", Machine: "c-d", Node: "other", Drive: "/d", Size: Size{2048, 1}, StorageMinMB: 5120, StorageMaxMB: 10240}, - } - v := buildView(info, owners, Size{1024, 1}) - if v.Used != (Size{4096 + 1024, 3}) { - t.Fatalf("used = %v", v.Used) - } - d := v.Drives[0] - if d.UsableMB != 90_000+11_100-10_000 { - t.Fatalf("usable = %d", d.UsableMB) - } - if d.CommittedMB != 8000+5120+5120+3000 || d.BookedMB != 3*10240+3000 || d.Homes != 4 { - t.Fatalf("committed %d booked %d homes %d", d.CommittedMB, d.BookedMB, d.Homes) - } -} - -type fakeNode struct { - name string - mu sync.Mutex - runs map[string]bool - last StartSpec - homes map[string][]byte - exports map[string]*fakeExport - imports map[string]*fakeImport - passes int - failWrites bool - usable Size -} - -func newFakeNode(name string) *fakeNode { - return &fakeNode{name: name, runs: map[string]bool{}, homes: map[string][]byte{}, - exports: map[string]*fakeExport{}, imports: map[string]*fakeImport{}, usable: Size{8192, 4}} -} - -func (f *fakeNode) Name() string { return f.name } -func (f *fakeNode) Info(ctx context.Context) (NodeInfo, error) { - running, _ := f.Running(ctx) - f.mu.Lock() - defer f.mu.Unlock() - homes := map[string]int64{} - for k, v := range f.homes { - homes[strings.TrimPrefix(k, "/d/")] = int64(len(v)) >> 20 - } - return NodeInfo{Name: f.name, Usable: f.usable, Running: running, - Drives: []DriveInfo{{Path: "/d", FreeMB: 50_000, HomesMB: homes}}}, nil -} -func (f *fakeNode) Running(context.Context) ([]RunningMachine, error) { - f.mu.Lock() - defer f.mu.Unlock() - var out []RunningMachine - for m := range f.runs { - out = append(out, RunningMachine{Name: m}) - } - return out, nil -} -func (f *fakeNode) IsRunning(machine string) bool { - f.mu.Lock() - defer f.mu.Unlock() - return f.runs[machine] -} -func (f *fakeNode) Start(_ context.Context, spec StartSpec) error { - f.mu.Lock() - defer f.mu.Unlock() - f.last, f.runs[spec.Machine] = spec, true - return nil -} -func (f *fakeNode) Poweroff(_ context.Context, machine string) error { - f.mu.Lock() - defer f.mu.Unlock() - delete(f.runs, machine) - return nil -} -func (f *fakeNode) SetLimits(context.Context, string, Size) error { return nil } -func (f *fakeNode) SetQuota(context.Context, string, string, int64) error { return nil } -func (f *fakeNode) DialAgent(_ context.Context, machine, secret string) (msgConn, error) { - if secret != "agent-secret" { - return nil, errors.New("bad agent secret") - } - return newEcho(), nil -} - -func (f *fakeNode) WaitStopped(_ context.Context, machine string) error { - if f.IsRunning(machine) { - return errors.New("still running") - } - return nil -} - -func (f *fakeNode) ExportHome(_ context.Context, drive, machine, parent string) (string, error) { - f.mu.Lock() - defer f.mu.Unlock() - data, ok := f.homes[drive+"/"+machine] - if !ok { - return "", errors.New("no such home") - } - kind := "FULL:" - if parent != "" { - kind = "INCR:" - } - id := randomHex(4) - f.exports[id] = &fakeExport{data: append([]byte(kind), data...)} - return id, nil -} - -func (f *fakeNode) ReadExport(_ context.Context, id string, max int) ([]byte, bool, error) { - f.mu.Lock() - defer f.mu.Unlock() - x := f.exports[id] - if x == nil { - return nil, false, errors.New("no export") - } - n := min(max, len(x.data)-x.pos) - chunk := x.data[x.pos : x.pos+n] - x.pos += n - return chunk, x.pos == len(x.data), nil -} - -func (f *fakeNode) CloseExport(_ context.Context, id string) error { - f.mu.Lock() - defer f.mu.Unlock() - delete(f.exports, id) - return nil -} - -func (f *fakeNode) ImportHome(_ context.Context, drive, machine string) (string, error) { - f.mu.Lock() - defer f.mu.Unlock() - if _, exists := f.homes[drive+"/"+machine]; exists { - return "", errors.New("home already exists") - } - id := randomHex(4) - f.imports[id] = &fakeImport{key: drive + "/" + machine} - return id, nil -} - -func (f *fakeNode) WriteImport(_ context.Context, id string, data []byte) error { - f.mu.Lock() - defer f.mu.Unlock() - if f.failWrites { - return errors.New("disk on fire") - } - x := f.imports[id] - if x == nil { - return errors.New("no import") - } - x.cur = append(x.cur, data...) - return nil -} - -func (f *fakeNode) EndReceive(_ context.Context, id string) error { - f.mu.Lock() - defer f.mu.Unlock() - x := f.imports[id] - if x == nil || len(x.cur) < 5 { - return errors.New("nothing received") - } - if string(x.cur[:5]) == "INCR:" && x.latest == nil { - return errors.New("incremental send without a parent") - } - x.latest, x.cur = x.cur[5:], nil - x.passes++ - return nil -} - -func (f *fakeNode) FinishImport(_ context.Context, id string, quotaMB int64) error { - f.mu.Lock() - defer f.mu.Unlock() - x := f.imports[id] - delete(f.imports, id) - f.homes[x.key] = x.latest - f.passes = x.passes - return nil -} - -func (f *fakeNode) AbortImport(_ context.Context, id string) error { - f.mu.Lock() - defer f.mu.Unlock() - delete(f.imports, id) - return nil -} - -func (f *fakeNode) DeleteHome(_ context.Context, drive, machine string) error { - f.mu.Lock() - defer f.mu.Unlock() - delete(f.homes, drive+"/"+machine) - return nil -} - -type fakeExport struct { - data []byte - pos int -} - -type fakeImport struct { - key string - cur, latest []byte - passes int -} - -type echo struct { - ch chan []byte - closed chan struct{} - once sync.Once -} - -func newEcho() *echo { return &echo{ch: make(chan []byte, 16), closed: make(chan struct{})} } -func (e *echo) Read(ctx context.Context) ([]byte, error) { - select { - case b := <-e.ch: - return b, nil - case <-e.closed: - return nil, errors.New("closed") - case <-ctx.Done(): - return nil, ctx.Err() - } -} -func (e *echo) Write(_ context.Context, b []byte) error { - e.ch <- append([]byte("echo: "), b...) - return nil -} -func (e *echo) Close() error { e.once.Do(func() { close(e.closed) }); return nil } - -func freePort(t *testing.T) string { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer l.Close() - return l.Addr().String() -} - -func waitFor(t *testing.T, what string, cond func() bool) { - deadline := time.Now().Add(10 * time.Second) - for !cond() { - if time.Now().After(deadline) { - t.Fatalf("timed out waiting for %s", what) - } - time.Sleep(20 * time.Millisecond) - } -} - -func TestMasterAndSlave(t *testing.T) { - addr := freePort(t) - secret := "cluster-secret-1234567890" - masterDir, slaveDir := t.TempDir(), t.TempDir() - - masterCfg := &Config{StateDir: masterDir, Container: Size{1024, 1}} - masterCfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr} - store, err := OpenStore(filepath.Join(masterDir, "state.json")) - if err != nil { - t.Fatal(err) - } - mgr := NewManager(masterCfg, store, newFakeNode("master")) - backCh := make(chan string, 4) - mgr.onNodeBack = func(n string) { backCh <- n } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go mgr.ServeCluster(ctx) - - slaveCfg := &Config{StateDir: slaveDir} - slaveCfg.Cluster = ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr} - slave := newFakeNode("slave-1") - slaveCtx, stopSlave := context.WithCancel(ctx) - go RunSlave(slaveCtx, slaveCfg, slave) - - waitFor(t, "slave to come online", func() bool { _, err := mgr.node("slave-1"); return err == nil }) - if got := <-backCh; got != "slave-1" { - t.Fatalf("back callback for %q", got) - } - pinned, err := os.ReadFile(filepath.Join(slaveDir, "master.fingerprint")) - if err != nil || len(strings.TrimSpace(string(pinned))) != 64 { - t.Fatalf("slave should pin the master's certificate, got %q %v", pinned, err) - } - - n, _ := mgr.node("slave-1") - spec := StartSpec{Machine: "c-abc", Secret: "agent-secret", Size: Size{1024, 1}, Drive: "/d"} - if err := n.Start(ctx, spec); err != nil { - t.Fatal(err) - } - if !slave.IsRunning("c-abc") || slave.last.Drive != "/d" { - t.Fatal("start didn't reach the slave") - } - if !n.IsRunning("c-abc") { - t.Fatal("master should count the container as running right away") - } - - conn, err := n.DialAgent(ctx, "c-abc", "agent-secret") - if err != nil { - t.Fatal(err) - } - if err := conn.Write(ctx, []byte(`{"jsonrpc":"2.0"}`)); err != nil { - t.Fatal(err) - } - readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second) - got, err := conn.Read(readCtx) - readCancel() - if err != nil || string(got) != `echo: {"jsonrpc":"2.0"}` { - t.Fatalf("relay got %q %v", got, err) - } - conn.Close() - if _, err := n.DialAgent(ctx, "c-abc", "wrong"); err == nil { - t.Fatal("a bad agent secret should fail through the relay") - } - - if err := n.Poweroff(ctx, "c-abc"); err != nil { - t.Fatal(err) - } - if slave.IsRunning("c-abc") || n.IsRunning("c-abc") { - t.Fatal("poweroff didn't take") - } - - stopSlave() - waitFor(t, "slave to go offline", func() bool { - _, err := mgr.node("slave-1") - var off *NodeOfflineError - return errors.As(err, &off) && off.Node == "slave-1" - }) - if off := mgr.OfflineNodes(); len(off) != 1 || off[0] != "slave-1" { - t.Fatalf("offline nodes = %v", off) - } -} - -func TestSlaveRejectsWrongMasterOrSecret(t *testing.T) { - addr := freePort(t) - secret := "cluster-secret-1234567890" - masterCfg := &Config{StateDir: t.TempDir()} - masterCfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr} - store, _ := OpenStore(filepath.Join(masterCfg.StateDir, "state.json")) - mgr := NewManager(masterCfg, store, newFakeNode("master")) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go mgr.ServeCluster(ctx) - waitFor(t, "master to listen", func() bool { - c, err := net.Dial("tcp", addr) - if err == nil { - c.Close() - } - return err == nil - }) - - wrongPin := &Config{StateDir: t.TempDir()} - wrongPin.Cluster = ClusterConfig{Name: "s", Secret: secret, Master: "wss://" + addr, MasterFingerprint: strings.Repeat("ab", 32)} - if err := slaveSession(ctx, wrongPin, newFakeNode("s")); err == nil { - t.Fatal("a master with a different certificate should be refused") - } - - wrongSecret := &Config{StateDir: t.TempDir()} - wrongSecret.Cluster = ClusterConfig{Name: "s", Secret: "not-the-secret-at-all", Master: "wss://" + addr} - if err := slaveSession(ctx, wrongSecret, newFakeNode("s")); err == nil { - t.Fatal("a slave with the wrong secret should be refused") - } - - sameName := &Config{StateDir: t.TempDir()} - sameName.Cluster = ClusterConfig{Name: "master", Secret: secret, Master: "wss://" + addr} - if err := slaveSession(ctx, sameName, newFakeNode("master")); err == nil { - t.Fatal("a slave can't take the master's name") - } -} diff --git a/commands.go b/commands.go deleted file mode 100644 index 804c3be..0000000 --- a/commands.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "math" - "regexp" - "strconv" - "strings" - "time" -) - -var errNotCommand = errors.New("not a command") - -var unitSpace = regexp.MustCompile(`(\d)\s+(tib|tb|t|gib|gb|g|mib|mb|m|cpus|cpu|cores|core)\b`) -var sizeToken = regexp.MustCompile(`^([+-]?)(\d+(?:\.\d+)?)(g|gb|gib|m|mb|mib|cpu|cpus|core|cores)$`) - -func parseScale(args string, cur Size) (Size, error) { - args = unitSpace.ReplaceAllString(strings.ToLower(args), "$1$2") - words := strings.Fields(args) - if len(words) == 0 { - return Size{cur.MemoryMB * 2, cur.CPUs * 2}, nil - } - if len(words) == 1 { - switch words[0] { - case "up", "out": - return Size{cur.MemoryMB * 2, cur.CPUs * 2}, nil - case "down", "in": - return Size{cur.MemoryMB / 2, cur.CPUs / 2}, nil - } - } - - next := cur - var memDone, cpuDone bool - for i, w := range words { - parts := sizeToken.FindStringSubmatch(w) - if parts == nil { - if i == 0 { - return cur, errNotCommand - } - return cur, fmt.Errorf("I didn't understand `%s`", w) - } - sign, num, unit := parts[1], parts[2], parts[3] - n, _ := strconv.ParseFloat(num, 64) - - if unit == "cpu" || unit == "cpus" || unit == "core" || unit == "cores" { - if cpuDone { - return cur, errors.New("CPU was given twice") - } - cpuDone = true - next.CPUs = apply(sign, cur.CPUs, n) - continue - } - if memDone { - return cur, errors.New("memory was given twice") - } - memDone = true - mb := n - if strings.HasPrefix(unit, "g") { - mb = n * 1024 - } - next.MemoryMB = int(math.Round(apply(sign, float64(cur.MemoryMB), mb))) - } - return next, nil -} - -func apply(sign string, cur, n float64) float64 { - switch sign { - case "+": - return cur + n - case "-": - return cur - n - } - return n -} - -type keepSetting struct { - show bool - reset bool - indefinite bool - limit time.Duration -} - -var dayToken = regexp.MustCompile(`(\d+(?:\.\d+)?)([dw])`) - -var wordUnit = regexp.MustCompile(`(\d+(?:\.\d+)?)\s*(weeks?|wks?|days?|hours?|hrs?|minutes?|mins?)\b`) - -func parseKeep(args string) (keepSetting, error) { - a := strings.ToLower(strings.TrimSpace(args)) - switch a { - case "": - return keepSetting{show: true}, nil - case "default", "reset", "normal": - return keepSetting{reset: true}, nil - case "indefinite", "indefinitely", "forever", "alive", "running", "on": - return keepSetting{indefinite: true}, nil - } - d, err := parseDuration(a) - if err != nil { - return keepSetting{}, errNotCommand - } - if d <= 0 { - return keepSetting{}, errors.New("the time has to be more than zero") - } - return keepSetting{limit: d}, nil -} - -func parseDuration(s string) (time.Duration, error) { - s = wordUnit.ReplaceAllStringFunc(s, func(m string) string { - p := wordUnit.FindStringSubmatch(m) - return p[1] + map[byte]string{'w': "w", 'd': "d", 'h': "h", 'm': "m"}[p[2][0]] - }) - s = strings.ReplaceAll(s, " ", "") - s = dayToken.ReplaceAllStringFunc(s, func(m string) string { - p := dayToken.FindStringSubmatch(m) - n, _ := strconv.ParseFloat(p[1], 64) - hours := n * 24 - if p[2] == "w" { - hours = n * 24 * 7 - } - return strconv.FormatFloat(hours, 'f', -1, 64) + "h" - }) - return time.ParseDuration(s) -} - -func human(d time.Duration) string { - d = d.Round(time.Minute) - days := d / (24 * time.Hour) - d -= days * 24 * time.Hour - out := "" - if days > 0 { - out = fmt.Sprintf("%dd", days) - } - if d > 0 { - r := strings.TrimSuffix(d.String(), "0s") - if strings.HasSuffix(r, "h0m") { - r = strings.TrimSuffix(r, "0m") - } - out += r - } - if out == "" { - out = "0m" - } - return out -} diff --git a/config.example.toml b/config.example.toml index 3c4a832..a10e30a 100644 --- a/config.example.toml +++ b/config.example.toml @@ -1,6 +1,6 @@ -system = "/var/lib/slackspawn/agent-system" +system = "/var/lib/slackerbot/agent-system" machines_dir = "/var/lib/machines" -state_dir = "/var/lib/slackspawn" +state_dir = "/var/lib/slackerbot" agent_cwd = "/root" idle_timeout = "6h" @@ -42,11 +42,11 @@ limit = "50G" # Folders on btrfs drives. reserve is kept free for everything else on that drive. [[drives]] -path = "/mnt/disk1/slackspawn" +path = "/mnt/disk1/slackerbot" reserve = "20G" [[drives]] -path = "/mnt/disk2/slackspawn" +path = "/mnt/disk2/slackerbot" reserve = "20G" # Master: set listen to accept slaves. Slaves: set master (and run with -mode slave). diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..d777f2e --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1793341285, + "narHash": "sha256-Y1CiZ4SoBUjLQyQM/pY+rS2dXD3CHLPstF0lmOc73D4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1c89083e451e891152365b799595cf7790a43325", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index b6ef122..81b0142 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "slackspawn: a Slack bot that runs goose agents in NixOS containers"; + description = "slackerbot: a Slack bot that runs goose agents in NixOS containers"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; @@ -7,13 +7,22 @@ let system = "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; + agent = self.nixosConfigurations.agent.config.system.build.toplevel; in { - packages.${system}.default = pkgs.buildGoModule { - pname = "slackspawn"; - version = "0.1.0"; - src = ./.; - vendorHash = pkgs.lib.fakeHash; # replace with the hash the first build prints + packages.${system} = { + slackerbot = pkgs.buildGoModule { + pname = "slackerbot"; + version = "0.1.0"; + src = ./slackerbot; + vendorHash = "sha256-DFNUnPMsfcUHDlIxcxsDvuqjoHcp9EDTXWVy5t/oxEk="; + # tests need network ports and ../config.example.toml + doCheck = false; + meta.mainProgram = "slackerbot"; + }; + default = self.packages.${system}.slackerbot; + + inherit agent; }; nixosConfigurations.agent = nixpkgs.lib.nixosSystem { @@ -22,5 +31,18 @@ }; nixosModules.host = import ./nix/host.nix self; + + devShells.${system}.default = pkgs.mkShell { + inputsFrom = [ self.packages.${system}.slackerbot ]; + packages = with pkgs; [ + go + gopls + gotools + go-tools + delve + btrfs-progs + nixpkgs-fmt + ]; + }; }; } diff --git a/go.sum b/go.sum deleted file mode 100644 index f9cfc1a..0000000 --- a/go.sum +++ /dev/null @@ -1,10 +0,0 @@ -github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= -github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= -github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= -github.com/godbus/dbus/v5 v5.2.4 h1:F2gUCy++NMIxGmo9J+z5le/+ZC8W1OgeR51P4i9ZB/U= -github.com/godbus/dbus/v5 v5.2.4/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/slack-go/slack v0.17.3 h1:zV5qO3Q+WJAQ/XwbGfNFrRMaJ5T/naqaonyPV/1TP4g= -github.com/slack-go/slack v0.17.3/go.mod h1:X+UqOufi3LYQHDnMG1vxf0J8asC6+WllXrVrhl8/Prk= diff --git a/migrate_test.go b/migrate_test.go deleted file mode 100644 index dd8bb71..0000000 --- a/migrate_test.go +++ /dev/null @@ -1,285 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/http/httptest" - "path/filepath" - "strings" - "sync" - "testing" - - "github.com/slack-go/slack" -) - -type testCluster struct { - mgr *Manager - store *Store - master, slave *fakeNode - ctx context.Context -} - -func newTestCluster(t *testing.T) *testCluster { - addr := freePort(t) - secret := "cluster-secret-1234567890" - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - cfg := &Config{StateDir: t.TempDir(), Container: Size{1024, 1}} - cfg.Cluster = ClusterConfig{Name: "master", Secret: secret, Listen: addr} - store, err := OpenStore(filepath.Join(cfg.StateDir, "state.json")) - if err != nil { - t.Fatal(err) - } - c := &testCluster{store: store, master: newFakeNode("master"), slave: newFakeNode("slave-1"), ctx: ctx} - c.mgr = NewManager(cfg, store, c.master) - go c.mgr.ServeCluster(ctx) - - slaveCfg := &Config{StateDir: t.TempDir()} - slaveCfg.Cluster = ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr} - go RunSlave(ctx, slaveCfg, c.slave) - waitFor(t, "slave to come online", func() bool { _, err := c.mgr.node("slave-1"); return err == nil }) - return c -} - -func (c *testCluster) addOwner(t *testing.T, data []byte, running bool) { - u := &Owner{ID: "C1", Kind: OwnerChannel, Machine: "c-c1", Secret: "s", Size: Size{1024, 1}, - Node: "master", Drive: "/d", StorageMinMB: 1024, StorageMaxMB: 2048} - if err := c.store.PutOwner(u); err != nil { - t.Fatal(err) - } - c.master.mu.Lock() - defer c.master.mu.Unlock() - c.master.homes["/d/c-c1"] = data - if running { - c.master.runs["c-c1"] = true - } -} - -func homeData() []byte { return bytes.Repeat([]byte("0123456789abcdef"), 600_000) } - -func TestMoveLiveContainer(t *testing.T) { - c := newTestCluster(t) - c.addOwner(t, homeData(), true) - - var phases []MovePhase - res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{ - Target: "slave-1", - WaitIdle: func(context.Context) { - c.master.mu.Lock() - c.master.homes["/d/c-c1"] = append(c.master.homes["/d/c-c1"], []byte("written during the bulk copy")...) - c.master.mu.Unlock() - }, - Progress: func(p MoveProgress) { - if len(phases) == 0 || phases[len(phases)-1] != p.Phase { - phases = append(phases, p.Phase) - } - }, - }) - if err != nil { - t.Fatal(err) - } - - want := append(homeData(), []byte("written during the bulk copy")...) - if !bytes.Equal(c.slave.homes["/d/c-c1"], want) { - t.Fatal("the late write didn't make it to the new node") - } - if c.slave.passes != 2 { - t.Fatalf("a live move should copy twice (bulk + final sync), got %d", c.slave.passes) - } - if _, still := c.master.homes["/d/c-c1"]; still { - t.Fatal("old home should be deleted after a good move") - } - if !c.slave.IsRunning("c-c1") || c.master.IsRunning("c-c1") || !res.Started { - t.Fatal("container should be running on the new node only") - } - if u := c.store.Owner("C1"); u.Node != "slave-1" || u.Drive != "/d" { - t.Fatalf("placement not switched: %+v", u) - } - if len(c.master.exports) != 0 { - t.Fatal("snapshots on the old node should be cleaned up") - } - wantPhases := []MovePhase{PhaseCopy, PhaseFinal, PhaseCleanup, PhaseStart} - if len(phases) != len(wantPhases) { - t.Fatalf("phases %v, want %v", phases, wantPhases) - } -} - -func TestMoveAutoStoppedContainer(t *testing.T) { - c := newTestCluster(t) - c.addOwner(t, homeData(), false) - - res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{}) - if err != nil { - t.Fatal(err) - } - if res.To != "slave-1" || res.Started || c.slave.IsRunning("c-c1") { - t.Fatalf("a stopped container should move without starting: %+v", res) - } - if c.slave.passes != 1 { - t.Fatalf("a stopped container needs one copy, got %d", c.slave.passes) - } -} - -func TestMoveFullNodeStartsElsewhere(t *testing.T) { - c := newTestCluster(t) - c.addOwner(t, homeData(), false) - c.master.mu.Lock() - c.master.usable = Size{512, 0.5} - c.master.mu.Unlock() - - _, err := c.mgr.Ensure(c.ctx, "C1", OwnerChannel) - var full *NodeFullError - if !errors.As(err, &full) || full.Node != "master" { - t.Fatalf("want node full on master, got %v", err) - } - res, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{StartAfter: true}) - if err != nil || !res.Started || !c.slave.IsRunning("c-c1") { - t.Fatalf("auto-move should start it on the new node: %+v %v", res, err) - } -} - -func TestMoveFailureRollsBack(t *testing.T) { - c := newTestCluster(t) - c.addOwner(t, homeData(), true) - - _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{ - Target: "slave-1", - WaitIdle: func(context.Context) { - c.slave.mu.Lock() - c.slave.failWrites = true - c.slave.mu.Unlock() - }, - }) - if err == nil { - t.Fatal("expected the final sync to fail") - } - if u := c.store.Owner("C1"); u.Node != "master" { - t.Fatalf("container should stay on its old node, got %s", u.Node) - } - if !c.master.IsRunning("c-c1") { - t.Fatal("container should be restarted on its old node") - } - if _, ok := c.master.homes["/d/c-c1"]; !ok { - t.Fatal("old home must survive a failed move") - } - if _, ok := c.slave.homes["/d/c-c1"]; ok || len(c.slave.imports) != 0 { - t.Fatal("half-copied home on the target should be cleaned up") - } -} - -func TestMoveRefusals(t *testing.T) { - c := newTestCluster(t) - c.addOwner(t, homeData(), false) - - if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "master"}); !errors.Is(err, ErrSameNode) { - t.Fatalf("same node: %v", err) - } - var unknown *UnknownNodeError - if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "nope"}); !errors.As(err, &unknown) { - t.Fatalf("unknown node: %v", err) - } - if _, err := c.mgr.Migrate(c.ctx, "nobody", MoveOptions{}); !errors.Is(err, ErrNoContainer) { - t.Fatalf("no container: %v", err) - } - c.slave.mu.Lock() - c.slave.usable = Size{512, 0.5} - c.slave.mu.Unlock() - c.master.mu.Lock() - c.master.runs["c-c1"] = true - c.master.mu.Unlock() - slave, _ := c.mgr.node("slave-1") - waitFor(t, "slave report", func() bool { - info, _ := slave.Info(c.ctx) - return info.Usable.MemoryMB == 512 - }) - var full *NodeFullError - if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "slave-1"}); !errors.As(err, &full) { - t.Fatalf("full target: %v", err) - } -} - -type slackLog struct { - mu sync.Mutex - n int - text map[string]string - under map[string]string - order []string -} - -func (s *slackLog) ServeHTTP(w http.ResponseWriter, r *http.Request) { - _ = r.ParseForm() - s.mu.Lock() - defer s.mu.Unlock() - ts := r.Form.Get("ts") - if strings.HasSuffix(r.URL.Path, "chat.postMessage") { - s.n++ - ts = fmt.Sprintf("%d.0", s.n) - s.order = append(s.order, ts) - s.under[ts] = r.Form.Get("thread_ts") - } - if !strings.HasSuffix(r.URL.Path, "chat.getPermalink") { - s.text[ts] = r.Form.Get("text") - } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "C1", "ts": ts, "permalink": "https://slack.example/" + ts}) -} - -func TestEveryMoveStartsAnAnnouncedThread(t *testing.T) { - c := newTestCluster(t) - c.addOwner(t, homeData(), false) - log := &slackLog{text: map[string]string{}, under: map[string]string{}} - srv := httptest.NewServer(log) - defer srv.Close() - b := &Bot{cfg: c.mgr.cfg, store: c.store, mgr: c.mgr, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/")), - acps: map[string]*ACP{}, busy: map[string]int{}} - - b.moveFor(c.ctx, "C1", "C1", "slave-1", "<@U1>", "100.0") - if _, err := c.mgr.Migrate(c.ctx, "C1", MoveOptions{Target: "master"}); err != nil { - t.Fatal(err) - } - b.runMove(c.ctx, "C1", "C1", MoveOptions{}, "auto process (master was full), for <@U2>'s message", nil) - - log.mu.Lock() - defer log.mu.Unlock() - var roots []string - for _, ts := range log.order { - if log.under[ts] == "" { - roots = append(roots, ts) - } - } - linked := 0 - for ts, parent := range log.under { - if parent == "100.0" && strings.Contains(log.text[ts], "Follow the move here") { - linked++ - } - } - if linked != 1 { - t.Fatalf("the thread where the move started should get one link to the move thread, got %d", linked) - } - if len(roots) != 2 { - t.Fatalf("want one new top-level message per move, got %d", len(roots)) - } - checks := []string{"started by <@U1>: master → slave-1", "started by auto process (master was full), for <@U2>'s message"} - for i, root := range roots { - if !strings.Contains(log.text[root], checks[i]) || !strings.Contains(log.text[root], ":white_check_mark: done") { - t.Fatalf("root %d = %q", i, log.text[root]) - } - replies := 0 - for ts, parent := range log.under { - if parent == root { - replies++ - if !strings.Contains(log.text[ts], "Moved from") { - t.Fatalf("progress reply = %q", log.text[ts]) - } - } - } - if replies != 1 { - t.Fatalf("want progress in one thread reply, got %d", replies) - } - } -} diff --git a/nix/container.nix b/nix/container.nix index fb8aee0..d779735 100644 --- a/nix/container.nix +++ b/nix/container.nix @@ -17,7 +17,7 @@ path = [ "/run/current-system/sw" ]; environment.HOME = "/root"; serviceConfig = { - EnvironmentFile = "/var/lib/slackspawn/agent.env"; + EnvironmentFile = "/var/lib/slackerbot/agent.env"; ExecStart = "${pkgs.goose-cli}/bin/goose serve --host 0.0.0.0 --port 1984 --with-builtin developer"; WorkingDirectory = "/root"; Restart = "always"; diff --git a/nix/host.nix b/nix/host.nix index 1f4e552..185b316 100644 --- a/nix/host.nix +++ b/nix/host.nix @@ -1,11 +1,11 @@ self: { config, lib, pkgs, ... }: let - cfg = config.services.slackspawn; + cfg = config.services.slackerbot; in { - options.services.slackspawn = { - enable = lib.mkEnableOption "slackspawn"; + options.services.slackerbot = { + enable = lib.mkEnableOption "slackerbot"; mode = lib.mkOption { type = lib.types.enum [ "master" "slave" ]; default = "master"; @@ -39,15 +39,15 @@ in networking.firewall.trustedInterfaces = [ "ve-+" ]; networking.firewall.allowedTCPPorts = lib.optional (cfg.clusterPort != null) cfg.clusterPort; - systemd.services.slackspawn = { + systemd.services.slackerbot = { wantedBy = [ "multi-user.target" ]; after = [ "network-online.target" "systemd-machined.service" ]; wants = [ "network-online.target" ]; path = [ pkgs.systemd pkgs.btrfs-progs ]; serviceConfig = { - ExecStart = "${self.packages.${pkgs.stdenv.hostPlatform.system}.default}/bin/slackspawn -mode ${cfg.mode} -config ${cfg.configFile}"; + ExecStart = "${self.packages.${pkgs.stdenv.hostPlatform.system}.default}/bin/slackerbot -mode ${cfg.mode} -config ${cfg.configFile}"; EnvironmentFile = lib.optional (cfg.environmentFile != null) cfg.environmentFile; - StateDirectory = "slackspawn"; + StateDirectory = "slackerbot"; Restart = "on-failure"; }; }; diff --git a/permissions_test.go b/permissions_test.go deleted file mode 100644 index 9950460..0000000 --- a/permissions_test.go +++ /dev/null @@ -1,230 +0,0 @@ -package main - -import ( - "context" - "errors" - "net/http/httptest" - "path/filepath" - "strings" - "testing" - - "github.com/slack-go/slack" -) - -func TestPermissions(t *testing.T) { - t.Setenv("SLACK_APP_TOKEN", "xapp-test") - t.Setenv("SLACK_BOT_TOKEN", "xoxb-test") - t.Setenv("CLUSTER_SECRET", "cluster-secret-1234567890") - c, err := LoadConfig("config.example.toml", ModeMaster) - if err != nil { - t.Fatal(err) - } - p := c.Permissions - - cases := []struct { - kind OwnerKind - channel, user string - action Action - want bool - }{ - {OwnerChannel, "C07ABC123", "U0111", ActMove, true}, - {OwnerChannel, "C07ABC123", "U0111", ActScale, false}, - {OwnerChannel, "C07ABC123", "U0222", ActKeep, true}, - {OwnerChannel, "C07ABC123", "U0ADMIN", ActMove, false}, - {OwnerChannel, "C0OTHER", "U0ADMIN", ActStorage, true}, - {OwnerChannel, "C0OTHER", "U0111", ActMove, false}, - {OwnerUser, "D1", "U9", ActScale, true}, - {OwnerUser, "D1", "U9", ActStorage, false}, - } - for _, c := range cases { - if got := p.Allowed(c.kind, c.channel, c.user, c.action); got != c.want { - t.Errorf("%s %s %s %s = %v, want %v", c.kind, c.channel, c.user, c.action, got, c.want) - } - } - if h := p.Holders("C07ABC123", ActMove); len(h) != 2 { - t.Errorf("move holders = %v", h) - } - if a := p.ActionsFor("C0OTHER", "U0ADMIN"); len(a) != len(allActions) { - t.Errorf("all should expand to every action, got %v", a) - } - - open := PermissionsConfig{Default: map[string][]string{"U1": {"all"}}} - if !open.Allowed(OwnerUser, "D1", "U9", ActMove) { - t.Error("DM permissions should default to allowed") - } - if (PermissionsConfig{}).validate() == nil { - t.Error("a missing default super user should be refused") - } - bad := PermissionsConfig{Default: map[string][]string{"U1": {"delete-everything"}}} - if bad.validate() == nil { - t.Error("unknown permission names should be refused") - } -} - -func TestNeededPermission(t *testing.T) { - for in, want := range map[[2]string]Action{ - {"scale", "up"}: ActScale, - {"scale", "8G 4cpu"}: ActScale, - {"scale", "the db to 3 replicas"}: "", - {"keep", ""}: "", - {"keep", "24h"}: ActKeep, - {"keep", "indefinite"}: ActKeep, - {"keep", "the tests green"}: "", - {"storage", ""}: "", - {"storage", "20G"}: ActStorage, - {"storage", "is full, clean it up"}: "", - {"status", ""}: "", - } { - if got := neededPermission(in[0], in[1]); got != want { - t.Errorf("%s %q needs %q, want %q", in[0], in[1], got, want) - } - } -} - -func TestPermissionsSavedAndSeededOnce(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - s, _ := OpenStore(path) - initial := PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UBOSS": {"all"}}} - if seeded, err := s.SeedPermissions(initial); !seeded || err != nil { - t.Fatalf("first start should seed: %v %v", seeded, err) - } - if err := s.UpdatePermissions(func(p *PermissionsConfig) error { - p.Grant("C1", []string{"UANA"}, []Action{ActMove}) - return nil - }); err != nil { - t.Fatal(err) - } - - s2, _ := OpenStore(path) - changed := PermissionsConfig{Managers: []string{"USOMEONE"}, Default: map[string][]string{"USOMEONE": {"all"}}} - if seeded, _ := s2.SeedPermissions(changed); seeded { - t.Fatal("config must not override saved permissions") - } - p := s2.Permissions() - if !p.IsManager("UBOSS") || p.IsManager("USOMEONE") || !p.Allowed(OwnerChannel, "C1", "UANA", ActMove) { - t.Fatalf("saved permissions lost: %+v", p) - } - if err := s2.UpdatePermissions(func(p *PermissionsConfig) error { - p.Default["UX"] = []string{"teleport"} - return nil - }); err == nil { - t.Fatal("invalid changes must be refused") - } -} - -func TestGrantRevoke(t *testing.T) { - p := PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UDEF": {"all"}}}.clone() - - p.Grant("C1", []string{"UANA"}, []Action{ActMove}) - if !p.Allowed(OwnerChannel, "C1", "UDEF", ActScale) { - t.Fatal("first grant in a channel should keep everyone from the default list") - } - if !p.Allowed(OwnerChannel, "C1", "UANA", ActMove) || p.Allowed(OwnerChannel, "C1", "UANA", ActScale) { - t.Fatal("grant should add exactly what was asked") - } - if p.Allowed(OwnerChannel, "C2", "UANA", ActMove) { - t.Fatal("grants stay in their channel") - } - - p.Grant("C1", []string{"UANA"}, []Action{"all"}) - if p.CanManage("C1", "UANA") { - t.Fatal("all must not include manage") - } - p.Grant("C1", []string{"UANA"}, []Action{ActManage}) - if !p.CanManage("C1", "UANA") || p.CanManage("C2", "UANA") { - t.Fatal("manage is per channel") - } - - p.Revoke("C1", []string{"UDEF"}, []Action{ActScale}) - if p.Allowed(OwnerChannel, "C1", "UDEF", ActScale) || !p.Allowed(OwnerChannel, "C1", "UDEF", ActMove) { - t.Fatal("revoking one action out of all should keep the rest") - } - p.Revoke("C1", []string{"UANA"}, nil) - if len(p.ActionsFor("C1", "UANA")) != 0 { - t.Fatal("revoke with no actions removes the person") - } - - p.Grant("", []string{"UNEW"}, []Action{ActKeep}) - if !p.Allowed(OwnerChannel, "C9", "UNEW", ActKeep) { - t.Fatal("default grants apply to channels without their own list") - } - - p.AddManagers([]string{"UTWO"}) - if err := p.RemoveManagers([]string{"UBOSS", "UTWO"}); !errors.Is(err, ErrLastManager) { - t.Fatal("removing every manager must be refused") - } - if err := p.RemoveManagers([]string{"UBOSS"}); err != nil || p.IsManager("UBOSS") { - t.Fatal("removing one of two managers should work") - } -} - -func TestParseGrant(t *testing.T) { - g, err := parseGrant("<@U1> <@U2|bob>, move scale") - if err != nil || len(g.users) != 2 || g.users[1] != "U2" || len(g.actions) != 2 { - t.Fatalf("%+v %v", g, err) - } - if g, _ := parseGrant("default <@U1> all"); !g.toDefault || g.actions[0] != "all" { - t.Fatalf("%+v", g) - } - for _, in := range []string{"me access to the repo", "all the things", ""} { - if _, err := parseGrant(in); !errors.Is(err, errNotCommand) { - t.Errorf("%q should go to the agent, got %v", in, err) - } - } - if _, err := parseGrant("<@U1> teleport"); err == nil || errors.Is(err, errNotCommand) { - t.Error("an unknown permission after a mention should be a usage error") - } -} - -func TestManageCommandsInSlack(t *testing.T) { - log := &slackLog{text: map[string]string{}, under: map[string]string{}} - srv := httptest.NewServer(log) - defer srv.Close() - store, _ := OpenStore(filepath.Join(t.TempDir(), "state.json")) - store.SeedPermissions(PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UBOSS": {"all"}}}) - b := &Bot{store: store, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/"))} - ctx := context.Background() - say := func(user, text string) string { - m := incoming{user: user, channel: "C1", ts: "1.0", text: text} - if !b.command(ctx, m) { - return "(went to the agent)" - } - log.mu.Lock() - defer log.mu.Unlock() - return log.text[log.order[len(log.order)-1]] - } - - if got := say("UANA", "grant <@UANA> all"); !strings.Contains(got, ":lock:") { - t.Fatalf("non-managers can't grant: %q", got) - } - if got := say("UBOSS", "grant <@UANA> move manage"); !strings.Contains(got, "<@UANA> can now: move, manage") { - t.Fatalf("manager grant: %q", got) - } - if got := say("UANA", "grant <@UCAL> scale"); !strings.Contains(got, "<@UCAL> can now: scale") { - t.Fatalf("channel manager grant: %q", got) - } - if got := say("UANA", "grant default <@UCAL> all"); !strings.Contains(got, ":lock:") { - t.Fatalf("channel managers can't touch the default list: %q", got) - } - if got := say("UANA", "managers add <@UCAL>"); !strings.Contains(got, ":lock:") { - t.Fatalf("only user managers add managers: %q", got) - } - if got := say("UBOSS", "managers remove <@UBOSS>"); !strings.Contains(got, "nobody able to manage") { - t.Fatalf("last manager guard: %q", got) - } - if got := say("UBOSS", "dm storage off"); !strings.Contains(got, "now turned off") { - t.Fatalf("dm toggle: %q", got) - } - if store.Permissions().Allowed(OwnerUser, "D1", "UX", ActStorage) { - t.Fatal("dm toggle not saved") - } - if got := say("UANA", "grant me access to the repo"); got != "(went to the agent)" { - t.Fatalf("plain sentence: %q", got) - } - if got := say("UANA", "permissions reset"); !strings.Contains(got, "default super-user list again") { - t.Fatalf("reset: %q", got) - } - if store.Permissions().Allowed(OwnerChannel, "C1", "UCAL", ActScale) { - t.Fatal("reset should drop the channel's own list") - } -} diff --git a/slack-manifest.yaml b/slack-manifest.yaml index 82b3a41..fb2b875 100644 --- a/slack-manifest.yaml +++ b/slack-manifest.yaml @@ -1,10 +1,10 @@ # Paste into https://api.slack.com/apps → Create New App → From a manifest. display_information: - name: Robotnix + name: slacker description: Your own agent in your own container. features: bot_user: - display_name: agent-box + display_name: Slacker always_online: true app_home: messages_tab_enabled: true diff --git a/go.mod b/slackerbot/go.mod similarity index 75% rename from go.mod rename to slackerbot/go.mod index ce1f97b..8c2c406 100644 --- a/go.mod +++ b/slackerbot/go.mod @@ -1,4 +1,4 @@ -module slackspawn +module slackerbot go 1.24.7 @@ -6,6 +6,10 @@ require ( github.com/BurntSushi/toml v1.6.0 github.com/coder/websocket v1.8.15 github.com/godbus/dbus/v5 v5.2.4 - github.com/gorilla/websocket v1.5.3 // indirect github.com/slack-go/slack v0.17.3 ) + +require ( + github.com/gorilla/websocket v1.5.3 // indirect + golang.org/x/sys v0.27.0 // indirect +) diff --git a/slackerbot/go.sum b/slackerbot/go.sum new file mode 100644 index 0000000..1838dd1 --- /dev/null +++ b/slackerbot/go.sum @@ -0,0 +1,22 @@ +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/godbus/dbus/v5 v5.2.4 h1:F2gUCy++NMIxGmo9J+z5le/+ZC8W1OgeR51P4i9ZB/U= +github.com/godbus/dbus/v5 v5.2.4/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/slack-go/slack v0.17.3 h1:zV5qO3Q+WJAQ/XwbGfNFrRMaJ5T/naqaonyPV/1TP4g= +github.com/slack-go/slack v0.17.3/go.mod h1:X+UqOufi3LYQHDnMG1vxf0J8asC6+WllXrVrhl8/Prk= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/acp.go b/slackerbot/internal/acp/acp.go similarity index 97% rename from acp.go rename to slackerbot/internal/acp/acp.go index cdc0061..ec54361 100644 --- a/acp.go +++ b/slackerbot/internal/acp/acp.go @@ -1,4 +1,4 @@ -package main +package acp import ( "context" @@ -7,6 +7,8 @@ import ( "fmt" "sync" "sync/atomic" + + "slackerbot/internal/node" ) type Update struct { @@ -33,7 +35,7 @@ type rpcMsg struct { } type ACP struct { - conn msgConn + conn node.MsgConn nextID atomic.Int64 writeMu sync.Mutex @@ -50,14 +52,14 @@ type ACP struct { } func DialACP(ctx context.Context, addr, secret string) (*ACP, error) { - conn, err := dialAgentWS(ctx, addr, secret) + conn, err := node.DialAgentWS(ctx, addr, secret) if err != nil { return nil, err } - return NewACP(ctx, conn) + return New(ctx, conn) } -func NewACP(ctx context.Context, conn msgConn) (*ACP, error) { +func New(ctx context.Context, conn node.MsgConn) (*ACP, error) { a := &ACP{ conn: conn, pending: map[int64]chan rpcMsg{}, diff --git a/slackerbot/internal/acp/acp_test.go b/slackerbot/internal/acp/acp_test.go new file mode 100644 index 0000000..1aa9685 --- /dev/null +++ b/slackerbot/internal/acp/acp_test.go @@ -0,0 +1,121 @@ +package acp + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/coder/websocket" +) + +func fakeGoose(t *testing.T) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/acp" || r.Header.Get("X-Secret-Key") != "s3cret" { + http.Error(w, "nope", http.StatusUnauthorized) + return + } + c, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + ctx := context.Background() + send := func(v any) { b, _ := json.Marshal(v); _ = c.Write(ctx, websocket.MessageText, b) } + var promptID json.RawMessage + + for { + _, data, err := c.Read(ctx) + if err != nil { + return + } + var m rpcMsg + _ = json.Unmarshal(data, &m) + switch m.Method { + case "initialize": + send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": map[string]any{ + "protocolVersion": 1, "agentCapabilities": map[string]any{"loadSession": true}}}) + case "session/new": + send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": map[string]any{"sessionId": "sess-1"}}) + case "session/load": + send(map[string]any{"jsonrpc": "2.0", "method": "session/update", "params": map[string]any{ + "sessionId": "sess-old", "update": map[string]any{"sessionUpdate": "agent_message_chunk", + "content": map[string]any{"type": "text", "text": "old history"}}}}) + send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": nil}) + case "session/prompt": + promptID = m.ID + upd := func(u map[string]any) { + send(map[string]any{"jsonrpc": "2.0", "method": "session/update", + "params": map[string]any{"sessionId": "sess-1", "update": u}}) + } + upd(map[string]any{"sessionUpdate": "agent_message_chunk", "content": map[string]any{"type": "text", "text": "Hello "}}) + upd(map[string]any{"sessionUpdate": "tool_call", "toolCallId": "t1", "title": "ls /root", "status": "pending"}) + send(map[string]any{"jsonrpc": "2.0", "id": "perm-1", "method": "session/request_permission", + "params": map[string]any{"sessionId": "sess-1", "options": []map[string]any{ + {"optionId": "no", "name": "Reject", "kind": "reject_once"}, + {"optionId": "yes", "name": "Allow", "kind": "allow_once"}}}}) + case "": + var res struct { + Outcome struct{ Outcome, OptionID string } `json:"outcome"` + } + _ = json.Unmarshal(m.Result, &res) + text := "world" + if res.Outcome.OptionID != "yes" { + text = "PERMISSION WRONG" + } + send(map[string]any{"jsonrpc": "2.0", "method": "session/update", "params": map[string]any{ + "sessionId": "sess-1", "update": map[string]any{"sessionUpdate": "agent_message_chunk", + "content": map[string]any{"type": "text", "text": text}}}}) + send(map[string]any{"jsonrpc": "2.0", "id": promptID, "result": map[string]any{"stopReason": "end_turn"}}) + } + } + })) +} + +func TestACP(t *testing.T) { + srv := fakeGoose(t) + defer srv.Close() + addr := strings.TrimPrefix(srv.URL, "http://") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if _, err := DialACP(ctx, addr, "wrong"); err == nil { + t.Fatal("wrong secret should fail") + } + a, err := DialACP(ctx, addr, "s3cret") + if err != nil { + t.Fatal(err) + } + defer a.Close() + if !a.CanLoad { + t.Fatal("loadSession capability not read") + } + id, err := a.NewSession(ctx, "/root") + if err != nil || id != "sess-1" { + t.Fatalf("new session: %q %v", id, err) + } + if err := a.LoadSession(ctx, "sess-old", "/root"); err != nil { + t.Fatal(err) + } + + var mu sync.Mutex + var text strings.Builder + var tools []string + stop, err := a.Prompt(ctx, id, "hi", func(u Update) { + mu.Lock() + defer mu.Unlock() + text.WriteString(u.Text) + if u.Kind == "tool_call" { + tools = append(tools, u.Title) + } + }) + if err != nil || stop != "end_turn" { + t.Fatalf("prompt: %q %v", stop, err) + } + if text.String() != "Hello world" || len(tools) != 1 || tools[0] != "ls /root" { + t.Fatalf("got text %q tools %v", text.String(), tools) + } +} diff --git a/balance.go b/slackerbot/internal/balance/balance.go similarity index 76% rename from balance.go rename to slackerbot/internal/balance/balance.go index 1da7087..0c4efb0 100644 --- a/balance.go +++ b/slackerbot/internal/balance/balance.go @@ -1,15 +1,22 @@ -package main +package balance import ( + "errors" "math" "sort" + + "slackerbot/internal/model" + "slackerbot/internal/node" + "slackerbot/internal/storage" ) +var ErrMaxedOut = errors.New("no node has room for this container") + type NodeView struct { Name string - Usable Size - Used Size - Drives []DriveReport + Usable model.Size + Used model.Size + Drives []storage.DriveReport Running int } @@ -46,7 +53,7 @@ func (v NodeView) Load() (cpu, ram, storage float64) { // The fullest of CPU, RAM and storage after adding the container. Placing // where this stays lowest keeps all three level across nodes. -func (v NodeView) loadAfter(want Size, storageMB int64) float64 { +func (v NodeView) loadAfter(want model.Size, storageMB int64) float64 { us, cm, _ := v.Storage() return max( ratio(v.Used.CPUs+want.CPUs, v.Usable.CPUs), @@ -55,7 +62,7 @@ func (v NodeView) loadAfter(want Size, storageMB int64) float64 { ) } -func choosePlacement(views []NodeView, want Size, storageMB int64) (node, drive string, err error) { +func ChoosePlacement(views []NodeView, want model.Size, storageMB int64) (node, drive string, err error) { sorted := append([]NodeView(nil), views...) sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name }) @@ -66,7 +73,7 @@ func choosePlacement(views []NodeView, want Size, storageMB int64) (node, drive continue } roomForCompute = true - d, err := pickDrive(v.Drives, storageMB) + d, err := storage.PickDrive(v.Drives, storageMB) if err != nil { continue } @@ -78,15 +85,15 @@ func choosePlacement(views []NodeView, want Size, storageMB int64) (node, drive case node != "": return node, drive, nil case roomForCompute: - return "", "", ErrStorageFull + return "", "", storage.ErrStorageFull } return "", "", ErrMaxedOut } -func buildView(info NodeInfo, owners []Owner, defaultSize Size) NodeView { +func BuildView(info node.NodeInfo, owners []model.Owner, defaultSize model.Size) NodeView { v := NodeView{Name: info.Name, Usable: info.Usable} - byMachine := map[string]*Owner{} + byMachine := map[string]*model.Owner{} for i := range owners { if owners[i].Node == info.Name { byMachine[owners[i].Machine] = &owners[i] @@ -102,7 +109,7 @@ func buildView(info NodeInfo, owners []Owner, defaultSize Size) NodeView { } for _, d := range info.Drives { - r := DriveReport{Path: d.Path} + r := storage.DriveReport{Path: d.Path} var homesUsed int64 for machine, used := range d.HomesMB { homesUsed += used @@ -118,7 +125,7 @@ func buildView(info NodeInfo, owners []Owner, defaultSize Size) NodeView { continue } used := d.HomesMB[u.Machine] - r.CommittedMB += committed(u.StorageMinMB, used) + r.CommittedMB += storage.Committed(u.StorageMinMB, used) r.BookedMB += u.StorageMaxMB r.Homes++ } diff --git a/slackerbot/internal/balance/balance_test.go b/slackerbot/internal/balance/balance_test.go new file mode 100644 index 0000000..81c3f84 --- /dev/null +++ b/slackerbot/internal/balance/balance_test.go @@ -0,0 +1,90 @@ +package balance + +import ( + "errors" + "testing" + + "slackerbot/internal/model" + "slackerbot/internal/node" + "slackerbot/internal/storage" +) + +func TestPlacementBalancesAllThree(t *testing.T) { + drive := func(usable, committed int64) []storage.DriveReport { + return []storage.DriveReport{{Path: "/d", UsableMB: usable, CommittedMB: committed}} + } + want := model.Size{MemoryMB: 2048, CPUs: 1} + views := []NodeView{ + {Name: "cpu-busy", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 2048, CPUs: 6}, Drives: drive(100_000, 10_000)}, + {Name: "ram-busy", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 12288, CPUs: 1}, Drives: drive(100_000, 10_000)}, + {Name: "disk-busy", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 2048, CPUs: 1}, Drives: drive(100_000, 80_000)}, + {Name: "calm", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 4096, CPUs: 2}, Drives: drive(100_000, 20_000)}, + } + node, d, err := ChoosePlacement(views, want, 5120) + if err != nil || node != "calm" || d != "/d" { + t.Fatalf("got %s %s %v, want calm", node, d, err) + } +} + +func TestPlacementSpreadsAcrossEqualNodes(t *testing.T) { + views := []NodeView{ + {Name: "a", Usable: model.Size{MemoryMB: 8192, CPUs: 4}, Drives: []storage.DriveReport{{Path: "/a", UsableMB: 50_000}}}, + {Name: "b", Usable: model.Size{MemoryMB: 8192, CPUs: 4}, Drives: []storage.DriveReport{{Path: "/b", UsableMB: 50_000}}}, + } + want := model.Size{MemoryMB: 2048, CPUs: 1} + counts := map[string]int{} + for i := 0; i < 4; i++ { + node, _, err := ChoosePlacement(views, want, 5120) + if err != nil { + t.Fatal(err) + } + counts[node]++ + for j := range views { + if views[j].Name == node { + views[j].Used = views[j].Used.Add(want) + views[j].Drives[0].CommittedMB += 5120 + } + } + } + if counts["a"] != 2 || counts["b"] != 2 { + t.Fatalf("uneven spread: %v", counts) + } +} + +func TestPlacementWhenFull(t *testing.T) { + full := []NodeView{{Name: "a", Usable: model.Size{MemoryMB: 4096, CPUs: 2}, Used: model.Size{MemoryMB: 4096, CPUs: 2}, Drives: []storage.DriveReport{{Path: "/a", UsableMB: 50_000}}}} + if _, _, err := ChoosePlacement(full, model.Size{MemoryMB: 1024, CPUs: 1}, 1024); !errors.Is(err, ErrMaxedOut) { + t.Fatalf("want maxed out, got %v", err) + } + noDisk := []NodeView{{Name: "a", Usable: model.Size{MemoryMB: 4096, CPUs: 2}, Drives: []storage.DriveReport{{Path: "/a", UsableMB: 1000}}}} + if _, _, err := ChoosePlacement(noDisk, model.Size{MemoryMB: 1024, CPUs: 1}, 5120); !errors.Is(err, storage.ErrStorageFull) { + t.Fatalf("want storage full, got %v", err) + } +} + +func TestBuildView(t *testing.T) { + info := node.NodeInfo{ + Name: "n1", + Usable: model.Size{MemoryMB: 16384, CPUs: 8}, + Running: []node.RunningMachine{{Name: "c-a"}, {Name: "u-stranger"}}, + Drives: []node.DriveInfo{{Path: "/d", FreeMB: 90_000, ReserveMB: 10_000, + HomesMB: map[string]int64{"c-a": 8000, "c-b": 100, "u-stranger": 3000}}}, + } + owners := []model.Owner{ + {ID: "A", Machine: "c-a", Node: "n1", Drive: "/d", Size: model.Size{MemoryMB: 4096, CPUs: 2}, StorageMinMB: 5120, StorageMaxMB: 10240}, + {ID: "B", Machine: "c-b", Node: "n1", Drive: "/d", Size: model.Size{MemoryMB: 2048, CPUs: 1}, StorageMinMB: 5120, StorageMaxMB: 10240}, + {ID: "C", Machine: "c-c", Node: "n1", Drive: "/d", Size: model.Size{MemoryMB: 2048, CPUs: 1}, StorageMinMB: 5120, StorageMaxMB: 10240}, + {ID: "D", Machine: "c-d", Node: "other", Drive: "/d", Size: model.Size{MemoryMB: 2048, CPUs: 1}, StorageMinMB: 5120, StorageMaxMB: 10240}, + } + v := BuildView(info, owners, model.Size{MemoryMB: 1024, CPUs: 1}) + if v.Used != (model.Size{MemoryMB: 4096 + 1024, CPUs: 3}) { + t.Fatalf("used = %v", v.Used) + } + d := v.Drives[0] + if d.UsableMB != 90_000+11_100-10_000 { + t.Fatalf("usable = %d", d.UsableMB) + } + if d.CommittedMB != 8000+5120+5120+3000 || d.BookedMB != 3*10240+3000 || d.Homes != 4 { + t.Fatalf("committed %d booked %d homes %d", d.CommittedMB, d.BookedMB, d.Homes) + } +} diff --git a/bot.go b/slackerbot/internal/bot/bot.go similarity index 81% rename from bot.go rename to slackerbot/internal/bot/bot.go index d3bccb2..0229f1d 100644 --- a/bot.go +++ b/slackerbot/internal/bot/bot.go @@ -1,4 +1,4 @@ -package main +package bot import ( "context" @@ -16,6 +16,18 @@ import ( "github.com/slack-go/slack" "github.com/slack-go/slack/slackevents" "github.com/slack-go/slack/socketmode" + + "slackerbot/internal/acp" + "slackerbot/internal/balance" + "slackerbot/internal/commands" + "slackerbot/internal/config" + "slackerbot/internal/manager" + "slackerbot/internal/model" + "slackerbot/internal/node" + "slackerbot/internal/state" + "slackerbot/internal/storage" + "slackerbot/internal/units" + "slackerbot/internal/util" ) const helpText = "I pass your messages to an agent running in its own container.\n" + @@ -40,17 +52,17 @@ const helpText = "I pass your messages to an agent running in its own container. "• `help`: this message" type Bot struct { - cfg *Config - store *Store - mgr *Manager + cfg *config.Config + store *state.Store + mgr *manager.Manager api *slack.Client sm *socketmode.Client botID string acpMu sync.Mutex - acps map[string]*ACP - dialMu keyedMutex - sessMu keyedMutex + acps map[string]*acp.ACP + dialMu util.KeyedMutex + sessMu util.KeyedMutex busyMu sync.Mutex busy map[string]int names sync.Map @@ -64,14 +76,14 @@ type incoming struct { dm bool } -func (m incoming) owner() (string, OwnerKind) { +func (m incoming) owner() (string, model.OwnerKind) { if m.dm { - return m.user, OwnerUser + return m.user, model.OwnerUser } - return m.channel, OwnerChannel + return m.channel, model.OwnerChannel } -func NewBot(cfg *Config, store *Store, mgr *Manager) (*Bot, error) { +func New(cfg *config.Config, store *state.Store, mgr *manager.Manager) (*Bot, error) { api := slack.New(cfg.Slack.BotToken, slack.OptionAppLevelToken(cfg.Slack.AppToken)) auth, err := api.AuthTest() if err != nil { @@ -81,11 +93,11 @@ func NewBot(cfg *Config, store *Store, mgr *Manager) (*Bot, error) { cfg: cfg, store: store, mgr: mgr, api: api, sm: socketmode.New(api), botID: auth.UserID, - acps: map[string]*ACP{}, + acps: map[string]*acp.ACP{}, busy: map[string]int{}, waiting: map[string]map[string]bool{}, } - mgr.onNodeBack = b.nodeBack + mgr.SetNodeBackHandler(b.nodeBack) return b, nil } @@ -169,13 +181,13 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) { } ownerID, kind := m.owner() - var off *NodeOfflineError - var full *NodeFullError + var off *node.NodeOfflineError + var full *node.NodeFullError u, err := b.mgr.Ensure(ctx, ownerID, kind) if errors.As(err, &full) { edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room…", full.Node)) trigger := fmt.Sprintf("auto process (%s was full), for <@%s>'s message", full.Node, m.user) - _, merr := b.runMove(ctx, m.channel, ownerID, MoveOptions{StartAfter: true}, trigger, func(link string) { + _, merr := b.runMove(ctx, m.channel, ownerID, manager.MoveOptions{StartAfter: true}, trigger, func(link string) { edit(fmt.Sprintf(":truck: %s is full, so this container is being moved to a node with room. <%s|Follow the move here>.", full.Node, link)) }) if merr != nil { @@ -213,7 +225,7 @@ func (b *Bot) onMessage(ctx context.Context, m incoming) { edit(":warning: Couldn't start a session: " + err.Error()) return } - _ = b.store.AddSession(&Session{ + _ = b.store.AddSession(&model.Session{ Channel: m.channel, ThreadTS: root, OwnerID: u.ID, ACPID: id, Title: title(m.text), CreatedAt: time.Now(), }) @@ -249,7 +261,7 @@ func (b *Bot) command(ctx context.Context, m incoming) bool { if verb == "grant" || verb == "revoke" || verb == "dm" || verb == "managers" || (verb == "permissions" && args != "") { return b.manageCommand(ctx, m, verb, args) } - if need := neededPermission(verb, args); need != "" && !b.permitted(ctx, m.channel, "", m.user, kind, need) { + if need := commands.NeededPermission(verb, args); need != "" && !b.permitted(ctx, m.channel, "", m.user, kind, need) { return true } if verb == "stop" || verb == "scale" || verb == "storage" || verb == "disk" { @@ -304,12 +316,12 @@ func (b *Bot) onAction(ctx context.Context, cb slack.InteractionCallback) { if ownerID != cb.User.ID && ownerID != cb.Channel.ID { continue } - kind := OwnerChannel + kind := model.OwnerChannel if ownerID == cb.User.ID { - kind = OwnerUser + kind = model.OwnerUser } u := b.store.Owner(ownerID) - if need := map[string]Action{"recreate": ActMove, "scale": ActScale}[a.ActionID]; need != "" && + if need := map[string]model.Action{"recreate": model.ActMove, "scale": model.ActScale}[a.ActionID]; need != "" && !b.permitted(ctx, cb.Channel.ID, "", cb.User.ID, kind, need) { continue } @@ -336,7 +348,7 @@ func (b *Bot) onAction(ctx context.Context, cb slack.InteractionCallback) { b.waitMu.Unlock() b.say(ctx, cb.Channel.ID, "", fmt.Sprintf("OK, waiting for %s. I'll post here when it's back up.", u.Node)) case "stop", "scale": - if b.mgr.State(u) == StateOffline { + if b.mgr.State(u) == manager.StateOffline { b.deadSlave(ctx, cb.Channel.ID, "", ownerID, u.Node) continue } @@ -368,7 +380,7 @@ func (b *Bot) sessionList(ctx context.Context, ownerID string) string { return sb.String() } -func (b *Bot) postStatus(ctx context.Context, channel, ownerID string, kind OwnerKind) { +func (b *Bot) postStatus(ctx context.Context, channel, ownerID string, kind model.OwnerKind) { u := b.store.Owner(ownerID) state := b.mgr.State(u) size := b.cfg.Container @@ -377,15 +389,15 @@ func (b *Bot) postStatus(ctx context.Context, channel, ownerID string, kind Owne } var sb strings.Builder whose := "This channel's container" - if kind == OwnerUser { + if kind == model.OwnerUser { whose = "Your container" } - fmt.Fprintf(&sb, "*%s:* %s\n*Size:* %s\n", whose, state, size) + fmt.Fprintf(&sb, "*%s:* %s\n*model.Size:* %s\n", whose, state, size) if u != nil && u.Node != "" { fmt.Fprintf(&sb, "*Runs on:* %s\n", u.Node) } - if u != nil && state == StateRunning { - fmt.Fprintf(&sb, "*Unused for:* %s\n", human(time.Since(u.LastActive))) + if u != nil && state == manager.StateRunning { + fmt.Fprintf(&sb, "*Unused for:* %s\n", commands.Human(time.Since(u.LastActive))) } fmt.Fprintf(&sb, "*Stops when unused for:* %s\n", b.describeIdle(u)) fmt.Fprintf(&sb, "*Home folder:* %s\n", b.describeStorage(ctx, u)) @@ -402,7 +414,7 @@ func (b *Bot) postStatus(ctx context.Context, channel, ownerID string, kind Owne blocks := []slack.Block{slack.NewSectionBlock(slack.NewTextBlockObject(slack.MarkdownType, text, false, false), nil, nil)} var buttons []slack.BlockElement - if state == StateRunning { + if state == manager.StateRunning { stop := slack.NewButtonBlockElement("stop", ownerID, slack.NewTextBlockObject(slack.PlainTextType, "Stop container", false, false)) stop.Style = slack.StyleDanger buttons = append(buttons, stop) @@ -421,7 +433,7 @@ func (b *Bot) postStatus(ctx context.Context, channel, ownerID string, kind Owne func (b *Bot) stopFor(ctx context.Context, ownerID string) string { u := b.store.Owner(ownerID) - if b.mgr.State(u) != StateRunning { + if b.mgr.State(u) != manager.StateRunning { return "The container isn't running." } if err := b.mgr.Poweroff(ctx, u); err != nil { @@ -431,18 +443,18 @@ func (b *Bot) stopFor(ctx context.Context, ownerID string) string { return "Shutting down the container. Its files are kept. Message me anytime to start it again." } -func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind OwnerKind, args string) (reply string, ok bool) { - old, next, err := b.mgr.Resize(ctx, ownerID, kind, func(cur Size) (Size, error) { - return parseScale(args, cur) +func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) { + old, next, err := b.mgr.Resize(ctx, ownerID, kind, func(cur model.Size) (model.Size, error) { + return commands.ParseScale(args, cur) }) switch { - case errors.Is(err, errNotCommand): + case errors.Is(err, commands.ErrNotCommand): return "", false - case errors.Is(err, ErrTooBig): + case errors.Is(err, manager.ErrTooBig): return fmt.Sprintf("That's bigger than allowed. The max is %s.", b.cfg.MaxContainer), true - case errors.Is(err, ErrTooSmall): - return fmt.Sprintf("That's too small. The minimum is %s.", MinContainer), true - case errors.Is(err, ErrMaxedOut) || errors.As(err, new(*NodeFullError)): + case errors.Is(err, manager.ErrTooSmall): + return fmt.Sprintf("That's too small. The minimum is %s.", manager.MinContainer), true + case errors.Is(err, balance.ErrMaxedOut) || errors.As(err, new(*node.NodeFullError)): return b.explain(err), true case err != nil: return fmt.Sprintf(":warning: %s. Try `scale 8G 4cpu`, `scale +2G`, `scale -1cpu` or `scale up`.", err), true @@ -450,7 +462,7 @@ func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind OwnerKind, args return fmt.Sprintf("No change. The container already has %s.", old), true } - if b.mgr.State(b.store.Owner(ownerID)) != StateRunning { + if b.mgr.State(b.store.Owner(ownerID)) != manager.StateRunning { return fmt.Sprintf("Done! The container will start with %s next time (was %s).", next, old), true } reply = fmt.Sprintf("Done! The container now has %s (was %s).", next, old) @@ -460,10 +472,10 @@ func (b *Bot) scaleFor(ctx context.Context, ownerID string, kind OwnerKind, args return reply, true } -func (b *Bot) keepFor(ownerID string, kind OwnerKind, args string) (reply string, ok bool) { - k, err := parseKeep(args) +func (b *Bot) keepFor(ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) { + k, err := commands.ParseKeep(args) switch { - case errors.Is(err, errNotCommand): + case errors.Is(err, commands.ErrNotCommand): return "", false case err != nil: return ":warning: " + err.Error(), true @@ -476,22 +488,22 @@ func (b *Bot) keepFor(ownerID string, kind OwnerKind, args string) (reply string var d time.Duration switch { - case k.show: + case k.Show: return "This container stops when unused for " + b.describeIdle(u) + ".", true - case k.reset: + case k.Reset: d = 0 - case k.indefinite: + case k.Indefinite: if !b.cfg.AllowIndefinite { return "Keeping containers indefinitely is turned off on this server.", true } - d = Indefinite + d = model.Indefinite default: - d = k.limit + d = k.Limit if d < 10*time.Minute { return "That's too short. The minimum is 10m.", true } if max := b.cfg.MaxIdleTimeout; max > 0 && d > max { - return fmt.Sprintf("That's longer than allowed. The max is %s.", human(max)), true + return fmt.Sprintf("That's longer than allowed. The max is %s.", commands.Human(max)), true } } @@ -500,14 +512,14 @@ func (b *Bot) keepFor(ownerID string, kind OwnerKind, args string) (reply string } _ = b.store.Touch(ownerID) u.IdleTimeout = d - if d == Indefinite { + if d == model.Indefinite { return "Done! This container won't be stopped for being unused. Send `stop` when you're finished, " + "or `keep default` to go back to normal.", true } return "Done! This container stops when unused for " + b.describeIdle(u) + ".", true } -func (b *Bot) storageFor(ctx context.Context, ownerID string, kind OwnerKind, args string) (reply string, ok bool) { +func (b *Bot) storageFor(ctx context.Context, ownerID string, kind model.OwnerKind, args string) (reply string, ok bool) { if strings.TrimSpace(args) == "" { u, err := b.mgr.Owner(ownerID, kind) if err != nil { @@ -517,42 +529,42 @@ func (b *Bot) storageFor(ctx context.Context, ownerID string, kind OwnerKind, ar } c, err := b.mgr.SetStorage(ctx, ownerID, kind, func(min, max int64) (int64, int64, error) { - return parseStorage(args, min, max) + return commands.ParseStorage(args, min, max) }) switch { - case errors.Is(err, errNotCommand): + case errors.Is(err, commands.ErrNotCommand): return "", false - case errors.Is(err, ErrStorageTooSmall): - return fmt.Sprintf("That's too small. The minimum is %s.", formatMB(MinStorageMB)), true - case errors.Is(err, ErrMinOverMax): - return fmt.Sprintf("The guaranteed space (%s) can't be more than the max (%s).", formatMB(c.NewMin), formatMB(c.NewMax)), true - case errors.Is(err, ErrStorageTooBig): - return fmt.Sprintf("The max can't go above %s.", formatMB(b.cfg.Storage.LimitMB)), true - case errors.Is(err, ErrBelowUsage): - return fmt.Sprintf("The home folder already holds %s, so the max can't go below that.", formatMB(c.UsedMB)), true - case errors.Is(err, ErrStorageFull): - return fmt.Sprintf(":no_entry: There isn't enough free space on the drive to guarantee %s.", formatMB(c.NewMin)), true + case errors.Is(err, storage.ErrStorageTooSmall): + return fmt.Sprintf("That's too small. The minimum is %s.", units.FormatMB(config.MinStorageMB)), true + case errors.Is(err, storage.ErrMinOverMax): + return fmt.Sprintf("The guaranteed space (%s) can't be more than the max (%s).", units.FormatMB(c.NewMin), units.FormatMB(c.NewMax)), true + case errors.Is(err, storage.ErrStorageTooBig): + return fmt.Sprintf("The max can't go above %s.", units.FormatMB(b.cfg.Storage.LimitMB)), true + case errors.Is(err, storage.ErrBelowUsage): + return fmt.Sprintf("The home folder already holds %s, so the max can't go below that.", units.FormatMB(c.UsedMB)), true + case errors.Is(err, storage.ErrStorageFull): + return fmt.Sprintf(":no_entry: There isn't enough free space on the drive to guarantee %s.", units.FormatMB(c.NewMin)), true case err != nil: return fmt.Sprintf(":warning: %s. Try `storage 20G`, `storage +5G` or `storage min 8G max 20G`.", err), true } reply = fmt.Sprintf("Done! Home folder: %s guaranteed, up to %s (was %s guaranteed, up to %s).", - formatMB(c.NewMin), formatMB(c.NewMax), formatMB(c.OldMin), formatMB(c.OldMax)) + units.FormatMB(c.NewMin), units.FormatMB(c.NewMax), units.FormatMB(c.OldMin), units.FormatMB(c.OldMax)) if c.Drive != nil && c.Drive.Overbooked() { reply += fmt.Sprintf("\n:warning: Heads up: drive %s is overbooked. Its containers may use up to %s in total, "+ "but only %s is available. Guaranteed space is safe, but if they all fill up, writes will start failing.", - c.Drive.Path, formatMB(c.Drive.BookedMB), formatMB(c.Drive.UsableMB)) + c.Drive.Path, units.FormatMB(c.Drive.BookedMB), units.FormatMB(c.Drive.UsableMB)) } return reply, true } -func (b *Bot) describeStorage(ctx context.Context, u *Owner) string { +func (b *Bot) describeStorage(ctx context.Context, u *model.Owner) string { if u == nil { - return fmt.Sprintf("%s guaranteed, up to %s", formatMB(b.cfg.Storage.MinMB), formatMB(b.cfg.Storage.MaxMB)) + return fmt.Sprintf("%s guaranteed, up to %s", units.FormatMB(b.cfg.Storage.MinMB), units.FormatMB(b.cfg.Storage.MaxMB)) } - text := fmt.Sprintf("%s guaranteed, up to %s", formatMB(u.StorageMinMB), formatMB(u.StorageMaxMB)) + text := fmt.Sprintf("%s guaranteed, up to %s", units.FormatMB(u.StorageMinMB), units.FormatMB(u.StorageMaxMB)) if used, ok := b.mgr.StorageUsed(ctx, u); ok { - text = formatMB(used) + " used, " + text + text = units.FormatMB(used) + " used, " + text } if u.Node != "" { text += fmt.Sprintf(" (on %s:%s)", u.Node, u.Drive) @@ -560,16 +572,16 @@ func (b *Bot) describeStorage(ctx context.Context, u *Owner) string { return text } -func describeNode(v NodeView) string { +func describeNode(v balance.NodeView) string { us, cm, _ := v.Storage() free := v.Usable.Sub(v.Used) text := fmt.Sprintf("*%s*: %s of %s CPUs free · %s of %s RAM free · %s of %s storage free · %d running, %d total", v.Name, strconv.FormatFloat(math.Max(0, free.CPUs), 'f', -1, 64), strconv.FormatFloat(v.Usable.CPUs, 'f', -1, 64), - formatMB(int64(max(0, free.MemoryMB))), formatMB(int64(v.Usable.MemoryMB)), - formatMB(max(0, us-cm)), formatMB(us), v.Running, v.Homes()) + units.FormatMB(int64(max(0, free.MemoryMB))), units.FormatMB(int64(v.Usable.MemoryMB)), + units.FormatMB(max(0, us-cm)), units.FormatMB(us), v.Running, v.Homes()) for _, d := range v.Drives { if d.Overbooked() { - text += fmt.Sprintf(" · :warning: %s overbooked (caps add up to %s of %s)", d.Path, formatMB(d.BookedMB), formatMB(d.UsableMB)) + text += fmt.Sprintf(" · :warning: %s overbooked (caps add up to %s of %s)", d.Path, units.FormatMB(d.BookedMB), units.FormatMB(d.UsableMB)) } } return text @@ -588,34 +600,13 @@ func (b *Bot) moveTarget(args string) (target string, ok bool) { return "", false } -func neededPermission(verb, args string) Action { - switch verb { - case "scale": - if _, err := parseScale(args, Size{1024, 1}); !errors.Is(err, errNotCommand) { - return ActScale - } - case "keep": - if k, err := parseKeep(args); !errors.Is(err, errNotCommand) && !k.show { - return ActKeep - } - case "storage", "disk": - if args == "" { - return "" - } - if _, _, err := parseStorage(args, 1024, 1024); !errors.Is(err, errNotCommand) { - return ActStorage - } - } - return "" -} - -func (b *Bot) permitted(ctx context.Context, channel, thread, user string, kind OwnerKind, a Action) bool { +func (b *Bot) permitted(ctx context.Context, channel, thread, user string, kind model.OwnerKind, a model.Action) bool { p := b.store.Permissions() if p.Allowed(kind, channel, user, a) { return true } var text string - if kind == OwnerUser { + if kind == model.OwnerUser { text = fmt.Sprintf(":lock: Changing *%s* from DMs is turned off on this server.", a) } else if holders := p.Holders(channel, a); len(holders) == 0 { text = fmt.Sprintf(":lock: Nobody here has *%s* permission. An admin can grant it in the config.", a) @@ -630,13 +621,13 @@ func (b *Bot) permitted(ctx context.Context, channel, thread, user string, kind return false } -func (b *Bot) permissionsList(ctx context.Context, channel string, kind OwnerKind) string { +func (b *Bot) permissionsList(ctx context.Context, channel string, kind model.OwnerKind) string { p := b.store.Permissions() - if kind == OwnerUser { + if kind == model.OwnerUser { var lines []string - for _, a := range allActions { + for _, a := range model.AllActions { state := "allowed" - if !p.Allowed(OwnerUser, channel, "", a) { + if !p.Allowed(model.OwnerUser, channel, "", a) { state = "turned off" } lines = append(lines, fmt.Sprintf("• *%s*: %s", a, state)) @@ -644,7 +635,7 @@ func (b *Bot) permissionsList(ctx context.Context, channel string, kind OwnerKin return "*In DMs, you can change your own container:*\n" + strings.Join(lines, "\n") + "\nUser managers can switch these with `dm on|off`." } - list, own := p.superUsers(channel) + list, own := p.SuperUsers(channel) source := "the server-wide default list (this channel has no list of its own)" if own { source = "this channel's own list" @@ -674,11 +665,11 @@ func (b *Bot) permissionsList(ctx context.Context, channel string, kind OwnerKin } func (b *Bot) blocked(ctx context.Context, channel, thread, ownerID string) bool { - if b.mgr.isMigrating(ownerID) { - b.say(ctx, channel, thread, b.explain(ErrMigrating)) + if b.mgr.IsMigrating(ownerID) { + b.say(ctx, channel, thread, b.explain(manager.ErrMigrating)) return true } - if u := b.store.Owner(ownerID); b.mgr.State(u) == StateOffline { + if u := b.store.Owner(ownerID); b.mgr.State(u) == manager.StateOffline { b.deadSlave(ctx, channel, "", ownerID, u.Node) return true } @@ -696,7 +687,7 @@ func (b *Bot) moveCommand(ctx context.Context, m incoming, replyThread string) b return false } ownerID, kind := m.owner() - if !b.permitted(ctx, m.channel, replyThread, m.user, kind, ActMove) { + if !b.permitted(ctx, m.channel, replyThread, m.user, kind, model.ActMove) { return true } if b.blocked(ctx, m.channel, replyThread, ownerID) { @@ -707,14 +698,14 @@ func (b *Bot) moveCommand(ctx context.Context, m incoming, replyThread string) b } func (b *Bot) moveFor(ctx context.Context, channel, ownerID, target, trigger, replyThread string) { - b.runMove(ctx, channel, ownerID, MoveOptions{Target: target}, trigger, func(link string) { + b.runMove(ctx, channel, ownerID, manager.MoveOptions{Target: target}, trigger, func(link string) { b.say(ctx, channel, replyThread, fmt.Sprintf(":truck: Moving this container. <%s|Follow the move here>.", link)) }) } // Every move gets its own top-level message saying who started it, with // progress in that message's thread. -func (b *Bot) runMove(ctx context.Context, channel, ownerID string, opts MoveOptions, trigger string, announce func(link string)) (*MoveResult, error) { +func (b *Bot) runMove(ctx context.Context, channel, ownerID string, opts manager.MoveOptions, trigger string, announce func(link string)) (*manager.MoveResult, error) { before := b.store.Owner(ownerID) from := "its node" if before != nil && before.Node != "" { @@ -756,15 +747,15 @@ func (b *Bot) runMove(ctx context.Context, channel, ownerID string, opts MoveOpt return res, err } -func (b *Bot) moveOutcome(ownerID, target string, res *MoveResult, err error) (text, outcome string) { +func (b *Bot) moveOutcome(ownerID, target string, res *manager.MoveResult, err error) (text, outcome string) { failed := ":x: didn't move" - var unknown *UnknownNodeError - var full *NodeFullError - var off *NodeOfflineError + var unknown *manager.UnknownNodeError + var full *node.NodeFullError + var off *node.NodeOfflineError switch { - case errors.Is(err, ErrNoContainer): + case errors.Is(err, manager.ErrNoContainer): return "There's no container here yet. Send me a message to create one.", failed - case errors.Is(err, ErrSameNode): + case errors.Is(err, manager.ErrSameNode): return "The container is already on " + target + ".", failed case errors.As(err, &unknown): return fmt.Sprintf("There's no node called %s. Send `nodes` to see them.", unknown.Node), failed @@ -772,9 +763,9 @@ func (b *Bot) moveOutcome(ownerID, target string, res *MoveResult, err error) (t return fmt.Sprintf("%s is offline right now, so nothing can move there.", off.Node), failed case errors.As(err, &full) && res == nil: return fmt.Sprintf(":no_entry: %s doesn't have room for this container. Send `nodes` to see what's free.", full.Node), failed - case errors.Is(err, ErrMaxedOut) || errors.Is(err, ErrStorageFull): + case errors.Is(err, balance.ErrMaxedOut) || errors.Is(err, storage.ErrStorageFull): return ":no_entry: No other node has room for this container right now. Send `nodes` to see what's free.", failed - case errors.Is(err, ErrMigrating): + case errors.Is(err, manager.ErrMigrating): return b.explain(err), failed } if err != nil { @@ -783,7 +774,7 @@ func (b *Bot) moveOutcome(ownerID, target string, res *MoveResult, err error) (t } return ":warning: The move didn't work, so the container stays on its old node: " + err.Error(), failed } - text = fmt.Sprintf(":white_check_mark: Moved from %s to %s (%s copied).", res.From, res.To, formatMB(res.CopiedMB)) + text = fmt.Sprintf(":white_check_mark: Moved from %s to %s (%s copied).", res.From, res.To, units.FormatMB(res.CopiedMB)) if res.Started { text += " It's running there now, and its sessions came along." } @@ -793,27 +784,27 @@ func (b *Bot) moveOutcome(ownerID, target string, res *MoveResult, err error) (t return text, ":white_check_mark: done, now on " + res.To } -func (b *Bot) progressEditor(ctx context.Context, channel, ts string) func(MoveProgress) { +func (b *Bot) progressEditor(ctx context.Context, channel, ts string) func(manager.MoveProgress) { var last time.Time - var lastPhase MovePhase - return func(p MoveProgress) { + var lastPhase manager.MovePhase + return func(p manager.MoveProgress) { if p.Phase == lastPhase && time.Since(last) < 3*time.Second { return } last, lastPhase = time.Now(), p.Phase - copied := formatMB(p.CopiedMB) + copied := units.FormatMB(p.CopiedMB) if p.TotalMB > 0 { - copied += " of ~" + formatMB(p.TotalMB) + copied += " of ~" + units.FormatMB(p.TotalMB) } var text string switch p.Phase { - case PhaseCopy: + case manager.PhaseCopy: text = fmt.Sprintf(":truck: Moving to %s… copying the home folder (%s). The container keeps running for now.", p.To, copied) - case PhaseFinal: + case manager.PhaseFinal: text = fmt.Sprintf(":truck: Moving to %s… final sync of recent changes. The container is briefly stopped.", p.To) - case PhaseCleanup: + case manager.PhaseCleanup: text = fmt.Sprintf(":truck: Moving to %s… copied %s, switching over.", p.To, copied) - case PhaseStart: + case manager.PhaseStart: text = fmt.Sprintf(":truck: Moving to %s… starting it there.", p.To) } _, _, _, _ = b.api.UpdateMessageContext(ctx, channel, ts, slack.MsgOptionText(text, false)) @@ -889,48 +880,48 @@ func (b *Bot) nodeBack(node string) { } } -func (b *Bot) idleLimit(u *Owner) (limit time.Duration, indefinite bool) { +func (b *Bot) idleLimit(u *model.Owner) (limit time.Duration, indefinite bool) { switch { case u == nil || u.IdleTimeout == 0: return b.cfg.IdleTimeout, false - case u.IdleTimeout == Indefinite: + case u.IdleTimeout == model.Indefinite: return 0, true } return u.IdleTimeout, false } -func (b *Bot) describeIdle(u *Owner) string { +func (b *Bot) describeIdle(u *model.Owner) string { limit, indefinite := b.idleLimit(u) switch { case indefinite: return "never (kept indefinitely)" case u == nil || u.IdleTimeout == 0: - return human(limit) + " (default)" + return commands.Human(limit) + " (default)" } - return human(limit) + return commands.Human(limit) } func (b *Bot) explain(err error) string { - var full *NodeFullError + var full *node.NodeFullError switch { - case errors.Is(err, ErrMigrating): + case errors.Is(err, manager.ErrMigrating): return ":truck: This container is being moved right now. Try again in a moment." case errors.As(err, &full): return fmt.Sprintf(":no_entry: Sorry, %s (where this container's files live) is maxed out right now. "+ "Try again a bit later.", full.Node) - case errors.Is(err, ErrMaxedOut): + case errors.Is(err, balance.ErrMaxedOut): used, usable := b.mgr.Totals(context.Background()) return fmt.Sprintf(":no_entry: Sorry, every node is maxed out right now (%s of %s in use). "+ "Try again a bit later.", used, usable) - case errors.Is(err, ErrStorageFull): + case errors.Is(err, storage.ErrStorageFull): return ":no_entry: Sorry, there isn't enough storage left on any node to set up a new container. Try again later." } return ":warning: " + err.Error() } -func clientKey(u *Owner) string { return u.Node + "/" + u.Machine } +func clientKey(u *model.Owner) string { return u.Node + "/" + u.Machine } -func (b *Bot) client(ctx context.Context, u *Owner) (*ACP, error) { +func (b *Bot) client(ctx context.Context, u *model.Owner) (*acp.ACP, error) { key := clientKey(u) unlock := b.dialMu.Lock(key) defer unlock() @@ -952,7 +943,7 @@ func (b *Bot) client(ctx context.Context, u *Owner) (*ACP, error) { } initCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - if c, err = NewACP(initCtx, conn); err != nil { + if c, err = acp.New(initCtx, conn); err != nil { return nil, err } b.acpMu.Lock() @@ -961,7 +952,7 @@ func (b *Bot) client(ctx context.Context, u *Owner) (*ACP, error) { return c, nil } -func (b *Bot) dropClient(u *Owner) { +func (b *Bot) dropClient(u *model.Owner) { b.acpMu.Lock() defer b.acpMu.Unlock() if c := b.acps[clientKey(u)]; c != nil { @@ -1012,7 +1003,7 @@ func (b *Bot) reapOnce(ctx context.Context) { } b.dropClient(&u) msg := fmt.Sprintf("I stopped the container after %s without use. Its files are kept. "+ - "Message me anytime to start it again, or use `keep` to change how long it waits.", human(limit)) + "Message me anytime to start it again, or use `keep` to change how long it waits.", commands.Human(limit)) b.say(ctx, u.ID, "", msg) // a user ID posts to their DM, a channel ID to the channel } } diff --git a/slackerbot/internal/bot/bot_test.go b/slackerbot/internal/bot/bot_test.go new file mode 100644 index 0000000..ea3f9c0 --- /dev/null +++ b/slackerbot/internal/bot/bot_test.go @@ -0,0 +1,249 @@ +package bot + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/slack-go/slack" + + "slackerbot/internal/acp" + "slackerbot/internal/manager" + "slackerbot/internal/model" + "slackerbot/internal/state" + "slackerbot/internal/testutil" + "slackerbot/internal/testutil/clustertest" +) + +func TestOwnership(t *testing.T) { + dm := incoming{user: "U42", channel: "D9", dm: true} + if id, kind := dm.owner(); id != "U42" || kind != model.OwnerUser { + t.Fatalf("DM should be owned by the user, got %s %s", id, kind) + } + ch := incoming{user: "U42", channel: "C7"} + if id, kind := ch.owner(); id != "C7" || kind != model.OwnerChannel { + t.Fatalf("channel message should be owned by the channel, got %s %s", id, kind) + } + other := incoming{user: "U99", channel: "C7"} + a, ka := ch.owner() + b, kb := other.owner() + if model.MachineName(a, ka) != model.MachineName(b, kb) || model.MachineName(a, ka) != "c-c7" { + t.Fatal("two people in one channel should share a container") + } + if model.MachineName("U42", model.OwnerUser) != "u-u42" || !model.IsOurs("u-u42") || !model.IsOurs("c-c7") || model.IsOurs("web") { + t.Fatal("machine naming is off") + } +} + +func TestTitle(t *testing.T) { + if got := title(" fix the build\nplease"); got != "fix the build" { + t.Fatal(got) + } + if got := title(strings.Repeat("é", 80)); !strings.HasSuffix(got, "…") || len([]rune(got)) != 61 { + t.Fatal(got) + } +} + +type fakeSlack struct { + mu sync.Mutex + msgs map[string]string + order []string + n int +} + +func (f *fakeSlack) handler(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + f.mu.Lock() + defer f.mu.Unlock() + ts := r.Form.Get("ts") + if strings.HasSuffix(r.URL.Path, "chat.postMessage") { + f.n++ + ts = "ts" + string(rune('0'+f.n)) + f.order = append(f.order, ts) + } + f.msgs[ts] = r.Form.Get("text") + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "D1", "ts": ts, "text": f.msgs[ts]}) +} + +func TestStreamerSplitsLongReplies(t *testing.T) { + f := &fakeSlack{msgs: map[string]string{"ph": ""}, order: []string{"ph"}} + srv := httptest.NewServer(http.HandlerFunc(f.handler)) + defer srv.Close() + api := slack.New("xoxb-test", slack.OptionAPIURL(srv.URL+"/")) + + st := NewStreamer(api, "D1", "root", "ph") + line := strings.Repeat("x", 99) + "\n" + for i := 0; i < 80; i++ { + st.Update(acp.Update{Kind: "agent_message_chunk", Text: line}) + } + st.Finish("end_turn", nil) + + f.mu.Lock() + defer f.mu.Unlock() + if len(f.order) != 3 { + t.Fatalf("want 3 messages, got %d", len(f.order)) + } + var all strings.Builder + for _, ts := range f.order { + if len(f.msgs[ts]) > slackMsgLimit { + t.Fatalf("message %s too long: %d", ts, len(f.msgs[ts])) + } + all.WriteString(f.msgs[ts]) + } + if all.Len() != 8000 { + t.Fatalf("text lost or duplicated: %d chars", all.Len()) + } +} + +func TestMoveTarget(t *testing.T) { + b := &Bot{} + for in, want := range map[string]string{"": "", "auto": "", "Slave-1": "slave-1", "to node-b": "node-b"} { + if got, ok := b.moveTarget(in); !ok || got != want { + t.Fatalf("move %q = %q %v, want %q", in, got, ok, want) + } + } + if _, ok := b.moveTarget("the files into the build folder"); ok { + t.Fatal("a normal sentence should go to the agent") + } +} + +type slackLog struct { + mu sync.Mutex + n int + text map[string]string + under map[string]string + order []string +} + +func (s *slackLog) ServeHTTP(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + s.mu.Lock() + defer s.mu.Unlock() + ts := r.Form.Get("ts") + if strings.HasSuffix(r.URL.Path, "chat.postMessage") { + s.n++ + ts = fmt.Sprintf("%d.0", s.n) + s.order = append(s.order, ts) + s.under[ts] = r.Form.Get("thread_ts") + } + if !strings.HasSuffix(r.URL.Path, "chat.getPermalink") { + s.text[ts] = r.Form.Get("text") + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "C1", "ts": ts, "permalink": "https://slack.example/" + ts}) +} + +func TestEveryMoveStartsAnAnnouncedThread(t *testing.T) { + c := clustertest.New(t) + c.AddOwner(t, testutil.HomeData(), false) + log := &slackLog{text: map[string]string{}, under: map[string]string{}} + srv := httptest.NewServer(log) + defer srv.Close() + b := &Bot{cfg: c.Cfg, store: c.Store, mgr: c.Mgr, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/")), + acps: map[string]*acp.ACP{}, busy: map[string]int{}} + + b.moveFor(c.Ctx, "C1", "C1", "slave-1", "<@U1>", "100.0") + if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "master"}); err != nil { + t.Fatal(err) + } + b.runMove(c.Ctx, "C1", "C1", manager.MoveOptions{}, "auto process (master was full), for <@U2>'s message", nil) + + log.mu.Lock() + defer log.mu.Unlock() + var roots []string + for _, ts := range log.order { + if log.under[ts] == "" { + roots = append(roots, ts) + } + } + linked := 0 + for ts, parent := range log.under { + if parent == "100.0" && strings.Contains(log.text[ts], "Follow the move here") { + linked++ + } + } + if linked != 1 { + t.Fatalf("the thread where the move started should get one link to the move thread, got %d", linked) + } + if len(roots) != 2 { + t.Fatalf("want one new top-level message per move, got %d", len(roots)) + } + checks := []string{"started by <@U1>: master → slave-1", "started by auto process (master was full), for <@U2>'s message"} + for i, root := range roots { + if !strings.Contains(log.text[root], checks[i]) || !strings.Contains(log.text[root], ":white_check_mark: done") { + t.Fatalf("root %d = %q", i, log.text[root]) + } + replies := 0 + for ts, parent := range log.under { + if parent == root { + replies++ + if !strings.Contains(log.text[ts], "Moved from") { + t.Fatalf("progress reply = %q", log.text[ts]) + } + } + } + if replies != 1 { + t.Fatalf("want progress in one thread reply, got %d", replies) + } + } +} + +func TestManageCommandsInSlack(t *testing.T) { + log := &slackLog{text: map[string]string{}, under: map[string]string{}} + srv := httptest.NewServer(log) + defer srv.Close() + store, _ := state.Open(filepath.Join(t.TempDir(), "state.json")) + store.SeedPermissions(model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UBOSS": {"all"}}}) + b := &Bot{store: store, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/"))} + ctx := context.Background() + say := func(user, text string) string { + m := incoming{user: user, channel: "C1", ts: "1.0", text: text} + if !b.command(ctx, m) { + return "(went to the agent)" + } + log.mu.Lock() + defer log.mu.Unlock() + return log.text[log.order[len(log.order)-1]] + } + + if got := say("UANA", "grant <@UANA> all"); !strings.Contains(got, ":lock:") { + t.Fatalf("non-managers can't grant: %q", got) + } + if got := say("UBOSS", "grant <@UANA> move manage"); !strings.Contains(got, "<@UANA> can now: move, manage") { + t.Fatalf("manager grant: %q", got) + } + if got := say("UANA", "grant <@UCAL> scale"); !strings.Contains(got, "<@UCAL> can now: scale") { + t.Fatalf("channel manager grant: %q", got) + } + if got := say("UANA", "grant default <@UCAL> all"); !strings.Contains(got, ":lock:") { + t.Fatalf("channel managers can't touch the default list: %q", got) + } + if got := say("UANA", "managers add <@UCAL>"); !strings.Contains(got, ":lock:") { + t.Fatalf("only user managers add managers: %q", got) + } + if got := say("UBOSS", "managers remove <@UBOSS>"); !strings.Contains(got, "nobody able to manage") { + t.Fatalf("last manager guard: %q", got) + } + if got := say("UBOSS", "dm storage off"); !strings.Contains(got, "now turned off") { + t.Fatalf("dm toggle: %q", got) + } + if store.Permissions().Allowed(model.OwnerUser, "D1", "UX", model.ActStorage) { + t.Fatal("dm toggle not saved") + } + if got := say("UANA", "grant me access to the repo"); got != "(went to the agent)" { + t.Fatalf("plain sentence: %q", got) + } + if got := say("UANA", "permissions reset"); !strings.Contains(got, "default super-user list again") { + t.Fatalf("reset: %q", got) + } + if store.Permissions().Allowed(model.OwnerChannel, "C1", "UCAL", model.ActScale) { + t.Fatal("reset should drop the channel's own list") + } +} diff --git a/manage.go b/slackerbot/internal/bot/manage.go similarity index 58% rename from manage.go rename to slackerbot/internal/bot/manage.go index 332418a..ee9cbd8 100644 --- a/manage.go +++ b/slackerbot/internal/bot/manage.go @@ -1,63 +1,16 @@ -package main +package bot import ( "context" "errors" "fmt" - "regexp" "slices" "strings" + + "slackerbot/internal/commands" + "slackerbot/internal/model" ) -var mentionRe = regexp.MustCompile(`^<@([UW][A-Z0-9]+)(?:\|[^>]*)?>[,.]?$`) - -type grantCmd struct { - toDefault bool - users []string - actions []Action -} - -func parseGrant(args string) (grantCmd, error) { - var g grantCmd - words := strings.Fields(args) - i := 0 - if len(words) > 0 && strings.ToLower(words[0]) == "default" { - g.toDefault, i = true, 1 - } - for ; i < len(words); i++ { - if m := mentionRe.FindStringSubmatch(words[i]); m != nil { - g.users = append(g.users, m[1]) - continue - } - w := strings.ToLower(strings.Trim(words[i], ",.")) - if w == "all" || slices.Contains(grantActions, Action(w)) { - g.actions = append(g.actions, Action(w)) - continue - } - if len(g.users) == 0 && !g.toDefault { - return g, errNotCommand - } - return g, fmt.Errorf("I didn't understand `%s`", words[i]) - } - switch { - case len(g.users) == 0 && !g.toDefault: - return g, errNotCommand - case len(g.users) == 0: - return g, errors.New("name at least one person, like `grant default @alice all`") - } - return g, nil -} - -func parseToggle(word string) (on, ok bool) { - switch strings.ToLower(word) { - case "on", "yes", "true", "allow", "allowed", "enable": - return true, true - case "off", "no", "false", "deny", "denied", "disable": - return false, true - } - return false, false -} - func mentions(users []string) string { out := make([]string, len(users)) for i, u := range users { @@ -66,7 +19,7 @@ func mentions(users []string) string { return strings.Join(out, ", ") } -func actionList(actions []Action) string { +func actionList(actions []model.Action) string { if len(actions) == 0 { return "nothing" } @@ -97,44 +50,44 @@ func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) } return reply(fmt.Sprintf(":lock: Only %s can do that. User managers: %s.", who, strings.Join(names, ", "))) } - update := func(change func(p *PermissionsConfig) error) error { + update := func(change func(p *model.PermissionsConfig) error) error { return b.store.UpdatePermissions(change) } switch verb { case "grant", "revoke": - g, err := parseGrant(args) - if errors.Is(err, errNotCommand) { + g, err := commands.ParseGrant(args) + if errors.Is(err, commands.ErrNotCommand) { return false } if err != nil { return reply(":warning: " + err.Error() + ". Try `grant @alice move scale` or `revoke @alice`.") } channel := m.channel - if g.toDefault { + if g.ToDefault { channel = "" if !p.IsManager(m.user) { return denied(true) } } else { - if kind == OwnerUser { + if kind == model.OwnerUser { return reply("DMs don't have a super-user list. User managers can switch DM permissions with `dm on|off`.") } if !p.CanManage(channel, m.user) { return denied(false) } } - if verb == "grant" && len(g.actions) == 0 { + if verb == "grant" && len(g.Actions) == 0 { return reply(":warning: Say what to grant, like `grant @alice move scale` (or `all`, or `manage`).") } - if slices.Contains(g.actions, ActManage) && !g.toDefault && !p.CanManage(channel, m.user) { + if slices.Contains(g.Actions, model.ActManage) && !g.ToDefault && !p.CanManage(channel, m.user) { return denied(false) } - err = update(func(p *PermissionsConfig) error { + err = update(func(p *model.PermissionsConfig) error { if verb == "grant" { - p.Grant(channel, g.users, g.actions) + p.Grant(channel, g.Users, g.Actions) } else { - p.Revoke(channel, g.users, g.actions) + p.Revoke(channel, g.Users, g.Actions) } return nil }) @@ -142,17 +95,17 @@ func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) return reply(":warning: " + err.Error()) } where := "in this channel" - if g.toDefault { + if g.ToDefault { where = "in every channel that uses the default list" } after := b.store.Permissions() var lines []string - for _, u := range g.users { - var actions []Action - if g.toDefault { + for _, u := range g.Users { + var actions []model.Action + if g.ToDefault { list := after.Default - for _, a := range grantActions { - if grants(list[u], a) { + for _, a := range model.GrantActions { + if model.Grants(list[u], a) { actions = append(actions, a) } } @@ -165,17 +118,17 @@ func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) case "dm": words := strings.Fields(strings.ToLower(args)) - if len(words) != 2 || !slices.Contains(allActions, Action(words[0])) { + if len(words) != 2 || !slices.Contains(model.AllActions, model.Action(words[0])) { return false } - on, ok := parseToggle(words[1]) + on, ok := commands.ParseToggle(words[1]) if !ok { return false } if !p.IsManager(m.user) { return denied(true) } - if err := update(func(p *PermissionsConfig) error { p.DM[words[0]] = on; return nil }); err != nil { + if err := update(func(p *model.PermissionsConfig) error { p.DM[words[0]] = on; return nil }); err != nil { return reply(":warning: " + err.Error()) } state := "turned off" @@ -190,21 +143,21 @@ func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) return reply("*User managers:* " + mentions(p.Managers)) } op := strings.ToLower(words[0]) - g, err := parseGrant(strings.Join(words[1:], " ")) - if (op != "add" && op != "remove") || err != nil || len(g.actions) > 0 { + g, err := commands.ParseGrant(strings.Join(words[1:], " ")) + if (op != "add" && op != "remove") || err != nil || len(g.Actions) > 0 { return false } if !p.IsManager(m.user) { return denied(true) } - err = update(func(p *PermissionsConfig) error { + err = update(func(p *model.PermissionsConfig) error { if op == "add" { - p.AddManagers(g.users) + p.AddManagers(g.Users) return nil } - return p.RemoveManagers(g.users) + return p.RemoveManagers(g.Users) }) - if errors.Is(err, ErrLastManager) { + if errors.Is(err, model.ErrLastManager) { return reply(":warning: That would leave nobody able to manage permissions. Add someone else first.") } if err != nil { @@ -214,19 +167,19 @@ func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) if op == "remove" { verbText = "are no longer user managers" } - return reply(fmt.Sprintf("Done. %s %s.", mentions(g.users), verbText)) + return reply(fmt.Sprintf("Done. %s %s.", mentions(g.Users), verbText)) case "permissions": if strings.ToLower(args) != "reset" { return false } - if kind == OwnerUser { + if kind == model.OwnerUser { return reply("DMs don't have a super-user list to reset.") } if !p.CanManage(m.channel, m.user) { return denied(false) } - if err := update(func(p *PermissionsConfig) error { delete(p.Channels, m.channel); return nil }); err != nil { + if err := update(func(p *model.PermissionsConfig) error { delete(p.Channels, m.channel); return nil }); err != nil { return reply(":warning: " + err.Error()) } return reply("Done. This channel uses the default super-user list again.") diff --git a/stream.go b/slackerbot/internal/bot/stream.go similarity index 97% rename from stream.go rename to slackerbot/internal/bot/stream.go index 67242cc..00ff272 100644 --- a/stream.go +++ b/slackerbot/internal/bot/stream.go @@ -1,4 +1,4 @@ -package main +package bot import ( "context" @@ -9,6 +9,8 @@ import ( "unicode/utf8" "github.com/slack-go/slack" + + "slackerbot/internal/acp" ) const ( @@ -40,7 +42,7 @@ func NewStreamer(api *slack.Client, channel, thread, placeholderTS string) *Stre return s } -func (s *Streamer) Update(u Update) { +func (s *Streamer) Update(u acp.Update) { s.mu.Lock() defer s.mu.Unlock() switch u.Kind { diff --git a/slackerbot/internal/commands/commands.go b/slackerbot/internal/commands/commands.go new file mode 100644 index 0000000..e574ba8 --- /dev/null +++ b/slackerbot/internal/commands/commands.go @@ -0,0 +1,262 @@ +package commands + +import ( + "errors" + "fmt" + "math" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "slackerbot/internal/model" + "slackerbot/internal/units" +) + +var ErrNotCommand = errors.New("not a command") + +var unitSpace = regexp.MustCompile(`(\d)\s+(tib|tb|t|gib|gb|g|mib|mb|m|cpus|cpu|cores|core)\b`) +var sizeToken = regexp.MustCompile(`^([+-]?)(\d+(?:\.\d+)?)(g|gb|gib|m|mb|mib|cpu|cpus|core|cores)$`) + +func ParseScale(args string, cur model.Size) (model.Size, error) { + args = unitSpace.ReplaceAllString(strings.ToLower(args), "$1$2") + words := strings.Fields(args) + if len(words) == 0 { + return model.Size{MemoryMB: cur.MemoryMB * 2, CPUs: cur.CPUs * 2}, nil + } + if len(words) == 1 { + switch words[0] { + case "up", "out": + return model.Size{MemoryMB: cur.MemoryMB * 2, CPUs: cur.CPUs * 2}, nil + case "down", "in": + return model.Size{MemoryMB: cur.MemoryMB / 2, CPUs: cur.CPUs / 2}, nil + } + } + + next := cur + var memDone, cpuDone bool + for i, w := range words { + parts := sizeToken.FindStringSubmatch(w) + if parts == nil { + if i == 0 { + return cur, ErrNotCommand + } + return cur, fmt.Errorf("I didn't understand `%s`", w) + } + sign, num, unit := parts[1], parts[2], parts[3] + n, _ := strconv.ParseFloat(num, 64) + + if unit == "cpu" || unit == "cpus" || unit == "core" || unit == "cores" { + if cpuDone { + return cur, errors.New("CPU was given twice") + } + cpuDone = true + next.CPUs = apply(sign, cur.CPUs, n) + continue + } + if memDone { + return cur, errors.New("memory was given twice") + } + memDone = true + mb := n + if strings.HasPrefix(unit, "g") { + mb = n * 1024 + } + next.MemoryMB = int(math.Round(apply(sign, float64(cur.MemoryMB), mb))) + } + return next, nil +} + +func apply(sign string, cur, n float64) float64 { + switch sign { + case "+": + return cur + n + case "-": + return cur - n + } + return n +} + +type KeepSetting struct { + Show bool + Reset bool + Indefinite bool + Limit time.Duration +} + +var dayToken = regexp.MustCompile(`(\d+(?:\.\d+)?)([dw])`) + +var wordUnit = regexp.MustCompile(`(\d+(?:\.\d+)?)\s*(weeks?|wks?|days?|hours?|hrs?|minutes?|mins?)\b`) + +func ParseKeep(args string) (KeepSetting, error) { + a := strings.ToLower(strings.TrimSpace(args)) + switch a { + case "": + return KeepSetting{Show: true}, nil + case "default", "reset", "normal": + return KeepSetting{Reset: true}, nil + case "indefinite", "indefinitely", "forever", "alive", "running", "on": + return KeepSetting{Indefinite: true}, nil + } + d, err := parseDuration(a) + if err != nil { + return KeepSetting{}, ErrNotCommand + } + if d <= 0 { + return KeepSetting{}, errors.New("the time has to be more than zero") + } + return KeepSetting{Limit: d}, nil +} + +func parseDuration(s string) (time.Duration, error) { + s = wordUnit.ReplaceAllStringFunc(s, func(m string) string { + p := wordUnit.FindStringSubmatch(m) + return p[1] + map[byte]string{'w': "w", 'd': "d", 'h': "h", 'm': "m"}[p[2][0]] + }) + s = strings.ReplaceAll(s, " ", "") + s = dayToken.ReplaceAllStringFunc(s, func(m string) string { + p := dayToken.FindStringSubmatch(m) + n, _ := strconv.ParseFloat(p[1], 64) + hours := n * 24 + if p[2] == "w" { + hours = n * 24 * 7 + } + return strconv.FormatFloat(hours, 'f', -1, 64) + "h" + }) + return time.ParseDuration(s) +} + +func Human(d time.Duration) string { + d = d.Round(time.Minute) + days := d / (24 * time.Hour) + d -= days * 24 * time.Hour + out := "" + if days > 0 { + out = fmt.Sprintf("%dd", days) + } + if d > 0 { + r := strings.TrimSuffix(d.String(), "0s") + if strings.HasSuffix(r, "h0m") { + r = strings.TrimSuffix(r, "0m") + } + out += r + } + if out == "" { + out = "0m" + } + return out +} + +func ParseStorage(args string, curMin, curMax int64) (min, max int64, err error) { + args = unitSpace.ReplaceAllString(strings.ToLower(args), "$1$2") + words := strings.Fields(args) + min, max = curMin, curMax + var minDone, maxDone bool + + for i := 0; i < len(words); i++ { + target := "max" + switch words[i] { + case "min", "guarantee", "guaranteed", "reserve": + target = "min" + i++ + case "max", "cap", "limit": + i++ + } + if i >= len(words) { + return curMin, curMax, fmt.Errorf("%s needs a size after it, like `%s 10G`", target, target) + } + p := units.StorageToken.FindStringSubmatch(words[i]) + if p == nil { + if i == 0 { + return curMin, curMax, ErrNotCommand + } + return curMin, curMax, fmt.Errorf("I didn't understand `%s`", words[i]) + } + n := units.ToMB(p[2], p[3]) + if target == "min" { + if minDone { + return curMin, curMax, errors.New("min was given twice") + } + minDone = true + min = int64(apply(p[1], float64(curMin), float64(n))) + } else { + if maxDone { + return curMin, curMax, errors.New("max was given twice") + } + maxDone = true + max = int64(apply(p[1], float64(curMax), float64(n))) + } + } + return min, max, nil +} + +var mentionRe = regexp.MustCompile(`^<@([UW][A-Z0-9]+)(?:\|[^>]*)?>[,.]?$`) + +type GrantCmd struct { + ToDefault bool + Users []string + Actions []model.Action +} + +func ParseGrant(args string) (GrantCmd, error) { + var g GrantCmd + words := strings.Fields(args) + i := 0 + if len(words) > 0 && strings.ToLower(words[0]) == "default" { + g.ToDefault, i = true, 1 + } + for ; i < len(words); i++ { + if m := mentionRe.FindStringSubmatch(words[i]); m != nil { + g.Users = append(g.Users, m[1]) + continue + } + w := strings.ToLower(strings.Trim(words[i], ",.")) + if w == "all" || slices.Contains(model.GrantActions, model.Action(w)) { + g.Actions = append(g.Actions, model.Action(w)) + continue + } + if len(g.Users) == 0 && !g.ToDefault { + return g, ErrNotCommand + } + return g, fmt.Errorf("I didn't understand `%s`", words[i]) + } + switch { + case len(g.Users) == 0 && !g.ToDefault: + return g, ErrNotCommand + case len(g.Users) == 0: + return g, errors.New("name at least one person, like `grant default @alice all`") + } + return g, nil +} + +func ParseToggle(word string) (on, ok bool) { + switch strings.ToLower(word) { + case "on", "yes", "true", "allow", "allowed", "enable": + return true, true + case "off", "no", "false", "deny", "denied", "disable": + return false, true + } + return false, false +} + +func NeededPermission(verb, args string) model.Action { + switch verb { + case "scale": + if _, err := ParseScale(args, model.Size{MemoryMB: 1024, CPUs: 1}); !errors.Is(err, ErrNotCommand) { + return model.ActScale + } + case "keep": + if k, err := ParseKeep(args); !errors.Is(err, ErrNotCommand) && !k.Show { + return model.ActKeep + } + case "storage", "disk": + if args == "" { + return "" + } + if _, _, err := ParseStorage(args, 1024, 1024); !errors.Is(err, ErrNotCommand) { + return model.ActStorage + } + } + return "" +} diff --git a/slackerbot/internal/commands/commands_test.go b/slackerbot/internal/commands/commands_test.go new file mode 100644 index 0000000..70057de --- /dev/null +++ b/slackerbot/internal/commands/commands_test.go @@ -0,0 +1,154 @@ +package commands + +import ( + "errors" + "testing" + "time" + + "slackerbot/internal/model" +) + +func TestParseScale(t *testing.T) { + cur := model.Size{MemoryMB: 2048, CPUs: 1} + cases := []struct { + in string + want model.Size + }{ + {"", model.Size{MemoryMB: 4096, CPUs: 2}}, + {"up", model.Size{MemoryMB: 4096, CPUs: 2}}, + {"down", model.Size{MemoryMB: 1024, CPUs: 0.5}}, + {"8G 4cpu", model.Size{MemoryMB: 8192, CPUs: 4}}, + {"8gb", model.Size{MemoryMB: 8192, CPUs: 1}}, + {"+2G", model.Size{MemoryMB: 4096, CPUs: 1}}, + {"-1cpu", model.Size{MemoryMB: 2048, CPUs: 0}}, + {"512M +0.5cpu", model.Size{MemoryMB: 512, CPUs: 1.5}}, + {"1.5G 3 cores", model.Size{MemoryMB: 1536, CPUs: 3}}, + {"+2 GB", model.Size{MemoryMB: 4096, CPUs: 1}}, + {"+1cpus -512mb", model.Size{MemoryMB: 1536, CPUs: 2}}, + } + for _, c := range cases { + got, err := ParseScale(c.in, cur) + if err != nil || got != c.want { + t.Fatalf("scale %q = %v, %v; want %v", c.in, got, err, c.want) + } + } + if _, err := ParseScale("the database to 3 replicas", cur); !errors.Is(err, ErrNotCommand) { + t.Fatal("a normal sentence should go to the agent") + } + if _, err := ParseScale("4G lots", cur); err == nil || errors.Is(err, ErrNotCommand) { + t.Fatal("a bad second word should be a usage error") + } + if _, err := ParseScale("4G 8G", cur); err == nil { + t.Fatal("memory twice should fail") + } +} + +func TestParseKeep(t *testing.T) { + for in, want := range map[string]time.Duration{ + "24h": 24 * time.Hour, "3d": 72 * time.Hour, "1d12h": 36 * time.Hour, "2w": 336 * time.Hour, + "5 hours": 5 * time.Hour, "2 days": 48 * time.Hour, "1d 6h": 30 * time.Hour, "90 mins": 90 * time.Minute, + } { + k, err := ParseKeep(in) + if err != nil || k.Limit != want { + t.Fatalf("keep %q = %+v, %v; want %s", in, k, err, want) + } + } + for _, in := range []string{"indefinite", "Indefinitely", "forever"} { + if k, _ := ParseKeep(in); !k.Indefinite { + t.Fatalf("keep %q should be indefinite", in) + } + } + if k, _ := ParseKeep("default"); !k.Reset { + t.Fatal("keep default should reset") + } + if k, _ := ParseKeep(""); !k.Show { + t.Fatal("bare keep should show") + } + if _, err := ParseKeep("the tests passing"); !errors.Is(err, ErrNotCommand) { + t.Fatal("a normal sentence should go to the agent") + } +} + +func TestHuman(t *testing.T) { + for d, want := range map[time.Duration]string{ + 6 * time.Hour: "6h", 36 * time.Hour: "1d12h", 90 * time.Minute: "1h30m", + 45 * time.Minute: "45m", 168 * time.Hour: "7d", 0: "0m", + } { + if got := Human(d); got != want { + t.Fatalf("Human(%s) = %q, want %q", d, got, want) + } + } + if got := (model.Size{MemoryMB: 8192, CPUs: 4}).String(); got != "8 GB RAM, 4 CPUs" { + t.Fatal(got) + } + if got := (model.Size{MemoryMB: 512, CPUs: 0.25}).String(); got != "512 MB RAM, 0.25 CPUs" { + t.Fatal(got) + } +} + +func TestParseStorage(t *testing.T) { + cases := []struct { + in string + min, max int64 + }{ + {"20G", 5120, 20480}, + {"+5G", 5120, 15360}, + {"-2G", 5120, 8192}, + {"max 30G", 5120, 30720}, + {"min 8G", 8192, 10240}, + {"min +1G max 30 GB", 6144, 30720}, + {"cap 1T", 5120, 1048576}, + } + for _, c := range cases { + min, max, err := ParseStorage(c.in, 5120, 10240) + if err != nil || min != c.min || max != c.max { + t.Fatalf("storage %q = %d/%d, %v; want %d/%d", c.in, min, max, err, c.min, c.max) + } + } + if _, _, err := ParseStorage("is full, clean up the logs", 5120, 10240); !errors.Is(err, ErrNotCommand) { + t.Fatal("a normal sentence should go to the agent") + } + for _, in := range []string{"min", "20G 30G", "min 5G min 6G", "20G lots"} { + if _, _, err := ParseStorage(in, 5120, 10240); err == nil || errors.Is(err, ErrNotCommand) { + t.Fatalf("%q should be a usage error, got %v", in, err) + } + } +} + +func TestNeededPermission(t *testing.T) { + for in, want := range map[[2]string]model.Action{ + {"scale", "up"}: model.ActScale, + {"scale", "8G 4cpu"}: model.ActScale, + {"scale", "the db to 3 replicas"}: "", + {"keep", ""}: "", + {"keep", "24h"}: model.ActKeep, + {"keep", "indefinite"}: model.ActKeep, + {"keep", "the tests green"}: "", + {"storage", ""}: "", + {"storage", "20G"}: model.ActStorage, + {"storage", "is full, clean it up"}: "", + {"status", ""}: "", + } { + if got := NeededPermission(in[0], in[1]); got != want { + t.Errorf("%s %q needs %q, want %q", in[0], in[1], got, want) + } + } +} + +func TestParseGrant(t *testing.T) { + g, err := ParseGrant("<@U1> <@U2|bob>, move scale") + if err != nil || len(g.Users) != 2 || g.Users[1] != "U2" || len(g.Actions) != 2 { + t.Fatalf("%+v %v", g, err) + } + if g, _ := ParseGrant("default <@U1> all"); !g.ToDefault || g.Actions[0] != "all" { + t.Fatalf("%+v", g) + } + for _, in := range []string{"me access to the repo", "all the things", ""} { + if _, err := ParseGrant(in); !errors.Is(err, ErrNotCommand) { + t.Errorf("%q should go to the agent, got %v", in, err) + } + } + if _, err := ParseGrant("<@U1> teleport"); err == nil || errors.Is(err, ErrNotCommand) { + t.Error("an unknown permission after a mention should be a usage error") + } +} diff --git a/config.go b/slackerbot/internal/config/config.go similarity index 74% rename from config.go rename to slackerbot/internal/config/config.go index 89a0664..16e0afe 100644 --- a/config.go +++ b/slackerbot/internal/config/config.go @@ -1,36 +1,33 @@ -package main +package config import ( "fmt" "os" "path/filepath" - "strconv" "strings" "time" "github.com/BurntSushi/toml" + + "slackerbot/internal/model" + "slackerbot/internal/units" ) -type Size struct { - MemoryMB int `toml:"memory_mb"` - CPUs float64 `toml:"cpus"` +const MinStorageMB = 512 + +type StorageConfig struct { + Min string `toml:"min"` + Max string `toml:"max"` + Limit string `toml:"limit"` + + MinMB, MaxMB, LimitMB int64 `toml:"-"` } -func (s Size) Add(o Size) Size { return Size{s.MemoryMB + o.MemoryMB, s.CPUs + o.CPUs} } -func (s Size) Sub(o Size) Size { return Size{s.MemoryMB - o.MemoryMB, s.CPUs - o.CPUs} } -func (s Size) Fits(in Size) bool { - return s.MemoryMB <= in.MemoryMB && s.CPUs <= in.CPUs+1e-9 -} -func (s Size) String() string { - mem := fmt.Sprintf("%d MB", s.MemoryMB) - if s.MemoryMB >= 1024 && s.MemoryMB%1024 == 0 { - mem = fmt.Sprintf("%d GB", s.MemoryMB/1024) - } - cpu := strconv.FormatFloat(s.CPUs, 'f', -1, 64) + " CPUs" - if s.CPUs == 1 { - cpu = "1 CPU" - } - return mem + " RAM, " + cpu +type DriveConfig struct { + Path string `toml:"path"` + Reserve string `toml:"reserve"` + + ReserveMB int64 `toml:"-"` } type Config struct { @@ -55,16 +52,16 @@ type Config struct { ROBinds []string `toml:"ro_binds"` RWBinds []string `toml:"rw_binds"` - Container Size `toml:"container"` - MaxContainer Size `toml:"max_container"` - HostReserve Size `toml:"host_reserve"` + Container model.Size `toml:"container"` + MaxContainer model.Size `toml:"max_container"` + HostReserve model.Size `toml:"host_reserve"` Storage StorageConfig `toml:"storage"` Drives []DriveConfig `toml:"drives"` Cluster ClusterConfig `toml:"cluster"` - Permissions PermissionsConfig `toml:"permissions"` + Permissions model.PermissionsConfig `toml:"permissions"` } type ClusterConfig struct { @@ -83,14 +80,14 @@ const ( func LoadConfig(path, mode string) (*Config, error) { c := &Config{ MachinesDir: "/var/lib/machines", - StateDir: "/var/lib/slackspawn", + StateDir: "/var/lib/slackerbot", IdleTimeout: 6 * time.Hour, AllowIndefinite: true, BootTimeout: 2 * time.Minute, AgentCwd: "/root", - Container: Size{MemoryMB: 2048, CPUs: 1}, - MaxContainer: Size{MemoryMB: 8192, CPUs: 4}, - HostReserve: Size{MemoryMB: 2048, CPUs: 1}, + Container: model.Size{MemoryMB: 2048, CPUs: 1}, + MaxContainer: model.Size{MemoryMB: 8192, CPUs: 4}, + HostReserve: model.Size{MemoryMB: 2048, CPUs: 1}, Storage: StorageConfig{Min: "5G", Max: "10G", Limit: "50G"}, } if _, err := toml.DecodeFile(path, c); err != nil { @@ -124,8 +121,8 @@ func LoadConfig(path, mode string) (*Config, error) { return nil, fmt.Errorf("mode must be %q or %q", ModeMaster, ModeSlave) case mode == ModeMaster && (c.Slack.AppToken == "" || c.Slack.BotToken == ""): return nil, fmt.Errorf("slack app_token and bot_token are required") - case mode == ModeMaster && c.Permissions.validate() != nil: - return nil, c.Permissions.validate() + case mode == ModeMaster && c.Permissions.Validate() != nil: + return nil, c.Permissions.Validate() case mode == ModeSlave && !strings.HasPrefix(c.Cluster.Master, "wss://"): return nil, fmt.Errorf("cluster.master must be the master's address, like wss://master.example:7946") case clustered && len(c.Cluster.Secret) < 16: @@ -147,18 +144,18 @@ func LoadConfig(path, mode string) (*Config, error) { func (c *Config) loadStorage() error { var err error s := &c.Storage - if s.MinMB, err = parseSizeMB(s.Min); err != nil { + if s.MinMB, err = units.ParseSizeMB(s.Min); err != nil { return fmt.Errorf("storage.min: %w", err) } - if s.MaxMB, err = parseSizeMB(s.Max); err != nil { + if s.MaxMB, err = units.ParseSizeMB(s.Max); err != nil { return fmt.Errorf("storage.max: %w", err) } - if s.LimitMB, err = parseSizeMB(s.Limit); err != nil { + if s.LimitMB, err = units.ParseSizeMB(s.Limit); err != nil { return fmt.Errorf("storage.limit: %w", err) } switch { case s.MinMB < MinStorageMB: - return fmt.Errorf("storage.min must be at least %s", formatMB(MinStorageMB)) + return fmt.Errorf("storage.min must be at least %s", units.FormatMB(MinStorageMB)) case s.MinMB > s.MaxMB: return fmt.Errorf("storage.min (%s) can't be more than storage.max (%s)", s.Min, s.Max) case s.MaxMB > s.LimitMB: @@ -180,7 +177,7 @@ func (c *Config) loadStorage() error { if d.Reserve == "" { d.Reserve = "0G" } - if d.ReserveMB, err = parseSizeMB(d.Reserve); err != nil { + if d.ReserveMB, err = units.ParseSizeMB(d.Reserve); err != nil { return fmt.Errorf("drive %s reserve: %w", d.Path, err) } } diff --git a/slackerbot/internal/config/config_test.go b/slackerbot/internal/config/config_test.go new file mode 100644 index 0000000..f65a4cb --- /dev/null +++ b/slackerbot/internal/config/config_test.go @@ -0,0 +1,26 @@ +package config + +import ( + "testing" + "time" +) + +func TestConfigExample(t *testing.T) { + t.Setenv("SLACK_APP_TOKEN", "xapp-test") + t.Setenv("SLACK_BOT_TOKEN", "xoxb-test") + t.Setenv("ANTHROPIC_API_KEY", "sk-test") + t.Setenv("CLUSTER_SECRET", "cluster-secret-1234567890") + c, err := LoadConfig("../../../config.example.toml", ModeMaster) + if err != nil { + t.Fatal(err) + } + if c.IdleTimeout != 6*time.Hour || c.Container.MemoryMB != 2048 || c.MaxContainer.CPUs != 4 { + t.Fatalf("unexpected config: %+v", c) + } + if c.Storage.MinMB != 5120 || c.Storage.MaxMB != 10240 || len(c.Drives) != 2 || c.Drives[0].ReserveMB != 20480 { + t.Fatalf("unexpected storage config: %+v %+v", c.Storage, c.Drives) + } + if c.AgentEnv["ANTHROPIC_API_KEY"] != "sk-test" { + t.Fatal("env var not expanded") + } +} diff --git a/capacity.go b/slackerbot/internal/local/capacity.go similarity index 76% rename from capacity.go rename to slackerbot/internal/local/capacity.go index 5d267aa..4fd4cc3 100644 --- a/capacity.go +++ b/slackerbot/internal/local/capacity.go @@ -1,4 +1,4 @@ -package main +package local import ( "bufio" @@ -7,19 +7,21 @@ import ( "runtime" "strconv" "strings" + + "slackerbot/internal/model" ) type Capacity struct { - Total Size - Usable Size + Total model.Size + Usable model.Size } -func DetectCapacity(reserve Size) (Capacity, error) { +func DetectCapacity(reserve model.Size) (Capacity, error) { memMB, err := totalMemoryMB() if err != nil { return Capacity{}, err } - total := Size{MemoryMB: memMB, CPUs: float64(runtime.NumCPU())} + total := model.Size{MemoryMB: memMB, CPUs: float64(runtime.NumCPU())} usable := total.Sub(reserve) if usable.MemoryMB <= 0 || usable.CPUs <= 0 { return Capacity{}, fmt.Errorf("host_reserve (%s) leaves nothing on this node (%s)", reserve, total) @@ -27,11 +29,11 @@ func DetectCapacity(reserve Size) (Capacity, error) { return Capacity{Total: total, Usable: usable}, nil } -func (c Capacity) Room(used, want Size) bool { +func (c Capacity) Room(used, want model.Size) bool { return used.Add(want).Fits(c.Usable) } -func (c Capacity) Slots(size Size) int { +func (c Capacity) Slots(size model.Size) int { byMem := c.Usable.MemoryMB / size.MemoryMB byCPU := int(c.Usable.CPUs/size.CPUs + 1e-9) return min(byMem, byCPU) diff --git a/slackerbot/internal/local/capacity_test.go b/slackerbot/internal/local/capacity_test.go new file mode 100644 index 0000000..28b9b1b --- /dev/null +++ b/slackerbot/internal/local/capacity_test.go @@ -0,0 +1,24 @@ +package local + +import ( + "testing" + + "slackerbot/internal/model" +) + +func TestCapacity(t *testing.T) { + c := Capacity{Usable: model.Size{MemoryMB: 6144, CPUs: 3}} + def := model.Size{MemoryMB: 2048, CPUs: 1} + if got := c.Slots(def); got != 3 { + t.Fatalf("slots = %d, want 3", got) + } + if !c.Room(model.Size{MemoryMB: 4096, CPUs: 2}, def) { + t.Fatal("third container should fit") + } + if c.Room(model.Size{MemoryMB: 6144, CPUs: 3}, def) { + t.Fatal("fourth container should not fit") + } + if c.Room(model.Size{MemoryMB: 2048, CPUs: 2.5}, def) { + t.Fatal("should be blocked by CPU") + } +} diff --git a/local.go b/slackerbot/internal/local/local.go similarity index 79% rename from local.go rename to slackerbot/internal/local/local.go index 2e9f9b4..1295d31 100644 --- a/local.go +++ b/slackerbot/internal/local/local.go @@ -1,4 +1,4 @@ -package main +package local import ( "context" @@ -15,8 +15,12 @@ import ( "syscall" "time" - "github.com/coder/websocket" "github.com/godbus/dbus/v5" + + "slackerbot/internal/config" + "slackerbot/internal/model" + "slackerbot/internal/node" + "slackerbot/internal/storage" ) const ( @@ -27,11 +31,11 @@ const ( machinedMgr = "org.freedesktop.machine1.Manager" sigPoweroff = 38 // SIGRTMIN+4, what `machinectl poweroff` sends - agentEnvInside = "/var/lib/slackspawn/agent.env" + agentEnvInside = "/var/lib/slackerbot/agent.env" ) type Local struct { - cfg *Config + cfg *config.Config name string cap Capacity bus *dbus.Conn @@ -41,7 +45,7 @@ type Local struct { imports transfers } -func NewLocal(cfg *Config) (*Local, error) { +func New(cfg *config.Config) (*Local, error) { system, err := filepath.EvalSymlinks(cfg.System) if err != nil { return nil, err @@ -50,7 +54,7 @@ func NewLocal(cfg *Config) (*Local, error) { return nil, fmt.Errorf("%s has no init. Is it a NixOS build with boot.isContainer = true?", system) } for _, d := range cfg.Drives { - if err := prepareDrive(d.Path); err != nil { + if err := storage.PrepareDrive(d.Path); err != nil { return nil, err } } @@ -65,36 +69,27 @@ func NewLocal(cfg *Config) (*Local, error) { return &Local{cfg: cfg, name: cfg.Cluster.Name, cap: capacity, bus: bus, system: system}, nil } -func MachineName(id string, kind OwnerKind) string { - if kind == OwnerChannel { - return "c-" + strings.ToLower(id) - } - return "u-" + strings.ToLower(id) -} - -func ours(name string) bool { return strings.HasPrefix(name, "u-") || strings.HasPrefix(name, "c-") } - -func unitName(machine string) string { return "spawn-" + machine + ".service" } +func unitName(machine string) string { return "slackerbot-" + machine + ".service" } func (l *Local) Name() string { return l.name } func (l *Local) machined() dbus.BusObject { return l.bus.Object(machinedName, machinedPath) } -func (l *Local) Info(ctx context.Context) (NodeInfo, error) { - info := NodeInfo{Name: l.name, Total: l.cap.Total, Usable: l.cap.Usable} +func (l *Local) Info(ctx context.Context) (node.NodeInfo, error) { + info := node.NodeInfo{Name: l.name, Total: l.cap.Total, Usable: l.cap.Usable} for _, d := range l.cfg.Drives { - free, err := freeMB(d.Path) + free, err := storage.FreeMB(d.Path) if err != nil { return info, err } - di := DriveInfo{Path: d.Path, FreeMB: free, ReserveMB: d.ReserveMB, HomesMB: map[string]int64{}} + di := node.DriveInfo{Path: d.Path, FreeMB: free, ReserveMB: d.ReserveMB, HomesMB: map[string]int64{}} entries, err := os.ReadDir(d.Path) if err != nil { return info, err } for _, e := range entries { - if e.IsDir() && ours(e.Name()) { - used, _ := usedMB(filepath.Join(d.Path, e.Name())) + if e.IsDir() && model.IsOurs(e.Name()) { + used, _ := storage.UsedMB(filepath.Join(d.Path, e.Name())) di.HomesMB[e.Name()] = used } } @@ -105,7 +100,7 @@ func (l *Local) Info(ctx context.Context) (NodeInfo, error) { return info, err } -func (l *Local) Running(ctx context.Context) ([]RunningMachine, error) { +func (l *Local) Running(ctx context.Context) ([]node.RunningMachine, error) { var list []struct { Name, Class, Service string Path dbus.ObjectPath @@ -113,12 +108,12 @@ func (l *Local) Running(ctx context.Context) ([]RunningMachine, error) { if err := l.machined().Call(machinedMgr+".ListMachines", 0).Store(&list); err != nil { return nil, err } - var out []RunningMachine + var out []node.RunningMachine for _, x := range list { - if !ours(x.Name) { + if !model.IsOurs(x.Name) { continue } - r := RunningMachine{Name: x.Name} + r := node.RunningMachine{Name: x.Name} if v, err := l.bus.Object(machinedName, x.Path).GetProperty("org.freedesktop.machine1.Machine.Timestamp"); err == nil { if us, ok := v.Value().(uint64); ok { r.Since = time.UnixMicro(int64(us)) @@ -134,17 +129,17 @@ func (l *Local) IsRunning(machine string) bool { return l.machined().Call(machinedMgr+".GetMachine", 0, machine).Store(&p) == nil } -func (l *Local) drive(path string) (DriveConfig, error) { +func (l *Local) drive(path string) (config.DriveConfig, error) { for _, d := range l.cfg.Drives { if d.Path == path { return d, nil } } - return DriveConfig{}, fmt.Errorf("drive %s isn't configured on node %s", path, l.name) + return config.DriveConfig{}, fmt.Errorf("drive %s isn't configured on node %s", path, l.name) } -func (l *Local) Start(ctx context.Context, spec StartSpec) error { - if !ours(spec.Machine) { +func (l *Local) Start(ctx context.Context, spec node.StartSpec) error { + if !model.IsOurs(spec.Machine) { return fmt.Errorf("unexpected machine name %q", spec.Machine) } if _, err := l.drive(spec.Drive); err != nil { @@ -152,13 +147,13 @@ func (l *Local) Start(ctx context.Context, spec StartSpec) error { } home := filepath.Join(spec.Drive, spec.Machine) if _, err := os.Stat(home); errors.Is(err, os.ErrNotExist) { - if err := createSubvolume(home); err != nil { + if err := storage.CreateSubvolume(home); err != nil { return err } } else if err != nil { return err } - if err := setQuota(home, spec.StorageMaxMB); err != nil { + if err := storage.SetQuota(home, spec.StorageMaxMB); err != nil { return err } @@ -227,7 +222,7 @@ func (l *Local) Start(ctx context.Context, spec StartSpec) error { return fmt.Errorf("%s started but never registered with machined", spec.Machine) } -func (l *Local) writeEnvFile(spec StartSpec) (string, error) { +func (l *Local) writeEnvFile(spec node.StartSpec) (string, error) { dir := filepath.Join(l.cfg.StateDir, "env") if err := os.MkdirAll(dir, 0o700); err != nil { return "", err @@ -250,7 +245,7 @@ func (l *Local) Poweroff(ctx context.Context, machine string) error { return l.machined().CallWithContext(ctx, machinedMgr+".KillMachine", 0, machine, "leader", int32(sigPoweroff)).Err } -func (l *Local) SetLimits(ctx context.Context, machine string, s Size) error { +func (l *Local) SetLimits(ctx context.Context, machine string, s model.Size) error { type prop struct { Name string Value dbus.Variant @@ -267,29 +262,18 @@ func (l *Local) SetQuota(ctx context.Context, drive, machine string, mb int64) e if _, err := l.drive(drive); err != nil { return err } - if !ours(machine) { + if !model.IsOurs(machine) { return fmt.Errorf("unexpected machine name %q", machine) } - return setQuota(filepath.Join(drive, machine), mb) + return storage.SetQuota(filepath.Join(drive, machine), mb) } -func (l *Local) DialAgent(ctx context.Context, machine, secret string) (msgConn, error) { +func (l *Local) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) { addr, err := l.waitReady(ctx, machine) if err != nil { return nil, err } - return dialAgentWS(ctx, addr, secret) -} - -func dialAgentWS(ctx context.Context, addr, secret string) (msgConn, error) { - conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/acp", &websocket.DialOptions{ - HTTPHeader: http.Header{"X-Secret-Key": []string{secret}}, - }) - if err != nil { - return nil, fmt.Errorf("connecting to goose: %w", err) - } - conn.SetReadLimit(32 << 20) - return wsConn{conn}, nil + return node.DialAgentWS(ctx, addr, secret) } func (l *Local) address(machine string) (string, error) { @@ -337,3 +321,5 @@ func (l *Local) waitReady(ctx context.Context, machine string) (string, error) { } } } + +func (l *Local) Capacity() Capacity { return l.cap } diff --git a/transfer.go b/slackerbot/internal/local/transfer.go similarity index 90% rename from transfer.go rename to slackerbot/internal/local/transfer.go index 57a091f..f85c1e0 100644 --- a/transfer.go +++ b/slackerbot/internal/local/transfer.go @@ -1,4 +1,4 @@ -package main +package local import ( "bytes" @@ -12,9 +12,11 @@ import ( "strings" "sync" "time" -) -const transferChunk = 4 << 20 + "slackerbot/internal/model" + "slackerbot/internal/storage" + "slackerbot/internal/util" +) type transfer struct { cmd *exec.Cmd @@ -36,7 +38,7 @@ type transfers struct { } func (t *transfers) add(x *transfer) string { - id := randomHex(8) + id := util.RandomHex(8) t.mu.Lock() defer t.mu.Unlock() if t.m == nil { @@ -67,7 +69,7 @@ func (l *Local) home(drive, machine string) (string, error) { if _, err := l.drive(drive); err != nil { return "", err } - if !ours(machine) { + if !model.IsOurs(machine) { return "", fmt.Errorf("unexpected machine name %q", machine) } return filepath.Join(drive, machine), nil @@ -100,7 +102,7 @@ func (l *Local) ExportHome(ctx context.Context, drive, machine, parent string) ( } args = append(args, "-p", p.snapshot) } - snapshot := filepath.Join(drive, ".move-"+machine+"-"+randomHex(4)) + snapshot := filepath.Join(drive, ".move-"+machine+"-"+util.RandomHex(4)) if out, err := exec.Command("btrfs", "subvolume", "snapshot", "-r", home, snapshot).CombinedOutput(); err != nil { return "", fmt.Errorf("snapshot: %v: %s", err, strings.TrimSpace(string(out))) } @@ -111,7 +113,7 @@ func (l *Local) ExportHome(ctx context.Context, drive, machine, parent string) ( err = x.cmd.Start() } if err != nil { - deleteSubvolume(snapshot) + storage.DeleteSubvolume(snapshot) return "", err } return l.exports.add(x), nil @@ -142,7 +144,7 @@ func (l *Local) CloseExport(ctx context.Context, id string) error { _ = x.cmd.Process.Kill() _ = x.cmd.Wait() } - return deleteSubvolume(x.snapshot) + return storage.DeleteSubvolume(x.snapshot) } func (l *Local) ImportHome(ctx context.Context, drive, machine string) (string, error) { @@ -154,7 +156,7 @@ func (l *Local) ImportHome(ctx context.Context, drive, machine string) (string, return "", fmt.Errorf("%s already has a home folder for %s; remove it first", l.name, machine) } x := &transfer{drive: drive, machine: machine, seen: map[string]bool{}, - tmpDir: filepath.Join(drive, ".incoming-"+machine+"-"+randomHex(4))} + tmpDir: filepath.Join(drive, ".incoming-"+machine+"-"+util.RandomHex(4))} if err := os.Mkdir(x.tmpDir, 0o700); err != nil { return "", err } @@ -226,7 +228,7 @@ func (l *Local) FinishImport(ctx context.Context, id string, quotaMB int64) erro if out, err := exec.Command("btrfs", "subvolume", "snapshot", filepath.Join(x.tmpDir, x.latest), home).CombinedOutput(); err != nil { return fmt.Errorf("snapshot: %v: %s", err, strings.TrimSpace(string(out))) } - return setQuota(home, quotaMB) + return storage.SetQuota(home, quotaMB) } func (l *Local) AbortImport(ctx context.Context, id string) error { @@ -251,20 +253,13 @@ func (l *Local) DeleteHome(ctx context.Context, drive, machine string) error { if l.IsRunning(machine) { return fmt.Errorf("%s is running", machine) } - return deleteSubvolume(home) -} - -func deleteSubvolume(path string) error { - if out, err := exec.Command("btrfs", "subvolume", "delete", path).CombinedOutput(); err != nil { - return fmt.Errorf("deleting %s: %v: %s", path, err, strings.TrimSpace(string(out))) - } - return nil + return storage.DeleteSubvolume(home) } func cleanupIncoming(dir string) { entries, _ := os.ReadDir(dir) for _, e := range entries { - _ = deleteSubvolume(filepath.Join(dir, e.Name())) + _ = storage.DeleteSubvolume(filepath.Join(dir, e.Name())) } _ = os.RemoveAll(dir) } diff --git a/cluster.go b/slackerbot/internal/manager/cluster.go similarity index 90% rename from cluster.go rename to slackerbot/internal/manager/cluster.go index 7b736c6..6d0fadd 100644 --- a/cluster.go +++ b/slackerbot/internal/manager/cluster.go @@ -1,4 +1,4 @@ -package main +package manager import ( "context" @@ -26,6 +26,10 @@ import ( "time" "github.com/coder/websocket" + + "slackerbot/internal/config" + "slackerbot/internal/model" + "slackerbot/internal/node" ) const ( @@ -56,7 +60,7 @@ type link struct { streams map[int64]*stream onCall func(f frame) - onReport func(info NodeInfo) + onReport func(info node.NodeInfo) done chan struct{} err error @@ -120,7 +124,7 @@ func (l *link) run() { ch <- f } case "report": - var info NodeInfo + var info node.NodeInfo if l.onReport != nil && json.Unmarshal(f.Params, &info) == nil { l.onReport(info) } @@ -188,7 +192,7 @@ func (l *link) reply(id int64, result any, err error) { _ = l.send(context.Background(), f) } -func (l *link) report(ctx context.Context, info NodeInfo) error { +func (l *link) report(ctx context.Context, info node.NodeInfo) error { p, err := json.Marshal(info) if err != nil { return err @@ -283,7 +287,7 @@ type Remote struct { mu sync.Mutex link *link - info NodeInfo + info node.NodeInfo infoAt time.Time starting map[string]time.Time stopping map[string]time.Time @@ -307,14 +311,14 @@ func (r *Remote) Online() bool { func (r *Remote) currentLink() (*link, error) { if !r.Online() { - return nil, &NodeOfflineError{Node: r.name} + return nil, &node.NodeOfflineError{Node: r.name} } r.mu.Lock() defer r.mu.Unlock() return r.link, nil } -func (r *Remote) setInfo(info NodeInfo) { +func (r *Remote) setInfo(info node.NodeInfo) { r.mu.Lock() defer r.mu.Unlock() info.Name = r.name @@ -329,7 +333,7 @@ func (r *Remote) setInfo(info NodeInfo) { } } -func containsMachine(list []RunningMachine, name string) bool { +func containsMachine(list []node.RunningMachine, name string) bool { for _, m := range list { if m.Name == name { return true @@ -340,8 +344,8 @@ func containsMachine(list []RunningMachine, name string) bool { // Reports can lag behind a start or stop the master just asked for, // so recent requests override them for a short while. -func (r *Remote) runningLocked() []RunningMachine { - var out []RunningMachine +func (r *Remote) runningLocked() []node.RunningMachine { + var out []node.RunningMachine for _, m := range r.info.Running { if t, ok := r.stopping[m.Name]; ok && time.Since(t) < pendingTimeout { continue @@ -350,15 +354,15 @@ func (r *Remote) runningLocked() []RunningMachine { } for m, t := range r.starting { if time.Since(t) < pendingTimeout && !containsMachine(out, m) { - out = append(out, RunningMachine{Name: m, Since: t}) + out = append(out, node.RunningMachine{Name: m, Since: t}) } } return out } -func (r *Remote) Info(ctx context.Context) (NodeInfo, error) { +func (r *Remote) Info(ctx context.Context) (node.NodeInfo, error) { if !r.Online() { - return NodeInfo{}, &NodeOfflineError{Node: r.name} + return node.NodeInfo{}, &node.NodeOfflineError{Node: r.name} } r.mu.Lock() defer r.mu.Unlock() @@ -367,7 +371,7 @@ func (r *Remote) Info(ctx context.Context) (NodeInfo, error) { return info, nil } -func (r *Remote) Running(ctx context.Context) ([]RunningMachine, error) { +func (r *Remote) Running(ctx context.Context) ([]node.RunningMachine, error) { info, err := r.Info(ctx) return info.Running, err } @@ -388,7 +392,7 @@ func (r *Remote) callNode(ctx context.Context, method string, params, result any return l.call(ctx, method, params, result) } -func (r *Remote) Start(ctx context.Context, spec StartSpec) error { +func (r *Remote) Start(ctx context.Context, spec node.StartSpec) error { if err := r.callNode(ctx, "start", spec, nil); err != nil { return err } @@ -410,7 +414,7 @@ func (r *Remote) Poweroff(ctx context.Context, machine string) error { return nil } -func (r *Remote) SetLimits(ctx context.Context, machine string, size Size) error { +func (r *Remote) SetLimits(ctx context.Context, machine string, size model.Size) error { return r.callNode(ctx, "set_limits", callParams{Machine: machine, Size: size}, nil) } @@ -464,7 +468,7 @@ func (r *Remote) DeleteHome(ctx context.Context, drive, machine string) error { return r.callNode(ctx, "delete_home", callParams{Drive: drive, Machine: machine}, nil) } -func (r *Remote) DialAgent(ctx context.Context, machine, secret string) (msgConn, error) { +func (r *Remote) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) { l, err := r.currentLink() if err != nil { return nil, err @@ -523,7 +527,7 @@ func (m *Manager) ServeCluster(ctx context.Context) error { return } back := false - l.onReport = func(info NodeInfo) { + l.onReport = func(info node.NodeInfo) { r.setInfo(info) if !back { back = true @@ -566,7 +570,7 @@ func loadOrCreateCert(dir string) (tls.Certificate, string, error) { serial, _ := rand.Int(rand.Reader, big.NewInt(1<<62)) tmpl := &x509.Certificate{ SerialNumber: serial, - Subject: pkix.Name{CommonName: "slackspawn master"}, + Subject: pkix.Name{CommonName: "slackerbot master"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().AddDate(20, 0, 0), KeyUsage: x509.KeyUsageDigitalSignature, @@ -602,7 +606,7 @@ func normalizeFingerprint(s string) string { return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), ":", "")) } -func RunSlave(ctx context.Context, cfg *Config, local Node) error { +func RunSlave(ctx context.Context, cfg *config.Config, local node.Node) error { backoff := time.Second for { started := time.Now() @@ -623,7 +627,7 @@ func RunSlave(ctx context.Context, cfg *Config, local Node) error { } } -func pinnedTLS(cfg *Config) *tls.Config { +func pinnedTLS(cfg *config.Config) *tls.Config { pinned := normalizeFingerprint(cfg.Cluster.MasterFingerprint) savedPath := filepath.Join(cfg.StateDir, "master.fingerprint") if pinned == "" { @@ -654,7 +658,7 @@ func pinnedTLS(cfg *Config) *tls.Config { } } -func slaveSession(ctx context.Context, cfg *Config, local Node) error { +func slaveSession(ctx context.Context, cfg *config.Config, local node.Node) error { client := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(cfg), Proxy: http.ProxyFromEnvironment}} dialCtx, cancel := context.WithTimeout(ctx, 30*time.Second) conn, _, err := websocket.Dial(dialCtx, strings.TrimSuffix(cfg.Cluster.Master, "/")+"/cluster", &websocket.DialOptions{ @@ -694,25 +698,25 @@ func slaveSession(ctx context.Context, cfg *Config, local Node) error { } type callParams struct { - Machine string `json:"machine,omitempty"` - Secret string `json:"secret,omitempty"` - Drive string `json:"drive,omitempty"` - MB int64 `json:"mb,omitempty"` - Size Size `json:"size"` - ID string `json:"id,omitempty"` - Parent string `json:"parent,omitempty"` - Max int `json:"max,omitempty"` - Data []byte `json:"data,omitempty"` - EOF bool `json:"eof,omitempty"` + Machine string `json:"machine,omitempty"` + Secret string `json:"secret,omitempty"` + Drive string `json:"drive,omitempty"` + MB int64 `json:"mb,omitempty"` + Size model.Size `json:"size"` + ID string `json:"id,omitempty"` + Parent string `json:"parent,omitempty"` + Max int `json:"max,omitempty"` + Data []byte `json:"data,omitempty"` + EOF bool `json:"eof,omitempty"` } -func serveCall(ctx context.Context, l *link, local Node, f frame) { +func serveCall(ctx context.Context, l *link, local node.Node, f frame) { var p callParams _ = json.Unmarshal(f.Params, &p) switch f.Method { case "start": - var spec StartSpec + var spec node.StartSpec if err := json.Unmarshal(f.Params, &spec); err != nil { l.reply(f.ID, nil, err) return @@ -761,9 +765,9 @@ func serveCall(ctx context.Context, l *link, local Node, f frame) { } } -func pipe(ctx context.Context, a, b msgConn) { +func pipe(ctx context.Context, a, b node.MsgConn) { ctx, cancel := context.WithCancel(ctx) - copyTo := func(dst, src msgConn) { + copyTo := func(dst, src node.MsgConn) { defer cancel() for { msg, err := src.Read(ctx) diff --git a/slackerbot/internal/manager/cluster_test.go b/slackerbot/internal/manager/cluster_test.go new file mode 100644 index 0000000..de9c263 --- /dev/null +++ b/slackerbot/internal/manager/cluster_test.go @@ -0,0 +1,137 @@ +package manager + +import ( + "context" + "errors" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "slackerbot/internal/config" + "slackerbot/internal/model" + "slackerbot/internal/node" + "slackerbot/internal/state" + "slackerbot/internal/testutil" +) + +func TestMasterAndSlave(t *testing.T) { + addr := testutil.FreePort(t) + secret := "cluster-secret-1234567890" + masterDir, slaveDir := t.TempDir(), t.TempDir() + + masterCfg := &config.Config{StateDir: masterDir, Container: model.Size{MemoryMB: 1024, CPUs: 1}} + masterCfg.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Listen: addr} + store, err := state.Open(filepath.Join(masterDir, "state.json")) + if err != nil { + t.Fatal(err) + } + mgr := New(masterCfg, store, testutil.NewFakeNode("master")) + backCh := make(chan string, 4) + mgr.onNodeBack = func(n string) { backCh <- n } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go mgr.ServeCluster(ctx) + + slaveCfg := &config.Config{StateDir: slaveDir} + slaveCfg.Cluster = config.ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr} + slave := testutil.NewFakeNode("slave-1") + slaveCtx, stopSlave := context.WithCancel(ctx) + go RunSlave(slaveCtx, slaveCfg, slave) + + testutil.WaitFor(t, "slave to come online", func() bool { _, err := mgr.nodeByName("slave-1"); return err == nil }) + if got := <-backCh; got != "slave-1" { + t.Fatalf("back callback for %q", got) + } + pinned, err := os.ReadFile(filepath.Join(slaveDir, "master.fingerprint")) + if err != nil || len(strings.TrimSpace(string(pinned))) != 64 { + t.Fatalf("slave should pin the master's certificate, got %q %v", pinned, err) + } + + n, _ := mgr.nodeByName("slave-1") + spec := node.StartSpec{Machine: "c-abc", Secret: "agent-secret", Size: model.Size{MemoryMB: 1024, CPUs: 1}, Drive: "/d"} + if err := n.Start(ctx, spec); err != nil { + t.Fatal(err) + } + if !slave.IsRunning("c-abc") || slave.Last.Drive != "/d" { + t.Fatal("start didn't reach the slave") + } + if !n.IsRunning("c-abc") { + t.Fatal("master should count the container as running right away") + } + + conn, err := n.DialAgent(ctx, "c-abc", "agent-secret") + if err != nil { + t.Fatal(err) + } + if err := conn.Write(ctx, []byte(`{"jsonrpc":"2.0"}`)); err != nil { + t.Fatal(err) + } + readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second) + got, err := conn.Read(readCtx) + readCancel() + if err != nil || string(got) != `echo: {"jsonrpc":"2.0"}` { + t.Fatalf("relay got %q %v", got, err) + } + conn.Close() + if _, err := n.DialAgent(ctx, "c-abc", "wrong"); err == nil { + t.Fatal("a bad agent secret should fail through the relay") + } + + if err := n.Poweroff(ctx, "c-abc"); err != nil { + t.Fatal(err) + } + if slave.IsRunning("c-abc") || n.IsRunning("c-abc") { + t.Fatal("poweroff didn't take") + } + + stopSlave() + testutil.WaitFor(t, "slave to go offline", func() bool { + _, err := mgr.nodeByName("slave-1") + var off *node.NodeOfflineError + return errors.As(err, &off) && off.Node == "slave-1" + }) + if off := mgr.OfflineNodes(); len(off) != 1 || off[0] != "slave-1" { + t.Fatalf("offline nodes = %v", off) + } +} + +func TestSlaveRejectsWrongMasterOrSecret(t *testing.T) { + addr := testutil.FreePort(t) + secret := "cluster-secret-1234567890" + masterCfg := &config.Config{StateDir: t.TempDir()} + masterCfg.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Listen: addr} + store, _ := state.Open(filepath.Join(masterCfg.StateDir, "state.json")) + mgr := New(masterCfg, store, testutil.NewFakeNode("master")) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go mgr.ServeCluster(ctx) + testutil.WaitFor(t, "master to listen", func() bool { + c, err := net.Dial("tcp", addr) + if err == nil { + c.Close() + } + return err == nil + }) + + wrongPin := &config.Config{StateDir: t.TempDir()} + wrongPin.Cluster = config.ClusterConfig{Name: "s", Secret: secret, Master: "wss://" + addr, MasterFingerprint: strings.Repeat("ab", 32)} + if err := slaveSession(ctx, wrongPin, testutil.NewFakeNode("s")); err == nil { + t.Fatal("a master with a different certificate should be refused") + } + + wrongSecret := &config.Config{StateDir: t.TempDir()} + wrongSecret.Cluster = config.ClusterConfig{Name: "s", Secret: "not-the-secret-at-all", Master: "wss://" + addr} + if err := slaveSession(ctx, wrongSecret, testutil.NewFakeNode("s")); err == nil { + t.Fatal("a slave with the wrong secret should be refused") + } + + sameName := &config.Config{StateDir: t.TempDir()} + sameName.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Master: "wss://" + addr} + if err := slaveSession(ctx, sameName, testutil.NewFakeNode("master")); err == nil { + t.Fatal("a slave can't take the master's name") + } +} diff --git a/manager.go b/slackerbot/internal/manager/manager.go similarity index 63% rename from manager.go rename to slackerbot/internal/manager/manager.go index a035fdd..5c979b1 100644 --- a/manager.go +++ b/slackerbot/internal/manager/manager.go @@ -1,23 +1,28 @@ -package main +package manager import ( "context" - "crypto/rand" - "encoding/hex" "errors" "fmt" "math" "sort" "sync" "time" + + "slackerbot/internal/balance" + "slackerbot/internal/config" + "slackerbot/internal/model" + "slackerbot/internal/node" + "slackerbot/internal/state" + "slackerbot/internal/storage" + "slackerbot/internal/util" ) var ( - ErrMaxedOut = errors.New("no node has room for this container") ErrTooBig = errors.New("bigger than max_container allows") ErrTooSmall = errors.New("smaller than the minimum size") - MinContainer = Size{MemoryMB: 256, CPUs: 0.25} + MinContainer = model.Size{MemoryMB: 256, CPUs: 0.25} ) type State string @@ -30,24 +35,26 @@ const ( ) type Manager struct { - cfg *Config - store *Store - local Node + cfg *config.Config + store *state.Store + local node.Node mu sync.RWMutex remotes map[string]*Remote onNodeBack func(node string) startMu sync.Mutex - ownerMu keyedMutex + ownerMu util.KeyedMutex migrating sync.Map } -func NewManager(cfg *Config, store *Store, local Node) *Manager { +func New(cfg *config.Config, store *state.Store, local node.Node) *Manager { return &Manager{cfg: cfg, store: store, local: local, remotes: map[string]*Remote{}} } -func (m *Manager) node(name string) (Node, error) { +func (m *Manager) SetNodeBackHandler(fn func(node string)) { m.onNodeBack = fn } + +func (m *Manager) nodeByName(name string) (node.Node, error) { if name == m.local.Name() { return m.local, nil } @@ -57,11 +64,11 @@ func (m *Manager) node(name string) (Node, error) { if r != nil && r.Online() { return r, nil } - return nil, &NodeOfflineError{Node: name} + return nil, &node.NodeOfflineError{Node: name} } -func (m *Manager) onlineNodes() []Node { - nodes := []Node{m.local} +func (m *Manager) onlineNodes() []node.Node { + nodes := []node.Node{m.local} m.mu.RLock() defer m.mu.RUnlock() for _, r := range m.remotes { @@ -85,9 +92,9 @@ func (m *Manager) OfflineNodes() []string { return out } -func (m *Manager) Views(ctx context.Context) ([]NodeView, error) { +func (m *Manager) Views(ctx context.Context) ([]balance.NodeView, error) { owners := m.store.Owners() - var views []NodeView + var views []balance.NodeView for _, n := range m.onlineNodes() { info, err := n.Info(ctx) if err != nil { @@ -96,21 +103,21 @@ func (m *Manager) Views(ctx context.Context) ([]NodeView, error) { } continue } - views = append(views, buildView(info, owners, m.cfg.Container)) + views = append(views, balance.BuildView(info, owners, m.cfg.Container)) } sort.Slice(views, func(i, j int) bool { return views[i].Name < views[j].Name }) return views, nil } -func (m *Manager) view(ctx context.Context, n Node) (NodeView, NodeInfo, error) { +func (m *Manager) view(ctx context.Context, n node.Node) (balance.NodeView, node.NodeInfo, error) { info, err := n.Info(ctx) if err != nil { - return NodeView{}, info, err + return balance.NodeView{}, info, err } - return buildView(info, m.store.Owners(), m.cfg.Container), info, nil + return balance.BuildView(info, m.store.Owners(), m.cfg.Container), info, nil } -func (m *Manager) Totals(ctx context.Context) (used, usable Size) { +func (m *Manager) Totals(ctx context.Context) (used, usable model.Size) { views, _ := m.Views(ctx) for _, v := range views { used, usable = used.Add(v.Used), usable.Add(v.Usable) @@ -118,15 +125,15 @@ func (m *Manager) Totals(ctx context.Context) (used, usable Size) { return } -func (m *Manager) ownerRecord(id string, kind OwnerKind) (*Owner, error) { +func (m *Manager) ownerRecord(id string, kind model.OwnerKind) (*model.Owner, error) { if u := m.store.Owner(id); u != nil { return u, nil } - u := &Owner{ + u := &model.Owner{ ID: id, Kind: kind, - Machine: MachineName(id, kind), - Secret: randomHex(32), + Machine: model.MachineName(id, kind), + Secret: util.RandomHex(32), Size: m.cfg.Container, StorageMinMB: m.cfg.Storage.MinMB, StorageMaxMB: m.cfg.Storage.MaxMB, @@ -135,16 +142,16 @@ func (m *Manager) ownerRecord(id string, kind OwnerKind) (*Owner, error) { return u, m.store.PutOwner(u) } -func (m *Manager) Owner(id string, kind OwnerKind) (*Owner, error) { +func (m *Manager) Owner(id string, kind model.OwnerKind) (*model.Owner, error) { unlock := m.ownerMu.Lock(id) defer unlock() return m.ownerRecord(id, kind) } -func (m *Manager) spec(u *Owner) StartSpec { - return StartSpec{ +func (m *Manager) spec(u *model.Owner) node.StartSpec { + return node.StartSpec{ Machine: u.Machine, - Description: "slackspawn container for " + string(u.Kind) + " " + u.ID, + Description: "slackerbot container for " + string(u.Kind) + " " + u.ID, Secret: u.Secret, Size: u.Size, Drive: u.Drive, @@ -153,8 +160,8 @@ func (m *Manager) spec(u *Owner) StartSpec { } } -func (m *Manager) Ensure(ctx context.Context, id string, kind OwnerKind) (*Owner, error) { - if m.isMigrating(id) { +func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (*model.Owner, error) { + if m.IsMigrating(id) { return nil, ErrMigrating } unlock := m.ownerMu.Lock(id) @@ -165,7 +172,7 @@ func (m *Manager) Ensure(ctx context.Context, id string, kind OwnerKind) (*Owner return nil, err } if u.Node != "" { - n, err := m.node(u.Node) + n, err := m.nodeByName(u.Node) if err != nil { return u, err } @@ -182,17 +189,17 @@ func (m *Manager) Ensure(ctx context.Context, id string, kind OwnerKind) (*Owner if err != nil { return u, err } - node, drive, err := choosePlacement(views, u.Size, u.StorageMinMB) + target, drive, err := balance.ChoosePlacement(views, u.Size, u.StorageMinMB) if err != nil { return u, err } - u.Node, u.Drive = node, drive - if err := m.store.SetPlacement(u.ID, node, drive); err != nil { + u.Node, u.Drive = target, drive + if err := m.store.SetPlacement(u.ID, target, drive); err != nil { return u, err } } - n, err := m.node(u.Node) + n, err := m.nodeByName(u.Node) if err != nil { return u, err } @@ -201,14 +208,14 @@ func (m *Manager) Ensure(ctx context.Context, id string, kind OwnerKind) (*Owner return u, err } if !v.Used.Add(u.Size).Fits(v.Usable) { - return u, &NodeFullError{Node: u.Node} + return u, &node.NodeFullError{Node: u.Node} } return u, n.Start(ctx, m.spec(u)) } // The old home folder stays on its node, untouched. func (m *Manager) Recreate(id string) error { - if m.isMigrating(id) { + if m.IsMigrating(id) { return ErrMigrating } unlock := m.ownerMu.Lock(id) @@ -219,19 +226,19 @@ func (m *Manager) Recreate(id string) error { return m.store.DeleteSessions(id) } -func (m *Manager) DialAgent(ctx context.Context, u *Owner) (msgConn, error) { - n, err := m.node(u.Node) +func (m *Manager) DialAgent(ctx context.Context, u *model.Owner) (node.MsgConn, error) { + n, err := m.nodeByName(u.Node) if err != nil { return nil, err } return n.DialAgent(ctx, u.Machine, u.Secret) } -func (m *Manager) State(u *Owner) State { +func (m *Manager) State(u *model.Owner) State { if u == nil || u.Node == "" { return StateNone } - n, err := m.node(u.Node) + n, err := m.nodeByName(u.Node) if err != nil { return StateOffline } @@ -241,20 +248,20 @@ func (m *Manager) State(u *Owner) State { return StateStopped } -func (m *Manager) Poweroff(ctx context.Context, u *Owner) error { - n, err := m.node(u.Node) +func (m *Manager) Poweroff(ctx context.Context, u *model.Owner) error { + n, err := m.nodeByName(u.Node) if err != nil { return err } return n.Poweroff(ctx, u.Machine) } -func (m *Manager) RunningOwners(ctx context.Context) []Owner { - byNodeMachine := map[string]Owner{} +func (m *Manager) RunningOwners(ctx context.Context) []model.Owner { + byNodeMachine := map[string]model.Owner{} for _, u := range m.store.Owners() { byNodeMachine[u.Node+"/"+u.Machine] = u } - var out []Owner + var out []model.Owner for _, n := range m.onlineNodes() { running, err := n.Running(ctx) if err != nil { @@ -269,8 +276,8 @@ func (m *Manager) RunningOwners(ctx context.Context) []Owner { return out } -func (m *Manager) Resize(ctx context.Context, id string, kind OwnerKind, change func(Size) (Size, error)) (old, next Size, err error) { - if m.isMigrating(id) { +func (m *Manager) Resize(ctx context.Context, id string, kind model.OwnerKind, change func(model.Size) (model.Size, error)) (old, next model.Size, err error) { + if m.IsMigrating(id) { return old, old, ErrMigrating } unlock := m.ownerMu.Lock(id) @@ -280,7 +287,7 @@ func (m *Manager) Resize(ctx context.Context, id string, kind OwnerKind, change u, err := m.ownerRecord(id, kind) if err != nil { - return Size{}, Size{}, err + return model.Size{}, model.Size{}, err } old = u.Size if next, err = change(old); err != nil { @@ -297,7 +304,7 @@ func (m *Manager) Resize(ctx context.Context, id string, kind OwnerKind, change } if u.Node != "" { - n, err := m.node(u.Node) + n, err := m.nodeByName(u.Node) if err != nil { return old, old, err } @@ -306,9 +313,9 @@ func (m *Manager) Resize(ctx context.Context, id string, kind OwnerKind, change if err != nil { return old, old, err } - growth := Size{max(0, next.MemoryMB-old.MemoryMB), math.Max(0, next.CPUs-old.CPUs)} + growth := model.Size{MemoryMB: max(0, next.MemoryMB-old.MemoryMB), CPUs: math.Max(0, next.CPUs-old.CPUs)} if !v.Used.Add(growth).Fits(v.Usable) { - return old, old, &NodeFullError{Node: u.Node} + return old, old, &node.NodeFullError{Node: u.Node} } if err := n.SetLimits(ctx, u.Machine, next); err != nil { return old, old, err @@ -321,11 +328,11 @@ func (m *Manager) Resize(ctx context.Context, id string, kind OwnerKind, change type StorageChange struct { OldMin, OldMax, NewMin, NewMax int64 UsedMB int64 - Drive *DriveReport + Drive *storage.DriveReport } -func (m *Manager) SetStorage(ctx context.Context, id string, kind OwnerKind, change func(min, max int64) (int64, int64, error)) (*StorageChange, error) { - if m.isMigrating(id) { +func (m *Manager) SetStorage(ctx context.Context, id string, kind model.OwnerKind, change func(min, max int64) (int64, int64, error)) (*StorageChange, error) { + if m.IsMigrating(id) { return &StorageChange{}, ErrMigrating } unlock := m.ownerMu.Lock(id) @@ -342,12 +349,12 @@ func (m *Manager) SetStorage(ctx context.Context, id string, kind OwnerKind, cha return c, err } switch { - case c.NewMin < MinStorageMB || c.NewMax < MinStorageMB: - return c, ErrStorageTooSmall + case c.NewMin < config.MinStorageMB || c.NewMax < config.MinStorageMB: + return c, storage.ErrStorageTooSmall case c.NewMin > c.NewMax: - return c, ErrMinOverMax + return c, storage.ErrMinOverMax case c.NewMax > m.cfg.Storage.LimitMB: - return c, ErrStorageTooBig + return c, storage.ErrStorageTooBig } if u.Node == "" { @@ -355,13 +362,13 @@ func (m *Manager) SetStorage(ctx context.Context, id string, kind OwnerKind, cha if err != nil { return c, err } - if _, _, err := choosePlacement(views, u.Size, c.NewMin); err != nil { + if _, _, err := balance.ChoosePlacement(views, u.Size, c.NewMin); err != nil { return c, err } return c, m.store.SetStorage(id, c.NewMin, c.NewMax) } - n, err := m.node(u.Node) + n, err := m.nodeByName(u.Node) if err != nil { return c, err } @@ -375,9 +382,9 @@ func (m *Manager) SetStorage(ctx context.Context, id string, kind OwnerKind, cha } } if c.NewMax < c.UsedMB { - return c, ErrBelowUsage + return c, storage.ErrBelowUsage } - var r *DriveReport + var r *storage.DriveReport for i := range v.Drives { if v.Drives[i].Path == u.Drive { r = &v.Drives[i] @@ -386,9 +393,9 @@ func (m *Manager) SetStorage(ctx context.Context, id string, kind OwnerKind, cha if r == nil { return c, fmt.Errorf("drive %s is no longer configured on %s", u.Drive, u.Node) } - extra := committed(c.NewMin, c.UsedMB) - committed(c.OldMin, c.UsedMB) + extra := storage.Committed(c.NewMin, c.UsedMB) - storage.Committed(c.OldMin, c.UsedMB) if extra > 0 && extra > r.HeadroomMB() { - return c, ErrStorageFull + return c, storage.ErrStorageFull } if err := n.SetQuota(ctx, u.Drive, u.Machine, c.NewMax); err != nil { return c, err @@ -399,11 +406,11 @@ func (m *Manager) SetStorage(ctx context.Context, id string, kind OwnerKind, cha return c, m.store.SetStorage(id, c.NewMin, c.NewMax) } -func (m *Manager) StorageUsed(ctx context.Context, u *Owner) (int64, bool) { +func (m *Manager) StorageUsed(ctx context.Context, u *model.Owner) (int64, bool) { if u == nil || u.Node == "" { return 0, false } - n, err := m.node(u.Node) + n, err := m.nodeByName(u.Node) if err != nil { return 0, false } @@ -418,20 +425,3 @@ func (m *Manager) StorageUsed(ctx context.Context, u *Owner) (int64, bool) { } return 0, false } - -type keyedMutex struct{ m sync.Map } - -func (k *keyedMutex) Lock(key string) (unlock func()) { - v, _ := k.m.LoadOrStore(key, &sync.Mutex{}) - mu := v.(*sync.Mutex) - mu.Lock() - return mu.Unlock -} - -func randomHex(n int) string { - b := make([]byte, n) - if _, err := rand.Read(b); err != nil { - panic(err) - } - return hex.EncodeToString(b) -} diff --git a/migrate.go b/slackerbot/internal/manager/migrate.go similarity index 87% rename from migrate.go rename to slackerbot/internal/manager/migrate.go index 8c3b049..134e574 100644 --- a/migrate.go +++ b/slackerbot/internal/manager/migrate.go @@ -1,12 +1,19 @@ -package main +package manager import ( "context" "errors" "fmt" "log" + + "slackerbot/internal/balance" + "slackerbot/internal/model" + "slackerbot/internal/node" + "slackerbot/internal/storage" ) +const transferChunk = 4 << 20 + var ( ErrMigrating = errors.New("this container is being moved") ErrNoContainer = errors.New("there's no container here yet") @@ -48,7 +55,7 @@ type MoveResult struct { LeftOld error } -func (m *Manager) isMigrating(id string) bool { +func (m *Manager) IsMigrating(id string) bool { _, ok := m.migrating.Load(id) return ok } @@ -68,7 +75,7 @@ func (m *Manager) Migrate(ctx context.Context, id string, opts MoveOptions) (*Mo if u == nil || u.Node == "" { return nil, ErrNoContainer } - src, err := m.node(u.Node) + src, err := m.nodeByName(u.Node) if err != nil { return nil, err } @@ -76,12 +83,12 @@ func (m *Manager) Migrate(ctx context.Context, id string, opts MoveOptions) (*Mo needStart := res.WasRunning || opts.StartAfter usedMB, _ := m.StorageUsed(ctx, u) - target, drive, err := m.pickTarget(ctx, u, opts.Target, needStart, committed(u.StorageMinMB, usedMB)) + target, drive, err := m.pickTarget(ctx, u, opts.Target, needStart, storage.Committed(u.StorageMinMB, usedMB)) if err != nil { return nil, err } res.To = target - dst, err := m.node(target) + dst, err := m.nodeByName(target) if err != nil { return nil, err } @@ -185,7 +192,7 @@ func (m *Manager) Migrate(ctx context.Context, id string, opts MoveOptions) (*Mo return res, nil } -func (m *Manager) pickTarget(ctx context.Context, u *Owner, want string, needStart bool, storageMB int64) (node, drive string, err error) { +func (m *Manager) pickTarget(ctx context.Context, u *model.Owner, want string, needStart bool, storageMB int64) (target, drive string, err error) { if want == u.Node { return "", "", ErrSameNode } @@ -196,11 +203,11 @@ func (m *Manager) pickTarget(ctx context.Context, u *Owner, want string, needSta if err != nil { return "", "", err } - compute := Size{} + compute := model.Size{} if needStart { compute = u.Size } - var candidates []NodeView + var candidates []balance.NodeView for _, v := range views { if v.Name != u.Node && (want == "" || v.Name == want) { candidates = append(candidates, v) @@ -208,15 +215,15 @@ func (m *Manager) pickTarget(ctx context.Context, u *Owner, want string, needSta } if want != "" && len(candidates) == 0 { if m.KnownNode(want) { - return "", "", &NodeOfflineError{Node: want} + return "", "", &node.NodeOfflineError{Node: want} } return "", "", &UnknownNodeError{Node: want} } - node, drive, err = choosePlacement(candidates, compute, storageMB) + target, drive, err = balance.ChoosePlacement(candidates, compute, storageMB) if err != nil && want != "" { - return "", "", &NodeFullError{Node: want} + return "", "", &node.NodeFullError{Node: want} } - return node, drive, err + return target, drive, err } func (m *Manager) KnownNode(name string) bool { diff --git a/slackerbot/internal/manager/migrate_test.go b/slackerbot/internal/manager/migrate_test.go new file mode 100644 index 0000000..fb676fd --- /dev/null +++ b/slackerbot/internal/manager/migrate_test.go @@ -0,0 +1,159 @@ +package manager_test + +import ( + "bytes" + "context" + "errors" + "testing" + + "slackerbot/internal/manager" + "slackerbot/internal/model" + "slackerbot/internal/node" + "slackerbot/internal/testutil" + "slackerbot/internal/testutil/clustertest" +) + +func TestMoveLiveContainer(t *testing.T) { + c := clustertest.New(t) + c.AddOwner(t, testutil.HomeData(), true) + + var phases []manager.MovePhase + res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{ + Target: "slave-1", + WaitIdle: func(context.Context) { + c.Master.Mu.Lock() + c.Master.Homes["/d/c-c1"] = append(c.Master.Homes["/d/c-c1"], []byte("written during the bulk copy")...) + c.Master.Mu.Unlock() + }, + Progress: func(p manager.MoveProgress) { + if len(phases) == 0 || phases[len(phases)-1] != p.Phase { + phases = append(phases, p.Phase) + } + }, + }) + if err != nil { + t.Fatal(err) + } + + want := append(testutil.HomeData(), []byte("written during the bulk copy")...) + if !bytes.Equal(c.Slave.Homes["/d/c-c1"], want) { + t.Fatal("the late write didn't make it to the new node") + } + if c.Slave.Passes != 2 { + t.Fatalf("a live move should copy twice (bulk + final sync), got %d", c.Slave.Passes) + } + if _, still := c.Master.Homes["/d/c-c1"]; still { + t.Fatal("old home should be deleted after a good move") + } + if !c.Slave.IsRunning("c-c1") || c.Master.IsRunning("c-c1") || !res.Started { + t.Fatal("container should be running on the new node only") + } + if u := c.Store.Owner("C1"); u.Node != "slave-1" || u.Drive != "/d" { + t.Fatalf("placement not switched: %+v", u) + } + if len(c.Master.Exports) != 0 { + t.Fatal("snapshots on the old node should be cleaned up") + } + wantPhases := []manager.MovePhase{manager.PhaseCopy, manager.PhaseFinal, manager.PhaseCleanup, manager.PhaseStart} + if len(phases) != len(wantPhases) { + t.Fatalf("phases %v, want %v", phases, wantPhases) + } +} + +func TestMoveAutoStoppedContainer(t *testing.T) { + c := clustertest.New(t) + c.AddOwner(t, testutil.HomeData(), false) + + res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{}) + if err != nil { + t.Fatal(err) + } + if res.To != "slave-1" || res.Started || c.Slave.IsRunning("c-c1") { + t.Fatalf("a stopped container should move without starting: %+v", res) + } + if c.Slave.Passes != 1 { + t.Fatalf("a stopped container needs one copy, got %d", c.Slave.Passes) + } +} + +func TestMoveFullNodeStartsElsewhere(t *testing.T) { + c := clustertest.New(t) + c.AddOwner(t, testutil.HomeData(), false) + c.Master.Mu.Lock() + c.Master.Usable = model.Size{MemoryMB: 512, CPUs: 0.5} + c.Master.Mu.Unlock() + + _, err := c.Mgr.Ensure(c.Ctx, "C1", model.OwnerChannel) + var full *node.NodeFullError + if !errors.As(err, &full) || full.Node != "master" { + t.Fatalf("want node full on master, got %v", err) + } + res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{StartAfter: true}) + if err != nil || !res.Started || !c.Slave.IsRunning("c-c1") { + t.Fatalf("auto-move should start it on the new node: %+v %v", res, err) + } +} + +func TestMoveFailureRollsBack(t *testing.T) { + c := clustertest.New(t) + c.AddOwner(t, testutil.HomeData(), true) + + _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{ + Target: "slave-1", + WaitIdle: func(context.Context) { + c.Slave.Mu.Lock() + c.Slave.FailWrites = true + c.Slave.Mu.Unlock() + }, + }) + if err == nil { + t.Fatal("expected the final sync to fail") + } + if u := c.Store.Owner("C1"); u.Node != "master" { + t.Fatalf("container should stay on its old node, got %s", u.Node) + } + if !c.Master.IsRunning("c-c1") { + t.Fatal("container should be restarted on its old node") + } + if _, ok := c.Master.Homes["/d/c-c1"]; !ok { + t.Fatal("old home must survive a failed move") + } + if _, ok := c.Slave.Homes["/d/c-c1"]; ok || len(c.Slave.Imports) != 0 { + t.Fatal("half-copied home on the target should be cleaned up") + } +} + +func TestMoveRefusals(t *testing.T) { + c := clustertest.New(t) + c.AddOwner(t, testutil.HomeData(), false) + + if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "master"}); !errors.Is(err, manager.ErrSameNode) { + t.Fatalf("same node: %v", err) + } + var unknown *manager.UnknownNodeError + if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "nope"}); !errors.As(err, &unknown) { + t.Fatalf("unknown node: %v", err) + } + if _, err := c.Mgr.Migrate(c.Ctx, "nobody", manager.MoveOptions{}); !errors.Is(err, manager.ErrNoContainer) { + t.Fatalf("no container: %v", err) + } + c.Slave.Mu.Lock() + c.Slave.Usable = model.Size{MemoryMB: 512, CPUs: 0.5} + c.Slave.Mu.Unlock() + c.Master.Mu.Lock() + c.Master.Runs["c-c1"] = true + c.Master.Mu.Unlock() + testutil.WaitFor(t, "slave report", func() bool { + views, _ := c.Mgr.Views(c.Ctx) + for _, v := range views { + if v.Name == "slave-1" { + return v.Usable.MemoryMB == 512 + } + } + return false + }) + var full *node.NodeFullError + if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "slave-1"}); !errors.As(err, &full) { + t.Fatalf("full target: %v", err) + } +} diff --git a/slackerbot/internal/model/model.go b/slackerbot/internal/model/model.go new file mode 100644 index 0000000..5672b29 --- /dev/null +++ b/slackerbot/internal/model/model.go @@ -0,0 +1,74 @@ +package model + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +type Size struct { + MemoryMB int `toml:"memory_mb"` + CPUs float64 `toml:"cpus"` +} + +func (s Size) Add(o Size) Size { return Size{s.MemoryMB + o.MemoryMB, s.CPUs + o.CPUs} } +func (s Size) Sub(o Size) Size { return Size{s.MemoryMB - o.MemoryMB, s.CPUs - o.CPUs} } +func (s Size) Fits(in Size) bool { + return s.MemoryMB <= in.MemoryMB && s.CPUs <= in.CPUs+1e-9 +} +func (s Size) String() string { + mem := fmt.Sprintf("%d MB", s.MemoryMB) + if s.MemoryMB >= 1024 && s.MemoryMB%1024 == 0 { + mem = fmt.Sprintf("%d GB", s.MemoryMB/1024) + } + cpu := strconv.FormatFloat(s.CPUs, 'f', -1, 64) + " CPUs" + if s.CPUs == 1 { + cpu = "1 CPU" + } + return mem + " RAM, " + cpu +} + +type OwnerKind string + +const ( + OwnerUser OwnerKind = "user" + OwnerChannel OwnerKind = "channel" +) + +type Owner struct { + ID string `json:"id"` + Kind OwnerKind `json:"kind"` + Machine string `json:"machine"` + Secret string `json:"secret"` + Size Size `json:"size"` + LastActive time.Time `json:"last_active"` + + Node string `json:"node,omitempty"` + Drive string `json:"drive,omitempty"` + StorageMinMB int64 `json:"storage_min_mb"` + StorageMaxMB int64 `json:"storage_max_mb"` + + // 0 means the config default. + IdleTimeout time.Duration `json:"idle_timeout,omitempty"` +} + +const Indefinite time.Duration = -1 + +type Session struct { + Channel string `json:"channel"` + ThreadTS string `json:"thread_ts"` + OwnerID string `json:"owner_id"` + ACPID string `json:"acp_id"` + Title string `json:"title"` + CreatedAt time.Time `json:"created_at"` +} + +func MachineName(id string, kind OwnerKind) string { + if kind == OwnerChannel { + return "c-" + strings.ToLower(id) + } + return "u-" + strings.ToLower(id) +} + +func IsOurs(name string) bool { return strings.HasPrefix(name, "u-") || strings.HasPrefix(name, "c-") } diff --git a/permissions.go b/slackerbot/internal/model/permissions.go similarity index 87% rename from permissions.go rename to slackerbot/internal/model/permissions.go index 1da6f03..16cfc29 100644 --- a/permissions.go +++ b/slackerbot/internal/model/permissions.go @@ -1,4 +1,4 @@ -package main +package model import ( "errors" @@ -20,8 +20,8 @@ const ( ) var ( - allActions = []Action{ActScale, ActStorage, ActMove, ActKeep} - grantActions = append(slices.Clone(allActions), ActManage) + AllActions = []Action{ActScale, ActStorage, ActMove, ActKeep} + GrantActions = append(slices.Clone(AllActions), ActManage) ErrLastManager = errors.New("that would leave nobody able to manage permissions") ) @@ -33,7 +33,7 @@ type PermissionsConfig struct { DM map[string]bool `toml:"dm" json:"dm"` } -func (p PermissionsConfig) validate() error { +func (p PermissionsConfig) Validate() error { if len(p.Managers) == 0 { return fmt.Errorf("permissions.managers must name at least one user manager") } @@ -43,7 +43,7 @@ func (p PermissionsConfig) validate() error { check := func(where string, list map[string][]string) error { for user, actions := range list { for _, a := range actions { - if a != "all" && !slices.Contains(grantActions, Action(a)) { + if a != "all" && !slices.Contains(GrantActions, Action(a)) { return fmt.Errorf("%s: unknown permission %q for %s (use %s, manage or all)", where, a, user, actionNames()) } } @@ -59,14 +59,14 @@ func (p PermissionsConfig) validate() error { } } for a := range p.DM { - if !slices.Contains(allActions, Action(a)) { + if !slices.Contains(AllActions, Action(a)) { return fmt.Errorf("permissions.dm: unknown permission %q (use %s)", a, actionNames()) } } return nil } -func (p PermissionsConfig) clone() PermissionsConfig { +func (p PermissionsConfig) Clone() PermissionsConfig { c := PermissionsConfig{ Managers: slices.Clone(p.Managers), Default: cloneList(p.Default), @@ -91,14 +91,14 @@ func cloneList(list map[string][]string) map[string][]string { } func actionNames() string { - names := make([]string, len(allActions)) - for i, a := range allActions { + names := make([]string, len(AllActions)) + for i, a := range AllActions { names[i] = string(a) } return strings.Join(names, ", ") } -func (p PermissionsConfig) superUsers(channel string) (list map[string][]string, own bool) { +func (p PermissionsConfig) SuperUsers(channel string) (list map[string][]string, own bool) { if l := p.Channels[channel]; len(l) > 0 { return l, true } @@ -106,7 +106,7 @@ func (p PermissionsConfig) superUsers(channel string) (list map[string][]string, } // "all" covers the container actions only; managing permissions is always granted by name. -func grants(actions []string, a Action) bool { +func Grants(actions []string, a Action) bool { if a == ActManage { return slices.Contains(actions, string(ActManage)) } @@ -118,8 +118,8 @@ func (p PermissionsConfig) Allowed(kind OwnerKind, channel, user string, a Actio allowed, set := p.DM[string(a)] return !set || allowed } - list, _ := p.superUsers(channel) - return grants(list[user], a) + list, _ := p.SuperUsers(channel) + return Grants(list[user], a) } func (p PermissionsConfig) IsManager(user string) bool { return slices.Contains(p.Managers, user) } @@ -129,10 +129,10 @@ func (p PermissionsConfig) CanManage(channel, user string) bool { } func (p PermissionsConfig) Holders(channel string, a Action) []string { - list, _ := p.superUsers(channel) + list, _ := p.SuperUsers(channel) var out []string for user, actions := range list { - if grants(actions, a) { + if Grants(actions, a) { out = append(out, user) } } @@ -141,10 +141,10 @@ func (p PermissionsConfig) Holders(channel string, a Action) []string { } func (p PermissionsConfig) ActionsFor(channel, user string) []Action { - list, _ := p.superUsers(channel) + list, _ := p.SuperUsers(channel) var out []Action - for _, a := range grantActions { - if grants(list[user], a) { + for _, a := range GrantActions { + if Grants(list[user], a) { out = append(out, a) } } @@ -153,7 +153,7 @@ func (p PermissionsConfig) ActionsFor(channel, user string) []Action { func expand(actions []Action) []Action { if slices.Contains(actions, "all") { - out := slices.Clone(allActions) + out := slices.Clone(AllActions) for _, a := range actions { if a != "all" && !slices.Contains(out, a) { out = append(out, a) diff --git a/slackerbot/internal/model/permissions_test.go b/slackerbot/internal/model/permissions_test.go new file mode 100644 index 0000000..a25286d --- /dev/null +++ b/slackerbot/internal/model/permissions_test.go @@ -0,0 +1,105 @@ +package model_test + +import ( + "errors" + "testing" + + "slackerbot/internal/config" + "slackerbot/internal/model" +) + +func TestPermissions(t *testing.T) { + t.Setenv("SLACK_APP_TOKEN", "xapp-test") + t.Setenv("SLACK_BOT_TOKEN", "xoxb-test") + t.Setenv("CLUSTER_SECRET", "cluster-secret-1234567890") + c, err := config.LoadConfig("../../../config.example.toml", config.ModeMaster) + if err != nil { + t.Fatal(err) + } + p := c.Permissions + + cases := []struct { + kind model.OwnerKind + channel, user string + action model.Action + want bool + }{ + {model.OwnerChannel, "C07ABC123", "U0111", model.ActMove, true}, + {model.OwnerChannel, "C07ABC123", "U0111", model.ActScale, false}, + {model.OwnerChannel, "C07ABC123", "U0222", model.ActKeep, true}, + {model.OwnerChannel, "C07ABC123", "U0ADMIN", model.ActMove, false}, + {model.OwnerChannel, "C0OTHER", "U0ADMIN", model.ActStorage, true}, + {model.OwnerChannel, "C0OTHER", "U0111", model.ActMove, false}, + {model.OwnerUser, "D1", "U9", model.ActScale, true}, + {model.OwnerUser, "D1", "U9", model.ActStorage, false}, + } + for _, c := range cases { + if got := p.Allowed(c.kind, c.channel, c.user, c.action); got != c.want { + t.Errorf("%s %s %s %s = %v, want %v", c.kind, c.channel, c.user, c.action, got, c.want) + } + } + if h := p.Holders("C07ABC123", model.ActMove); len(h) != 2 { + t.Errorf("move holders = %v", h) + } + if a := p.ActionsFor("C0OTHER", "U0ADMIN"); len(a) != len(model.AllActions) { + t.Errorf("all should expand to every action, got %v", a) + } + + open := model.PermissionsConfig{Default: map[string][]string{"U1": {"all"}}} + if !open.Allowed(model.OwnerUser, "D1", "U9", model.ActMove) { + t.Error("DM permissions should default to allowed") + } + if (model.PermissionsConfig{}).Validate() == nil { + t.Error("a missing default super user should be refused") + } + bad := model.PermissionsConfig{Default: map[string][]string{"U1": {"delete-everything"}}} + if bad.Validate() == nil { + t.Error("unknown permission names should be refused") + } +} + +func TestGrantRevoke(t *testing.T) { + p := model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UDEF": {"all"}}}.Clone() + + p.Grant("C1", []string{"UANA"}, []model.Action{model.ActMove}) + if !p.Allowed(model.OwnerChannel, "C1", "UDEF", model.ActScale) { + t.Fatal("first grant in a channel should keep everyone from the default list") + } + if !p.Allowed(model.OwnerChannel, "C1", "UANA", model.ActMove) || p.Allowed(model.OwnerChannel, "C1", "UANA", model.ActScale) { + t.Fatal("grant should add exactly what was asked") + } + if p.Allowed(model.OwnerChannel, "C2", "UANA", model.ActMove) { + t.Fatal("grants stay in their channel") + } + + p.Grant("C1", []string{"UANA"}, []model.Action{"all"}) + if p.CanManage("C1", "UANA") { + t.Fatal("all must not include manage") + } + p.Grant("C1", []string{"UANA"}, []model.Action{model.ActManage}) + if !p.CanManage("C1", "UANA") || p.CanManage("C2", "UANA") { + t.Fatal("manage is per channel") + } + + p.Revoke("C1", []string{"UDEF"}, []model.Action{model.ActScale}) + if p.Allowed(model.OwnerChannel, "C1", "UDEF", model.ActScale) || !p.Allowed(model.OwnerChannel, "C1", "UDEF", model.ActMove) { + t.Fatal("revoking one action out of all should keep the rest") + } + p.Revoke("C1", []string{"UANA"}, nil) + if len(p.ActionsFor("C1", "UANA")) != 0 { + t.Fatal("revoke with no actions removes the person") + } + + p.Grant("", []string{"UNEW"}, []model.Action{model.ActKeep}) + if !p.Allowed(model.OwnerChannel, "C9", "UNEW", model.ActKeep) { + t.Fatal("default grants apply to channels without their own list") + } + + p.AddManagers([]string{"UTWO"}) + if err := p.RemoveManagers([]string{"UBOSS", "UTWO"}); !errors.Is(err, model.ErrLastManager) { + t.Fatal("removing every manager must be refused") + } + if err := p.RemoveManagers([]string{"UBOSS"}); err != nil || p.IsManager("UBOSS") { + t.Fatal("removing one of two managers should work") + } +} diff --git a/node.go b/slackerbot/internal/node/node.go similarity index 78% rename from node.go rename to slackerbot/internal/node/node.go index db86226..bfe843e 100644 --- a/node.go +++ b/slackerbot/internal/node/node.go @@ -1,11 +1,14 @@ -package main +package node import ( "context" "fmt" + "net/http" "time" "github.com/coder/websocket" + + "slackerbot/internal/model" ) type Node interface { @@ -15,9 +18,9 @@ type Node interface { IsRunning(machine string) bool Start(ctx context.Context, spec StartSpec) error Poweroff(ctx context.Context, machine string) error - SetLimits(ctx context.Context, machine string, size Size) error + SetLimits(ctx context.Context, machine string, size model.Size) error SetQuota(ctx context.Context, drive, machine string, mb int64) error - DialAgent(ctx context.Context, machine, secret string) (msgConn, error) + DialAgent(ctx context.Context, machine, secret string) (MsgConn, error) WaitStopped(ctx context.Context, machine string) error ExportHome(ctx context.Context, drive, machine, parent string) (id string, err error) @@ -33,8 +36,8 @@ type Node interface { type NodeInfo struct { Name string `json:"name"` - Total Size `json:"total"` - Usable Size `json:"usable"` + Total model.Size `json:"total"` + Usable model.Size `json:"usable"` Drives []DriveInfo `json:"drives"` Running []RunningMachine `json:"running"` } @@ -55,7 +58,7 @@ type StartSpec struct { Machine string `json:"machine"` Description string `json:"description"` Secret string `json:"secret"` - Size Size `json:"size"` + Size model.Size `json:"size"` Drive string `json:"drive"` StorageMaxMB int64 `json:"storage_max_mb"` Env map[string]string `json:"env"` @@ -69,7 +72,7 @@ type NodeFullError struct{ Node string } func (e *NodeFullError) Error() string { return fmt.Sprintf("node %s is full", e.Node) } -type msgConn interface { +type MsgConn interface { Read(ctx context.Context) ([]byte, error) Write(ctx context.Context, b []byte) error Close() error @@ -87,3 +90,14 @@ func (w wsConn) Write(ctx context.Context, b []byte) error { } func (w wsConn) Close() error { return w.c.Close(websocket.StatusNormalClosure, "") } + +func DialAgentWS(ctx context.Context, addr, secret string) (MsgConn, error) { + conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/acp", &websocket.DialOptions{ + HTTPHeader: http.Header{"X-Secret-Key": []string{secret}}, + }) + if err != nil { + return nil, fmt.Errorf("connecting to goose: %w", err) + } + conn.SetReadLimit(32 << 20) + return wsConn{conn}, nil +} diff --git a/store.go b/slackerbot/internal/state/store.go similarity index 61% rename from store.go rename to slackerbot/internal/state/store.go index af0c638..a73ff72 100644 --- a/store.go +++ b/slackerbot/internal/state/store.go @@ -1,4 +1,4 @@ -package main +package state import ( "encoding/json" @@ -8,57 +8,24 @@ import ( "sort" "sync" "time" + + "slackerbot/internal/model" ) -type OwnerKind string - -const ( - OwnerUser OwnerKind = "user" - OwnerChannel OwnerKind = "channel" -) - -type Owner struct { - ID string `json:"id"` - Kind OwnerKind `json:"kind"` - Machine string `json:"machine"` - Secret string `json:"secret"` - Size Size `json:"size"` - LastActive time.Time `json:"last_active"` - - Node string `json:"node,omitempty"` - Drive string `json:"drive,omitempty"` - StorageMinMB int64 `json:"storage_min_mb"` - StorageMaxMB int64 `json:"storage_max_mb"` - - // 0 means the config default. - IdleTimeout time.Duration `json:"idle_timeout,omitempty"` -} - -const Indefinite time.Duration = -1 - -type Session struct { - Channel string `json:"channel"` - ThreadTS string `json:"thread_ts"` - OwnerID string `json:"owner_id"` - ACPID string `json:"acp_id"` - Title string `json:"title"` - CreatedAt time.Time `json:"created_at"` -} - type Store struct { path string mu sync.Mutex data struct { - Owners map[string]*Owner `json:"owners"` - Sessions map[string]*Session `json:"sessions"` - Permissions *PermissionsConfig `json:"permissions,omitempty"` + Owners map[string]*model.Owner `json:"owners"` + Sessions map[string]*model.Session `json:"sessions"` + Permissions *model.PermissionsConfig `json:"permissions,omitempty"` } } -func OpenStore(path string) (*Store, error) { +func Open(path string) (*Store, error) { s := &Store{path: path} - s.data.Owners = map[string]*Owner{} - s.data.Sessions = map[string]*Session{} + s.data.Owners = map[string]*model.Owner{} + s.data.Sessions = map[string]*model.Session{} b, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { @@ -71,10 +38,10 @@ func OpenStore(path string) (*Store, error) { return nil, err } if s.data.Owners == nil { - s.data.Owners = map[string]*Owner{} + s.data.Owners = map[string]*model.Owner{} } if s.data.Sessions == nil { - s.data.Sessions = map[string]*Session{} + s.data.Sessions = map[string]*model.Session{} } return s, nil } @@ -91,7 +58,7 @@ func (s *Store) save() error { return os.Rename(tmp, s.path) } -func (s *Store) Owner(id string) *Owner { +func (s *Store) Owner(id string) *model.Owner { s.mu.Lock() defer s.mu.Unlock() if u, ok := s.data.Owners[id]; ok { @@ -101,7 +68,7 @@ func (s *Store) Owner(id string) *Owner { return nil } -func (s *Store) OwnerByMachine(machine string) *Owner { +func (s *Store) OwnerByMachine(machine string) *model.Owner { s.mu.Lock() defer s.mu.Unlock() for _, u := range s.data.Owners { @@ -113,7 +80,7 @@ func (s *Store) OwnerByMachine(machine string) *Owner { return nil } -func (s *Store) PutOwner(u *Owner) error { +func (s *Store) PutOwner(u *model.Owner) error { s.mu.Lock() defer s.mu.Unlock() c := *u @@ -131,7 +98,7 @@ func (s *Store) Touch(id string) error { return nil } -func (s *Store) SetSize(id string, size Size) error { +func (s *Store) SetSize(id string, size model.Size) error { s.mu.Lock() defer s.mu.Unlock() if u, ok := s.data.Owners[id]; ok { @@ -141,10 +108,10 @@ func (s *Store) SetSize(id string, size Size) error { return nil } -func (s *Store) Owners() []Owner { +func (s *Store) Owners() []model.Owner { s.mu.Lock() defer s.mu.Unlock() - out := make([]Owner, 0, len(s.data.Owners)) + out := make([]model.Owner, 0, len(s.data.Owners)) for _, u := range s.data.Owners { out = append(out, *u) } @@ -194,39 +161,39 @@ func (s *Store) SetIdleTimeout(id string, d time.Duration) error { // The config's permissions are only a starting point: once saved here, // changes made in Slack win over later config edits. -func (s *Store) SeedPermissions(initial PermissionsConfig) (seeded bool, err error) { +func (s *Store) SeedPermissions(initial model.PermissionsConfig) (seeded bool, err error) { s.mu.Lock() defer s.mu.Unlock() if s.data.Permissions != nil { return false, nil } - p := initial.clone() + p := initial.Clone() s.data.Permissions = &p return true, s.save() } -func (s *Store) Permissions() PermissionsConfig { +func (s *Store) Permissions() model.PermissionsConfig { s.mu.Lock() defer s.mu.Unlock() if s.data.Permissions == nil { - return PermissionsConfig{} + return model.PermissionsConfig{} } - return s.data.Permissions.clone() + return s.data.Permissions.Clone() } -func (s *Store) UpdatePermissions(change func(p *PermissionsConfig) error) error { +func (s *Store) UpdatePermissions(change func(p *model.PermissionsConfig) error) error { s.mu.Lock() defer s.mu.Unlock() - var p PermissionsConfig + var p model.PermissionsConfig if s.data.Permissions != nil { - p = s.data.Permissions.clone() + p = s.data.Permissions.Clone() } else { - p = PermissionsConfig{}.clone() + p = model.PermissionsConfig{}.Clone() } if err := change(&p); err != nil { return err } - if err := p.validate(); err != nil { + if err := p.Validate(); err != nil { return err } s.data.Permissions = &p @@ -235,7 +202,7 @@ func (s *Store) UpdatePermissions(change func(p *PermissionsConfig) error) error func sessionKey(channel, threadTS string) string { return channel + "/" + threadTS } -func (s *Store) Session(channel, threadTS string) *Session { +func (s *Store) Session(channel, threadTS string) *model.Session { s.mu.Lock() defer s.mu.Unlock() if x, ok := s.data.Sessions[sessionKey(channel, threadTS)]; ok { @@ -245,7 +212,7 @@ func (s *Store) Session(channel, threadTS string) *Session { return nil } -func (s *Store) AddSession(x *Session) error { +func (s *Store) AddSession(x *model.Session) error { s.mu.Lock() defer s.mu.Unlock() c := *x @@ -253,10 +220,10 @@ func (s *Store) AddSession(x *Session) error { return s.save() } -func (s *Store) Sessions(ownerID string, limit int) []Session { +func (s *Store) Sessions(ownerID string, limit int) []model.Session { s.mu.Lock() defer s.mu.Unlock() - var out []Session + var out []model.Session for _, x := range s.data.Sessions { if x.OwnerID == ownerID { out = append(out, *x) diff --git a/slackerbot/internal/state/store_test.go b/slackerbot/internal/state/store_test.go new file mode 100644 index 0000000..e517199 --- /dev/null +++ b/slackerbot/internal/state/store_test.go @@ -0,0 +1,66 @@ +package state + +import ( + "path/filepath" + "testing" + "time" + + "slackerbot/internal/model" +) + +func TestStore(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + _ = s.PutOwner(&model.Owner{ID: "C1", Kind: model.OwnerChannel, Machine: "c-c1", Size: model.Size{MemoryMB: 1024, CPUs: 1}}) + _ = s.AddSession(&model.Session{Channel: "D1", ThreadTS: "1.0", OwnerID: "C1", ACPID: "a", CreatedAt: time.Now().Add(-time.Hour)}) + _ = s.AddSession(&model.Session{Channel: "D1", ThreadTS: "2.0", OwnerID: "C1", ACPID: "b", CreatedAt: time.Now()}) + + s2, err := Open(path) + if err != nil { + t.Fatal(err) + } + if u := s2.OwnerByMachine("c-c1"); u == nil || u.Size.MemoryMB != 1024 { + t.Fatal("user not saved") + } + list := s2.Sessions("C1", 0) + if len(list) != 2 || list[0].ACPID != "b" { + t.Fatalf("sessions not newest-first: %+v", list) + } + if s2.Session("D1", "1.0").ACPID != "a" { + t.Fatal("session lookup failed") + } +} + +func TestPermissionsSavedAndSeededOnce(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + s, _ := Open(path) + initial := model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UBOSS": {"all"}}} + if seeded, err := s.SeedPermissions(initial); !seeded || err != nil { + t.Fatalf("first start should seed: %v %v", seeded, err) + } + if err := s.UpdatePermissions(func(p *model.PermissionsConfig) error { + p.Grant("C1", []string{"UANA"}, []model.Action{model.ActMove}) + return nil + }); err != nil { + t.Fatal(err) + } + + s2, _ := Open(path) + changed := model.PermissionsConfig{Managers: []string{"USOMEONE"}, Default: map[string][]string{"USOMEONE": {"all"}}} + if seeded, _ := s2.SeedPermissions(changed); seeded { + t.Fatal("config must not override saved permissions") + } + p := s2.Permissions() + if !p.IsManager("UBOSS") || p.IsManager("USOMEONE") || !p.Allowed(model.OwnerChannel, "C1", "UANA", model.ActMove) { + t.Fatalf("saved permissions lost: %+v", p) + } + if err := s2.UpdatePermissions(func(p *model.PermissionsConfig) error { + p.Default["UX"] = []string{"teleport"} + return nil + }); err == nil { + t.Fatal("invalid changes must be refused") + } +} diff --git a/storage.go b/slackerbot/internal/storage/storage.go similarity index 51% rename from storage.go rename to slackerbot/internal/storage/storage.go index fd51a6e..35f2294 100644 --- a/storage.go +++ b/slackerbot/internal/storage/storage.go @@ -1,12 +1,10 @@ -package main +package storage import ( "errors" "fmt" - "math" "os" "os/exec" - "regexp" "strconv" "strings" "syscall" @@ -22,99 +20,6 @@ var ( ErrStorageTooSmall = errors.New("too small") ) -const MinStorageMB = 512 - -type StorageConfig struct { - Min string `toml:"min"` - Max string `toml:"max"` - Limit string `toml:"limit"` - - MinMB, MaxMB, LimitMB int64 `toml:"-"` -} - -type DriveConfig struct { - Path string `toml:"path"` - Reserve string `toml:"reserve"` - - ReserveMB int64 `toml:"-"` -} - -var storageToken = regexp.MustCompile(`^([+-]?)(\d+(?:\.\d+)?)(tib|tb|t|gib|gb|g|mib|mb|m)$`) - -func parseSizeMB(s string) (int64, error) { - t := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), " ", "")) - p := storageToken.FindStringSubmatch(t) - if p == nil || p[1] != "" { - return 0, fmt.Errorf("%q isn't a size like 5G or 500M", s) - } - return toMB(p[2], p[3]), nil -} - -func toMB(num, unit string) int64 { - n, _ := strconv.ParseFloat(num, 64) - switch unit[0] { - case 't': - n *= 1024 * 1024 - case 'g': - n *= 1024 - } - return int64(math.Round(n)) -} - -func formatMB(mb int64) string { - trim := func(f float64) string { return strconv.FormatFloat(math.Round(f*10)/10, 'f', -1, 64) } - switch { - case mb >= 1024*1024: - return trim(float64(mb)/(1024*1024)) + " TB" - case mb >= 1024: - return trim(float64(mb)/1024) + " GB" - } - return fmt.Sprintf("%d MB", mb) -} - -func parseStorage(args string, curMin, curMax int64) (min, max int64, err error) { - args = unitSpace.ReplaceAllString(strings.ToLower(args), "$1$2") - words := strings.Fields(args) - min, max = curMin, curMax - var minDone, maxDone bool - - for i := 0; i < len(words); i++ { - target := "max" - switch words[i] { - case "min", "guarantee", "guaranteed", "reserve": - target = "min" - i++ - case "max", "cap", "limit": - i++ - } - if i >= len(words) { - return curMin, curMax, fmt.Errorf("%s needs a size after it, like `%s 10G`", target, target) - } - p := storageToken.FindStringSubmatch(words[i]) - if p == nil { - if i == 0 { - return curMin, curMax, errNotCommand - } - return curMin, curMax, fmt.Errorf("I didn't understand `%s`", words[i]) - } - n := toMB(p[2], p[3]) - if target == "min" { - if minDone { - return curMin, curMax, errors.New("min was given twice") - } - minDone = true - min = int64(apply(p[1], float64(curMin), float64(n))) - } else { - if maxDone { - return curMin, curMax, errors.New("max was given twice") - } - maxDone = true - max = int64(apply(p[1], float64(curMax), float64(n))) - } - } - return min, max, nil -} - type DriveReport struct { Path string UsableMB int64 @@ -127,9 +32,9 @@ func (d DriveReport) HeadroomMB() int64 { return d.UsableMB - d.CommittedMB } func (d DriveReport) Overbooked() bool { return d.BookedMB > d.UsableMB } -func committed(minMB, usedMB int64) int64 { return max(minMB, usedMB) } +func Committed(minMB, usedMB int64) int64 { return max(minMB, usedMB) } -func pickDrive(reports []DriveReport, needMB int64) (string, error) { +func PickDrive(reports []DriveReport, needMB int64) (string, error) { best := -1 for i, r := range reports { if r.HeadroomMB() >= needMB && (best < 0 || r.HeadroomMB() > reports[best].HeadroomMB()) { @@ -142,7 +47,7 @@ func pickDrive(reports []DriveReport, needMB int64) (string, error) { return reports[best].Path, nil } -func prepareDrive(path string) error { +func PrepareDrive(path string) error { if err := os.MkdirAll(path, 0o700); err != nil { return err } @@ -159,7 +64,7 @@ func prepareDrive(path string) error { return nil } -func freeMB(path string) (int64, error) { +func FreeMB(path string) (int64, error) { var st syscall.Statfs_t if err := syscall.Statfs(path, &st); err != nil { return 0, err @@ -167,14 +72,21 @@ func freeMB(path string) (int64, error) { return int64(st.Bavail) * int64(st.Bsize) / (1 << 20), nil } -func createSubvolume(path string) error { +func CreateSubvolume(path string) error { if out, err := exec.Command("btrfs", "subvolume", "create", path).CombinedOutput(); err != nil { return fmt.Errorf("creating %s: %v: %s", path, err, strings.TrimSpace(string(out))) } return nil } -func setQuota(path string, mb int64) error { +func DeleteSubvolume(path string) error { + if out, err := exec.Command("btrfs", "subvolume", "delete", path).CombinedOutput(); err != nil { + return fmt.Errorf("deleting %s: %v: %s", path, err, strings.TrimSpace(string(out))) + } + return nil +} + +func SetQuota(path string, mb int64) error { bytes := strconv.FormatInt(mb<<20, 10) if out, err := exec.Command("btrfs", "qgroup", "limit", bytes, path).CombinedOutput(); err != nil { return fmt.Errorf("setting quota on %s: %v: %s", path, err, strings.TrimSpace(string(out))) @@ -183,7 +95,7 @@ func setQuota(path string, mb int64) error { } // Quota numbers can lag a few seconds behind writes. -func usedMB(path string) (int64, error) { +func UsedMB(path string) (int64, error) { out, err := exec.Command("btrfs", "qgroup", "show", "-f", "--raw", path).CombinedOutput() if err != nil { return 0, fmt.Errorf("reading usage of %s: %v: %s", path, err, strings.TrimSpace(string(out))) diff --git a/slackerbot/internal/storage/storage_test.go b/slackerbot/internal/storage/storage_test.go new file mode 100644 index 0000000..d16f225 --- /dev/null +++ b/slackerbot/internal/storage/storage_test.go @@ -0,0 +1,32 @@ +package storage + +import ( + "errors" + "testing" +) + +func TestDriveMath(t *testing.T) { + d := DriveReport{Path: "/a", UsableMB: 100_000, CommittedMB: 90_000, BookedMB: 150_000} + if d.HeadroomMB() != 10_000 || !d.Overbooked() { + t.Fatalf("headroom %d overbooked %v", d.HeadroomMB(), d.Overbooked()) + } + if Committed(5120, 8000) != 8000 || Committed(5120, 100) != 5120 { + t.Fatal("a home holds its guarantee or its real use, whichever is bigger") + } + drives := []DriveReport{d, {Path: "/b", UsableMB: 50_000, CommittedMB: 20_000}} + if got, _ := PickDrive(drives, 5120); got != "/b" { + t.Fatalf("should pick the drive with the most room, got %s", got) + } + if _, err := PickDrive(drives, 40_000); !errors.Is(err, ErrStorageFull) { + t.Fatal("a guarantee that fits nowhere should be rejected") + } +} + +func TestParseQgroupShow(t *testing.T) { + out := "Qgroupid Referenced Exclusive Max referenced Max exclusive Path\n" + + "-------- ---------- --------- -------------- ------------- ----\n" + + "0/257 3221225472 3221225472 10737418240 none c-c07abc\n" + if mb, err := parseQgroupShow(out); err != nil || mb != 3072 { + t.Fatalf("got %d, %v", mb, err) + } +} diff --git a/slackerbot/internal/testutil/clustertest/clustertest.go b/slackerbot/internal/testutil/clustertest/clustertest.go new file mode 100644 index 0000000..046ee60 --- /dev/null +++ b/slackerbot/internal/testutil/clustertest/clustertest.go @@ -0,0 +1,59 @@ +package clustertest + +import ( + "context" + "path/filepath" + "testing" + + "slackerbot/internal/config" + "slackerbot/internal/manager" + "slackerbot/internal/model" + "slackerbot/internal/state" + "slackerbot/internal/testutil" +) + +const ClusterSecret = "cluster-secret-1234567890" + +type Cluster struct { + Cfg *config.Config + Mgr *manager.Manager + Store *state.Store + Master, Slave *testutil.FakeNode + Ctx context.Context +} + +func New(t testing.TB) *Cluster { + addr := testutil.FreePort(t) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + cfg := &config.Config{StateDir: t.TempDir(), Container: model.Size{MemoryMB: 1024, CPUs: 1}} + cfg.Cluster = config.ClusterConfig{Name: "master", Secret: ClusterSecret, Listen: addr} + store, err := state.Open(filepath.Join(cfg.StateDir, "state.json")) + if err != nil { + t.Fatal(err) + } + c := &Cluster{Cfg: cfg, Store: store, Master: testutil.NewFakeNode("master"), Slave: testutil.NewFakeNode("slave-1"), Ctx: ctx} + c.Mgr = manager.New(cfg, store, c.Master) + go c.Mgr.ServeCluster(ctx) + + slaveCfg := &config.Config{StateDir: t.TempDir()} + slaveCfg.Cluster = config.ClusterConfig{Name: "slave-1", Secret: ClusterSecret, Master: "wss://" + addr} + go manager.RunSlave(ctx, slaveCfg, c.Slave) + testutil.WaitFor(t, "slave to come online", func() bool { return c.Mgr.State(&model.Owner{Node: "slave-1"}) != manager.StateOffline }) + return c +} + +func (c *Cluster) AddOwner(t testing.TB, data []byte, running bool) { + u := &model.Owner{ID: "C1", Kind: model.OwnerChannel, Machine: "c-c1", Secret: "s", Size: model.Size{MemoryMB: 1024, CPUs: 1}, + Node: "master", Drive: "/d", StorageMinMB: 1024, StorageMaxMB: 2048} + if err := c.Store.PutOwner(u); err != nil { + t.Fatal(err) + } + c.Master.Mu.Lock() + defer c.Master.Mu.Unlock() + c.Master.Homes["/d/c-c1"] = data + if running { + c.Master.Runs["c-c1"] = true + } +} diff --git a/slackerbot/internal/testutil/testutil.go b/slackerbot/internal/testutil/testutil.go new file mode 100644 index 0000000..44a7511 --- /dev/null +++ b/slackerbot/internal/testutil/testutil.go @@ -0,0 +1,243 @@ +package testutil + +import ( + "bytes" + "context" + "errors" + "net" + "strings" + "sync" + "testing" + "time" + + "slackerbot/internal/model" + "slackerbot/internal/node" + "slackerbot/internal/util" +) + +type FakeNode struct { + name string + Mu sync.Mutex + Runs map[string]bool + Last node.StartSpec + Homes map[string][]byte + Exports map[string]*fakeExport + Imports map[string]*fakeImport + Passes int + FailWrites bool + Usable model.Size +} + +func NewFakeNode(name string) *FakeNode { + return &FakeNode{name: name, Runs: map[string]bool{}, Homes: map[string][]byte{}, + Exports: map[string]*fakeExport{}, Imports: map[string]*fakeImport{}, Usable: model.Size{MemoryMB: 8192, CPUs: 4}} +} + +func (f *FakeNode) Name() string { return f.name } +func (f *FakeNode) Info(ctx context.Context) (node.NodeInfo, error) { + running, _ := f.Running(ctx) + f.Mu.Lock() + defer f.Mu.Unlock() + homes := map[string]int64{} + for k, v := range f.Homes { + homes[strings.TrimPrefix(k, "/d/")] = int64(len(v)) >> 20 + } + return node.NodeInfo{Name: f.name, Usable: f.Usable, Running: running, + Drives: []node.DriveInfo{{Path: "/d", FreeMB: 50_000, HomesMB: homes}}}, nil +} +func (f *FakeNode) Running(context.Context) ([]node.RunningMachine, error) { + f.Mu.Lock() + defer f.Mu.Unlock() + var out []node.RunningMachine + for m := range f.Runs { + out = append(out, node.RunningMachine{Name: m}) + } + return out, nil +} +func (f *FakeNode) IsRunning(machine string) bool { + f.Mu.Lock() + defer f.Mu.Unlock() + return f.Runs[machine] +} +func (f *FakeNode) Start(_ context.Context, spec node.StartSpec) error { + f.Mu.Lock() + defer f.Mu.Unlock() + f.Last, f.Runs[spec.Machine] = spec, true + return nil +} +func (f *FakeNode) Poweroff(_ context.Context, machine string) error { + f.Mu.Lock() + defer f.Mu.Unlock() + delete(f.Runs, machine) + return nil +} +func (f *FakeNode) SetLimits(context.Context, string, model.Size) error { return nil } +func (f *FakeNode) SetQuota(context.Context, string, string, int64) error { return nil } +func (f *FakeNode) DialAgent(_ context.Context, machine, secret string) (node.MsgConn, error) { + if secret != "agent-secret" { + return nil, errors.New("bad agent secret") + } + return newEcho(), nil +} + +func (f *FakeNode) WaitStopped(_ context.Context, machine string) error { + if f.IsRunning(machine) { + return errors.New("still running") + } + return nil +} + +func (f *FakeNode) ExportHome(_ context.Context, drive, machine, parent string) (string, error) { + f.Mu.Lock() + defer f.Mu.Unlock() + data, ok := f.Homes[drive+"/"+machine] + if !ok { + return "", errors.New("no such home") + } + kind := "FULL:" + if parent != "" { + kind = "INCR:" + } + id := util.RandomHex(4) + f.Exports[id] = &fakeExport{data: append([]byte(kind), data...)} + return id, nil +} + +func (f *FakeNode) ReadExport(_ context.Context, id string, max int) ([]byte, bool, error) { + f.Mu.Lock() + defer f.Mu.Unlock() + x := f.Exports[id] + if x == nil { + return nil, false, errors.New("no export") + } + n := min(max, len(x.data)-x.pos) + chunk := x.data[x.pos : x.pos+n] + x.pos += n + return chunk, x.pos == len(x.data), nil +} + +func (f *FakeNode) CloseExport(_ context.Context, id string) error { + f.Mu.Lock() + defer f.Mu.Unlock() + delete(f.Exports, id) + return nil +} + +func (f *FakeNode) ImportHome(_ context.Context, drive, machine string) (string, error) { + f.Mu.Lock() + defer f.Mu.Unlock() + if _, exists := f.Homes[drive+"/"+machine]; exists { + return "", errors.New("home already exists") + } + id := util.RandomHex(4) + f.Imports[id] = &fakeImport{key: drive + "/" + machine} + return id, nil +} + +func (f *FakeNode) WriteImport(_ context.Context, id string, data []byte) error { + f.Mu.Lock() + defer f.Mu.Unlock() + if f.FailWrites { + return errors.New("disk on fire") + } + x := f.Imports[id] + if x == nil { + return errors.New("no import") + } + x.cur = append(x.cur, data...) + return nil +} + +func (f *FakeNode) EndReceive(_ context.Context, id string) error { + f.Mu.Lock() + defer f.Mu.Unlock() + x := f.Imports[id] + if x == nil || len(x.cur) < 5 { + return errors.New("nothing received") + } + if string(x.cur[:5]) == "INCR:" && x.latest == nil { + return errors.New("incremental send without a parent") + } + x.latest, x.cur = x.cur[5:], nil + x.passes++ + return nil +} + +func (f *FakeNode) FinishImport(_ context.Context, id string, quotaMB int64) error { + f.Mu.Lock() + defer f.Mu.Unlock() + x := f.Imports[id] + delete(f.Imports, id) + f.Homes[x.key] = x.latest + f.Passes = x.passes + return nil +} + +func (f *FakeNode) AbortImport(_ context.Context, id string) error { + f.Mu.Lock() + defer f.Mu.Unlock() + delete(f.Imports, id) + return nil +} + +func (f *FakeNode) DeleteHome(_ context.Context, drive, machine string) error { + f.Mu.Lock() + defer f.Mu.Unlock() + delete(f.Homes, drive+"/"+machine) + return nil +} + +type fakeExport struct { + data []byte + pos int +} + +type fakeImport struct { + key string + cur, latest []byte + passes int +} + +type echo struct { + ch chan []byte + closed chan struct{} + once sync.Once +} + +func newEcho() *echo { return &echo{ch: make(chan []byte, 16), closed: make(chan struct{})} } +func (e *echo) Read(ctx context.Context) ([]byte, error) { + select { + case b := <-e.ch: + return b, nil + case <-e.closed: + return nil, errors.New("closed") + case <-ctx.Done(): + return nil, ctx.Err() + } +} +func (e *echo) Write(_ context.Context, b []byte) error { + e.ch <- append([]byte("echo: "), b...) + return nil +} +func (e *echo) Close() error { e.once.Do(func() { close(e.closed) }); return nil } + +func FreePort(t testing.TB) string { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + return l.Addr().String() +} + +func WaitFor(t testing.TB, what string, cond func() bool) { + deadline := time.Now().Add(10 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(20 * time.Millisecond) + } +} + +func HomeData() []byte { return bytes.Repeat([]byte("0123456789abcdef"), 600_000) } diff --git a/slackerbot/internal/units/units.go b/slackerbot/internal/units/units.go new file mode 100644 index 0000000..1a3aff2 --- /dev/null +++ b/slackerbot/internal/units/units.go @@ -0,0 +1,42 @@ +package units + +import ( + "fmt" + "math" + "regexp" + "strconv" + "strings" +) + +var StorageToken = regexp.MustCompile(`^([+-]?)(\d+(?:\.\d+)?)(tib|tb|t|gib|gb|g|mib|mb|m)$`) + +func ParseSizeMB(s string) (int64, error) { + t := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), " ", "")) + p := StorageToken.FindStringSubmatch(t) + if p == nil || p[1] != "" { + return 0, fmt.Errorf("%q isn't a size like 5G or 500M", s) + } + return ToMB(p[2], p[3]), nil +} + +func ToMB(num, unit string) int64 { + n, _ := strconv.ParseFloat(num, 64) + switch unit[0] { + case 't': + n *= 1024 * 1024 + case 'g': + n *= 1024 + } + return int64(math.Round(n)) +} + +func FormatMB(mb int64) string { + trim := func(f float64) string { return strconv.FormatFloat(math.Round(f*10)/10, 'f', -1, 64) } + switch { + case mb >= 1024*1024: + return trim(float64(mb)/(1024*1024)) + " TB" + case mb >= 1024: + return trim(float64(mb)/1024) + " GB" + } + return fmt.Sprintf("%d MB", mb) +} diff --git a/slackerbot/internal/units/units_test.go b/slackerbot/internal/units/units_test.go new file mode 100644 index 0000000..1733bae --- /dev/null +++ b/slackerbot/internal/units/units_test.go @@ -0,0 +1,19 @@ +package units + +import "testing" + +func TestSizes(t *testing.T) { + for in, want := range map[string]int64{"5G": 5120, "500M": 500, "1.5gb": 1536, "1T": 1048576, "20 GB": 20480} { + if got, err := ParseSizeMB(in); err != nil || got != want { + t.Fatalf("ParseSizeMB(%q) = %d, %v; want %d", in, got, err, want) + } + } + if _, err := ParseSizeMB("+5G"); err == nil { + t.Fatal("a signed size isn't a config size") + } + for mb, want := range map[int64]string{512: "512 MB", 5120: "5 GB", 2560: "2.5 GB", 1258291: "1.2 TB"} { + if got := FormatMB(mb); got != want { + t.Fatalf("FormatMB(%d) = %q, want %q", mb, got, want) + } + } +} diff --git a/slackerbot/internal/util/util.go b/slackerbot/internal/util/util.go new file mode 100644 index 0000000..7c47299 --- /dev/null +++ b/slackerbot/internal/util/util.go @@ -0,0 +1,24 @@ +package util + +import ( + "crypto/rand" + "encoding/hex" + "sync" +) + +type KeyedMutex struct{ m sync.Map } + +func (k *KeyedMutex) Lock(key string) (unlock func()) { + v, _ := k.m.LoadOrStore(key, &sync.Mutex{}) + mu := v.(*sync.Mutex) + mu.Lock() + return mu.Unlock +} + +func RandomHex(n int) string { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + panic(err) + } + return hex.EncodeToString(b) +} diff --git a/main.go b/slackerbot/main.go similarity index 54% rename from main.go rename to slackerbot/main.go index fafa0b9..6b501f4 100644 --- a/main.go +++ b/slackerbot/main.go @@ -8,14 +8,20 @@ import ( "os/signal" "path/filepath" "syscall" + + "slackerbot/internal/bot" + "slackerbot/internal/config" + "slackerbot/internal/local" + "slackerbot/internal/manager" + "slackerbot/internal/state" ) func main() { - cfgPath := flag.String("config", "/etc/slackspawn/config.toml", "path to the config file") - mode := flag.String("mode", ModeMaster, "master (runs the Slack bot) or slave (runs containers for a master)") + cfgPath := flag.String("config", "/etc/slackerbot/config.toml", "path to the config file") + mode := flag.String("mode", config.ModeMaster, "master (runs the Slack bot) or slave (runs containers for a master)") flag.Parse() - cfg, err := LoadConfig(*cfgPath, *mode) + cfg, err := config.LoadConfig(*cfgPath, *mode) if err != nil { log.Fatal(err) } @@ -23,23 +29,23 @@ func main() { log.Print("warning: not running as root; starting containers will fail") } - local, err := NewLocal(cfg) + node, err := local.New(cfg) if err != nil { log.Fatal(err) } - log.Printf("node %s: %s total, %s for containers", local.Name(), local.cap.Total, local.cap.Usable) + log.Printf("node %s: %s total, %s for containers", node.Name(), node.Capacity().Total, node.Capacity().Usable) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - if *mode == ModeSlave { - if err := RunSlave(ctx, cfg, local); err != nil { + if *mode == config.ModeSlave { + if err := manager.RunSlave(ctx, cfg, node); err != nil { log.Fatal(err) } return } - store, err := OpenStore(filepath.Join(cfg.StateDir, "state.json")) + store, err := state.Open(filepath.Join(cfg.StateDir, "state.json")) if err != nil { log.Fatal(err) } @@ -48,8 +54,8 @@ func main() { } else if !seeded { log.Print("permissions: using the saved ones; [permissions] in the config only seeds a fresh state") } - mgr := NewManager(cfg, store, local) - bot, err := NewBot(cfg, store, mgr) + mgr := manager.New(cfg, store, node) + b, err := bot.New(cfg, store, mgr) if err != nil { log.Fatal(err) } @@ -62,7 +68,7 @@ func main() { } log.Print("connected to Slack, waiting for messages") - if err := bot.Run(ctx); err != nil && ctx.Err() == nil { + if err := b.Run(ctx); err != nil && ctx.Err() == nil { log.Fatal(err) } } diff --git a/slackspawn b/slackspawn deleted file mode 100755 index e79017e..0000000 Binary files a/slackspawn and /dev/null differ diff --git a/slackspawn_test.go b/slackspawn_test.go deleted file mode 100644 index 4525618..0000000 --- a/slackspawn_test.go +++ /dev/null @@ -1,429 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "net/http" - "net/http/httptest" - "path/filepath" - "strings" - "sync" - "testing" - "time" - - "github.com/coder/websocket" - "github.com/slack-go/slack" -) - -func TestCapacity(t *testing.T) { - c := Capacity{Usable: Size{MemoryMB: 6144, CPUs: 3}} - def := Size{MemoryMB: 2048, CPUs: 1} - if got := c.Slots(def); got != 3 { - t.Fatalf("slots = %d, want 3", got) - } - if !c.Room(Size{4096, 2}, def) { - t.Fatal("third container should fit") - } - if c.Room(Size{6144, 3}, def) { - t.Fatal("fourth container should not fit") - } - if c.Room(Size{2048, 2.5}, def) { - t.Fatal("should be blocked by CPU") - } -} - -func TestConfigExample(t *testing.T) { - t.Setenv("SLACK_APP_TOKEN", "xapp-test") - t.Setenv("SLACK_BOT_TOKEN", "xoxb-test") - t.Setenv("ANTHROPIC_API_KEY", "sk-test") - t.Setenv("CLUSTER_SECRET", "cluster-secret-1234567890") - c, err := LoadConfig("config.example.toml", ModeMaster) - if err != nil { - t.Fatal(err) - } - if c.IdleTimeout != 6*time.Hour || c.Container.MemoryMB != 2048 || c.MaxContainer.CPUs != 4 { - t.Fatalf("unexpected config: %+v", c) - } - if c.Storage.MinMB != 5120 || c.Storage.MaxMB != 10240 || len(c.Drives) != 2 || c.Drives[0].ReserveMB != 20480 { - t.Fatalf("unexpected storage config: %+v %+v", c.Storage, c.Drives) - } - if c.AgentEnv["ANTHROPIC_API_KEY"] != "sk-test" { - t.Fatal("env var not expanded") - } -} - -func TestStore(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - s, err := OpenStore(path) - if err != nil { - t.Fatal(err) - } - _ = s.PutOwner(&Owner{ID: "C1", Kind: OwnerChannel, Machine: "c-c1", Size: Size{1024, 1}}) - _ = s.AddSession(&Session{Channel: "D1", ThreadTS: "1.0", OwnerID: "C1", ACPID: "a", CreatedAt: time.Now().Add(-time.Hour)}) - _ = s.AddSession(&Session{Channel: "D1", ThreadTS: "2.0", OwnerID: "C1", ACPID: "b", CreatedAt: time.Now()}) - - s2, err := OpenStore(path) - if err != nil { - t.Fatal(err) - } - if u := s2.OwnerByMachine("c-c1"); u == nil || u.Size.MemoryMB != 1024 { - t.Fatal("user not saved") - } - list := s2.Sessions("C1", 0) - if len(list) != 2 || list[0].ACPID != "b" { - t.Fatalf("sessions not newest-first: %+v", list) - } - if s2.Session("D1", "1.0").ACPID != "a" { - t.Fatal("session lookup failed") - } -} - -func TestOwnership(t *testing.T) { - dm := incoming{user: "U42", channel: "D9", dm: true} - if id, kind := dm.owner(); id != "U42" || kind != OwnerUser { - t.Fatalf("DM should be owned by the user, got %s %s", id, kind) - } - ch := incoming{user: "U42", channel: "C7"} - if id, kind := ch.owner(); id != "C7" || kind != OwnerChannel { - t.Fatalf("channel message should be owned by the channel, got %s %s", id, kind) - } - other := incoming{user: "U99", channel: "C7"} - a, ka := ch.owner() - b, kb := other.owner() - if MachineName(a, ka) != MachineName(b, kb) || MachineName(a, ka) != "c-c7" { - t.Fatal("two people in one channel should share a container") - } - if MachineName("U42", OwnerUser) != "u-u42" || !ours("u-u42") || !ours("c-c7") || ours("web") { - t.Fatal("machine naming is off") - } -} - -func TestParseScale(t *testing.T) { - cur := Size{MemoryMB: 2048, CPUs: 1} - cases := []struct { - in string - want Size - }{ - {"", Size{4096, 2}}, - {"up", Size{4096, 2}}, - {"down", Size{1024, 0.5}}, - {"8G 4cpu", Size{8192, 4}}, - {"8gb", Size{8192, 1}}, - {"+2G", Size{4096, 1}}, - {"-1cpu", Size{2048, 0}}, - {"512M +0.5cpu", Size{512, 1.5}}, - {"1.5G 3 cores", Size{1536, 3}}, - {"+2 GB", Size{4096, 1}}, - {"+1cpus -512mb", Size{1536, 2}}, - } - for _, c := range cases { - got, err := parseScale(c.in, cur) - if err != nil || got != c.want { - t.Fatalf("scale %q = %v, %v; want %v", c.in, got, err, c.want) - } - } - if _, err := parseScale("the database to 3 replicas", cur); !errors.Is(err, errNotCommand) { - t.Fatal("a normal sentence should go to the agent") - } - if _, err := parseScale("4G lots", cur); err == nil || errors.Is(err, errNotCommand) { - t.Fatal("a bad second word should be a usage error") - } - if _, err := parseScale("4G 8G", cur); err == nil { - t.Fatal("memory twice should fail") - } -} - -func TestParseKeep(t *testing.T) { - for in, want := range map[string]time.Duration{ - "24h": 24 * time.Hour, "3d": 72 * time.Hour, "1d12h": 36 * time.Hour, "2w": 336 * time.Hour, - "5 hours": 5 * time.Hour, "2 days": 48 * time.Hour, "1d 6h": 30 * time.Hour, "90 mins": 90 * time.Minute, - } { - k, err := parseKeep(in) - if err != nil || k.limit != want { - t.Fatalf("keep %q = %+v, %v; want %s", in, k, err, want) - } - } - for _, in := range []string{"indefinite", "Indefinitely", "forever"} { - if k, _ := parseKeep(in); !k.indefinite { - t.Fatalf("keep %q should be indefinite", in) - } - } - if k, _ := parseKeep("default"); !k.reset { - t.Fatal("keep default should reset") - } - if k, _ := parseKeep(""); !k.show { - t.Fatal("bare keep should show") - } - if _, err := parseKeep("the tests passing"); !errors.Is(err, errNotCommand) { - t.Fatal("a normal sentence should go to the agent") - } -} - -func TestHuman(t *testing.T) { - for d, want := range map[time.Duration]string{ - 6 * time.Hour: "6h", 36 * time.Hour: "1d12h", 90 * time.Minute: "1h30m", - 45 * time.Minute: "45m", 168 * time.Hour: "7d", 0: "0m", - } { - if got := human(d); got != want { - t.Fatalf("human(%s) = %q, want %q", d, got, want) - } - } - if got := (Size{8192, 4}).String(); got != "8 GB RAM, 4 CPUs" { - t.Fatal(got) - } - if got := (Size{512, 0.25}).String(); got != "512 MB RAM, 0.25 CPUs" { - t.Fatal(got) - } -} - -func TestSizes(t *testing.T) { - for in, want := range map[string]int64{"5G": 5120, "500M": 500, "1.5gb": 1536, "1T": 1048576, "20 GB": 20480} { - if got, err := parseSizeMB(in); err != nil || got != want { - t.Fatalf("parseSizeMB(%q) = %d, %v; want %d", in, got, err, want) - } - } - if _, err := parseSizeMB("+5G"); err == nil { - t.Fatal("a signed size isn't a config size") - } - for mb, want := range map[int64]string{512: "512 MB", 5120: "5 GB", 2560: "2.5 GB", 1258291: "1.2 TB"} { - if got := formatMB(mb); got != want { - t.Fatalf("formatMB(%d) = %q, want %q", mb, got, want) - } - } -} - -func TestParseStorage(t *testing.T) { - cases := []struct { - in string - min, max int64 - }{ - {"20G", 5120, 20480}, - {"+5G", 5120, 15360}, - {"-2G", 5120, 8192}, - {"max 30G", 5120, 30720}, - {"min 8G", 8192, 10240}, - {"min +1G max 30 GB", 6144, 30720}, - {"cap 1T", 5120, 1048576}, - } - for _, c := range cases { - min, max, err := parseStorage(c.in, 5120, 10240) - if err != nil || min != c.min || max != c.max { - t.Fatalf("storage %q = %d/%d, %v; want %d/%d", c.in, min, max, err, c.min, c.max) - } - } - if _, _, err := parseStorage("is full, clean up the logs", 5120, 10240); !errors.Is(err, errNotCommand) { - t.Fatal("a normal sentence should go to the agent") - } - for _, in := range []string{"min", "20G 30G", "min 5G min 6G", "20G lots"} { - if _, _, err := parseStorage(in, 5120, 10240); err == nil || errors.Is(err, errNotCommand) { - t.Fatalf("%q should be a usage error, got %v", in, err) - } - } -} - -func TestDriveMath(t *testing.T) { - d := DriveReport{Path: "/a", UsableMB: 100_000, CommittedMB: 90_000, BookedMB: 150_000} - if d.HeadroomMB() != 10_000 || !d.Overbooked() { - t.Fatalf("headroom %d overbooked %v", d.HeadroomMB(), d.Overbooked()) - } - if committed(5120, 8000) != 8000 || committed(5120, 100) != 5120 { - t.Fatal("a home holds its guarantee or its real use, whichever is bigger") - } - drives := []DriveReport{d, {Path: "/b", UsableMB: 50_000, CommittedMB: 20_000}} - if got, _ := pickDrive(drives, 5120); got != "/b" { - t.Fatalf("should pick the drive with the most room, got %s", got) - } - if _, err := pickDrive(drives, 40_000); !errors.Is(err, ErrStorageFull) { - t.Fatal("a guarantee that fits nowhere should be rejected") - } -} - -func TestParseQgroupShow(t *testing.T) { - out := "Qgroupid Referenced Exclusive Max referenced Max exclusive Path\n" + - "-------- ---------- --------- -------------- ------------- ----\n" + - "0/257 3221225472 3221225472 10737418240 none c-c07abc\n" - if mb, err := parseQgroupShow(out); err != nil || mb != 3072 { - t.Fatalf("got %d, %v", mb, err) - } -} - -func TestTitle(t *testing.T) { - if got := title(" fix the build\nplease"); got != "fix the build" { - t.Fatal(got) - } - if got := title(strings.Repeat("é", 80)); !strings.HasSuffix(got, "…") || len([]rune(got)) != 61 { - t.Fatal(got) - } -} - -func fakeGoose(t *testing.T) *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/acp" || r.Header.Get("X-Secret-Key") != "s3cret" { - http.Error(w, "nope", http.StatusUnauthorized) - return - } - c, err := websocket.Accept(w, r, nil) - if err != nil { - return - } - ctx := context.Background() - send := func(v any) { b, _ := json.Marshal(v); _ = c.Write(ctx, websocket.MessageText, b) } - var promptID json.RawMessage - - for { - _, data, err := c.Read(ctx) - if err != nil { - return - } - var m rpcMsg - _ = json.Unmarshal(data, &m) - switch m.Method { - case "initialize": - send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": map[string]any{ - "protocolVersion": 1, "agentCapabilities": map[string]any{"loadSession": true}}}) - case "session/new": - send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": map[string]any{"sessionId": "sess-1"}}) - case "session/load": - send(map[string]any{"jsonrpc": "2.0", "method": "session/update", "params": map[string]any{ - "sessionId": "sess-old", "update": map[string]any{"sessionUpdate": "agent_message_chunk", - "content": map[string]any{"type": "text", "text": "old history"}}}}) - send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": nil}) - case "session/prompt": - promptID = m.ID - upd := func(u map[string]any) { - send(map[string]any{"jsonrpc": "2.0", "method": "session/update", - "params": map[string]any{"sessionId": "sess-1", "update": u}}) - } - upd(map[string]any{"sessionUpdate": "agent_message_chunk", "content": map[string]any{"type": "text", "text": "Hello "}}) - upd(map[string]any{"sessionUpdate": "tool_call", "toolCallId": "t1", "title": "ls /root", "status": "pending"}) - send(map[string]any{"jsonrpc": "2.0", "id": "perm-1", "method": "session/request_permission", - "params": map[string]any{"sessionId": "sess-1", "options": []map[string]any{ - {"optionId": "no", "name": "Reject", "kind": "reject_once"}, - {"optionId": "yes", "name": "Allow", "kind": "allow_once"}}}}) - case "": - var res struct { - Outcome struct{ Outcome, OptionID string } `json:"outcome"` - } - _ = json.Unmarshal(m.Result, &res) - text := "world" - if res.Outcome.OptionID != "yes" { - text = "PERMISSION WRONG" - } - send(map[string]any{"jsonrpc": "2.0", "method": "session/update", "params": map[string]any{ - "sessionId": "sess-1", "update": map[string]any{"sessionUpdate": "agent_message_chunk", - "content": map[string]any{"type": "text", "text": text}}}}) - send(map[string]any{"jsonrpc": "2.0", "id": promptID, "result": map[string]any{"stopReason": "end_turn"}}) - } - } - })) -} - -func TestACP(t *testing.T) { - srv := fakeGoose(t) - defer srv.Close() - addr := strings.TrimPrefix(srv.URL, "http://") - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - if _, err := DialACP(ctx, addr, "wrong"); err == nil { - t.Fatal("wrong secret should fail") - } - a, err := DialACP(ctx, addr, "s3cret") - if err != nil { - t.Fatal(err) - } - defer a.Close() - if !a.CanLoad { - t.Fatal("loadSession capability not read") - } - id, err := a.NewSession(ctx, "/root") - if err != nil || id != "sess-1" { - t.Fatalf("new session: %q %v", id, err) - } - if err := a.LoadSession(ctx, "sess-old", "/root"); err != nil { - t.Fatal(err) - } - - var mu sync.Mutex - var text strings.Builder - var tools []string - stop, err := a.Prompt(ctx, id, "hi", func(u Update) { - mu.Lock() - defer mu.Unlock() - text.WriteString(u.Text) - if u.Kind == "tool_call" { - tools = append(tools, u.Title) - } - }) - if err != nil || stop != "end_turn" { - t.Fatalf("prompt: %q %v", stop, err) - } - if text.String() != "Hello world" || len(tools) != 1 || tools[0] != "ls /root" { - t.Fatalf("got text %q tools %v", text.String(), tools) - } -} - -type fakeSlack struct { - mu sync.Mutex - msgs map[string]string - order []string - n int -} - -func (f *fakeSlack) handler(w http.ResponseWriter, r *http.Request) { - _ = r.ParseForm() - f.mu.Lock() - defer f.mu.Unlock() - ts := r.Form.Get("ts") - if strings.HasSuffix(r.URL.Path, "chat.postMessage") { - f.n++ - ts = "ts" + string(rune('0'+f.n)) - f.order = append(f.order, ts) - } - f.msgs[ts] = r.Form.Get("text") - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "D1", "ts": ts, "text": f.msgs[ts]}) -} - -func TestStreamerSplitsLongReplies(t *testing.T) { - f := &fakeSlack{msgs: map[string]string{"ph": ""}, order: []string{"ph"}} - srv := httptest.NewServer(http.HandlerFunc(f.handler)) - defer srv.Close() - api := slack.New("xoxb-test", slack.OptionAPIURL(srv.URL+"/")) - - st := NewStreamer(api, "D1", "root", "ph") - line := strings.Repeat("x", 99) + "\n" - for i := 0; i < 80; i++ { - st.Update(Update{Kind: "agent_message_chunk", Text: line}) - } - st.Finish("end_turn", nil) - - f.mu.Lock() - defer f.mu.Unlock() - if len(f.order) != 3 { - t.Fatalf("want 3 messages, got %d", len(f.order)) - } - var all strings.Builder - for _, ts := range f.order { - if len(f.msgs[ts]) > slackMsgLimit { - t.Fatalf("message %s too long: %d", ts, len(f.msgs[ts])) - } - all.WriteString(f.msgs[ts]) - } - if all.Len() != 8000 { - t.Fatalf("text lost or duplicated: %d chars", all.Len()) - } -} - -func TestMoveTarget(t *testing.T) { - b := &Bot{} - for in, want := range map[string]string{"": "", "auto": "", "Slave-1": "slave-1", "to node-b": "node-b"} { - if got, ok := b.moveTarget(in); !ok || got != want { - t.Fatalf("move %q = %q %v, want %q", in, got, ok, want) - } - } - if _, ok := b.moveTarget("the files into the build folder"); ok { - t.Fatal("a normal sentence should go to the agent") - } -}