clean up of the codebase

This commit is contained in:
ari committed 2026-11-01 13:01:54 -05:00
1 parent 7822529a38
commit 0ac61b4ece
52 files changed
+2545 -2274

No files matched your search

+292
View File
@@ -0,0 +1,292 @@
package acp
import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
"sync/atomic"
"slackerbot/internal/node"
)
type Update struct {
Kind string
Text string
Title string
Status string
}
type rpcError struct {
Code int `json:"code"`
Message string `json:"message"`
}
func (e *rpcError) Error() string { return fmt.Sprintf("agent error %d: %s", e.Code, e.Message) }
type rpcMsg struct {
JSONRPC string `json:"jsonrpc"`
ID json.RawMessage `json:"id,omitempty"`
Method string `json:"method,omitempty"`
Params json.RawMessage `json:"params,omitempty"`
Result json.RawMessage `json:"result,omitempty"`
Error *rpcError `json:"error,omitempty"`
}
type ACP struct {
conn node.MsgConn
nextID atomic.Int64
writeMu sync.Mutex
mu sync.Mutex
pending map[int64]chan rpcMsg
handlers map[string]func(Update)
loaded map[string]bool
CanLoad bool
done chan struct{}
err error
}
func DialACP(ctx context.Context, addr, secret string) (*ACP, error) {
conn, err := node.DialAgentWS(ctx, addr, secret)
if err != nil {
return nil, err
}
return New(ctx, conn)
}
func New(ctx context.Context, conn node.MsgConn) (*ACP, error) {
a := &ACP{
conn: conn,
pending: map[int64]chan rpcMsg{},
handlers: map[string]func(Update){},
loaded: map[string]bool{},
done: make(chan struct{}),
}
go a.readLoop()
var init struct {
AgentCapabilities struct {
LoadSession bool `json:"loadSession"`
} `json:"agentCapabilities"`
}
err := a.call(ctx, "initialize", map[string]any{
"protocolVersion": 1,
"clientCapabilities": map[string]any{
"fs": map[string]bool{"readTextFile": false, "writeTextFile": false},
"terminal": false,
},
}, &init)
if err != nil {
a.Close()
return nil, err
}
a.CanLoad = init.AgentCapabilities.LoadSession
return a, nil
}
func (a *ACP) Done() <-chan struct{} { return a.done }
func (a *ACP) Close() { a.conn.Close() }
func (a *ACP) NewSession(ctx context.Context, cwd string) (string, error) {
var res struct {
SessionID string `json:"sessionId"`
}
if err := a.call(ctx, "session/new", map[string]any{"cwd": cwd, "mcpServers": []any{}}, &res); err != nil {
return "", err
}
a.mu.Lock()
a.loaded[res.SessionID] = true
a.mu.Unlock()
return res.SessionID, nil
}
// The agent replays the session's history as updates; Slack already shows it, so they're dropped.
func (a *ACP) LoadSession(ctx context.Context, id, cwd string) error {
a.mu.Lock()
open := a.loaded[id]
a.mu.Unlock()
if open {
return nil
}
if !a.CanLoad {
return errors.New("this agent can't reopen old sessions")
}
if err := a.call(ctx, "session/load", map[string]any{"sessionId": id, "cwd": cwd, "mcpServers": []any{}}, nil); err != nil {
return err
}
a.mu.Lock()
a.loaded[id] = true
a.mu.Unlock()
return nil
}
func (a *ACP) Prompt(ctx context.Context, id, text string, onUpdate func(Update)) (stopReason string, err error) {
a.mu.Lock()
a.handlers[id] = onUpdate
a.mu.Unlock()
defer func() {
a.mu.Lock()
delete(a.handlers, id)
a.mu.Unlock()
}()
var res struct {
StopReason string `json:"stopReason"`
}
err = a.call(ctx, "session/prompt", map[string]any{
"sessionId": id,
"prompt": []map[string]string{{"type": "text", "text": text}},
}, &res)
if ctx.Err() != nil {
a.notify("session/cancel", map[string]string{"sessionId": id})
}
return res.StopReason, err
}
func (a *ACP) call(ctx context.Context, method string, params, result any) error {
id := a.nextID.Add(1)
ch := make(chan rpcMsg, 1)
a.mu.Lock()
a.pending[id] = ch
a.mu.Unlock()
defer func() {
a.mu.Lock()
delete(a.pending, id)
a.mu.Unlock()
}()
p, _ := json.Marshal(params)
idRaw, _ := json.Marshal(id)
if err := a.send(ctx, rpcMsg{JSONRPC: "2.0", ID: idRaw, Method: method, Params: p}); err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case <-a.done:
return fmt.Errorf("connection to goose closed: %w", a.err)
case msg := <-ch:
if msg.Error != nil {
return msg.Error
}
if result != nil && len(msg.Result) > 0 && string(msg.Result) != "null" {
return json.Unmarshal(msg.Result, result)
}
return nil
}
}
func (a *ACP) notify(method string, params any) {
p, _ := json.Marshal(params)
_ = a.send(context.Background(), rpcMsg{JSONRPC: "2.0", Method: method, Params: p})
}
func (a *ACP) send(ctx context.Context, msg rpcMsg) error {
b, err := json.Marshal(msg)
if err != nil {
return err
}
a.writeMu.Lock()
defer a.writeMu.Unlock()
return a.conn.Write(ctx, b)
}
func (a *ACP) readLoop() {
defer close(a.done)
for {
data, err := a.conn.Read(context.Background())
if err != nil {
a.err = err
return
}
var msg rpcMsg
if json.Unmarshal(data, &msg) != nil {
continue
}
switch {
case msg.Method != "" && len(msg.ID) > 0:
go a.handleRequest(msg)
case msg.Method == "session/update":
a.handleUpdate(msg.Params)
case msg.Method == "" && len(msg.ID) > 0:
var id int64
if json.Unmarshal(msg.ID, &id) == nil {
a.mu.Lock()
ch := a.pending[id]
a.mu.Unlock()
if ch != nil {
ch <- msg
}
}
}
}
}
func (a *ACP) handleUpdate(params json.RawMessage) {
var p struct {
SessionID string `json:"sessionId"`
Update struct {
Kind string `json:"sessionUpdate"`
Content json.RawMessage `json:"content"`
Title string `json:"title"`
Status string `json:"status"`
} `json:"update"`
}
if json.Unmarshal(params, &p) != nil {
return
}
u := Update{Kind: p.Update.Kind, Title: p.Update.Title, Status: p.Update.Status}
// Message chunks carry one content block; tool updates carry a list.
var block struct {
Type string `json:"type"`
Text string `json:"text"`
}
if json.Unmarshal(p.Update.Content, &block) == nil && block.Type == "text" {
u.Text = block.Text
}
a.mu.Lock()
h := a.handlers[p.SessionID]
a.mu.Unlock()
if h != nil {
h(u)
}
}
// Tool permission requests are auto-approved: the container is the sandbox.
func (a *ACP) handleRequest(msg rpcMsg) {
reply := rpcMsg{JSONRPC: "2.0", ID: msg.ID}
switch msg.Method {
case "session/request_permission":
var p struct {
Options []struct {
OptionID string `json:"optionId"`
Kind string `json:"kind"`
} `json:"options"`
}
_ = json.Unmarshal(msg.Params, &p)
choice := ""
for _, want := range []string{"allow_always", "allow_once"} {
for _, o := range p.Options {
if o.Kind == want && choice == "" {
choice = o.OptionID
}
}
}
var outcome any = map[string]string{"outcome": "cancelled"}
if choice != "" {
outcome = map[string]string{"outcome": "selected", "optionId": choice}
}
reply.Result, _ = json.Marshal(map[string]any{"outcome": outcome})
default:
reply.Error = &rpcError{Code: -32601, Message: "method not supported by this client: " + msg.Method}
}
_ = a.send(context.Background(), reply)
}
+121
View File
@@ -0,0 +1,121 @@
package acp
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/coder/websocket"
)
func fakeGoose(t *testing.T) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/acp" || r.Header.Get("X-Secret-Key") != "s3cret" {
http.Error(w, "nope", http.StatusUnauthorized)
return
}
c, err := websocket.Accept(w, r, nil)
if err != nil {
return
}
ctx := context.Background()
send := func(v any) { b, _ := json.Marshal(v); _ = c.Write(ctx, websocket.MessageText, b) }
var promptID json.RawMessage
for {
_, data, err := c.Read(ctx)
if err != nil {
return
}
var m rpcMsg
_ = json.Unmarshal(data, &m)
switch m.Method {
case "initialize":
send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": map[string]any{
"protocolVersion": 1, "agentCapabilities": map[string]any{"loadSession": true}}})
case "session/new":
send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": map[string]any{"sessionId": "sess-1"}})
case "session/load":
send(map[string]any{"jsonrpc": "2.0", "method": "session/update", "params": map[string]any{
"sessionId": "sess-old", "update": map[string]any{"sessionUpdate": "agent_message_chunk",
"content": map[string]any{"type": "text", "text": "old history"}}}})
send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": nil})
case "session/prompt":
promptID = m.ID
upd := func(u map[string]any) {
send(map[string]any{"jsonrpc": "2.0", "method": "session/update",
"params": map[string]any{"sessionId": "sess-1", "update": u}})
}
upd(map[string]any{"sessionUpdate": "agent_message_chunk", "content": map[string]any{"type": "text", "text": "Hello "}})
upd(map[string]any{"sessionUpdate": "tool_call", "toolCallId": "t1", "title": "ls /root", "status": "pending"})
send(map[string]any{"jsonrpc": "2.0", "id": "perm-1", "method": "session/request_permission",
"params": map[string]any{"sessionId": "sess-1", "options": []map[string]any{
{"optionId": "no", "name": "Reject", "kind": "reject_once"},
{"optionId": "yes", "name": "Allow", "kind": "allow_once"}}}})
case "":
var res struct {
Outcome struct{ Outcome, OptionID string } `json:"outcome"`
}
_ = json.Unmarshal(m.Result, &res)
text := "world"
if res.Outcome.OptionID != "yes" {
text = "PERMISSION WRONG"
}
send(map[string]any{"jsonrpc": "2.0", "method": "session/update", "params": map[string]any{
"sessionId": "sess-1", "update": map[string]any{"sessionUpdate": "agent_message_chunk",
"content": map[string]any{"type": "text", "text": text}}}})
send(map[string]any{"jsonrpc": "2.0", "id": promptID, "result": map[string]any{"stopReason": "end_turn"}})
}
}
}))
}
func TestACP(t *testing.T) {
srv := fakeGoose(t)
defer srv.Close()
addr := strings.TrimPrefix(srv.URL, "http://")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if _, err := DialACP(ctx, addr, "wrong"); err == nil {
t.Fatal("wrong secret should fail")
}
a, err := DialACP(ctx, addr, "s3cret")
if err != nil {
t.Fatal(err)
}
defer a.Close()
if !a.CanLoad {
t.Fatal("loadSession capability not read")
}
id, err := a.NewSession(ctx, "/root")
if err != nil || id != "sess-1" {
t.Fatalf("new session: %q %v", id, err)
}
if err := a.LoadSession(ctx, "sess-old", "/root"); err != nil {
t.Fatal(err)
}
var mu sync.Mutex
var text strings.Builder
var tools []string
stop, err := a.Prompt(ctx, id, "hi", func(u Update) {
mu.Lock()
defer mu.Unlock()
text.WriteString(u.Text)
if u.Kind == "tool_call" {
tools = append(tools, u.Title)
}
})
if err != nil || stop != "end_turn" {
t.Fatalf("prompt: %q %v", stop, err)
}
if text.String() != "Hello world" || len(tools) != 1 || tools[0] != "ls /root" {
t.Fatalf("got text %q tools %v", text.String(), tools)
}
}
+136
View File
@@ -0,0 +1,136 @@
package balance
import (
"errors"
"math"
"sort"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/storage"
)
var ErrMaxedOut = errors.New("no node has room for this container")
type NodeView struct {
Name string
Usable model.Size
Used model.Size
Drives []storage.DriveReport
Running int
}
func (v NodeView) Storage() (usableMB, committedMB, bookedMB int64) {
for _, d := range v.Drives {
usableMB += d.UsableMB
committedMB += d.CommittedMB
bookedMB += d.BookedMB
}
return
}
func ratio(used, total float64) float64 {
if total <= 0 {
return math.Inf(1)
}
return used / total
}
func (v NodeView) Homes() int {
n := 0
for _, d := range v.Drives {
n += d.Homes
}
return n
}
func (v NodeView) Load() (cpu, ram, storage float64) {
us, cm, _ := v.Storage()
return ratio(v.Used.CPUs, v.Usable.CPUs),
ratio(float64(v.Used.MemoryMB), float64(v.Usable.MemoryMB)),
ratio(float64(cm), float64(us))
}
// The fullest of CPU, RAM and storage after adding the container. Placing
// where this stays lowest keeps all three level across nodes.
func (v NodeView) loadAfter(want model.Size, storageMB int64) float64 {
us, cm, _ := v.Storage()
return max(
ratio(v.Used.CPUs+want.CPUs, v.Usable.CPUs),
ratio(float64(v.Used.MemoryMB+want.MemoryMB), float64(v.Usable.MemoryMB)),
ratio(float64(cm+storageMB), float64(us)),
)
}
func ChoosePlacement(views []NodeView, want model.Size, storageMB int64) (node, drive string, err error) {
sorted := append([]NodeView(nil), views...)
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
best := math.Inf(1)
roomForCompute := false
for _, v := range sorted {
if !v.Used.Add(want).Fits(v.Usable) {
continue
}
roomForCompute = true
d, err := storage.PickDrive(v.Drives, storageMB)
if err != nil {
continue
}
if score := v.loadAfter(want, storageMB); score < best {
best, node, drive = score, v.Name, d
}
}
switch {
case node != "":
return node, drive, nil
case roomForCompute:
return "", "", storage.ErrStorageFull
}
return "", "", ErrMaxedOut
}
func BuildView(info node.NodeInfo, owners []model.Owner, defaultSize model.Size) NodeView {
v := NodeView{Name: info.Name, Usable: info.Usable}
byMachine := map[string]*model.Owner{}
for i := range owners {
if owners[i].Node == info.Name {
byMachine[owners[i].Machine] = &owners[i]
}
}
v.Running = len(info.Running)
for _, r := range info.Running {
if u := byMachine[r.Name]; u != nil {
v.Used = v.Used.Add(u.Size)
} else {
v.Used = v.Used.Add(defaultSize)
}
}
for _, d := range info.Drives {
r := storage.DriveReport{Path: d.Path}
var homesUsed int64
for machine, used := range d.HomesMB {
homesUsed += used
if u := byMachine[machine]; u != nil && u.Drive == d.Path {
continue
}
r.CommittedMB += used
r.BookedMB += used
r.Homes++
}
for _, u := range byMachine {
if u.Drive != d.Path {
continue
}
used := d.HomesMB[u.Machine]
r.CommittedMB += storage.Committed(u.StorageMinMB, used)
r.BookedMB += u.StorageMaxMB
r.Homes++
}
r.UsableMB = d.FreeMB + homesUsed - d.ReserveMB
v.Drives = append(v.Drives, r)
}
return v
}
@@ -0,0 +1,90 @@
package balance
import (
"errors"
"testing"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/storage"
)
func TestPlacementBalancesAllThree(t *testing.T) {
drive := func(usable, committed int64) []storage.DriveReport {
return []storage.DriveReport{{Path: "/d", UsableMB: usable, CommittedMB: committed}}
}
want := model.Size{MemoryMB: 2048, CPUs: 1}
views := []NodeView{
{Name: "cpu-busy", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 2048, CPUs: 6}, Drives: drive(100_000, 10_000)},
{Name: "ram-busy", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 12288, CPUs: 1}, Drives: drive(100_000, 10_000)},
{Name: "disk-busy", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 2048, CPUs: 1}, Drives: drive(100_000, 80_000)},
{Name: "calm", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 4096, CPUs: 2}, Drives: drive(100_000, 20_000)},
}
node, d, err := ChoosePlacement(views, want, 5120)
if err != nil || node != "calm" || d != "/d" {
t.Fatalf("got %s %s %v, want calm", node, d, err)
}
}
func TestPlacementSpreadsAcrossEqualNodes(t *testing.T) {
views := []NodeView{
{Name: "a", Usable: model.Size{MemoryMB: 8192, CPUs: 4}, Drives: []storage.DriveReport{{Path: "/a", UsableMB: 50_000}}},
{Name: "b", Usable: model.Size{MemoryMB: 8192, CPUs: 4}, Drives: []storage.DriveReport{{Path: "/b", UsableMB: 50_000}}},
}
want := model.Size{MemoryMB: 2048, CPUs: 1}
counts := map[string]int{}
for i := 0; i < 4; i++ {
node, _, err := ChoosePlacement(views, want, 5120)
if err != nil {
t.Fatal(err)
}
counts[node]++
for j := range views {
if views[j].Name == node {
views[j].Used = views[j].Used.Add(want)
views[j].Drives[0].CommittedMB += 5120
}
}
}
if counts["a"] != 2 || counts["b"] != 2 {
t.Fatalf("uneven spread: %v", counts)
}
}
func TestPlacementWhenFull(t *testing.T) {
full := []NodeView{{Name: "a", Usable: model.Size{MemoryMB: 4096, CPUs: 2}, Used: model.Size{MemoryMB: 4096, CPUs: 2}, Drives: []storage.DriveReport{{Path: "/a", UsableMB: 50_000}}}}
if _, _, err := ChoosePlacement(full, model.Size{MemoryMB: 1024, CPUs: 1}, 1024); !errors.Is(err, ErrMaxedOut) {
t.Fatalf("want maxed out, got %v", err)
}
noDisk := []NodeView{{Name: "a", Usable: model.Size{MemoryMB: 4096, CPUs: 2}, Drives: []storage.DriveReport{{Path: "/a", UsableMB: 1000}}}}
if _, _, err := ChoosePlacement(noDisk, model.Size{MemoryMB: 1024, CPUs: 1}, 5120); !errors.Is(err, storage.ErrStorageFull) {
t.Fatalf("want storage full, got %v", err)
}
}
func TestBuildView(t *testing.T) {
info := node.NodeInfo{
Name: "n1",
Usable: model.Size{MemoryMB: 16384, CPUs: 8},
Running: []node.RunningMachine{{Name: "c-a"}, {Name: "u-stranger"}},
Drives: []node.DriveInfo{{Path: "/d", FreeMB: 90_000, ReserveMB: 10_000,
HomesMB: map[string]int64{"c-a": 8000, "c-b": 100, "u-stranger": 3000}}},
}
owners := []model.Owner{
{ID: "A", Machine: "c-a", Node: "n1", Drive: "/d", Size: model.Size{MemoryMB: 4096, CPUs: 2}, StorageMinMB: 5120, StorageMaxMB: 10240},
{ID: "B", Machine: "c-b", Node: "n1", Drive: "/d", Size: model.Size{MemoryMB: 2048, CPUs: 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
{ID: "C", Machine: "c-c", Node: "n1", Drive: "/d", Size: model.Size{MemoryMB: 2048, CPUs: 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
{ID: "D", Machine: "c-d", Node: "other", Drive: "/d", Size: model.Size{MemoryMB: 2048, CPUs: 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
}
v := BuildView(info, owners, model.Size{MemoryMB: 1024, CPUs: 1})
if v.Used != (model.Size{MemoryMB: 4096 + 1024, CPUs: 3}) {
t.Fatalf("used = %v", v.Used)
}
d := v.Drives[0]
if d.UsableMB != 90_000+11_100-10_000 {
t.Fatalf("usable = %d", d.UsableMB)
}
if d.CommittedMB != 8000+5120+5120+3000 || d.BookedMB != 3*10240+3000 || d.Homes != 4 {
t.Fatalf("committed %d booked %d homes %d", d.CommittedMB, d.BookedMB, d.Homes)
}
}
File diff suppressed because it is too large. Load diff
+249
View File
@@ -0,0 +1,249 @@
package bot
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"github.com/slack-go/slack"
"slackerbot/internal/acp"
"slackerbot/internal/manager"
"slackerbot/internal/model"
"slackerbot/internal/state"
"slackerbot/internal/testutil"
"slackerbot/internal/testutil/clustertest"
)
func TestOwnership(t *testing.T) {
dm := incoming{user: "U42", channel: "D9", dm: true}
if id, kind := dm.owner(); id != "U42" || kind != model.OwnerUser {
t.Fatalf("DM should be owned by the user, got %s %s", id, kind)
}
ch := incoming{user: "U42", channel: "C7"}
if id, kind := ch.owner(); id != "C7" || kind != model.OwnerChannel {
t.Fatalf("channel message should be owned by the channel, got %s %s", id, kind)
}
other := incoming{user: "U99", channel: "C7"}
a, ka := ch.owner()
b, kb := other.owner()
if model.MachineName(a, ka) != model.MachineName(b, kb) || model.MachineName(a, ka) != "c-c7" {
t.Fatal("two people in one channel should share a container")
}
if model.MachineName("U42", model.OwnerUser) != "u-u42" || !model.IsOurs("u-u42") || !model.IsOurs("c-c7") || model.IsOurs("web") {
t.Fatal("machine naming is off")
}
}
func TestTitle(t *testing.T) {
if got := title(" fix the build\nplease"); got != "fix the build" {
t.Fatal(got)
}
if got := title(strings.Repeat("é", 80)); !strings.HasSuffix(got, "…") || len([]rune(got)) != 61 {
t.Fatal(got)
}
}
type fakeSlack struct {
mu sync.Mutex
msgs map[string]string
order []string
n int
}
func (f *fakeSlack) handler(w http.ResponseWriter, r *http.Request) {
_ = r.ParseForm()
f.mu.Lock()
defer f.mu.Unlock()
ts := r.Form.Get("ts")
if strings.HasSuffix(r.URL.Path, "chat.postMessage") {
f.n++
ts = "ts" + string(rune('0'+f.n))
f.order = append(f.order, ts)
}
f.msgs[ts] = r.Form.Get("text")
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "D1", "ts": ts, "text": f.msgs[ts]})
}
func TestStreamerSplitsLongReplies(t *testing.T) {
f := &fakeSlack{msgs: map[string]string{"ph": ""}, order: []string{"ph"}}
srv := httptest.NewServer(http.HandlerFunc(f.handler))
defer srv.Close()
api := slack.New("xoxb-test", slack.OptionAPIURL(srv.URL+"/"))
st := NewStreamer(api, "D1", "root", "ph")
line := strings.Repeat("x", 99) + "\n"
for i := 0; i < 80; i++ {
st.Update(acp.Update{Kind: "agent_message_chunk", Text: line})
}
st.Finish("end_turn", nil)
f.mu.Lock()
defer f.mu.Unlock()
if len(f.order) != 3 {
t.Fatalf("want 3 messages, got %d", len(f.order))
}
var all strings.Builder
for _, ts := range f.order {
if len(f.msgs[ts]) > slackMsgLimit {
t.Fatalf("message %s too long: %d", ts, len(f.msgs[ts]))
}
all.WriteString(f.msgs[ts])
}
if all.Len() != 8000 {
t.Fatalf("text lost or duplicated: %d chars", all.Len())
}
}
func TestMoveTarget(t *testing.T) {
b := &Bot{}
for in, want := range map[string]string{"": "", "auto": "", "Slave-1": "slave-1", "to node-b": "node-b"} {
if got, ok := b.moveTarget(in); !ok || got != want {
t.Fatalf("move %q = %q %v, want %q", in, got, ok, want)
}
}
if _, ok := b.moveTarget("the files into the build folder"); ok {
t.Fatal("a normal sentence should go to the agent")
}
}
type slackLog struct {
mu sync.Mutex
n int
text map[string]string
under map[string]string
order []string
}
func (s *slackLog) ServeHTTP(w http.ResponseWriter, r *http.Request) {
_ = r.ParseForm()
s.mu.Lock()
defer s.mu.Unlock()
ts := r.Form.Get("ts")
if strings.HasSuffix(r.URL.Path, "chat.postMessage") {
s.n++
ts = fmt.Sprintf("%d.0", s.n)
s.order = append(s.order, ts)
s.under[ts] = r.Form.Get("thread_ts")
}
if !strings.HasSuffix(r.URL.Path, "chat.getPermalink") {
s.text[ts] = r.Form.Get("text")
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "C1", "ts": ts, "permalink": "https://slack.example/" + ts})
}
func TestEveryMoveStartsAnAnnouncedThread(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), false)
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
srv := httptest.NewServer(log)
defer srv.Close()
b := &Bot{cfg: c.Cfg, store: c.Store, mgr: c.Mgr, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/")),
acps: map[string]*acp.ACP{}, busy: map[string]int{}}
b.moveFor(c.Ctx, "C1", "C1", "slave-1", "<@U1>", "100.0")
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "master"}); err != nil {
t.Fatal(err)
}
b.runMove(c.Ctx, "C1", "C1", manager.MoveOptions{}, "auto process (master was full), for <@U2>'s message", nil)
log.mu.Lock()
defer log.mu.Unlock()
var roots []string
for _, ts := range log.order {
if log.under[ts] == "" {
roots = append(roots, ts)
}
}
linked := 0
for ts, parent := range log.under {
if parent == "100.0" && strings.Contains(log.text[ts], "Follow the move here") {
linked++
}
}
if linked != 1 {
t.Fatalf("the thread where the move started should get one link to the move thread, got %d", linked)
}
if len(roots) != 2 {
t.Fatalf("want one new top-level message per move, got %d", len(roots))
}
checks := []string{"started by <@U1>: master → slave-1", "started by auto process (master was full), for <@U2>'s message"}
for i, root := range roots {
if !strings.Contains(log.text[root], checks[i]) || !strings.Contains(log.text[root], ":white_check_mark: done") {
t.Fatalf("root %d = %q", i, log.text[root])
}
replies := 0
for ts, parent := range log.under {
if parent == root {
replies++
if !strings.Contains(log.text[ts], "Moved from") {
t.Fatalf("progress reply = %q", log.text[ts])
}
}
}
if replies != 1 {
t.Fatalf("want progress in one thread reply, got %d", replies)
}
}
}
func TestManageCommandsInSlack(t *testing.T) {
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
srv := httptest.NewServer(log)
defer srv.Close()
store, _ := state.Open(filepath.Join(t.TempDir(), "state.json"))
store.SeedPermissions(model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UBOSS": {"all"}}})
b := &Bot{store: store, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/"))}
ctx := context.Background()
say := func(user, text string) string {
m := incoming{user: user, channel: "C1", ts: "1.0", text: text}
if !b.command(ctx, m) {
return "(went to the agent)"
}
log.mu.Lock()
defer log.mu.Unlock()
return log.text[log.order[len(log.order)-1]]
}
if got := say("UANA", "grant <@UANA> all"); !strings.Contains(got, ":lock:") {
t.Fatalf("non-managers can't grant: %q", got)
}
if got := say("UBOSS", "grant <@UANA> move manage"); !strings.Contains(got, "<@UANA> can now: move, manage") {
t.Fatalf("manager grant: %q", got)
}
if got := say("UANA", "grant <@UCAL> scale"); !strings.Contains(got, "<@UCAL> can now: scale") {
t.Fatalf("channel manager grant: %q", got)
}
if got := say("UANA", "grant default <@UCAL> all"); !strings.Contains(got, ":lock:") {
t.Fatalf("channel managers can't touch the default list: %q", got)
}
if got := say("UANA", "managers add <@UCAL>"); !strings.Contains(got, ":lock:") {
t.Fatalf("only user managers add managers: %q", got)
}
if got := say("UBOSS", "managers remove <@UBOSS>"); !strings.Contains(got, "nobody able to manage") {
t.Fatalf("last manager guard: %q", got)
}
if got := say("UBOSS", "dm storage off"); !strings.Contains(got, "now turned off") {
t.Fatalf("dm toggle: %q", got)
}
if store.Permissions().Allowed(model.OwnerUser, "D1", "UX", model.ActStorage) {
t.Fatal("dm toggle not saved")
}
if got := say("UANA", "grant me access to the repo"); got != "(went to the agent)" {
t.Fatalf("plain sentence: %q", got)
}
if got := say("UANA", "permissions reset"); !strings.Contains(got, "default super-user list again") {
t.Fatalf("reset: %q", got)
}
if store.Permissions().Allowed(model.OwnerChannel, "C1", "UCAL", model.ActScale) {
t.Fatal("reset should drop the channel's own list")
}
}
+188
View File
@@ -0,0 +1,188 @@
package bot
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"slackerbot/internal/commands"
"slackerbot/internal/model"
)
func mentions(users []string) string {
out := make([]string, len(users))
for i, u := range users {
out[i] = "<@" + u + ">"
}
return strings.Join(out, ", ")
}
func actionList(actions []model.Action) string {
if len(actions) == 0 {
return "nothing"
}
names := make([]string, len(actions))
for i, a := range actions {
names[i] = string(a)
}
return strings.Join(names, ", ")
}
// manageCommand handles permission changes. It returns false if the message
// isn't one of these commands, so it goes to the agent.
func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) bool {
_, kind := m.owner()
p := b.store.Permissions()
reply := func(text string) bool {
b.say(ctx, m.channel, "", text)
return true
}
denied := func(global bool) bool {
who := "user managers"
if !global {
who = "user managers and people with *manage* permission here"
}
names := make([]string, len(p.Managers))
for i, u := range p.Managers {
names[i] = b.displayName(ctx, u)
}
return reply(fmt.Sprintf(":lock: Only %s can do that. User managers: %s.", who, strings.Join(names, ", ")))
}
update := func(change func(p *model.PermissionsConfig) error) error {
return b.store.UpdatePermissions(change)
}
switch verb {
case "grant", "revoke":
g, err := commands.ParseGrant(args)
if errors.Is(err, commands.ErrNotCommand) {
return false
}
if err != nil {
return reply(":warning: " + err.Error() + ". Try `grant @alice move scale` or `revoke @alice`.")
}
channel := m.channel
if g.ToDefault {
channel = ""
if !p.IsManager(m.user) {
return denied(true)
}
} else {
if kind == model.OwnerUser {
return reply("DMs don't have a super-user list. User managers can switch DM permissions with `dm <permission> on|off`.")
}
if !p.CanManage(channel, m.user) {
return denied(false)
}
}
if verb == "grant" && len(g.Actions) == 0 {
return reply(":warning: Say what to grant, like `grant @alice move scale` (or `all`, or `manage`).")
}
if slices.Contains(g.Actions, model.ActManage) && !g.ToDefault && !p.CanManage(channel, m.user) {
return denied(false)
}
err = update(func(p *model.PermissionsConfig) error {
if verb == "grant" {
p.Grant(channel, g.Users, g.Actions)
} else {
p.Revoke(channel, g.Users, g.Actions)
}
return nil
})
if err != nil {
return reply(":warning: " + err.Error())
}
where := "in this channel"
if g.ToDefault {
where = "in every channel that uses the default list"
}
after := b.store.Permissions()
var lines []string
for _, u := range g.Users {
var actions []model.Action
if g.ToDefault {
list := after.Default
for _, a := range model.GrantActions {
if model.Grants(list[u], a) {
actions = append(actions, a)
}
}
} else {
actions = after.ActionsFor(channel, u)
}
lines = append(lines, fmt.Sprintf("• <@%s> can now: %s", u, actionList(actions)))
}
return reply(fmt.Sprintf("Done, %s:\n%s", where, strings.Join(lines, "\n")))
case "dm":
words := strings.Fields(strings.ToLower(args))
if len(words) != 2 || !slices.Contains(model.AllActions, model.Action(words[0])) {
return false
}
on, ok := commands.ParseToggle(words[1])
if !ok {
return false
}
if !p.IsManager(m.user) {
return denied(true)
}
if err := update(func(p *model.PermissionsConfig) error { p.DM[words[0]] = on; return nil }); err != nil {
return reply(":warning: " + err.Error())
}
state := "turned off"
if on {
state = "allowed"
}
return reply(fmt.Sprintf("Done. Changing *%s* on your own DM container is now %s for everyone.", words[0], state))
case "managers":
words := strings.Fields(args)
if len(words) == 0 {
return reply("*User managers:* " + mentions(p.Managers))
}
op := strings.ToLower(words[0])
g, err := commands.ParseGrant(strings.Join(words[1:], " "))
if (op != "add" && op != "remove") || err != nil || len(g.Actions) > 0 {
return false
}
if !p.IsManager(m.user) {
return denied(true)
}
err = update(func(p *model.PermissionsConfig) error {
if op == "add" {
p.AddManagers(g.Users)
return nil
}
return p.RemoveManagers(g.Users)
})
if errors.Is(err, model.ErrLastManager) {
return reply(":warning: That would leave nobody able to manage permissions. Add someone else first.")
}
if err != nil {
return reply(":warning: " + err.Error())
}
verbText := "are now user managers"
if op == "remove" {
verbText = "are no longer user managers"
}
return reply(fmt.Sprintf("Done. %s %s.", mentions(g.Users), verbText))
case "permissions":
if strings.ToLower(args) != "reset" {
return false
}
if kind == model.OwnerUser {
return reply("DMs don't have a super-user list to reset.")
}
if !p.CanManage(m.channel, m.user) {
return denied(false)
}
if err := update(func(p *model.PermissionsConfig) error { delete(p.Channels, m.channel); return nil }); err != nil {
return reply(":warning: " + err.Error())
}
return reply("Done. This channel uses the default super-user list again.")
}
return false
}
+150
View File
@@ -0,0 +1,150 @@
package bot
import (
"context"
"log"
"strings"
"sync"
"time"
"unicode/utf8"
"github.com/slack-go/slack"
"slackerbot/internal/acp"
)
const (
slackMsgLimit = 3500
flushEvery = 1500 * time.Millisecond // under Slack's edit rate limit
)
type Streamer struct {
api *slack.Client
channel string
thread string
mu sync.Mutex
full strings.Builder
dirty bool
// Only the flush loop touches these.
ts string
offset int
quit chan struct{}
done chan struct{}
}
func NewStreamer(api *slack.Client, channel, thread, placeholderTS string) *Streamer {
s := &Streamer{api: api, channel: channel, thread: thread, ts: placeholderTS,
quit: make(chan struct{}), done: make(chan struct{})}
go s.loop()
return s
}
func (s *Streamer) Update(u acp.Update) {
s.mu.Lock()
defer s.mu.Unlock()
switch u.Kind {
case "agent_message_chunk":
s.full.WriteString(u.Text)
case "tool_call":
if u.Title == "" {
return
}
if t := s.full.String(); t != "" && !strings.HasSuffix(t, "\n") {
s.full.WriteString("\n")
}
s.full.WriteString("> :wrench: " + u.Title + "\n")
default:
return
}
s.dirty = true
}
func (s *Streamer) Finish(stopReason string, err error) {
close(s.quit)
<-s.done
s.mu.Lock()
note := ""
switch {
case err != nil:
note = ":warning: " + err.Error()
case stopReason == "max_tokens", stopReason == "max_turn_requests":
note = "_(stopped: hit the agent's length limit. Reply to keep going.)_"
case stopReason == "refusal":
note = "_(the agent declined this one)_"
case stopReason == "cancelled":
note = "_(cancelled)_"
}
if note != "" {
if s.full.Len() > 0 {
s.full.WriteString("\n\n")
}
s.full.WriteString(note)
}
if s.full.Len() == 0 {
s.full.WriteString("_(no reply)_")
}
s.dirty = true
s.mu.Unlock()
s.flush()
}
func (s *Streamer) loop() {
defer close(s.done)
t := time.NewTicker(flushEvery)
defer t.Stop()
for {
select {
case <-s.quit:
return
case <-t.C:
s.flush()
}
}
}
func (s *Streamer) flush() {
s.mu.Lock()
if !s.dirty {
s.mu.Unlock()
return
}
s.dirty = false
text := s.full.String()[s.offset:]
s.mu.Unlock()
ctx := context.Background()
for len(text) > slackMsgLimit {
cut := strings.LastIndex(text[:slackMsgLimit], "\n")
if cut < slackMsgLimit/2 {
cut = slackMsgLimit
for cut > 0 && !utf8.RuneStart(text[cut]) {
cut--
}
}
s.edit(ctx, text[:cut])
_, ts, err := s.api.PostMessageContext(ctx, s.channel,
slack.MsgOptionText("…", false), slack.MsgOptionTS(s.thread))
if err != nil {
log.Printf("slack post: %v", err)
return
}
s.ts = ts
s.offset += cut
text = text[cut:]
}
s.edit(ctx, text)
}
func (s *Streamer) edit(ctx context.Context, text string) {
if strings.TrimSpace(text) == "" {
text = "…"
}
if _, _, _, err := s.api.UpdateMessageContext(ctx, s.channel, s.ts, slack.MsgOptionText(text, false)); err != nil {
log.Printf("slack update: %v", err)
}
}
+262
View File
@@ -0,0 +1,262 @@
package commands
import (
"errors"
"fmt"
"math"
"regexp"
"slices"
"strconv"
"strings"
"time"
"slackerbot/internal/model"
"slackerbot/internal/units"
)
var ErrNotCommand = errors.New("not a command")
var unitSpace = regexp.MustCompile(`(\d)\s+(tib|tb|t|gib|gb|g|mib|mb|m|cpus|cpu|cores|core)\b`)
var sizeToken = regexp.MustCompile(`^([+-]?)(\d+(?:\.\d+)?)(g|gb|gib|m|mb|mib|cpu|cpus|core|cores)$`)
func ParseScale(args string, cur model.Size) (model.Size, error) {
args = unitSpace.ReplaceAllString(strings.ToLower(args), "$1$2")
words := strings.Fields(args)
if len(words) == 0 {
return model.Size{MemoryMB: cur.MemoryMB * 2, CPUs: cur.CPUs * 2}, nil
}
if len(words) == 1 {
switch words[0] {
case "up", "out":
return model.Size{MemoryMB: cur.MemoryMB * 2, CPUs: cur.CPUs * 2}, nil
case "down", "in":
return model.Size{MemoryMB: cur.MemoryMB / 2, CPUs: cur.CPUs / 2}, nil
}
}
next := cur
var memDone, cpuDone bool
for i, w := range words {
parts := sizeToken.FindStringSubmatch(w)
if parts == nil {
if i == 0 {
return cur, ErrNotCommand
}
return cur, fmt.Errorf("I didn't understand `%s`", w)
}
sign, num, unit := parts[1], parts[2], parts[3]
n, _ := strconv.ParseFloat(num, 64)
if unit == "cpu" || unit == "cpus" || unit == "core" || unit == "cores" {
if cpuDone {
return cur, errors.New("CPU was given twice")
}
cpuDone = true
next.CPUs = apply(sign, cur.CPUs, n)
continue
}
if memDone {
return cur, errors.New("memory was given twice")
}
memDone = true
mb := n
if strings.HasPrefix(unit, "g") {
mb = n * 1024
}
next.MemoryMB = int(math.Round(apply(sign, float64(cur.MemoryMB), mb)))
}
return next, nil
}
func apply(sign string, cur, n float64) float64 {
switch sign {
case "+":
return cur + n
case "-":
return cur - n
}
return n
}
type KeepSetting struct {
Show bool
Reset bool
Indefinite bool
Limit time.Duration
}
var dayToken = regexp.MustCompile(`(\d+(?:\.\d+)?)([dw])`)
var wordUnit = regexp.MustCompile(`(\d+(?:\.\d+)?)\s*(weeks?|wks?|days?|hours?|hrs?|minutes?|mins?)\b`)
func ParseKeep(args string) (KeepSetting, error) {
a := strings.ToLower(strings.TrimSpace(args))
switch a {
case "":
return KeepSetting{Show: true}, nil
case "default", "reset", "normal":
return KeepSetting{Reset: true}, nil
case "indefinite", "indefinitely", "forever", "alive", "running", "on":
return KeepSetting{Indefinite: true}, nil
}
d, err := parseDuration(a)
if err != nil {
return KeepSetting{}, ErrNotCommand
}
if d <= 0 {
return KeepSetting{}, errors.New("the time has to be more than zero")
}
return KeepSetting{Limit: d}, nil
}
func parseDuration(s string) (time.Duration, error) {
s = wordUnit.ReplaceAllStringFunc(s, func(m string) string {
p := wordUnit.FindStringSubmatch(m)
return p[1] + map[byte]string{'w': "w", 'd': "d", 'h': "h", 'm': "m"}[p[2][0]]
})
s = strings.ReplaceAll(s, " ", "")
s = dayToken.ReplaceAllStringFunc(s, func(m string) string {
p := dayToken.FindStringSubmatch(m)
n, _ := strconv.ParseFloat(p[1], 64)
hours := n * 24
if p[2] == "w" {
hours = n * 24 * 7
}
return strconv.FormatFloat(hours, 'f', -1, 64) + "h"
})
return time.ParseDuration(s)
}
func Human(d time.Duration) string {
d = d.Round(time.Minute)
days := d / (24 * time.Hour)
d -= days * 24 * time.Hour
out := ""
if days > 0 {
out = fmt.Sprintf("%dd", days)
}
if d > 0 {
r := strings.TrimSuffix(d.String(), "0s")
if strings.HasSuffix(r, "h0m") {
r = strings.TrimSuffix(r, "0m")
}
out += r
}
if out == "" {
out = "0m"
}
return out
}
func ParseStorage(args string, curMin, curMax int64) (min, max int64, err error) {
args = unitSpace.ReplaceAllString(strings.ToLower(args), "$1$2")
words := strings.Fields(args)
min, max = curMin, curMax
var minDone, maxDone bool
for i := 0; i < len(words); i++ {
target := "max"
switch words[i] {
case "min", "guarantee", "guaranteed", "reserve":
target = "min"
i++
case "max", "cap", "limit":
i++
}
if i >= len(words) {
return curMin, curMax, fmt.Errorf("%s needs a size after it, like `%s 10G`", target, target)
}
p := units.StorageToken.FindStringSubmatch(words[i])
if p == nil {
if i == 0 {
return curMin, curMax, ErrNotCommand
}
return curMin, curMax, fmt.Errorf("I didn't understand `%s`", words[i])
}
n := units.ToMB(p[2], p[3])
if target == "min" {
if minDone {
return curMin, curMax, errors.New("min was given twice")
}
minDone = true
min = int64(apply(p[1], float64(curMin), float64(n)))
} else {
if maxDone {
return curMin, curMax, errors.New("max was given twice")
}
maxDone = true
max = int64(apply(p[1], float64(curMax), float64(n)))
}
}
return min, max, nil
}
var mentionRe = regexp.MustCompile(`^<@([UW][A-Z0-9]+)(?:\|[^>]*)?>[,.]?$`)
type GrantCmd struct {
ToDefault bool
Users []string
Actions []model.Action
}
func ParseGrant(args string) (GrantCmd, error) {
var g GrantCmd
words := strings.Fields(args)
i := 0
if len(words) > 0 && strings.ToLower(words[0]) == "default" {
g.ToDefault, i = true, 1
}
for ; i < len(words); i++ {
if m := mentionRe.FindStringSubmatch(words[i]); m != nil {
g.Users = append(g.Users, m[1])
continue
}
w := strings.ToLower(strings.Trim(words[i], ",."))
if w == "all" || slices.Contains(model.GrantActions, model.Action(w)) {
g.Actions = append(g.Actions, model.Action(w))
continue
}
if len(g.Users) == 0 && !g.ToDefault {
return g, ErrNotCommand
}
return g, fmt.Errorf("I didn't understand `%s`", words[i])
}
switch {
case len(g.Users) == 0 && !g.ToDefault:
return g, ErrNotCommand
case len(g.Users) == 0:
return g, errors.New("name at least one person, like `grant default @alice all`")
}
return g, nil
}
func ParseToggle(word string) (on, ok bool) {
switch strings.ToLower(word) {
case "on", "yes", "true", "allow", "allowed", "enable":
return true, true
case "off", "no", "false", "deny", "denied", "disable":
return false, true
}
return false, false
}
func NeededPermission(verb, args string) model.Action {
switch verb {
case "scale":
if _, err := ParseScale(args, model.Size{MemoryMB: 1024, CPUs: 1}); !errors.Is(err, ErrNotCommand) {
return model.ActScale
}
case "keep":
if k, err := ParseKeep(args); !errors.Is(err, ErrNotCommand) && !k.Show {
return model.ActKeep
}
case "storage", "disk":
if args == "" {
return ""
}
if _, _, err := ParseStorage(args, 1024, 1024); !errors.Is(err, ErrNotCommand) {
return model.ActStorage
}
}
return ""
}
@@ -0,0 +1,154 @@
package commands
import (
"errors"
"testing"
"time"
"slackerbot/internal/model"
)
func TestParseScale(t *testing.T) {
cur := model.Size{MemoryMB: 2048, CPUs: 1}
cases := []struct {
in string
want model.Size
}{
{"", model.Size{MemoryMB: 4096, CPUs: 2}},
{"up", model.Size{MemoryMB: 4096, CPUs: 2}},
{"down", model.Size{MemoryMB: 1024, CPUs: 0.5}},
{"8G 4cpu", model.Size{MemoryMB: 8192, CPUs: 4}},
{"8gb", model.Size{MemoryMB: 8192, CPUs: 1}},
{"+2G", model.Size{MemoryMB: 4096, CPUs: 1}},
{"-1cpu", model.Size{MemoryMB: 2048, CPUs: 0}},
{"512M +0.5cpu", model.Size{MemoryMB: 512, CPUs: 1.5}},
{"1.5G 3 cores", model.Size{MemoryMB: 1536, CPUs: 3}},
{"+2 GB", model.Size{MemoryMB: 4096, CPUs: 1}},
{"+1cpus -512mb", model.Size{MemoryMB: 1536, CPUs: 2}},
}
for _, c := range cases {
got, err := ParseScale(c.in, cur)
if err != nil || got != c.want {
t.Fatalf("scale %q = %v, %v; want %v", c.in, got, err, c.want)
}
}
if _, err := ParseScale("the database to 3 replicas", cur); !errors.Is(err, ErrNotCommand) {
t.Fatal("a normal sentence should go to the agent")
}
if _, err := ParseScale("4G lots", cur); err == nil || errors.Is(err, ErrNotCommand) {
t.Fatal("a bad second word should be a usage error")
}
if _, err := ParseScale("4G 8G", cur); err == nil {
t.Fatal("memory twice should fail")
}
}
func TestParseKeep(t *testing.T) {
for in, want := range map[string]time.Duration{
"24h": 24 * time.Hour, "3d": 72 * time.Hour, "1d12h": 36 * time.Hour, "2w": 336 * time.Hour,
"5 hours": 5 * time.Hour, "2 days": 48 * time.Hour, "1d 6h": 30 * time.Hour, "90 mins": 90 * time.Minute,
} {
k, err := ParseKeep(in)
if err != nil || k.Limit != want {
t.Fatalf("keep %q = %+v, %v; want %s", in, k, err, want)
}
}
for _, in := range []string{"indefinite", "Indefinitely", "forever"} {
if k, _ := ParseKeep(in); !k.Indefinite {
t.Fatalf("keep %q should be indefinite", in)
}
}
if k, _ := ParseKeep("default"); !k.Reset {
t.Fatal("keep default should reset")
}
if k, _ := ParseKeep(""); !k.Show {
t.Fatal("bare keep should show")
}
if _, err := ParseKeep("the tests passing"); !errors.Is(err, ErrNotCommand) {
t.Fatal("a normal sentence should go to the agent")
}
}
func TestHuman(t *testing.T) {
for d, want := range map[time.Duration]string{
6 * time.Hour: "6h", 36 * time.Hour: "1d12h", 90 * time.Minute: "1h30m",
45 * time.Minute: "45m", 168 * time.Hour: "7d", 0: "0m",
} {
if got := Human(d); got != want {
t.Fatalf("Human(%s) = %q, want %q", d, got, want)
}
}
if got := (model.Size{MemoryMB: 8192, CPUs: 4}).String(); got != "8 GB RAM, 4 CPUs" {
t.Fatal(got)
}
if got := (model.Size{MemoryMB: 512, CPUs: 0.25}).String(); got != "512 MB RAM, 0.25 CPUs" {
t.Fatal(got)
}
}
func TestParseStorage(t *testing.T) {
cases := []struct {
in string
min, max int64
}{
{"20G", 5120, 20480},
{"+5G", 5120, 15360},
{"-2G", 5120, 8192},
{"max 30G", 5120, 30720},
{"min 8G", 8192, 10240},
{"min +1G max 30 GB", 6144, 30720},
{"cap 1T", 5120, 1048576},
}
for _, c := range cases {
min, max, err := ParseStorage(c.in, 5120, 10240)
if err != nil || min != c.min || max != c.max {
t.Fatalf("storage %q = %d/%d, %v; want %d/%d", c.in, min, max, err, c.min, c.max)
}
}
if _, _, err := ParseStorage("is full, clean up the logs", 5120, 10240); !errors.Is(err, ErrNotCommand) {
t.Fatal("a normal sentence should go to the agent")
}
for _, in := range []string{"min", "20G 30G", "min 5G min 6G", "20G lots"} {
if _, _, err := ParseStorage(in, 5120, 10240); err == nil || errors.Is(err, ErrNotCommand) {
t.Fatalf("%q should be a usage error, got %v", in, err)
}
}
}
func TestNeededPermission(t *testing.T) {
for in, want := range map[[2]string]model.Action{
{"scale", "up"}: model.ActScale,
{"scale", "8G 4cpu"}: model.ActScale,
{"scale", "the db to 3 replicas"}: "",
{"keep", ""}: "",
{"keep", "24h"}: model.ActKeep,
{"keep", "indefinite"}: model.ActKeep,
{"keep", "the tests green"}: "",
{"storage", ""}: "",
{"storage", "20G"}: model.ActStorage,
{"storage", "is full, clean it up"}: "",
{"status", ""}: "",
} {
if got := NeededPermission(in[0], in[1]); got != want {
t.Errorf("%s %q needs %q, want %q", in[0], in[1], got, want)
}
}
}
func TestParseGrant(t *testing.T) {
g, err := ParseGrant("<@U1> <@U2|bob>, move scale")
if err != nil || len(g.Users) != 2 || g.Users[1] != "U2" || len(g.Actions) != 2 {
t.Fatalf("%+v %v", g, err)
}
if g, _ := ParseGrant("default <@U1> all"); !g.ToDefault || g.Actions[0] != "all" {
t.Fatalf("%+v", g)
}
for _, in := range []string{"me access to the repo", "all the things", ""} {
if _, err := ParseGrant(in); !errors.Is(err, ErrNotCommand) {
t.Errorf("%q should go to the agent, got %v", in, err)
}
}
if _, err := ParseGrant("<@U1> teleport"); err == nil || errors.Is(err, ErrNotCommand) {
t.Error("an unknown permission after a mention should be a usage error")
}
}
+185
View File
@@ -0,0 +1,185 @@
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/BurntSushi/toml"
"slackerbot/internal/model"
"slackerbot/internal/units"
)
const MinStorageMB = 512
type StorageConfig struct {
Min string `toml:"min"`
Max string `toml:"max"`
Limit string `toml:"limit"`
MinMB, MaxMB, LimitMB int64 `toml:"-"`
}
type DriveConfig struct {
Path string `toml:"path"`
Reserve string `toml:"reserve"`
ReserveMB int64 `toml:"-"`
}
type Config struct {
Slack struct {
AppToken string `toml:"app_token"`
BotToken string `toml:"bot_token"`
} `toml:"slack"`
System string `toml:"system"`
MachinesDir string `toml:"machines_dir"`
StateDir string `toml:"state_dir"`
IdleTimeout time.Duration `toml:"idle_timeout"`
MaxIdleTimeout time.Duration `toml:"max_idle_timeout"`
AllowIndefinite bool `toml:"allow_indefinite"`
BootTimeout time.Duration `toml:"boot_timeout"`
AgentCwd string `toml:"agent_cwd"`
AgentEnv map[string]string `toml:"agent_env"`
ROBinds []string `toml:"ro_binds"`
RWBinds []string `toml:"rw_binds"`
Container model.Size `toml:"container"`
MaxContainer model.Size `toml:"max_container"`
HostReserve model.Size `toml:"host_reserve"`
Storage StorageConfig `toml:"storage"`
Drives []DriveConfig `toml:"drives"`
Cluster ClusterConfig `toml:"cluster"`
Permissions model.PermissionsConfig `toml:"permissions"`
}
type ClusterConfig struct {
Name string `toml:"name"`
Secret string `toml:"secret"`
Listen string `toml:"listen"`
Master string `toml:"master"`
MasterFingerprint string `toml:"master_fingerprint"`
}
const (
ModeMaster = "master"
ModeSlave = "slave"
)
func LoadConfig(path, mode string) (*Config, error) {
c := &Config{
MachinesDir: "/var/lib/machines",
StateDir: "/var/lib/slackerbot",
IdleTimeout: 6 * time.Hour,
AllowIndefinite: true,
BootTimeout: 2 * time.Minute,
AgentCwd: "/root",
Container: model.Size{MemoryMB: 2048, CPUs: 1},
MaxContainer: model.Size{MemoryMB: 8192, CPUs: 4},
HostReserve: model.Size{MemoryMB: 2048, CPUs: 1},
Storage: StorageConfig{Min: "5G", Max: "10G", Limit: "50G"},
}
if _, err := toml.DecodeFile(path, c); err != nil {
return nil, fmt.Errorf("reading %s: %w", path, err)
}
if v := os.Getenv("SLACK_APP_TOKEN"); v != "" {
c.Slack.AppToken = v
}
if v := os.Getenv("SLACK_BOT_TOKEN"); v != "" {
c.Slack.BotToken = v
}
for k, v := range c.AgentEnv {
c.AgentEnv[k] = os.ExpandEnv(v)
}
c.Cluster.Secret = os.ExpandEnv(c.Cluster.Secret)
if v := os.Getenv("CLUSTER_SECRET"); v != "" {
c.Cluster.Secret = v
}
if c.Cluster.Name == "" {
c.Cluster.Name, _ = os.Hostname()
}
if err := c.loadStorage(); err != nil {
return nil, err
}
clustered := (mode == ModeMaster && c.Cluster.Listen != "") || mode == ModeSlave
switch {
case mode != ModeMaster && mode != ModeSlave:
return nil, fmt.Errorf("mode must be %q or %q", ModeMaster, ModeSlave)
case mode == ModeMaster && (c.Slack.AppToken == "" || c.Slack.BotToken == ""):
return nil, fmt.Errorf("slack app_token and bot_token are required")
case mode == ModeMaster && c.Permissions.Validate() != nil:
return nil, c.Permissions.Validate()
case mode == ModeSlave && !strings.HasPrefix(c.Cluster.Master, "wss://"):
return nil, fmt.Errorf("cluster.master must be the master's address, like wss://master.example:7946")
case clustered && len(c.Cluster.Secret) < 16:
return nil, fmt.Errorf("cluster.secret must be at least 16 characters, and the same on every node")
case c.Cluster.Name == "":
return nil, fmt.Errorf("cluster.name is required")
case c.System == "":
return nil, fmt.Errorf("system (the NixOS build) is required")
case c.Container.MemoryMB <= 0 || c.Container.CPUs <= 0:
return nil, fmt.Errorf("container memory_mb and cpus must be above zero")
case c.IdleTimeout <= 0:
return nil, fmt.Errorf("idle_timeout must be above zero")
case !c.Container.Fits(c.MaxContainer):
return nil, fmt.Errorf("max_container must be at least as big as container")
}
return c, nil
}
func (c *Config) loadStorage() error {
var err error
s := &c.Storage
if s.MinMB, err = units.ParseSizeMB(s.Min); err != nil {
return fmt.Errorf("storage.min: %w", err)
}
if s.MaxMB, err = units.ParseSizeMB(s.Max); err != nil {
return fmt.Errorf("storage.max: %w", err)
}
if s.LimitMB, err = units.ParseSizeMB(s.Limit); err != nil {
return fmt.Errorf("storage.limit: %w", err)
}
switch {
case s.MinMB < MinStorageMB:
return fmt.Errorf("storage.min must be at least %s", units.FormatMB(MinStorageMB))
case s.MinMB > s.MaxMB:
return fmt.Errorf("storage.min (%s) can't be more than storage.max (%s)", s.Min, s.Max)
case s.MaxMB > s.LimitMB:
return fmt.Errorf("storage.max (%s) can't be more than storage.limit (%s)", s.Max, s.Limit)
case len(c.Drives) == 0:
return fmt.Errorf("add at least one [[drives]] entry (a folder on a btrfs drive)")
}
seen := map[string]bool{}
for i := range c.Drives {
d := &c.Drives[i]
if d.Path == "" || !filepath.IsAbs(d.Path) {
return fmt.Errorf("drives[%d].path must be a full path", i)
}
d.Path = filepath.Clean(d.Path)
if seen[d.Path] {
return fmt.Errorf("drive %s is listed twice", d.Path)
}
seen[d.Path] = true
if d.Reserve == "" {
d.Reserve = "0G"
}
if d.ReserveMB, err = units.ParseSizeMB(d.Reserve); err != nil {
return fmt.Errorf("drive %s reserve: %w", d.Path, err)
}
}
return nil
}
+26
View File
@@ -0,0 +1,26 @@
package config
import (
"testing"
"time"
)
func TestConfigExample(t *testing.T) {
t.Setenv("SLACK_APP_TOKEN", "xapp-test")
t.Setenv("SLACK_BOT_TOKEN", "xoxb-test")
t.Setenv("ANTHROPIC_API_KEY", "sk-test")
t.Setenv("CLUSTER_SECRET", "cluster-secret-1234567890")
c, err := LoadConfig("../../../config.example.toml", ModeMaster)
if err != nil {
t.Fatal(err)
}
if c.IdleTimeout != 6*time.Hour || c.Container.MemoryMB != 2048 || c.MaxContainer.CPUs != 4 {
t.Fatalf("unexpected config: %+v", c)
}
if c.Storage.MinMB != 5120 || c.Storage.MaxMB != 10240 || len(c.Drives) != 2 || c.Drives[0].ReserveMB != 20480 {
t.Fatalf("unexpected storage config: %+v %+v", c.Storage, c.Drives)
}
if c.AgentEnv["ANTHROPIC_API_KEY"] != "sk-test" {
t.Fatal("env var not expanded")
}
}
+60
View File
@@ -0,0 +1,60 @@
package local
import (
"bufio"
"fmt"
"os"
"runtime"
"strconv"
"strings"
"slackerbot/internal/model"
)
type Capacity struct {
Total model.Size
Usable model.Size
}
func DetectCapacity(reserve model.Size) (Capacity, error) {
memMB, err := totalMemoryMB()
if err != nil {
return Capacity{}, err
}
total := model.Size{MemoryMB: memMB, CPUs: float64(runtime.NumCPU())}
usable := total.Sub(reserve)
if usable.MemoryMB <= 0 || usable.CPUs <= 0 {
return Capacity{}, fmt.Errorf("host_reserve (%s) leaves nothing on this node (%s)", reserve, total)
}
return Capacity{Total: total, Usable: usable}, nil
}
func (c Capacity) Room(used, want model.Size) bool {
return used.Add(want).Fits(c.Usable)
}
func (c Capacity) Slots(size model.Size) int {
byMem := c.Usable.MemoryMB / size.MemoryMB
byCPU := int(c.Usable.CPUs/size.CPUs + 1e-9)
return min(byMem, byCPU)
}
func totalMemoryMB() (int, error) {
f, err := os.Open("/proc/meminfo")
if err != nil {
return 0, err
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
fields := strings.Fields(sc.Text())
if len(fields) >= 2 && fields[0] == "MemTotal:" {
kb, err := strconv.Atoi(fields[1])
if err != nil {
return 0, err
}
return kb / 1024, nil
}
}
return 0, fmt.Errorf("MemTotal not found in /proc/meminfo")
}
@@ -0,0 +1,24 @@
package local
import (
"testing"
"slackerbot/internal/model"
)
func TestCapacity(t *testing.T) {
c := Capacity{Usable: model.Size{MemoryMB: 6144, CPUs: 3}}
def := model.Size{MemoryMB: 2048, CPUs: 1}
if got := c.Slots(def); got != 3 {
t.Fatalf("slots = %d, want 3", got)
}
if !c.Room(model.Size{MemoryMB: 4096, CPUs: 2}, def) {
t.Fatal("third container should fit")
}
if c.Room(model.Size{MemoryMB: 6144, CPUs: 3}, def) {
t.Fatal("fourth container should not fit")
}
if c.Room(model.Size{MemoryMB: 2048, CPUs: 2.5}, def) {
t.Fatal("should be blocked by CPU")
}
}
+325
View File
@@ -0,0 +1,325 @@
package local
import (
"context"
"errors"
"fmt"
"math"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/godbus/dbus/v5"
"slackerbot/internal/config"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/storage"
)
const (
ServicePort = 1984
machinedName = "org.freedesktop.machine1"
machinedPath = "/org/freedesktop/machine1"
machinedMgr = "org.freedesktop.machine1.Manager"
sigPoweroff = 38 // SIGRTMIN+4, what `machinectl poweroff` sends
agentEnvInside = "/var/lib/slackerbot/agent.env"
)
type Local struct {
cfg *config.Config
name string
cap Capacity
bus *dbus.Conn
system string
exports transfers
imports transfers
}
func New(cfg *config.Config) (*Local, error) {
system, err := filepath.EvalSymlinks(cfg.System)
if err != nil {
return nil, err
}
if _, err := os.Stat(filepath.Join(system, "init")); err != nil {
return nil, fmt.Errorf("%s has no init. Is it a NixOS build with boot.isContainer = true?", system)
}
for _, d := range cfg.Drives {
if err := storage.PrepareDrive(d.Path); err != nil {
return nil, err
}
}
capacity, err := DetectCapacity(cfg.HostReserve)
if err != nil {
return nil, err
}
bus, err := dbus.ConnectSystemBus()
if err != nil {
return nil, fmt.Errorf("connecting to the system bus: %w", err)
}
return &Local{cfg: cfg, name: cfg.Cluster.Name, cap: capacity, bus: bus, system: system}, nil
}
func unitName(machine string) string { return "slackerbot-" + machine + ".service" }
func (l *Local) Name() string { return l.name }
func (l *Local) machined() dbus.BusObject { return l.bus.Object(machinedName, machinedPath) }
func (l *Local) Info(ctx context.Context) (node.NodeInfo, error) {
info := node.NodeInfo{Name: l.name, Total: l.cap.Total, Usable: l.cap.Usable}
for _, d := range l.cfg.Drives {
free, err := storage.FreeMB(d.Path)
if err != nil {
return info, err
}
di := node.DriveInfo{Path: d.Path, FreeMB: free, ReserveMB: d.ReserveMB, HomesMB: map[string]int64{}}
entries, err := os.ReadDir(d.Path)
if err != nil {
return info, err
}
for _, e := range entries {
if e.IsDir() && model.IsOurs(e.Name()) {
used, _ := storage.UsedMB(filepath.Join(d.Path, e.Name()))
di.HomesMB[e.Name()] = used
}
}
info.Drives = append(info.Drives, di)
}
running, err := l.Running(ctx)
info.Running = running
return info, err
}
func (l *Local) Running(ctx context.Context) ([]node.RunningMachine, error) {
var list []struct {
Name, Class, Service string
Path dbus.ObjectPath
}
if err := l.machined().Call(machinedMgr+".ListMachines", 0).Store(&list); err != nil {
return nil, err
}
var out []node.RunningMachine
for _, x := range list {
if !model.IsOurs(x.Name) {
continue
}
r := node.RunningMachine{Name: x.Name}
if v, err := l.bus.Object(machinedName, x.Path).GetProperty("org.freedesktop.machine1.Machine.Timestamp"); err == nil {
if us, ok := v.Value().(uint64); ok {
r.Since = time.UnixMicro(int64(us))
}
}
out = append(out, r)
}
return out, nil
}
func (l *Local) IsRunning(machine string) bool {
var p dbus.ObjectPath
return l.machined().Call(machinedMgr+".GetMachine", 0, machine).Store(&p) == nil
}
func (l *Local) drive(path string) (config.DriveConfig, error) {
for _, d := range l.cfg.Drives {
if d.Path == path {
return d, nil
}
}
return config.DriveConfig{}, fmt.Errorf("drive %s isn't configured on node %s", path, l.name)
}
func (l *Local) Start(ctx context.Context, spec node.StartSpec) error {
if !model.IsOurs(spec.Machine) {
return fmt.Errorf("unexpected machine name %q", spec.Machine)
}
if _, err := l.drive(spec.Drive); err != nil {
return err
}
home := filepath.Join(spec.Drive, spec.Machine)
if _, err := os.Stat(home); errors.Is(err, os.ErrNotExist) {
if err := storage.CreateSubvolume(home); err != nil {
return err
}
} else if err != nil {
return err
}
if err := storage.SetQuota(home, spec.StorageMaxMB); err != nil {
return err
}
// Only the home folder persists. NixOS rebuilds the rest on boot.
root := filepath.Join(l.cfg.MachinesDir, spec.Machine)
if err := os.RemoveAll(root); err != nil {
return fmt.Errorf("clearing old container files: %w", err)
}
if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil {
return err
}
if f, err := os.OpenFile(filepath.Join(root, "etc/os-release"), os.O_CREATE, 0o644); err == nil {
f.Close()
}
envFile, err := l.writeEnvFile(spec)
if err != nil {
return err
}
args := []string{
"--unit=" + unitName(spec.Machine),
"--collect",
"--quiet",
"--description=" + spec.Description,
// Same as systemd-nspawn@.service.
"--property=Type=notify",
"--property=KillMode=mixed",
"--property=Delegate=yes",
fmt.Sprintf("--property=MemoryMax=%dM", spec.Size.MemoryMB),
fmt.Sprintf("--property=CPUQuota=%d%%", int(math.Round(spec.Size.CPUs*100))),
"systemd-nspawn",
"--machine=" + spec.Machine,
"--directory=" + root,
"--keep-unit",
"--network-veth", // own network, so every container can use port 1984
"--bind-ro=/nix/store",
"--bind-ro=" + envFile + ":" + agentEnvInside,
"--bind=" + home + ":/root",
}
for _, b := range l.cfg.ROBinds {
args = append(args, "--bind-ro="+strings.ReplaceAll(b, "{machine}", spec.Machine))
}
for _, b := range l.cfg.RWBinds {
b = strings.ReplaceAll(b, "{machine}", spec.Machine)
src, _, _ := strings.Cut(b, ":")
if err := os.MkdirAll(src, 0o755); err != nil {
return err
}
args = append(args, "--bind="+b)
}
args = append(args, filepath.Join(l.system, "init"))
if out, err := exec.CommandContext(ctx, "systemd-run", args...).CombinedOutput(); err != nil {
return fmt.Errorf("systemd-run: %v: %s", err, strings.TrimSpace(string(out)))
}
// Wait until machined lists it, so the next capacity check counts it.
for i := 0; i < 50; i++ {
if l.IsRunning(spec.Machine) {
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("%s started but never registered with machined", spec.Machine)
}
func (l *Local) writeEnvFile(spec node.StartSpec) (string, error) {
dir := filepath.Join(l.cfg.StateDir, "env")
if err := os.MkdirAll(dir, 0o700); err != nil {
return "", err
}
var b strings.Builder
line := func(k, v string) {
v = strings.ReplaceAll(v, `\`, `\\`)
v = strings.ReplaceAll(v, `"`, `\"`)
fmt.Fprintf(&b, "%s=\"%s\"\n", k, v)
}
line("GOOSE_SERVER__SECRET_KEY", spec.Secret)
for k, v := range spec.Env {
line(k, v)
}
path := filepath.Join(dir, spec.Machine+".env")
return path, os.WriteFile(path, []byte(b.String()), 0o600)
}
func (l *Local) Poweroff(ctx context.Context, machine string) error {
return l.machined().CallWithContext(ctx, machinedMgr+".KillMachine", 0, machine, "leader", int32(sigPoweroff)).Err
}
func (l *Local) SetLimits(ctx context.Context, machine string, s model.Size) error {
type prop struct {
Name string
Value dbus.Variant
}
props := []prop{
{"MemoryMax", dbus.MakeVariant(uint64(s.MemoryMB) << 20)},
{"CPUQuotaPerSecUSec", dbus.MakeVariant(uint64(s.CPUs * 1e6))},
}
return l.bus.Object("org.freedesktop.systemd1", "/org/freedesktop/systemd1").
CallWithContext(ctx, "org.freedesktop.systemd1.Manager.SetUnitProperties", 0, unitName(machine), true, props).Err
}
func (l *Local) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
if _, err := l.drive(drive); err != nil {
return err
}
if !model.IsOurs(machine) {
return fmt.Errorf("unexpected machine name %q", machine)
}
return storage.SetQuota(filepath.Join(drive, machine), mb)
}
func (l *Local) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
addr, err := l.waitReady(ctx, machine)
if err != nil {
return nil, err
}
return node.DialAgentWS(ctx, addr, secret)
}
func (l *Local) address(machine string) (string, error) {
var addrs []struct {
Family int32
Addr []byte
}
if err := l.machined().Call(machinedMgr+".GetMachineAddresses", 0, machine).Store(&addrs); err != nil {
return "", err
}
for _, a := range addrs {
ip := net.IP(a.Addr)
if a.Family == syscall.AF_INET && !ip.IsLinkLocalUnicast() {
return net.JoinHostPort(ip.String(), strconv.Itoa(ServicePort)), nil
}
}
return "", fmt.Errorf("%s has no IPv4 address yet", machine)
}
func (l *Local) waitReady(ctx context.Context, machine string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, l.cfg.BootTimeout)
defer cancel()
client := http.Client{Timeout: 3 * time.Second}
var lastErr error
for {
addr, err := l.address(machine)
if err == nil {
var resp *http.Response
resp, err = client.Get("http://" + addr + "/status")
if err == nil {
resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return addr, nil
}
err = fmt.Errorf("/status returned %s", resp.Status)
}
}
lastErr = err
select {
case <-ctx.Done():
return "", fmt.Errorf("goose didn't answer on port %d within %s (last error: %v)", ServicePort, l.cfg.BootTimeout, lastErr)
case <-time.After(time.Second):
}
}
}
func (l *Local) Capacity() Capacity { return l.cap }
+265
View File
@@ -0,0 +1,265 @@
package local
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"slackerbot/internal/model"
"slackerbot/internal/storage"
"slackerbot/internal/util"
)
type transfer struct {
cmd *exec.Cmd
out io.ReadCloser
in io.WriteCloser
stderr bytes.Buffer
snapshot string
tmpDir string
drive string
machine string
seen map[string]bool
latest string
}
type transfers struct {
mu sync.Mutex
m map[string]*transfer
}
func (t *transfers) add(x *transfer) string {
id := util.RandomHex(8)
t.mu.Lock()
defer t.mu.Unlock()
if t.m == nil {
t.m = map[string]*transfer{}
}
t.m[id] = x
return id
}
func (t *transfers) get(id string) (*transfer, error) {
t.mu.Lock()
defer t.mu.Unlock()
if x := t.m[id]; x != nil {
return x, nil
}
return nil, fmt.Errorf("no transfer %s", id)
}
func (t *transfers) take(id string) *transfer {
t.mu.Lock()
defer t.mu.Unlock()
x := t.m[id]
delete(t.m, id)
return x
}
func (l *Local) home(drive, machine string) (string, error) {
if _, err := l.drive(drive); err != nil {
return "", err
}
if !model.IsOurs(machine) {
return "", fmt.Errorf("unexpected machine name %q", machine)
}
return filepath.Join(drive, machine), nil
}
func (l *Local) WaitStopped(ctx context.Context, machine string) error {
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
for l.IsRunning(machine) {
select {
case <-ctx.Done():
return fmt.Errorf("%s didn't stop in time", machine)
case <-time.After(500 * time.Millisecond):
}
}
return nil
}
// With a parent export, only the changes since the parent's snapshot are sent.
func (l *Local) ExportHome(ctx context.Context, drive, machine, parent string) (string, error) {
home, err := l.home(drive, machine)
if err != nil {
return "", err
}
args := []string{"send", "-q"}
if parent != "" {
p, err := l.exports.get(parent)
if err != nil {
return "", err
}
args = append(args, "-p", p.snapshot)
}
snapshot := filepath.Join(drive, ".move-"+machine+"-"+util.RandomHex(4))
if out, err := exec.Command("btrfs", "subvolume", "snapshot", "-r", home, snapshot).CombinedOutput(); err != nil {
return "", fmt.Errorf("snapshot: %v: %s", err, strings.TrimSpace(string(out)))
}
x := &transfer{snapshot: snapshot, drive: drive, machine: machine}
x.cmd = exec.Command("btrfs", append(args, snapshot)...)
x.cmd.Stderr = &x.stderr
if x.out, err = x.cmd.StdoutPipe(); err == nil {
err = x.cmd.Start()
}
if err != nil {
storage.DeleteSubvolume(snapshot)
return "", err
}
return l.exports.add(x), nil
}
func (l *Local) ReadExport(ctx context.Context, id string, max int) ([]byte, bool, error) {
x, err := l.exports.get(id)
if err != nil {
return nil, false, err
}
buf := make([]byte, max)
n, err := io.ReadFull(x.out, buf)
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
if werr := x.cmd.Wait(); werr != nil {
return nil, false, fmt.Errorf("btrfs send: %v: %s", werr, strings.TrimSpace(x.stderr.String()))
}
return buf[:n], true, nil
}
return buf[:n], false, err
}
func (l *Local) CloseExport(ctx context.Context, id string) error {
x := l.exports.take(id)
if x == nil {
return nil
}
if x.cmd.ProcessState == nil {
_ = x.cmd.Process.Kill()
_ = x.cmd.Wait()
}
return storage.DeleteSubvolume(x.snapshot)
}
func (l *Local) ImportHome(ctx context.Context, drive, machine string) (string, error) {
home, err := l.home(drive, machine)
if err != nil {
return "", err
}
if _, err := os.Stat(home); err == nil {
return "", fmt.Errorf("%s already has a home folder for %s; remove it first", l.name, machine)
}
x := &transfer{drive: drive, machine: machine, seen: map[string]bool{},
tmpDir: filepath.Join(drive, ".incoming-"+machine+"-"+util.RandomHex(4))}
if err := os.Mkdir(x.tmpDir, 0o700); err != nil {
return "", err
}
return l.imports.add(x), nil
}
func (l *Local) WriteImport(ctx context.Context, id string, data []byte) error {
x, err := l.imports.get(id)
if err != nil {
return err
}
if x.cmd == nil {
x.stderr.Reset()
x.cmd = exec.Command("btrfs", "receive", "-q", x.tmpDir)
x.cmd.Stderr = &x.stderr
if x.in, err = x.cmd.StdinPipe(); err == nil {
err = x.cmd.Start()
}
if err != nil {
x.cmd = nil
return err
}
}
if _, err := x.in.Write(data); err != nil {
return fmt.Errorf("btrfs receive: %v: %s", err, strings.TrimSpace(x.stderr.String()))
}
return nil
}
func (l *Local) EndReceive(ctx context.Context, id string) error {
x, err := l.imports.get(id)
if err != nil {
return err
}
if x.cmd == nil {
return errors.New("nothing was received")
}
x.in.Close()
err = x.cmd.Wait()
x.cmd = nil
if err != nil {
return fmt.Errorf("btrfs receive: %v: %s", err, strings.TrimSpace(x.stderr.String()))
}
entries, err := os.ReadDir(x.tmpDir)
if err != nil {
return err
}
for _, e := range entries {
if !x.seen[e.Name()] {
x.seen[e.Name()] = true
x.latest = e.Name()
}
}
return nil
}
// Received subvolumes are read-only; a writable snapshot of the latest one
// becomes the new home.
func (l *Local) FinishImport(ctx context.Context, id string, quotaMB int64) error {
x := l.imports.take(id)
if x == nil {
return fmt.Errorf("no transfer %s", id)
}
defer cleanupIncoming(x.tmpDir)
if x.latest == "" {
return errors.New("nothing was received")
}
home := filepath.Join(x.drive, x.machine)
if out, err := exec.Command("btrfs", "subvolume", "snapshot", filepath.Join(x.tmpDir, x.latest), home).CombinedOutput(); err != nil {
return fmt.Errorf("snapshot: %v: %s", err, strings.TrimSpace(string(out)))
}
return storage.SetQuota(home, quotaMB)
}
func (l *Local) AbortImport(ctx context.Context, id string) error {
x := l.imports.take(id)
if x == nil {
return nil
}
if x.cmd != nil {
x.in.Close()
_ = x.cmd.Process.Kill()
_ = x.cmd.Wait()
}
cleanupIncoming(x.tmpDir)
return nil
}
func (l *Local) DeleteHome(ctx context.Context, drive, machine string) error {
home, err := l.home(drive, machine)
if err != nil {
return err
}
if l.IsRunning(machine) {
return fmt.Errorf("%s is running", machine)
}
return storage.DeleteSubvolume(home)
}
func cleanupIncoming(dir string) {
entries, _ := os.ReadDir(dir)
for _, e := range entries {
_ = storage.DeleteSubvolume(filepath.Join(dir, e.Name()))
}
_ = os.RemoveAll(dir)
}
+787
View File
@@ -0,0 +1,787 @@
package manager
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/hex"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"log"
"math/big"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/coder/websocket"
"slackerbot/internal/config"
"slackerbot/internal/model"
"slackerbot/internal/node"
)
const (
reportEvery = 5 * time.Second
offlineAfter = 20 * time.Second
callTimeout = 3 * time.Minute
pendingTimeout = 30 * time.Second
streamBuffer = 4096
)
type frame struct {
Type string `json:"t"`
ID int64 `json:"id,omitempty"`
Method string `json:"m,omitempty"`
Params json.RawMessage `json:"p,omitempty"`
Result json.RawMessage `json:"r,omitempty"`
Error string `json:"e,omitempty"`
Data string `json:"d,omitempty"`
}
type link struct {
conn *websocket.Conn
writeMu sync.Mutex
nextID atomic.Int64
mu sync.Mutex
pending map[int64]chan frame
streams map[int64]*stream
onCall func(f frame)
onReport func(info node.NodeInfo)
done chan struct{}
err error
}
func newLink(conn *websocket.Conn) *link {
conn.SetReadLimit(64 << 20)
return &link{
conn: conn,
pending: map[int64]chan frame{},
streams: map[int64]*stream{},
done: make(chan struct{}),
}
}
func (l *link) Done() <-chan struct{} { return l.done }
func (l *link) Close() { l.conn.Close(websocket.StatusNormalClosure, "") }
func (l *link) send(ctx context.Context, f frame) error {
b, err := json.Marshal(f)
if err != nil {
return err
}
l.writeMu.Lock()
defer l.writeMu.Unlock()
return l.conn.Write(ctx, websocket.MessageText, b)
}
func (l *link) run() {
defer func() {
l.mu.Lock()
streams := l.streams
l.streams = map[int64]*stream{}
l.mu.Unlock()
for _, s := range streams {
s.closeLocal()
}
close(l.done)
}()
for {
_, data, err := l.conn.Read(context.Background())
if err != nil {
l.err = err
return
}
var f frame
if json.Unmarshal(data, &f) != nil {
continue
}
switch f.Type {
case "call":
if l.onCall != nil {
go l.onCall(f)
}
case "reply":
l.mu.Lock()
ch := l.pending[f.ID]
l.mu.Unlock()
if ch != nil {
ch <- f
}
case "report":
var info node.NodeInfo
if l.onReport != nil && json.Unmarshal(f.Params, &info) == nil {
l.onReport(info)
}
case "data":
if s := l.stream(f.ID); s != nil {
select {
case s.in <- []byte(f.Data):
default:
s.Close()
}
}
case "close":
if s := l.stream(f.ID); s != nil {
s.closeLocal()
}
}
}
}
func (l *link) call(ctx context.Context, method string, params, result any) error {
return l.callWithID(ctx, l.nextID.Add(1), method, params, result)
}
func (l *link) callWithID(ctx context.Context, id int64, method string, params, result any) error {
ch := make(chan frame, 1)
l.mu.Lock()
l.pending[id] = ch
l.mu.Unlock()
defer func() {
l.mu.Lock()
delete(l.pending, id)
l.mu.Unlock()
}()
p, err := json.Marshal(params)
if err != nil {
return err
}
if err := l.send(ctx, frame{Type: "call", ID: id, Method: method, Params: p}); err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case <-l.done:
return fmt.Errorf("link closed: %v", l.err)
case f := <-ch:
if f.Error != "" {
return errors.New(f.Error)
}
if result != nil && len(f.Result) > 0 {
return json.Unmarshal(f.Result, result)
}
return nil
}
}
func (l *link) reply(id int64, result any, err error) {
f := frame{Type: "reply", ID: id}
if err != nil {
f.Error = err.Error()
} else if result != nil {
f.Result, _ = json.Marshal(result)
}
_ = l.send(context.Background(), f)
}
func (l *link) report(ctx context.Context, info node.NodeInfo) error {
p, err := json.Marshal(info)
if err != nil {
return err
}
return l.send(ctx, frame{Type: "report", Params: p})
}
func (l *link) stream(id int64) *stream {
l.mu.Lock()
defer l.mu.Unlock()
return l.streams[id]
}
func (l *link) addStream(id int64) *stream {
s := &stream{l: l, id: id, in: make(chan []byte, streamBuffer), closed: make(chan struct{})}
l.mu.Lock()
l.streams[id] = s
l.mu.Unlock()
return s
}
func (l *link) dropStream(id int64) {
l.mu.Lock()
delete(l.streams, id)
l.mu.Unlock()
}
// The stream shares its ID with the "open_agent" call, so it's registered
// before the call goes out and no early data is lost.
func (l *link) openAgent(ctx context.Context, machine, secret string) (*stream, error) {
id := l.nextID.Add(1)
s := l.addStream(id)
err := l.callWithID(ctx, id, "open_agent", map[string]string{"machine": machine, "secret": secret}, nil)
if err != nil {
l.dropStream(id)
return nil, err
}
return s, nil
}
type stream struct {
l *link
id int64
in chan []byte
closed chan struct{}
once sync.Once
}
func (s *stream) Read(ctx context.Context) ([]byte, error) {
select {
case b := <-s.in:
return b, nil
default:
}
select {
case b := <-s.in:
return b, nil
case <-s.closed:
return nil, errors.New("stream closed")
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (s *stream) Write(ctx context.Context, b []byte) error {
select {
case <-s.closed:
return errors.New("stream closed")
default:
}
return s.l.send(ctx, frame{Type: "data", ID: s.id, Data: string(b)})
}
func (s *stream) Close() error {
s.once.Do(func() {
_ = s.l.send(context.Background(), frame{Type: "close", ID: s.id})
s.l.dropStream(s.id)
close(s.closed)
})
return nil
}
func (s *stream) closeLocal() {
s.once.Do(func() {
s.l.dropStream(s.id)
close(s.closed)
})
}
type Remote struct {
name string
mu sync.Mutex
link *link
info node.NodeInfo
infoAt time.Time
starting map[string]time.Time
stopping map[string]time.Time
}
func (r *Remote) Name() string { return r.name }
func (r *Remote) Online() bool {
r.mu.Lock()
defer r.mu.Unlock()
if r.link == nil || time.Since(r.infoAt) > offlineAfter {
return false
}
select {
case <-r.link.Done():
return false
default:
return true
}
}
func (r *Remote) currentLink() (*link, error) {
if !r.Online() {
return nil, &node.NodeOfflineError{Node: r.name}
}
r.mu.Lock()
defer r.mu.Unlock()
return r.link, nil
}
func (r *Remote) setInfo(info node.NodeInfo) {
r.mu.Lock()
defer r.mu.Unlock()
info.Name = r.name
r.info, r.infoAt = info, time.Now()
for _, m := range info.Running {
delete(r.starting, m.Name)
}
for m := range r.stopping {
if !containsMachine(info.Running, m) {
delete(r.stopping, m)
}
}
}
func containsMachine(list []node.RunningMachine, name string) bool {
for _, m := range list {
if m.Name == name {
return true
}
}
return false
}
// Reports can lag behind a start or stop the master just asked for,
// so recent requests override them for a short while.
func (r *Remote) runningLocked() []node.RunningMachine {
var out []node.RunningMachine
for _, m := range r.info.Running {
if t, ok := r.stopping[m.Name]; ok && time.Since(t) < pendingTimeout {
continue
}
out = append(out, m)
}
for m, t := range r.starting {
if time.Since(t) < pendingTimeout && !containsMachine(out, m) {
out = append(out, node.RunningMachine{Name: m, Since: t})
}
}
return out
}
func (r *Remote) Info(ctx context.Context) (node.NodeInfo, error) {
if !r.Online() {
return node.NodeInfo{}, &node.NodeOfflineError{Node: r.name}
}
r.mu.Lock()
defer r.mu.Unlock()
info := r.info
info.Running = r.runningLocked()
return info, nil
}
func (r *Remote) Running(ctx context.Context) ([]node.RunningMachine, error) {
info, err := r.Info(ctx)
return info.Running, err
}
func (r *Remote) IsRunning(machine string) bool {
r.mu.Lock()
defer r.mu.Unlock()
return containsMachine(r.runningLocked(), machine)
}
func (r *Remote) callNode(ctx context.Context, method string, params, result any) error {
l, err := r.currentLink()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, callTimeout)
defer cancel()
return l.call(ctx, method, params, result)
}
func (r *Remote) Start(ctx context.Context, spec node.StartSpec) error {
if err := r.callNode(ctx, "start", spec, nil); err != nil {
return err
}
r.mu.Lock()
r.starting[spec.Machine] = time.Now()
delete(r.stopping, spec.Machine)
r.mu.Unlock()
return nil
}
func (r *Remote) Poweroff(ctx context.Context, machine string) error {
if err := r.callNode(ctx, "poweroff", map[string]string{"machine": machine}, nil); err != nil {
return err
}
r.mu.Lock()
r.stopping[machine] = time.Now()
delete(r.starting, machine)
r.mu.Unlock()
return nil
}
func (r *Remote) SetLimits(ctx context.Context, machine string, size model.Size) error {
return r.callNode(ctx, "set_limits", callParams{Machine: machine, Size: size}, nil)
}
func (r *Remote) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
return r.callNode(ctx, "set_quota", callParams{Drive: drive, Machine: machine, MB: mb}, nil)
}
func (r *Remote) WaitStopped(ctx context.Context, machine string) error {
return r.callNode(ctx, "wait_stopped", callParams{Machine: machine}, nil)
}
func (r *Remote) ExportHome(ctx context.Context, drive, machine, parent string) (string, error) {
var res callParams
err := r.callNode(ctx, "export_home", callParams{Drive: drive, Machine: machine, Parent: parent}, &res)
return res.ID, err
}
func (r *Remote) ReadExport(ctx context.Context, id string, max int) ([]byte, bool, error) {
var res callParams
err := r.callNode(ctx, "read_export", callParams{ID: id, Max: max}, &res)
return res.Data, res.EOF, err
}
func (r *Remote) CloseExport(ctx context.Context, id string) error {
return r.callNode(ctx, "close_export", callParams{ID: id}, nil)
}
func (r *Remote) ImportHome(ctx context.Context, drive, machine string) (string, error) {
var res callParams
err := r.callNode(ctx, "import_home", callParams{Drive: drive, Machine: machine}, &res)
return res.ID, err
}
func (r *Remote) WriteImport(ctx context.Context, id string, data []byte) error {
return r.callNode(ctx, "write_import", callParams{ID: id, Data: data}, nil)
}
func (r *Remote) EndReceive(ctx context.Context, id string) error {
return r.callNode(ctx, "end_receive", callParams{ID: id}, nil)
}
func (r *Remote) FinishImport(ctx context.Context, id string, quotaMB int64) error {
return r.callNode(ctx, "finish_import", callParams{ID: id, MB: quotaMB}, nil)
}
func (r *Remote) AbortImport(ctx context.Context, id string) error {
return r.callNode(ctx, "abort_import", callParams{ID: id}, nil)
}
func (r *Remote) DeleteHome(ctx context.Context, drive, machine string) error {
return r.callNode(ctx, "delete_home", callParams{Drive: drive, Machine: machine}, nil)
}
func (r *Remote) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
l, err := r.currentLink()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(ctx, callTimeout)
defer cancel()
return l.openAgent(ctx, machine, secret)
}
func (m *Manager) attach(name string, l *link) (*Remote, error) {
if name == "" || name == m.local.Name() {
return nil, fmt.Errorf("node name %q is empty or taken by the master", name)
}
m.mu.Lock()
r := m.remotes[name]
if r == nil {
r = &Remote{name: name, starting: map[string]time.Time{}, stopping: map[string]time.Time{}}
m.remotes[name] = r
}
m.mu.Unlock()
r.mu.Lock()
old := r.link
r.link = l
r.infoAt = time.Time{}
r.mu.Unlock()
if old != nil {
old.Close()
}
return r, nil
}
func (m *Manager) ServeCluster(ctx context.Context) error {
cc := m.cfg.Cluster
cert, fingerprint, err := loadOrCreateCert(m.cfg.StateDir)
if err != nil {
return err
}
log.Printf("cluster: listening on %s, certificate fingerprint %s", cc.Listen, fingerprint)
mux := http.NewServeMux()
mux.HandleFunc("/cluster", func(w http.ResponseWriter, req *http.Request) {
if subtle.ConstantTimeCompare([]byte(req.Header.Get("X-Cluster-Secret")), []byte(cc.Secret)) != 1 {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
name := req.Header.Get("X-Node-Name")
conn, err := websocket.Accept(w, req, nil)
if err != nil {
return
}
l := newLink(conn)
r, err := m.attach(name, l)
if err != nil {
conn.Close(websocket.StatusPolicyViolation, err.Error())
return
}
back := false
l.onReport = func(info node.NodeInfo) {
r.setInfo(info)
if !back {
back = true
log.Printf("cluster: node %s connected", name)
if m.onNodeBack != nil {
go m.onNodeBack(name)
}
}
}
l.onCall = func(f frame) { l.reply(f.ID, nil, errors.New("the master takes no calls")) }
l.run()
log.Printf("cluster: node %s disconnected: %v", name, l.err)
})
srv := &http.Server{
Addr: cc.Listen,
Handler: mux,
TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13},
}
go func() {
<-ctx.Done()
srv.Close()
}()
if err := srv.ListenAndServeTLS("", ""); !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}
func loadOrCreateCert(dir string) (tls.Certificate, string, error) {
certPath, keyPath := filepath.Join(dir, "cluster.crt"), filepath.Join(dir, "cluster.key")
if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
return cert, fingerprintOf(cert.Certificate[0]), nil
}
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return tls.Certificate{}, "", err
}
serial, _ := rand.Int(rand.Reader, big.NewInt(1<<62))
tmpl := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: "slackerbot master"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().AddDate(20, 0, 0),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
return tls.Certificate{}, "", err
}
keyDER, err := x509.MarshalECPrivateKey(key)
if err != nil {
return tls.Certificate{}, "", err
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return tls.Certificate{}, "", err
}
if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
return tls.Certificate{}, "", err
}
if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
return tls.Certificate{}, "", err
}
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
return cert, fingerprintOf(der), err
}
func fingerprintOf(der []byte) string {
sum := sha256.Sum256(der)
return hex.EncodeToString(sum[:])
}
func normalizeFingerprint(s string) string {
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), ":", ""))
}
func RunSlave(ctx context.Context, cfg *config.Config, local node.Node) error {
backoff := time.Second
for {
started := time.Now()
err := slaveSession(ctx, cfg, local)
if ctx.Err() != nil {
return nil
}
if time.Since(started) > time.Minute {
backoff = time.Second
}
log.Printf("cluster: link to master lost (%v), retrying in %s", err, backoff)
select {
case <-ctx.Done():
return nil
case <-time.After(backoff):
}
backoff = min(backoff*2, 30*time.Second)
}
}
func pinnedTLS(cfg *config.Config) *tls.Config {
pinned := normalizeFingerprint(cfg.Cluster.MasterFingerprint)
savedPath := filepath.Join(cfg.StateDir, "master.fingerprint")
if pinned == "" {
if b, err := os.ReadFile(savedPath); err == nil {
pinned = normalizeFingerprint(string(b))
}
}
return &tls.Config{
MinVersion: tls.VersionTLS13,
// Not verified against a CA: the master's self-signed certificate is pinned instead.
InsecureSkipVerify: true,
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
if len(raw) == 0 {
return errors.New("master sent no certificate")
}
got := fingerprintOf(raw[0])
if pinned == "" {
log.Printf("cluster: trusting master certificate %s from now on", got)
pinned = got
_ = os.MkdirAll(cfg.StateDir, 0o700)
return os.WriteFile(savedPath, []byte(got+"\n"), 0o600)
}
if got != pinned {
return fmt.Errorf("master certificate %s doesn't match the pinned %s", got, pinned)
}
return nil
},
}
}
func slaveSession(ctx context.Context, cfg *config.Config, local node.Node) error {
client := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(cfg), Proxy: http.ProxyFromEnvironment}}
dialCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
conn, _, err := websocket.Dial(dialCtx, strings.TrimSuffix(cfg.Cluster.Master, "/")+"/cluster", &websocket.DialOptions{
HTTPClient: client,
HTTPHeader: http.Header{
"X-Cluster-Secret": []string{cfg.Cluster.Secret},
"X-Node-Name": []string{cfg.Cluster.Name},
},
})
cancel()
if err != nil {
return err
}
l := newLink(conn)
defer l.Close()
l.onCall = func(f frame) { serveCall(ctx, l, local, f) }
go l.run()
log.Printf("cluster: connected to master %s as %s", cfg.Cluster.Master, cfg.Cluster.Name)
t := time.NewTicker(reportEvery)
defer t.Stop()
for {
info, err := local.Info(ctx)
if err != nil {
log.Printf("cluster: reading node info: %v", err)
} else if err := l.report(ctx, info); err != nil {
return err
}
select {
case <-ctx.Done():
return nil
case <-l.Done():
return l.err
case <-t.C:
}
}
}
type callParams struct {
Machine string `json:"machine,omitempty"`
Secret string `json:"secret,omitempty"`
Drive string `json:"drive,omitempty"`
MB int64 `json:"mb,omitempty"`
Size model.Size `json:"size"`
ID string `json:"id,omitempty"`
Parent string `json:"parent,omitempty"`
Max int `json:"max,omitempty"`
Data []byte `json:"data,omitempty"`
EOF bool `json:"eof,omitempty"`
}
func serveCall(ctx context.Context, l *link, local node.Node, f frame) {
var p callParams
_ = json.Unmarshal(f.Params, &p)
switch f.Method {
case "start":
var spec node.StartSpec
if err := json.Unmarshal(f.Params, &spec); err != nil {
l.reply(f.ID, nil, err)
return
}
l.reply(f.ID, nil, local.Start(ctx, spec))
case "poweroff":
l.reply(f.ID, nil, local.Poweroff(ctx, p.Machine))
case "set_limits":
l.reply(f.ID, nil, local.SetLimits(ctx, p.Machine, p.Size))
case "set_quota":
l.reply(f.ID, nil, local.SetQuota(ctx, p.Drive, p.Machine, p.MB))
case "wait_stopped":
l.reply(f.ID, nil, local.WaitStopped(ctx, p.Machine))
case "export_home":
id, err := local.ExportHome(ctx, p.Drive, p.Machine, p.Parent)
l.reply(f.ID, callParams{ID: id}, err)
case "read_export":
data, eof, err := local.ReadExport(ctx, p.ID, min(p.Max, transferChunk))
l.reply(f.ID, callParams{Data: data, EOF: eof}, err)
case "close_export":
l.reply(f.ID, nil, local.CloseExport(ctx, p.ID))
case "import_home":
id, err := local.ImportHome(ctx, p.Drive, p.Machine)
l.reply(f.ID, callParams{ID: id}, err)
case "write_import":
l.reply(f.ID, nil, local.WriteImport(ctx, p.ID, p.Data))
case "end_receive":
l.reply(f.ID, nil, local.EndReceive(ctx, p.ID))
case "finish_import":
l.reply(f.ID, nil, local.FinishImport(ctx, p.ID, p.MB))
case "abort_import":
l.reply(f.ID, nil, local.AbortImport(ctx, p.ID))
case "delete_home":
l.reply(f.ID, nil, local.DeleteHome(ctx, p.Drive, p.Machine))
case "open_agent":
agent, err := local.DialAgent(ctx, p.Machine, p.Secret)
if err != nil {
l.reply(f.ID, nil, err)
return
}
s := l.addStream(f.ID)
l.reply(f.ID, nil, nil)
pipe(ctx, agent, s)
default:
l.reply(f.ID, nil, fmt.Errorf("unknown call %q", f.Method))
}
}
func pipe(ctx context.Context, a, b node.MsgConn) {
ctx, cancel := context.WithCancel(ctx)
copyTo := func(dst, src node.MsgConn) {
defer cancel()
for {
msg, err := src.Read(ctx)
if err != nil {
return
}
if err := dst.Write(ctx, msg); err != nil {
return
}
}
}
go copyTo(a, b)
copyTo(b, a)
<-ctx.Done()
a.Close()
b.Close()
}
+137
View File
@@ -0,0 +1,137 @@
package manager
import (
"context"
"errors"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
"slackerbot/internal/config"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/state"
"slackerbot/internal/testutil"
)
func TestMasterAndSlave(t *testing.T) {
addr := testutil.FreePort(t)
secret := "cluster-secret-1234567890"
masterDir, slaveDir := t.TempDir(), t.TempDir()
masterCfg := &config.Config{StateDir: masterDir, Container: model.Size{MemoryMB: 1024, CPUs: 1}}
masterCfg.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Listen: addr}
store, err := state.Open(filepath.Join(masterDir, "state.json"))
if err != nil {
t.Fatal(err)
}
mgr := New(masterCfg, store, testutil.NewFakeNode("master"))
backCh := make(chan string, 4)
mgr.onNodeBack = func(n string) { backCh <- n }
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go mgr.ServeCluster(ctx)
slaveCfg := &config.Config{StateDir: slaveDir}
slaveCfg.Cluster = config.ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr}
slave := testutil.NewFakeNode("slave-1")
slaveCtx, stopSlave := context.WithCancel(ctx)
go RunSlave(slaveCtx, slaveCfg, slave)
testutil.WaitFor(t, "slave to come online", func() bool { _, err := mgr.nodeByName("slave-1"); return err == nil })
if got := <-backCh; got != "slave-1" {
t.Fatalf("back callback for %q", got)
}
pinned, err := os.ReadFile(filepath.Join(slaveDir, "master.fingerprint"))
if err != nil || len(strings.TrimSpace(string(pinned))) != 64 {
t.Fatalf("slave should pin the master's certificate, got %q %v", pinned, err)
}
n, _ := mgr.nodeByName("slave-1")
spec := node.StartSpec{Machine: "c-abc", Secret: "agent-secret", Size: model.Size{MemoryMB: 1024, CPUs: 1}, Drive: "/d"}
if err := n.Start(ctx, spec); err != nil {
t.Fatal(err)
}
if !slave.IsRunning("c-abc") || slave.Last.Drive != "/d" {
t.Fatal("start didn't reach the slave")
}
if !n.IsRunning("c-abc") {
t.Fatal("master should count the container as running right away")
}
conn, err := n.DialAgent(ctx, "c-abc", "agent-secret")
if err != nil {
t.Fatal(err)
}
if err := conn.Write(ctx, []byte(`{"jsonrpc":"2.0"}`)); err != nil {
t.Fatal(err)
}
readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second)
got, err := conn.Read(readCtx)
readCancel()
if err != nil || string(got) != `echo: {"jsonrpc":"2.0"}` {
t.Fatalf("relay got %q %v", got, err)
}
conn.Close()
if _, err := n.DialAgent(ctx, "c-abc", "wrong"); err == nil {
t.Fatal("a bad agent secret should fail through the relay")
}
if err := n.Poweroff(ctx, "c-abc"); err != nil {
t.Fatal(err)
}
if slave.IsRunning("c-abc") || n.IsRunning("c-abc") {
t.Fatal("poweroff didn't take")
}
stopSlave()
testutil.WaitFor(t, "slave to go offline", func() bool {
_, err := mgr.nodeByName("slave-1")
var off *node.NodeOfflineError
return errors.As(err, &off) && off.Node == "slave-1"
})
if off := mgr.OfflineNodes(); len(off) != 1 || off[0] != "slave-1" {
t.Fatalf("offline nodes = %v", off)
}
}
func TestSlaveRejectsWrongMasterOrSecret(t *testing.T) {
addr := testutil.FreePort(t)
secret := "cluster-secret-1234567890"
masterCfg := &config.Config{StateDir: t.TempDir()}
masterCfg.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Listen: addr}
store, _ := state.Open(filepath.Join(masterCfg.StateDir, "state.json"))
mgr := New(masterCfg, store, testutil.NewFakeNode("master"))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go mgr.ServeCluster(ctx)
testutil.WaitFor(t, "master to listen", func() bool {
c, err := net.Dial("tcp", addr)
if err == nil {
c.Close()
}
return err == nil
})
wrongPin := &config.Config{StateDir: t.TempDir()}
wrongPin.Cluster = config.ClusterConfig{Name: "s", Secret: secret, Master: "wss://" + addr, MasterFingerprint: strings.Repeat("ab", 32)}
if err := slaveSession(ctx, wrongPin, testutil.NewFakeNode("s")); err == nil {
t.Fatal("a master with a different certificate should be refused")
}
wrongSecret := &config.Config{StateDir: t.TempDir()}
wrongSecret.Cluster = config.ClusterConfig{Name: "s", Secret: "not-the-secret-at-all", Master: "wss://" + addr}
if err := slaveSession(ctx, wrongSecret, testutil.NewFakeNode("s")); err == nil {
t.Fatal("a slave with the wrong secret should be refused")
}
sameName := &config.Config{StateDir: t.TempDir()}
sameName.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Master: "wss://" + addr}
if err := slaveSession(ctx, sameName, testutil.NewFakeNode("master")); err == nil {
t.Fatal("a slave can't take the master's name")
}
}
+427
View File
@@ -0,0 +1,427 @@
package manager
import (
"context"
"errors"
"fmt"
"math"
"sort"
"sync"
"time"
"slackerbot/internal/balance"
"slackerbot/internal/config"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/state"
"slackerbot/internal/storage"
"slackerbot/internal/util"
)
var (
ErrTooBig = errors.New("bigger than max_container allows")
ErrTooSmall = errors.New("smaller than the minimum size")
MinContainer = model.Size{MemoryMB: 256, CPUs: 0.25}
)
type State string
const (
StateRunning State = "running"
StateStopped State = "stopped"
StateNone State = "not created"
StateOffline State = "on an offline node"
)
type Manager struct {
cfg *config.Config
store *state.Store
local node.Node
mu sync.RWMutex
remotes map[string]*Remote
onNodeBack func(node string)
startMu sync.Mutex
ownerMu util.KeyedMutex
migrating sync.Map
}
func New(cfg *config.Config, store *state.Store, local node.Node) *Manager {
return &Manager{cfg: cfg, store: store, local: local, remotes: map[string]*Remote{}}
}
func (m *Manager) SetNodeBackHandler(fn func(node string)) { m.onNodeBack = fn }
func (m *Manager) nodeByName(name string) (node.Node, error) {
if name == m.local.Name() {
return m.local, nil
}
m.mu.RLock()
r := m.remotes[name]
m.mu.RUnlock()
if r != nil && r.Online() {
return r, nil
}
return nil, &node.NodeOfflineError{Node: name}
}
func (m *Manager) onlineNodes() []node.Node {
nodes := []node.Node{m.local}
m.mu.RLock()
defer m.mu.RUnlock()
for _, r := range m.remotes {
if r.Online() {
nodes = append(nodes, r)
}
}
return nodes
}
func (m *Manager) OfflineNodes() []string {
m.mu.RLock()
defer m.mu.RUnlock()
var out []string
for name, r := range m.remotes {
if !r.Online() {
out = append(out, name)
}
}
sort.Strings(out)
return out
}
func (m *Manager) Views(ctx context.Context) ([]balance.NodeView, error) {
owners := m.store.Owners()
var views []balance.NodeView
for _, n := range m.onlineNodes() {
info, err := n.Info(ctx)
if err != nil {
if n == m.local {
return nil, err
}
continue
}
views = append(views, balance.BuildView(info, owners, m.cfg.Container))
}
sort.Slice(views, func(i, j int) bool { return views[i].Name < views[j].Name })
return views, nil
}
func (m *Manager) view(ctx context.Context, n node.Node) (balance.NodeView, node.NodeInfo, error) {
info, err := n.Info(ctx)
if err != nil {
return balance.NodeView{}, info, err
}
return balance.BuildView(info, m.store.Owners(), m.cfg.Container), info, nil
}
func (m *Manager) Totals(ctx context.Context) (used, usable model.Size) {
views, _ := m.Views(ctx)
for _, v := range views {
used, usable = used.Add(v.Used), usable.Add(v.Usable)
}
return
}
func (m *Manager) ownerRecord(id string, kind model.OwnerKind) (*model.Owner, error) {
if u := m.store.Owner(id); u != nil {
return u, nil
}
u := &model.Owner{
ID: id,
Kind: kind,
Machine: model.MachineName(id, kind),
Secret: util.RandomHex(32),
Size: m.cfg.Container,
StorageMinMB: m.cfg.Storage.MinMB,
StorageMaxMB: m.cfg.Storage.MaxMB,
LastActive: time.Now(),
}
return u, m.store.PutOwner(u)
}
func (m *Manager) Owner(id string, kind model.OwnerKind) (*model.Owner, error) {
unlock := m.ownerMu.Lock(id)
defer unlock()
return m.ownerRecord(id, kind)
}
func (m *Manager) spec(u *model.Owner) node.StartSpec {
return node.StartSpec{
Machine: u.Machine,
Description: "slackerbot container for " + string(u.Kind) + " " + u.ID,
Secret: u.Secret,
Size: u.Size,
Drive: u.Drive,
StorageMaxMB: u.StorageMaxMB,
Env: m.cfg.AgentEnv,
}
}
func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (*model.Owner, error) {
if m.IsMigrating(id) {
return nil, ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
u, err := m.ownerRecord(id, kind)
if err != nil {
return nil, err
}
if u.Node != "" {
n, err := m.nodeByName(u.Node)
if err != nil {
return u, err
}
if n.IsRunning(u.Machine) {
return u, nil
}
}
m.startMu.Lock()
defer m.startMu.Unlock()
if u.Node == "" {
views, err := m.Views(ctx)
if err != nil {
return u, err
}
target, drive, err := balance.ChoosePlacement(views, u.Size, u.StorageMinMB)
if err != nil {
return u, err
}
u.Node, u.Drive = target, drive
if err := m.store.SetPlacement(u.ID, target, drive); err != nil {
return u, err
}
}
n, err := m.nodeByName(u.Node)
if err != nil {
return u, err
}
v, _, err := m.view(ctx, n)
if err != nil {
return u, err
}
if !v.Used.Add(u.Size).Fits(v.Usable) {
return u, &node.NodeFullError{Node: u.Node}
}
return u, n.Start(ctx, m.spec(u))
}
// The old home folder stays on its node, untouched.
func (m *Manager) Recreate(id string) error {
if m.IsMigrating(id) {
return ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
if err := m.store.SetPlacement(id, "", ""); err != nil {
return err
}
return m.store.DeleteSessions(id)
}
func (m *Manager) DialAgent(ctx context.Context, u *model.Owner) (node.MsgConn, error) {
n, err := m.nodeByName(u.Node)
if err != nil {
return nil, err
}
return n.DialAgent(ctx, u.Machine, u.Secret)
}
func (m *Manager) State(u *model.Owner) State {
if u == nil || u.Node == "" {
return StateNone
}
n, err := m.nodeByName(u.Node)
if err != nil {
return StateOffline
}
if n.IsRunning(u.Machine) {
return StateRunning
}
return StateStopped
}
func (m *Manager) Poweroff(ctx context.Context, u *model.Owner) error {
n, err := m.nodeByName(u.Node)
if err != nil {
return err
}
return n.Poweroff(ctx, u.Machine)
}
func (m *Manager) RunningOwners(ctx context.Context) []model.Owner {
byNodeMachine := map[string]model.Owner{}
for _, u := range m.store.Owners() {
byNodeMachine[u.Node+"/"+u.Machine] = u
}
var out []model.Owner
for _, n := range m.onlineNodes() {
running, err := n.Running(ctx)
if err != nil {
continue
}
for _, r := range running {
if u, ok := byNodeMachine[n.Name()+"/"+r.Name]; ok {
out = append(out, u)
}
}
}
return out
}
func (m *Manager) Resize(ctx context.Context, id string, kind model.OwnerKind, change func(model.Size) (model.Size, error)) (old, next model.Size, err error) {
if m.IsMigrating(id) {
return old, old, ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
m.startMu.Lock()
defer m.startMu.Unlock()
u, err := m.ownerRecord(id, kind)
if err != nil {
return model.Size{}, model.Size{}, err
}
old = u.Size
if next, err = change(old); err != nil {
return old, old, err
}
next.CPUs = math.Round(next.CPUs*100) / 100
switch {
case !MinContainer.Fits(next):
return old, old, ErrTooSmall
case !next.Fits(m.cfg.MaxContainer):
return old, old, ErrTooBig
case next == old:
return old, old, nil
}
if u.Node != "" {
n, err := m.nodeByName(u.Node)
if err != nil {
return old, old, err
}
if n.IsRunning(u.Machine) {
v, _, err := m.view(ctx, n)
if err != nil {
return old, old, err
}
growth := model.Size{MemoryMB: max(0, next.MemoryMB-old.MemoryMB), CPUs: math.Max(0, next.CPUs-old.CPUs)}
if !v.Used.Add(growth).Fits(v.Usable) {
return old, old, &node.NodeFullError{Node: u.Node}
}
if err := n.SetLimits(ctx, u.Machine, next); err != nil {
return old, old, err
}
}
}
return old, next, m.store.SetSize(id, next)
}
type StorageChange struct {
OldMin, OldMax, NewMin, NewMax int64
UsedMB int64
Drive *storage.DriveReport
}
func (m *Manager) SetStorage(ctx context.Context, id string, kind model.OwnerKind, change func(min, max int64) (int64, int64, error)) (*StorageChange, error) {
if m.IsMigrating(id) {
return &StorageChange{}, ErrMigrating
}
unlock := m.ownerMu.Lock(id)
defer unlock()
m.startMu.Lock()
defer m.startMu.Unlock()
u, err := m.ownerRecord(id, kind)
if err != nil {
return nil, err
}
c := &StorageChange{OldMin: u.StorageMinMB, OldMax: u.StorageMaxMB}
if c.NewMin, c.NewMax, err = change(c.OldMin, c.OldMax); err != nil {
return c, err
}
switch {
case c.NewMin < config.MinStorageMB || c.NewMax < config.MinStorageMB:
return c, storage.ErrStorageTooSmall
case c.NewMin > c.NewMax:
return c, storage.ErrMinOverMax
case c.NewMax > m.cfg.Storage.LimitMB:
return c, storage.ErrStorageTooBig
}
if u.Node == "" {
views, err := m.Views(ctx)
if err != nil {
return c, err
}
if _, _, err := balance.ChoosePlacement(views, u.Size, c.NewMin); err != nil {
return c, err
}
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
}
n, err := m.nodeByName(u.Node)
if err != nil {
return c, err
}
v, info, err := m.view(ctx, n)
if err != nil {
return c, err
}
for _, d := range info.Drives {
if d.Path == u.Drive {
c.UsedMB = d.HomesMB[u.Machine]
}
}
if c.NewMax < c.UsedMB {
return c, storage.ErrBelowUsage
}
var r *storage.DriveReport
for i := range v.Drives {
if v.Drives[i].Path == u.Drive {
r = &v.Drives[i]
}
}
if r == nil {
return c, fmt.Errorf("drive %s is no longer configured on %s", u.Drive, u.Node)
}
extra := storage.Committed(c.NewMin, c.UsedMB) - storage.Committed(c.OldMin, c.UsedMB)
if extra > 0 && extra > r.HeadroomMB() {
return c, storage.ErrStorageFull
}
if err := n.SetQuota(ctx, u.Drive, u.Machine, c.NewMax); err != nil {
return c, err
}
r.CommittedMB += extra
r.BookedMB += c.NewMax - c.OldMax
c.Drive = r
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
}
func (m *Manager) StorageUsed(ctx context.Context, u *model.Owner) (int64, bool) {
if u == nil || u.Node == "" {
return 0, false
}
n, err := m.nodeByName(u.Node)
if err != nil {
return 0, false
}
info, err := n.Info(ctx)
if err != nil {
return 0, false
}
for _, d := range info.Drives {
if used, ok := d.HomesMB[u.Machine]; ok && d.Path == u.Drive {
return used, true
}
}
return 0, false
}
+237
View File
@@ -0,0 +1,237 @@
package manager
import (
"context"
"errors"
"fmt"
"log"
"slackerbot/internal/balance"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/storage"
)
const transferChunk = 4 << 20
var (
ErrMigrating = errors.New("this container is being moved")
ErrNoContainer = errors.New("there's no container here yet")
ErrSameNode = errors.New("the container is already on that node")
)
type UnknownNodeError struct{ Node string }
func (e *UnknownNodeError) Error() string { return fmt.Sprintf("there's no node called %s", e.Node) }
type MovePhase string
const (
PhaseCopy MovePhase = "copy"
PhaseFinal MovePhase = "final"
PhaseStart MovePhase = "start"
PhaseCleanup MovePhase = "cleanup"
)
type MoveProgress struct {
To string
Phase MovePhase
CopiedMB int64
TotalMB int64
}
type MoveOptions struct {
Target string
StartAfter bool
WaitIdle func(ctx context.Context)
Progress func(MoveProgress)
}
type MoveResult struct {
From, To string
WasRunning bool
Started bool
CopiedMB int64
LeftOld error
}
func (m *Manager) IsMigrating(id string) bool {
_, ok := m.migrating.Load(id)
return ok
}
// Migrate copies the owner's home folder to another node and switches the
// container over. A running container keeps running through the bulk copy
// and is only stopped for a final incremental sync.
func (m *Manager) Migrate(ctx context.Context, id string, opts MoveOptions) (*MoveResult, error) {
if _, busy := m.migrating.LoadOrStore(id, true); busy {
return nil, ErrMigrating
}
defer m.migrating.Delete(id)
unlock := m.ownerMu.Lock(id)
defer unlock()
u := m.store.Owner(id)
if u == nil || u.Node == "" {
return nil, ErrNoContainer
}
src, err := m.nodeByName(u.Node)
if err != nil {
return nil, err
}
res := &MoveResult{From: u.Node, WasRunning: src.IsRunning(u.Machine)}
needStart := res.WasRunning || opts.StartAfter
usedMB, _ := m.StorageUsed(ctx, u)
target, drive, err := m.pickTarget(ctx, u, opts.Target, needStart, storage.Committed(u.StorageMinMB, usedMB))
if err != nil {
return nil, err
}
res.To = target
dst, err := m.nodeByName(target)
if err != nil {
return nil, err
}
progress := func(p MovePhase, copied int64) {
if opts.Progress != nil {
opts.Progress(MoveProgress{To: target, Phase: p, CopiedMB: copied >> 20, TotalMB: usedMB})
}
}
imp, err := dst.ImportHome(ctx, drive, u.Machine)
if err != nil {
return nil, err
}
var exports []string
stopped := false
rollback := func(cause error) (*MoveResult, error) {
_ = dst.AbortImport(context.Background(), imp)
for _, e := range exports {
_ = src.CloseExport(context.Background(), e)
}
if stopped {
if err := src.Start(context.Background(), m.spec(u)); err != nil {
return res, fmt.Errorf("%w (and restarting it on %s failed: %v)", cause, u.Node, err)
}
}
return res, cause
}
var copied int64
send := func(parent string, phase MovePhase) error {
exp, err := src.ExportHome(ctx, u.Drive, u.Machine, parent)
if err != nil {
return err
}
exports = append(exports, exp)
for {
data, eof, err := src.ReadExport(ctx, exp, transferChunk)
if err != nil {
return err
}
if len(data) > 0 {
if err := dst.WriteImport(ctx, imp, data); err != nil {
return err
}
copied += int64(len(data))
progress(phase, copied)
}
if eof {
return dst.EndReceive(ctx, imp)
}
}
}
progress(PhaseCopy, 0)
if err := send("", PhaseCopy); err != nil {
return rollback(err)
}
if res.WasRunning {
progress(PhaseFinal, copied)
if opts.WaitIdle != nil {
opts.WaitIdle(ctx)
}
if err := src.Poweroff(ctx, u.Machine); err != nil {
return rollback(err)
}
stopped = true
if err := src.WaitStopped(ctx, u.Machine); err != nil {
return rollback(err)
}
if err := send(exports[0], PhaseFinal); err != nil {
return rollback(err)
}
}
if err := dst.FinishImport(ctx, imp, u.StorageMaxMB); err != nil {
return rollback(err)
}
res.CopiedMB = copied >> 20
oldNode, oldDrive := u.Node, u.Drive
if err := m.store.SetPlacement(id, target, drive); err != nil {
return rollback(err)
}
u.Node, u.Drive = target, drive
progress(PhaseCleanup, copied)
for _, e := range exports {
_ = src.CloseExport(ctx, e)
}
if res.LeftOld = src.DeleteHome(ctx, oldDrive, u.Machine); res.LeftOld != nil {
log.Printf("move: old home of %s left on %s: %v", u.Machine, oldNode, res.LeftOld)
}
if !needStart {
return res, nil
}
progress(PhaseStart, copied)
if err := dst.Start(ctx, m.spec(u)); err != nil {
return res, fmt.Errorf("moved to %s, but starting it there failed: %w", target, err)
}
res.Started = true
return res, nil
}
func (m *Manager) pickTarget(ctx context.Context, u *model.Owner, want string, needStart bool, storageMB int64) (target, drive string, err error) {
if want == u.Node {
return "", "", ErrSameNode
}
m.startMu.Lock()
defer m.startMu.Unlock()
views, err := m.Views(ctx)
if err != nil {
return "", "", err
}
compute := model.Size{}
if needStart {
compute = u.Size
}
var candidates []balance.NodeView
for _, v := range views {
if v.Name != u.Node && (want == "" || v.Name == want) {
candidates = append(candidates, v)
}
}
if want != "" && len(candidates) == 0 {
if m.KnownNode(want) {
return "", "", &node.NodeOfflineError{Node: want}
}
return "", "", &UnknownNodeError{Node: want}
}
target, drive, err = balance.ChoosePlacement(candidates, compute, storageMB)
if err != nil && want != "" {
return "", "", &node.NodeFullError{Node: want}
}
return target, drive, err
}
func (m *Manager) KnownNode(name string) bool {
if name == m.local.Name() {
return true
}
m.mu.RLock()
defer m.mu.RUnlock()
_, ok := m.remotes[name]
return ok
}
+159
View File
@@ -0,0 +1,159 @@
package manager_test
import (
"bytes"
"context"
"errors"
"testing"
"slackerbot/internal/manager"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/testutil"
"slackerbot/internal/testutil/clustertest"
)
func TestMoveLiveContainer(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), true)
var phases []manager.MovePhase
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{
Target: "slave-1",
WaitIdle: func(context.Context) {
c.Master.Mu.Lock()
c.Master.Homes["/d/c-c1"] = append(c.Master.Homes["/d/c-c1"], []byte("written during the bulk copy")...)
c.Master.Mu.Unlock()
},
Progress: func(p manager.MoveProgress) {
if len(phases) == 0 || phases[len(phases)-1] != p.Phase {
phases = append(phases, p.Phase)
}
},
})
if err != nil {
t.Fatal(err)
}
want := append(testutil.HomeData(), []byte("written during the bulk copy")...)
if !bytes.Equal(c.Slave.Homes["/d/c-c1"], want) {
t.Fatal("the late write didn't make it to the new node")
}
if c.Slave.Passes != 2 {
t.Fatalf("a live move should copy twice (bulk + final sync), got %d", c.Slave.Passes)
}
if _, still := c.Master.Homes["/d/c-c1"]; still {
t.Fatal("old home should be deleted after a good move")
}
if !c.Slave.IsRunning("c-c1") || c.Master.IsRunning("c-c1") || !res.Started {
t.Fatal("container should be running on the new node only")
}
if u := c.Store.Owner("C1"); u.Node != "slave-1" || u.Drive != "/d" {
t.Fatalf("placement not switched: %+v", u)
}
if len(c.Master.Exports) != 0 {
t.Fatal("snapshots on the old node should be cleaned up")
}
wantPhases := []manager.MovePhase{manager.PhaseCopy, manager.PhaseFinal, manager.PhaseCleanup, manager.PhaseStart}
if len(phases) != len(wantPhases) {
t.Fatalf("phases %v, want %v", phases, wantPhases)
}
}
func TestMoveAutoStoppedContainer(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), false)
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{})
if err != nil {
t.Fatal(err)
}
if res.To != "slave-1" || res.Started || c.Slave.IsRunning("c-c1") {
t.Fatalf("a stopped container should move without starting: %+v", res)
}
if c.Slave.Passes != 1 {
t.Fatalf("a stopped container needs one copy, got %d", c.Slave.Passes)
}
}
func TestMoveFullNodeStartsElsewhere(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), false)
c.Master.Mu.Lock()
c.Master.Usable = model.Size{MemoryMB: 512, CPUs: 0.5}
c.Master.Mu.Unlock()
_, err := c.Mgr.Ensure(c.Ctx, "C1", model.OwnerChannel)
var full *node.NodeFullError
if !errors.As(err, &full) || full.Node != "master" {
t.Fatalf("want node full on master, got %v", err)
}
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{StartAfter: true})
if err != nil || !res.Started || !c.Slave.IsRunning("c-c1") {
t.Fatalf("auto-move should start it on the new node: %+v %v", res, err)
}
}
func TestMoveFailureRollsBack(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), true)
_, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{
Target: "slave-1",
WaitIdle: func(context.Context) {
c.Slave.Mu.Lock()
c.Slave.FailWrites = true
c.Slave.Mu.Unlock()
},
})
if err == nil {
t.Fatal("expected the final sync to fail")
}
if u := c.Store.Owner("C1"); u.Node != "master" {
t.Fatalf("container should stay on its old node, got %s", u.Node)
}
if !c.Master.IsRunning("c-c1") {
t.Fatal("container should be restarted on its old node")
}
if _, ok := c.Master.Homes["/d/c-c1"]; !ok {
t.Fatal("old home must survive a failed move")
}
if _, ok := c.Slave.Homes["/d/c-c1"]; ok || len(c.Slave.Imports) != 0 {
t.Fatal("half-copied home on the target should be cleaned up")
}
}
func TestMoveRefusals(t *testing.T) {
c := clustertest.New(t)
c.AddOwner(t, testutil.HomeData(), false)
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "master"}); !errors.Is(err, manager.ErrSameNode) {
t.Fatalf("same node: %v", err)
}
var unknown *manager.UnknownNodeError
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "nope"}); !errors.As(err, &unknown) {
t.Fatalf("unknown node: %v", err)
}
if _, err := c.Mgr.Migrate(c.Ctx, "nobody", manager.MoveOptions{}); !errors.Is(err, manager.ErrNoContainer) {
t.Fatalf("no container: %v", err)
}
c.Slave.Mu.Lock()
c.Slave.Usable = model.Size{MemoryMB: 512, CPUs: 0.5}
c.Slave.Mu.Unlock()
c.Master.Mu.Lock()
c.Master.Runs["c-c1"] = true
c.Master.Mu.Unlock()
testutil.WaitFor(t, "slave report", func() bool {
views, _ := c.Mgr.Views(c.Ctx)
for _, v := range views {
if v.Name == "slave-1" {
return v.Usable.MemoryMB == 512
}
}
return false
})
var full *node.NodeFullError
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "slave-1"}); !errors.As(err, &full) {
t.Fatalf("full target: %v", err)
}
}
+74
View File
@@ -0,0 +1,74 @@
package model
import (
"fmt"
"strconv"
"strings"
"time"
)
type Size struct {
MemoryMB int `toml:"memory_mb"`
CPUs float64 `toml:"cpus"`
}
func (s Size) Add(o Size) Size { return Size{s.MemoryMB + o.MemoryMB, s.CPUs + o.CPUs} }
func (s Size) Sub(o Size) Size { return Size{s.MemoryMB - o.MemoryMB, s.CPUs - o.CPUs} }
func (s Size) Fits(in Size) bool {
return s.MemoryMB <= in.MemoryMB && s.CPUs <= in.CPUs+1e-9
}
func (s Size) String() string {
mem := fmt.Sprintf("%d MB", s.MemoryMB)
if s.MemoryMB >= 1024 && s.MemoryMB%1024 == 0 {
mem = fmt.Sprintf("%d GB", s.MemoryMB/1024)
}
cpu := strconv.FormatFloat(s.CPUs, 'f', -1, 64) + " CPUs"
if s.CPUs == 1 {
cpu = "1 CPU"
}
return mem + " RAM, " + cpu
}
type OwnerKind string
const (
OwnerUser OwnerKind = "user"
OwnerChannel OwnerKind = "channel"
)
type Owner struct {
ID string `json:"id"`
Kind OwnerKind `json:"kind"`
Machine string `json:"machine"`
Secret string `json:"secret"`
Size Size `json:"size"`
LastActive time.Time `json:"last_active"`
Node string `json:"node,omitempty"`
Drive string `json:"drive,omitempty"`
StorageMinMB int64 `json:"storage_min_mb"`
StorageMaxMB int64 `json:"storage_max_mb"`
// 0 means the config default.
IdleTimeout time.Duration `json:"idle_timeout,omitempty"`
}
const Indefinite time.Duration = -1
type Session struct {
Channel string `json:"channel"`
ThreadTS string `json:"thread_ts"`
OwnerID string `json:"owner_id"`
ACPID string `json:"acp_id"`
Title string `json:"title"`
CreatedAt time.Time `json:"created_at"`
}
func MachineName(id string, kind OwnerKind) string {
if kind == OwnerChannel {
return "c-" + strings.ToLower(id)
}
return "u-" + strings.ToLower(id)
}
func IsOurs(name string) bool { return strings.HasPrefix(name, "u-") || strings.HasPrefix(name, "c-") }
+244
View File
@@ -0,0 +1,244 @@
package model
import (
"errors"
"fmt"
"maps"
"slices"
"sort"
"strings"
)
type Action string
const (
ActScale Action = "scale"
ActStorage Action = "storage"
ActMove Action = "move"
ActKeep Action = "keep"
ActManage Action = "manage"
)
var (
AllActions = []Action{ActScale, ActStorage, ActMove, ActKeep}
GrantActions = append(slices.Clone(AllActions), ActManage)
ErrLastManager = errors.New("that would leave nobody able to manage permissions")
)
type PermissionsConfig struct {
Managers []string `toml:"managers" json:"managers"`
Default map[string][]string `toml:"default" json:"default"`
Channels map[string]map[string][]string `toml:"channels" json:"channels"`
DM map[string]bool `toml:"dm" json:"dm"`
}
func (p PermissionsConfig) Validate() error {
if len(p.Managers) == 0 {
return fmt.Errorf("permissions.managers must name at least one user manager")
}
if len(p.Default) == 0 {
return fmt.Errorf("[permissions.default] must name at least one super user, e.g. U0ADMIN = [\"all\"]")
}
check := func(where string, list map[string][]string) error {
for user, actions := range list {
for _, a := range actions {
if a != "all" && !slices.Contains(GrantActions, Action(a)) {
return fmt.Errorf("%s: unknown permission %q for %s (use %s, manage or all)", where, a, user, actionNames())
}
}
}
return nil
}
if err := check("permissions.default", p.Default); err != nil {
return err
}
for channel, list := range p.Channels {
if err := check("permissions.channels."+channel, list); err != nil {
return err
}
}
for a := range p.DM {
if !slices.Contains(AllActions, Action(a)) {
return fmt.Errorf("permissions.dm: unknown permission %q (use %s)", a, actionNames())
}
}
return nil
}
func (p PermissionsConfig) Clone() PermissionsConfig {
c := PermissionsConfig{
Managers: slices.Clone(p.Managers),
Default: cloneList(p.Default),
Channels: map[string]map[string][]string{},
DM: maps.Clone(p.DM),
}
for ch, list := range p.Channels {
c.Channels[ch] = cloneList(list)
}
if c.DM == nil {
c.DM = map[string]bool{}
}
return c
}
func cloneList(list map[string][]string) map[string][]string {
out := map[string][]string{}
for user, actions := range list {
out[user] = slices.Clone(actions)
}
return out
}
func actionNames() string {
names := make([]string, len(AllActions))
for i, a := range AllActions {
names[i] = string(a)
}
return strings.Join(names, ", ")
}
func (p PermissionsConfig) SuperUsers(channel string) (list map[string][]string, own bool) {
if l := p.Channels[channel]; len(l) > 0 {
return l, true
}
return p.Default, false
}
// "all" covers the container actions only; managing permissions is always granted by name.
func Grants(actions []string, a Action) bool {
if a == ActManage {
return slices.Contains(actions, string(ActManage))
}
return slices.Contains(actions, "all") || slices.Contains(actions, string(a))
}
func (p PermissionsConfig) Allowed(kind OwnerKind, channel, user string, a Action) bool {
if kind == OwnerUser {
allowed, set := p.DM[string(a)]
return !set || allowed
}
list, _ := p.SuperUsers(channel)
return Grants(list[user], a)
}
func (p PermissionsConfig) IsManager(user string) bool { return slices.Contains(p.Managers, user) }
func (p PermissionsConfig) CanManage(channel, user string) bool {
return p.IsManager(user) || p.Allowed(OwnerChannel, channel, user, ActManage)
}
func (p PermissionsConfig) Holders(channel string, a Action) []string {
list, _ := p.SuperUsers(channel)
var out []string
for user, actions := range list {
if Grants(actions, a) {
out = append(out, user)
}
}
sort.Strings(out)
return out
}
func (p PermissionsConfig) ActionsFor(channel, user string) []Action {
list, _ := p.SuperUsers(channel)
var out []Action
for _, a := range GrantActions {
if Grants(list[user], a) {
out = append(out, a)
}
}
return out
}
func expand(actions []Action) []Action {
if slices.Contains(actions, "all") {
out := slices.Clone(AllActions)
for _, a := range actions {
if a != "all" && !slices.Contains(out, a) {
out = append(out, a)
}
}
return out
}
return actions
}
// An empty channel means the default list. The first grant in a channel that
// still uses the default list copies it, so nobody loses access there.
func (p *PermissionsConfig) Grant(channel string, users []string, actions []Action) {
list := p.Default
if channel != "" {
if len(p.Channels[channel]) == 0 {
p.Channels[channel] = cloneList(p.Default)
}
list = p.Channels[channel]
}
for _, user := range users {
have := expandStrings(list[user])
for _, a := range expand(actions) {
if !slices.Contains(have, string(a)) {
have = append(have, string(a))
}
}
list[user] = have
}
}
// No actions means everything.
func (p *PermissionsConfig) Revoke(channel string, users []string, actions []Action) {
list := p.Default
if channel != "" {
if len(p.Channels[channel]) == 0 {
p.Channels[channel] = cloneList(p.Default)
}
list = p.Channels[channel]
}
remove := expand(actions)
for _, user := range users {
if len(actions) == 0 {
delete(list, user)
continue
}
var keep []string
for _, a := range expandStrings(list[user]) {
if !slices.Contains(remove, Action(a)) {
keep = append(keep, a)
}
}
if len(keep) == 0 {
delete(list, user)
} else {
list[user] = keep
}
}
}
func expandStrings(actions []string) []string {
as := make([]Action, len(actions))
for i, a := range actions {
as[i] = Action(a)
}
var out []string
for _, a := range expand(as) {
out = append(out, string(a))
}
return out
}
func (p *PermissionsConfig) AddManagers(users []string) {
for _, u := range users {
if !p.IsManager(u) {
p.Managers = append(p.Managers, u)
}
}
}
func (p *PermissionsConfig) RemoveManagers(users []string) error {
keep := slices.DeleteFunc(slices.Clone(p.Managers), func(u string) bool { return slices.Contains(users, u) })
if len(keep) == 0 {
return ErrLastManager
}
p.Managers = keep
return nil
}
@@ -0,0 +1,105 @@
package model_test
import (
"errors"
"testing"
"slackerbot/internal/config"
"slackerbot/internal/model"
)
func TestPermissions(t *testing.T) {
t.Setenv("SLACK_APP_TOKEN", "xapp-test")
t.Setenv("SLACK_BOT_TOKEN", "xoxb-test")
t.Setenv("CLUSTER_SECRET", "cluster-secret-1234567890")
c, err := config.LoadConfig("../../../config.example.toml", config.ModeMaster)
if err != nil {
t.Fatal(err)
}
p := c.Permissions
cases := []struct {
kind model.OwnerKind
channel, user string
action model.Action
want bool
}{
{model.OwnerChannel, "C07ABC123", "U0111", model.ActMove, true},
{model.OwnerChannel, "C07ABC123", "U0111", model.ActScale, false},
{model.OwnerChannel, "C07ABC123", "U0222", model.ActKeep, true},
{model.OwnerChannel, "C07ABC123", "U0ADMIN", model.ActMove, false},
{model.OwnerChannel, "C0OTHER", "U0ADMIN", model.ActStorage, true},
{model.OwnerChannel, "C0OTHER", "U0111", model.ActMove, false},
{model.OwnerUser, "D1", "U9", model.ActScale, true},
{model.OwnerUser, "D1", "U9", model.ActStorage, false},
}
for _, c := range cases {
if got := p.Allowed(c.kind, c.channel, c.user, c.action); got != c.want {
t.Errorf("%s %s %s %s = %v, want %v", c.kind, c.channel, c.user, c.action, got, c.want)
}
}
if h := p.Holders("C07ABC123", model.ActMove); len(h) != 2 {
t.Errorf("move holders = %v", h)
}
if a := p.ActionsFor("C0OTHER", "U0ADMIN"); len(a) != len(model.AllActions) {
t.Errorf("all should expand to every action, got %v", a)
}
open := model.PermissionsConfig{Default: map[string][]string{"U1": {"all"}}}
if !open.Allowed(model.OwnerUser, "D1", "U9", model.ActMove) {
t.Error("DM permissions should default to allowed")
}
if (model.PermissionsConfig{}).Validate() == nil {
t.Error("a missing default super user should be refused")
}
bad := model.PermissionsConfig{Default: map[string][]string{"U1": {"delete-everything"}}}
if bad.Validate() == nil {
t.Error("unknown permission names should be refused")
}
}
func TestGrantRevoke(t *testing.T) {
p := model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UDEF": {"all"}}}.Clone()
p.Grant("C1", []string{"UANA"}, []model.Action{model.ActMove})
if !p.Allowed(model.OwnerChannel, "C1", "UDEF", model.ActScale) {
t.Fatal("first grant in a channel should keep everyone from the default list")
}
if !p.Allowed(model.OwnerChannel, "C1", "UANA", model.ActMove) || p.Allowed(model.OwnerChannel, "C1", "UANA", model.ActScale) {
t.Fatal("grant should add exactly what was asked")
}
if p.Allowed(model.OwnerChannel, "C2", "UANA", model.ActMove) {
t.Fatal("grants stay in their channel")
}
p.Grant("C1", []string{"UANA"}, []model.Action{"all"})
if p.CanManage("C1", "UANA") {
t.Fatal("all must not include manage")
}
p.Grant("C1", []string{"UANA"}, []model.Action{model.ActManage})
if !p.CanManage("C1", "UANA") || p.CanManage("C2", "UANA") {
t.Fatal("manage is per channel")
}
p.Revoke("C1", []string{"UDEF"}, []model.Action{model.ActScale})
if p.Allowed(model.OwnerChannel, "C1", "UDEF", model.ActScale) || !p.Allowed(model.OwnerChannel, "C1", "UDEF", model.ActMove) {
t.Fatal("revoking one action out of all should keep the rest")
}
p.Revoke("C1", []string{"UANA"}, nil)
if len(p.ActionsFor("C1", "UANA")) != 0 {
t.Fatal("revoke with no actions removes the person")
}
p.Grant("", []string{"UNEW"}, []model.Action{model.ActKeep})
if !p.Allowed(model.OwnerChannel, "C9", "UNEW", model.ActKeep) {
t.Fatal("default grants apply to channels without their own list")
}
p.AddManagers([]string{"UTWO"})
if err := p.RemoveManagers([]string{"UBOSS", "UTWO"}); !errors.Is(err, model.ErrLastManager) {
t.Fatal("removing every manager must be refused")
}
if err := p.RemoveManagers([]string{"UBOSS"}); err != nil || p.IsManager("UBOSS") {
t.Fatal("removing one of two managers should work")
}
}
+103
View File
@@ -0,0 +1,103 @@
package node
import (
"context"
"fmt"
"net/http"
"time"
"github.com/coder/websocket"
"slackerbot/internal/model"
)
type Node interface {
Name() string
Info(ctx context.Context) (NodeInfo, error)
Running(ctx context.Context) ([]RunningMachine, error)
IsRunning(machine string) bool
Start(ctx context.Context, spec StartSpec) error
Poweroff(ctx context.Context, machine string) error
SetLimits(ctx context.Context, machine string, size model.Size) error
SetQuota(ctx context.Context, drive, machine string, mb int64) error
DialAgent(ctx context.Context, machine, secret string) (MsgConn, error)
WaitStopped(ctx context.Context, machine string) error
ExportHome(ctx context.Context, drive, machine, parent string) (id string, err error)
ReadExport(ctx context.Context, id string, max int) (data []byte, eof bool, err error)
CloseExport(ctx context.Context, id string) error
ImportHome(ctx context.Context, drive, machine string) (id string, err error)
WriteImport(ctx context.Context, id string, data []byte) error
EndReceive(ctx context.Context, id string) error
FinishImport(ctx context.Context, id string, quotaMB int64) error
AbortImport(ctx context.Context, id string) error
DeleteHome(ctx context.Context, drive, machine string) error
}
type NodeInfo struct {
Name string `json:"name"`
Total model.Size `json:"total"`
Usable model.Size `json:"usable"`
Drives []DriveInfo `json:"drives"`
Running []RunningMachine `json:"running"`
}
type DriveInfo struct {
Path string `json:"path"`
FreeMB int64 `json:"free_mb"`
ReserveMB int64 `json:"reserve_mb"`
HomesMB map[string]int64 `json:"homes_mb"`
}
type RunningMachine struct {
Name string `json:"name"`
Since time.Time `json:"since"`
}
type StartSpec struct {
Machine string `json:"machine"`
Description string `json:"description"`
Secret string `json:"secret"`
Size model.Size `json:"size"`
Drive string `json:"drive"`
StorageMaxMB int64 `json:"storage_max_mb"`
Env map[string]string `json:"env"`
}
type NodeOfflineError struct{ Node string }
func (e *NodeOfflineError) Error() string { return fmt.Sprintf("node %s is offline", e.Node) }
type NodeFullError struct{ Node string }
func (e *NodeFullError) Error() string { return fmt.Sprintf("node %s is full", e.Node) }
type MsgConn interface {
Read(ctx context.Context) ([]byte, error)
Write(ctx context.Context, b []byte) error
Close() error
}
type wsConn struct{ c *websocket.Conn }
func (w wsConn) Read(ctx context.Context) ([]byte, error) {
_, b, err := w.c.Read(ctx)
return b, err
}
func (w wsConn) Write(ctx context.Context, b []byte) error {
return w.c.Write(ctx, websocket.MessageText, b)
}
func (w wsConn) Close() error { return w.c.Close(websocket.StatusNormalClosure, "") }
func DialAgentWS(ctx context.Context, addr, secret string) (MsgConn, error) {
conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/acp", &websocket.DialOptions{
HTTPHeader: http.Header{"X-Secret-Key": []string{secret}},
})
if err != nil {
return nil, fmt.Errorf("connecting to goose: %w", err)
}
conn.SetReadLimit(32 << 20)
return wsConn{conn}, nil
}
+237
View File
@@ -0,0 +1,237 @@
package state
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"sort"
"sync"
"time"
"slackerbot/internal/model"
)
type Store struct {
path string
mu sync.Mutex
data struct {
Owners map[string]*model.Owner `json:"owners"`
Sessions map[string]*model.Session `json:"sessions"`
Permissions *model.PermissionsConfig `json:"permissions,omitempty"`
}
}
func Open(path string) (*Store, error) {
s := &Store{path: path}
s.data.Owners = map[string]*model.Owner{}
s.data.Sessions = map[string]*model.Session{}
b, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return s, os.MkdirAll(filepath.Dir(path), 0o700)
}
if err != nil {
return nil, err
}
if err := json.Unmarshal(b, &s.data); err != nil {
return nil, err
}
if s.data.Owners == nil {
s.data.Owners = map[string]*model.Owner{}
}
if s.data.Sessions == nil {
s.data.Sessions = map[string]*model.Session{}
}
return s, nil
}
func (s *Store) save() error {
b, err := json.MarshalIndent(&s.data, "", " ")
if err != nil {
return err
}
tmp := s.path + ".tmp"
if err := os.WriteFile(tmp, b, 0o600); err != nil {
return err
}
return os.Rename(tmp, s.path)
}
func (s *Store) Owner(id string) *model.Owner {
s.mu.Lock()
defer s.mu.Unlock()
if u, ok := s.data.Owners[id]; ok {
c := *u
return &c
}
return nil
}
func (s *Store) OwnerByMachine(machine string) *model.Owner {
s.mu.Lock()
defer s.mu.Unlock()
for _, u := range s.data.Owners {
if u.Machine == machine {
c := *u
return &c
}
}
return nil
}
func (s *Store) PutOwner(u *model.Owner) error {
s.mu.Lock()
defer s.mu.Unlock()
c := *u
s.data.Owners[u.ID] = &c
return s.save()
}
func (s *Store) Touch(id string) error {
s.mu.Lock()
defer s.mu.Unlock()
if u, ok := s.data.Owners[id]; ok {
u.LastActive = time.Now()
return s.save()
}
return nil
}
func (s *Store) SetSize(id string, size model.Size) error {
s.mu.Lock()
defer s.mu.Unlock()
if u, ok := s.data.Owners[id]; ok {
u.Size = size
return s.save()
}
return nil
}
func (s *Store) Owners() []model.Owner {
s.mu.Lock()
defer s.mu.Unlock()
out := make([]model.Owner, 0, len(s.data.Owners))
for _, u := range s.data.Owners {
out = append(out, *u)
}
return out
}
func (s *Store) SetStorage(id string, minMB, maxMB int64) error {
s.mu.Lock()
defer s.mu.Unlock()
if u, ok := s.data.Owners[id]; ok {
u.StorageMinMB, u.StorageMaxMB = minMB, maxMB
return s.save()
}
return nil
}
func (s *Store) SetPlacement(id, node, drive string) error {
s.mu.Lock()
defer s.mu.Unlock()
if u, ok := s.data.Owners[id]; ok {
u.Node, u.Drive = node, drive
return s.save()
}
return nil
}
func (s *Store) DeleteSessions(ownerID string) error {
s.mu.Lock()
defer s.mu.Unlock()
for k, x := range s.data.Sessions {
if x.OwnerID == ownerID {
delete(s.data.Sessions, k)
}
}
return s.save()
}
func (s *Store) SetIdleTimeout(id string, d time.Duration) error {
s.mu.Lock()
defer s.mu.Unlock()
if u, ok := s.data.Owners[id]; ok {
u.IdleTimeout = d
return s.save()
}
return nil
}
// The config's permissions are only a starting point: once saved here,
// changes made in Slack win over later config edits.
func (s *Store) SeedPermissions(initial model.PermissionsConfig) (seeded bool, err error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.data.Permissions != nil {
return false, nil
}
p := initial.Clone()
s.data.Permissions = &p
return true, s.save()
}
func (s *Store) Permissions() model.PermissionsConfig {
s.mu.Lock()
defer s.mu.Unlock()
if s.data.Permissions == nil {
return model.PermissionsConfig{}
}
return s.data.Permissions.Clone()
}
func (s *Store) UpdatePermissions(change func(p *model.PermissionsConfig) error) error {
s.mu.Lock()
defer s.mu.Unlock()
var p model.PermissionsConfig
if s.data.Permissions != nil {
p = s.data.Permissions.Clone()
} else {
p = model.PermissionsConfig{}.Clone()
}
if err := change(&p); err != nil {
return err
}
if err := p.Validate(); err != nil {
return err
}
s.data.Permissions = &p
return s.save()
}
func sessionKey(channel, threadTS string) string { return channel + "/" + threadTS }
func (s *Store) Session(channel, threadTS string) *model.Session {
s.mu.Lock()
defer s.mu.Unlock()
if x, ok := s.data.Sessions[sessionKey(channel, threadTS)]; ok {
c := *x
return &c
}
return nil
}
func (s *Store) AddSession(x *model.Session) error {
s.mu.Lock()
defer s.mu.Unlock()
c := *x
s.data.Sessions[sessionKey(x.Channel, x.ThreadTS)] = &c
return s.save()
}
func (s *Store) Sessions(ownerID string, limit int) []model.Session {
s.mu.Lock()
defer s.mu.Unlock()
var out []model.Session
for _, x := range s.data.Sessions {
if x.OwnerID == ownerID {
out = append(out, *x)
}
}
sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) })
if limit > 0 && len(out) > limit {
out = out[:limit]
}
return out
}
+66
View File
@@ -0,0 +1,66 @@
package state
import (
"path/filepath"
"testing"
"time"
"slackerbot/internal/model"
)
func TestStore(t *testing.T) {
path := filepath.Join(t.TempDir(), "state.json")
s, err := Open(path)
if err != nil {
t.Fatal(err)
}
_ = s.PutOwner(&model.Owner{ID: "C1", Kind: model.OwnerChannel, Machine: "c-c1", Size: model.Size{MemoryMB: 1024, CPUs: 1}})
_ = s.AddSession(&model.Session{Channel: "D1", ThreadTS: "1.0", OwnerID: "C1", ACPID: "a", CreatedAt: time.Now().Add(-time.Hour)})
_ = s.AddSession(&model.Session{Channel: "D1", ThreadTS: "2.0", OwnerID: "C1", ACPID: "b", CreatedAt: time.Now()})
s2, err := Open(path)
if err != nil {
t.Fatal(err)
}
if u := s2.OwnerByMachine("c-c1"); u == nil || u.Size.MemoryMB != 1024 {
t.Fatal("user not saved")
}
list := s2.Sessions("C1", 0)
if len(list) != 2 || list[0].ACPID != "b" {
t.Fatalf("sessions not newest-first: %+v", list)
}
if s2.Session("D1", "1.0").ACPID != "a" {
t.Fatal("session lookup failed")
}
}
func TestPermissionsSavedAndSeededOnce(t *testing.T) {
path := filepath.Join(t.TempDir(), "state.json")
s, _ := Open(path)
initial := model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UBOSS": {"all"}}}
if seeded, err := s.SeedPermissions(initial); !seeded || err != nil {
t.Fatalf("first start should seed: %v %v", seeded, err)
}
if err := s.UpdatePermissions(func(p *model.PermissionsConfig) error {
p.Grant("C1", []string{"UANA"}, []model.Action{model.ActMove})
return nil
}); err != nil {
t.Fatal(err)
}
s2, _ := Open(path)
changed := model.PermissionsConfig{Managers: []string{"USOMEONE"}, Default: map[string][]string{"USOMEONE": {"all"}}}
if seeded, _ := s2.SeedPermissions(changed); seeded {
t.Fatal("config must not override saved permissions")
}
p := s2.Permissions()
if !p.IsManager("UBOSS") || p.IsManager("USOMEONE") || !p.Allowed(model.OwnerChannel, "C1", "UANA", model.ActMove) {
t.Fatalf("saved permissions lost: %+v", p)
}
if err := s2.UpdatePermissions(func(p *model.PermissionsConfig) error {
p.Default["UX"] = []string{"teleport"}
return nil
}); err == nil {
t.Fatal("invalid changes must be refused")
}
}
+119
View File
@@ -0,0 +1,119 @@
package storage
import (
"errors"
"fmt"
"os"
"os/exec"
"strconv"
"strings"
"syscall"
)
const btrfsMagic = 0x9123683E
var (
ErrStorageFull = errors.New("not enough storage left on the drives")
ErrStorageTooBig = errors.New("above the storage limit")
ErrMinOverMax = errors.New("the guaranteed amount can't be more than the max")
ErrBelowUsage = errors.New("below what's already in use")
ErrStorageTooSmall = errors.New("too small")
)
type DriveReport struct {
Path string
UsableMB int64
CommittedMB int64
BookedMB int64
Homes int
}
func (d DriveReport) HeadroomMB() int64 { return d.UsableMB - d.CommittedMB }
func (d DriveReport) Overbooked() bool { return d.BookedMB > d.UsableMB }
func Committed(minMB, usedMB int64) int64 { return max(minMB, usedMB) }
func PickDrive(reports []DriveReport, needMB int64) (string, error) {
best := -1
for i, r := range reports {
if r.HeadroomMB() >= needMB && (best < 0 || r.HeadroomMB() > reports[best].HeadroomMB()) {
best = i
}
}
if best < 0 {
return "", ErrStorageFull
}
return reports[best].Path, nil
}
func PrepareDrive(path string) error {
if err := os.MkdirAll(path, 0o700); err != nil {
return err
}
var st syscall.Statfs_t
if err := syscall.Statfs(path, &st); err != nil {
return err
}
if uint32(st.Type) != btrfsMagic {
return fmt.Errorf("%s is not on a btrfs filesystem", path)
}
if out, err := exec.Command("btrfs", "quota", "enable", path).CombinedOutput(); err != nil {
return fmt.Errorf("turning on btrfs quotas for %s: %v: %s", path, err, strings.TrimSpace(string(out)))
}
return nil
}
func FreeMB(path string) (int64, error) {
var st syscall.Statfs_t
if err := syscall.Statfs(path, &st); err != nil {
return 0, err
}
return int64(st.Bavail) * int64(st.Bsize) / (1 << 20), nil
}
func CreateSubvolume(path string) error {
if out, err := exec.Command("btrfs", "subvolume", "create", path).CombinedOutput(); err != nil {
return fmt.Errorf("creating %s: %v: %s", path, err, strings.TrimSpace(string(out)))
}
return nil
}
func DeleteSubvolume(path string) error {
if out, err := exec.Command("btrfs", "subvolume", "delete", path).CombinedOutput(); err != nil {
return fmt.Errorf("deleting %s: %v: %s", path, err, strings.TrimSpace(string(out)))
}
return nil
}
func SetQuota(path string, mb int64) error {
bytes := strconv.FormatInt(mb<<20, 10)
if out, err := exec.Command("btrfs", "qgroup", "limit", bytes, path).CombinedOutput(); err != nil {
return fmt.Errorf("setting quota on %s: %v: %s", path, err, strings.TrimSpace(string(out)))
}
return nil
}
// Quota numbers can lag a few seconds behind writes.
func UsedMB(path string) (int64, error) {
out, err := exec.Command("btrfs", "qgroup", "show", "-f", "--raw", path).CombinedOutput()
if err != nil {
return 0, fmt.Errorf("reading usage of %s: %v: %s", path, err, strings.TrimSpace(string(out)))
}
return parseQgroupShow(string(out))
}
// Parses `btrfs qgroup show --raw`: "0/257 <referenced bytes> <exclusive bytes> ...".
func parseQgroupShow(out string) (int64, error) {
for _, line := range strings.Split(out, "\n") {
f := strings.Fields(line)
if len(f) >= 2 && strings.HasPrefix(f[0], "0/") {
b, err := strconv.ParseInt(f[1], 10, 64)
if err != nil {
return 0, err
}
return b >> 20, nil
}
}
return 0, errors.New("no quota group found")
}
@@ -0,0 +1,32 @@
package storage
import (
"errors"
"testing"
)
func TestDriveMath(t *testing.T) {
d := DriveReport{Path: "/a", UsableMB: 100_000, CommittedMB: 90_000, BookedMB: 150_000}
if d.HeadroomMB() != 10_000 || !d.Overbooked() {
t.Fatalf("headroom %d overbooked %v", d.HeadroomMB(), d.Overbooked())
}
if Committed(5120, 8000) != 8000 || Committed(5120, 100) != 5120 {
t.Fatal("a home holds its guarantee or its real use, whichever is bigger")
}
drives := []DriveReport{d, {Path: "/b", UsableMB: 50_000, CommittedMB: 20_000}}
if got, _ := PickDrive(drives, 5120); got != "/b" {
t.Fatalf("should pick the drive with the most room, got %s", got)
}
if _, err := PickDrive(drives, 40_000); !errors.Is(err, ErrStorageFull) {
t.Fatal("a guarantee that fits nowhere should be rejected")
}
}
func TestParseQgroupShow(t *testing.T) {
out := "Qgroupid Referenced Exclusive Max referenced Max exclusive Path\n" +
"-------- ---------- --------- -------------- ------------- ----\n" +
"0/257 3221225472 3221225472 10737418240 none c-c07abc\n"
if mb, err := parseQgroupShow(out); err != nil || mb != 3072 {
t.Fatalf("got %d, %v", mb, err)
}
}
@@ -0,0 +1,59 @@
package clustertest
import (
"context"
"path/filepath"
"testing"
"slackerbot/internal/config"
"slackerbot/internal/manager"
"slackerbot/internal/model"
"slackerbot/internal/state"
"slackerbot/internal/testutil"
)
const ClusterSecret = "cluster-secret-1234567890"
type Cluster struct {
Cfg *config.Config
Mgr *manager.Manager
Store *state.Store
Master, Slave *testutil.FakeNode
Ctx context.Context
}
func New(t testing.TB) *Cluster {
addr := testutil.FreePort(t)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
cfg := &config.Config{StateDir: t.TempDir(), Container: model.Size{MemoryMB: 1024, CPUs: 1}}
cfg.Cluster = config.ClusterConfig{Name: "master", Secret: ClusterSecret, Listen: addr}
store, err := state.Open(filepath.Join(cfg.StateDir, "state.json"))
if err != nil {
t.Fatal(err)
}
c := &Cluster{Cfg: cfg, Store: store, Master: testutil.NewFakeNode("master"), Slave: testutil.NewFakeNode("slave-1"), Ctx: ctx}
c.Mgr = manager.New(cfg, store, c.Master)
go c.Mgr.ServeCluster(ctx)
slaveCfg := &config.Config{StateDir: t.TempDir()}
slaveCfg.Cluster = config.ClusterConfig{Name: "slave-1", Secret: ClusterSecret, Master: "wss://" + addr}
go manager.RunSlave(ctx, slaveCfg, c.Slave)
testutil.WaitFor(t, "slave to come online", func() bool { return c.Mgr.State(&model.Owner{Node: "slave-1"}) != manager.StateOffline })
return c
}
func (c *Cluster) AddOwner(t testing.TB, data []byte, running bool) {
u := &model.Owner{ID: "C1", Kind: model.OwnerChannel, Machine: "c-c1", Secret: "s", Size: model.Size{MemoryMB: 1024, CPUs: 1},
Node: "master", Drive: "/d", StorageMinMB: 1024, StorageMaxMB: 2048}
if err := c.Store.PutOwner(u); err != nil {
t.Fatal(err)
}
c.Master.Mu.Lock()
defer c.Master.Mu.Unlock()
c.Master.Homes["/d/c-c1"] = data
if running {
c.Master.Runs["c-c1"] = true
}
}
+243
View File
@@ -0,0 +1,243 @@
package testutil
import (
"bytes"
"context"
"errors"
"net"
"strings"
"sync"
"testing"
"time"
"slackerbot/internal/model"
"slackerbot/internal/node"
"slackerbot/internal/util"
)
type FakeNode struct {
name string
Mu sync.Mutex
Runs map[string]bool
Last node.StartSpec
Homes map[string][]byte
Exports map[string]*fakeExport
Imports map[string]*fakeImport
Passes int
FailWrites bool
Usable model.Size
}
func NewFakeNode(name string) *FakeNode {
return &FakeNode{name: name, Runs: map[string]bool{}, Homes: map[string][]byte{},
Exports: map[string]*fakeExport{}, Imports: map[string]*fakeImport{}, Usable: model.Size{MemoryMB: 8192, CPUs: 4}}
}
func (f *FakeNode) Name() string { return f.name }
func (f *FakeNode) Info(ctx context.Context) (node.NodeInfo, error) {
running, _ := f.Running(ctx)
f.Mu.Lock()
defer f.Mu.Unlock()
homes := map[string]int64{}
for k, v := range f.Homes {
homes[strings.TrimPrefix(k, "/d/")] = int64(len(v)) >> 20
}
return node.NodeInfo{Name: f.name, Usable: f.Usable, Running: running,
Drives: []node.DriveInfo{{Path: "/d", FreeMB: 50_000, HomesMB: homes}}}, nil
}
func (f *FakeNode) Running(context.Context) ([]node.RunningMachine, error) {
f.Mu.Lock()
defer f.Mu.Unlock()
var out []node.RunningMachine
for m := range f.Runs {
out = append(out, node.RunningMachine{Name: m})
}
return out, nil
}
func (f *FakeNode) IsRunning(machine string) bool {
f.Mu.Lock()
defer f.Mu.Unlock()
return f.Runs[machine]
}
func (f *FakeNode) Start(_ context.Context, spec node.StartSpec) error {
f.Mu.Lock()
defer f.Mu.Unlock()
f.Last, f.Runs[spec.Machine] = spec, true
return nil
}
func (f *FakeNode) Poweroff(_ context.Context, machine string) error {
f.Mu.Lock()
defer f.Mu.Unlock()
delete(f.Runs, machine)
return nil
}
func (f *FakeNode) SetLimits(context.Context, string, model.Size) error { return nil }
func (f *FakeNode) SetQuota(context.Context, string, string, int64) error { return nil }
func (f *FakeNode) DialAgent(_ context.Context, machine, secret string) (node.MsgConn, error) {
if secret != "agent-secret" {
return nil, errors.New("bad agent secret")
}
return newEcho(), nil
}
func (f *FakeNode) WaitStopped(_ context.Context, machine string) error {
if f.IsRunning(machine) {
return errors.New("still running")
}
return nil
}
func (f *FakeNode) ExportHome(_ context.Context, drive, machine, parent string) (string, error) {
f.Mu.Lock()
defer f.Mu.Unlock()
data, ok := f.Homes[drive+"/"+machine]
if !ok {
return "", errors.New("no such home")
}
kind := "FULL:"
if parent != "" {
kind = "INCR:"
}
id := util.RandomHex(4)
f.Exports[id] = &fakeExport{data: append([]byte(kind), data...)}
return id, nil
}
func (f *FakeNode) ReadExport(_ context.Context, id string, max int) ([]byte, bool, error) {
f.Mu.Lock()
defer f.Mu.Unlock()
x := f.Exports[id]
if x == nil {
return nil, false, errors.New("no export")
}
n := min(max, len(x.data)-x.pos)
chunk := x.data[x.pos : x.pos+n]
x.pos += n
return chunk, x.pos == len(x.data), nil
}
func (f *FakeNode) CloseExport(_ context.Context, id string) error {
f.Mu.Lock()
defer f.Mu.Unlock()
delete(f.Exports, id)
return nil
}
func (f *FakeNode) ImportHome(_ context.Context, drive, machine string) (string, error) {
f.Mu.Lock()
defer f.Mu.Unlock()
if _, exists := f.Homes[drive+"/"+machine]; exists {
return "", errors.New("home already exists")
}
id := util.RandomHex(4)
f.Imports[id] = &fakeImport{key: drive + "/" + machine}
return id, nil
}
func (f *FakeNode) WriteImport(_ context.Context, id string, data []byte) error {
f.Mu.Lock()
defer f.Mu.Unlock()
if f.FailWrites {
return errors.New("disk on fire")
}
x := f.Imports[id]
if x == nil {
return errors.New("no import")
}
x.cur = append(x.cur, data...)
return nil
}
func (f *FakeNode) EndReceive(_ context.Context, id string) error {
f.Mu.Lock()
defer f.Mu.Unlock()
x := f.Imports[id]
if x == nil || len(x.cur) < 5 {
return errors.New("nothing received")
}
if string(x.cur[:5]) == "INCR:" && x.latest == nil {
return errors.New("incremental send without a parent")
}
x.latest, x.cur = x.cur[5:], nil
x.passes++
return nil
}
func (f *FakeNode) FinishImport(_ context.Context, id string, quotaMB int64) error {
f.Mu.Lock()
defer f.Mu.Unlock()
x := f.Imports[id]
delete(f.Imports, id)
f.Homes[x.key] = x.latest
f.Passes = x.passes
return nil
}
func (f *FakeNode) AbortImport(_ context.Context, id string) error {
f.Mu.Lock()
defer f.Mu.Unlock()
delete(f.Imports, id)
return nil
}
func (f *FakeNode) DeleteHome(_ context.Context, drive, machine string) error {
f.Mu.Lock()
defer f.Mu.Unlock()
delete(f.Homes, drive+"/"+machine)
return nil
}
type fakeExport struct {
data []byte
pos int
}
type fakeImport struct {
key string
cur, latest []byte
passes int
}
type echo struct {
ch chan []byte
closed chan struct{}
once sync.Once
}
func newEcho() *echo { return &echo{ch: make(chan []byte, 16), closed: make(chan struct{})} }
func (e *echo) Read(ctx context.Context) ([]byte, error) {
select {
case b := <-e.ch:
return b, nil
case <-e.closed:
return nil, errors.New("closed")
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (e *echo) Write(_ context.Context, b []byte) error {
e.ch <- append([]byte("echo: "), b...)
return nil
}
func (e *echo) Close() error { e.once.Do(func() { close(e.closed) }); return nil }
func FreePort(t testing.TB) string {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer l.Close()
return l.Addr().String()
}
func WaitFor(t testing.TB, what string, cond func() bool) {
deadline := time.Now().Add(10 * time.Second)
for !cond() {
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for %s", what)
}
time.Sleep(20 * time.Millisecond)
}
}
func HomeData() []byte { return bytes.Repeat([]byte("0123456789abcdef"), 600_000) }
+42
View File
@@ -0,0 +1,42 @@
package units
import (
"fmt"
"math"
"regexp"
"strconv"
"strings"
)
var StorageToken = regexp.MustCompile(`^([+-]?)(\d+(?:\.\d+)?)(tib|tb|t|gib|gb|g|mib|mb|m)$`)
func ParseSizeMB(s string) (int64, error) {
t := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), " ", ""))
p := StorageToken.FindStringSubmatch(t)
if p == nil || p[1] != "" {
return 0, fmt.Errorf("%q isn't a size like 5G or 500M", s)
}
return ToMB(p[2], p[3]), nil
}
func ToMB(num, unit string) int64 {
n, _ := strconv.ParseFloat(num, 64)
switch unit[0] {
case 't':
n *= 1024 * 1024
case 'g':
n *= 1024
}
return int64(math.Round(n))
}
func FormatMB(mb int64) string {
trim := func(f float64) string { return strconv.FormatFloat(math.Round(f*10)/10, 'f', -1, 64) }
switch {
case mb >= 1024*1024:
return trim(float64(mb)/(1024*1024)) + " TB"
case mb >= 1024:
return trim(float64(mb)/1024) + " GB"
}
return fmt.Sprintf("%d MB", mb)
}
+19
View File
@@ -0,0 +1,19 @@
package units
import "testing"
func TestSizes(t *testing.T) {
for in, want := range map[string]int64{"5G": 5120, "500M": 500, "1.5gb": 1536, "1T": 1048576, "20 GB": 20480} {
if got, err := ParseSizeMB(in); err != nil || got != want {
t.Fatalf("ParseSizeMB(%q) = %d, %v; want %d", in, got, err, want)
}
}
if _, err := ParseSizeMB("+5G"); err == nil {
t.Fatal("a signed size isn't a config size")
}
for mb, want := range map[int64]string{512: "512 MB", 5120: "5 GB", 2560: "2.5 GB", 1258291: "1.2 TB"} {
if got := FormatMB(mb); got != want {
t.Fatalf("FormatMB(%d) = %q, want %q", mb, got, want)
}
}
}
+24
View File
@@ -0,0 +1,24 @@
package util
import (
"crypto/rand"
"encoding/hex"
"sync"
)
type KeyedMutex struct{ m sync.Map }
func (k *KeyedMutex) Lock(key string) (unlock func()) {
v, _ := k.m.LoadOrStore(key, &sync.Mutex{})
mu := v.(*sync.Mutex)
mu.Lock()
return mu.Unlock
}
func RandomHex(n int) string {
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
panic(err)
}
return hex.EncodeToString(b)
}