clean up of the codebase
This commit is contained in:
1 parent
7822529a38
commit
0ac61b4ece
52 files changed
+2545
-2274
No files matched your search
@@ -0,0 +1,292 @@
|
||||
package acp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"slackerbot/internal/node"
|
||||
)
|
||||
|
||||
type Update struct {
|
||||
Kind string
|
||||
Text string
|
||||
Title string
|
||||
Status string
|
||||
}
|
||||
|
||||
type rpcError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (e *rpcError) Error() string { return fmt.Sprintf("agent error %d: %s", e.Code, e.Message) }
|
||||
|
||||
type rpcMsg struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Params json.RawMessage `json:"params,omitempty"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *rpcError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type ACP struct {
|
||||
conn node.MsgConn
|
||||
|
||||
nextID atomic.Int64
|
||||
writeMu sync.Mutex
|
||||
|
||||
mu sync.Mutex
|
||||
pending map[int64]chan rpcMsg
|
||||
handlers map[string]func(Update)
|
||||
loaded map[string]bool
|
||||
|
||||
CanLoad bool
|
||||
|
||||
done chan struct{}
|
||||
err error
|
||||
}
|
||||
|
||||
func DialACP(ctx context.Context, addr, secret string) (*ACP, error) {
|
||||
conn, err := node.DialAgentWS(ctx, addr, secret)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(ctx, conn)
|
||||
}
|
||||
|
||||
func New(ctx context.Context, conn node.MsgConn) (*ACP, error) {
|
||||
a := &ACP{
|
||||
conn: conn,
|
||||
pending: map[int64]chan rpcMsg{},
|
||||
handlers: map[string]func(Update){},
|
||||
loaded: map[string]bool{},
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
go a.readLoop()
|
||||
|
||||
var init struct {
|
||||
AgentCapabilities struct {
|
||||
LoadSession bool `json:"loadSession"`
|
||||
} `json:"agentCapabilities"`
|
||||
}
|
||||
err := a.call(ctx, "initialize", map[string]any{
|
||||
"protocolVersion": 1,
|
||||
"clientCapabilities": map[string]any{
|
||||
"fs": map[string]bool{"readTextFile": false, "writeTextFile": false},
|
||||
"terminal": false,
|
||||
},
|
||||
}, &init)
|
||||
if err != nil {
|
||||
a.Close()
|
||||
return nil, err
|
||||
}
|
||||
a.CanLoad = init.AgentCapabilities.LoadSession
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (a *ACP) Done() <-chan struct{} { return a.done }
|
||||
func (a *ACP) Close() { a.conn.Close() }
|
||||
|
||||
func (a *ACP) NewSession(ctx context.Context, cwd string) (string, error) {
|
||||
var res struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
}
|
||||
if err := a.call(ctx, "session/new", map[string]any{"cwd": cwd, "mcpServers": []any{}}, &res); err != nil {
|
||||
return "", err
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.loaded[res.SessionID] = true
|
||||
a.mu.Unlock()
|
||||
return res.SessionID, nil
|
||||
}
|
||||
|
||||
// The agent replays the session's history as updates; Slack already shows it, so they're dropped.
|
||||
func (a *ACP) LoadSession(ctx context.Context, id, cwd string) error {
|
||||
a.mu.Lock()
|
||||
open := a.loaded[id]
|
||||
a.mu.Unlock()
|
||||
if open {
|
||||
return nil
|
||||
}
|
||||
if !a.CanLoad {
|
||||
return errors.New("this agent can't reopen old sessions")
|
||||
}
|
||||
if err := a.call(ctx, "session/load", map[string]any{"sessionId": id, "cwd": cwd, "mcpServers": []any{}}, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.loaded[id] = true
|
||||
a.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ACP) Prompt(ctx context.Context, id, text string, onUpdate func(Update)) (stopReason string, err error) {
|
||||
a.mu.Lock()
|
||||
a.handlers[id] = onUpdate
|
||||
a.mu.Unlock()
|
||||
defer func() {
|
||||
a.mu.Lock()
|
||||
delete(a.handlers, id)
|
||||
a.mu.Unlock()
|
||||
}()
|
||||
|
||||
var res struct {
|
||||
StopReason string `json:"stopReason"`
|
||||
}
|
||||
err = a.call(ctx, "session/prompt", map[string]any{
|
||||
"sessionId": id,
|
||||
"prompt": []map[string]string{{"type": "text", "text": text}},
|
||||
}, &res)
|
||||
if ctx.Err() != nil {
|
||||
a.notify("session/cancel", map[string]string{"sessionId": id})
|
||||
}
|
||||
return res.StopReason, err
|
||||
}
|
||||
|
||||
func (a *ACP) call(ctx context.Context, method string, params, result any) error {
|
||||
id := a.nextID.Add(1)
|
||||
ch := make(chan rpcMsg, 1)
|
||||
a.mu.Lock()
|
||||
a.pending[id] = ch
|
||||
a.mu.Unlock()
|
||||
defer func() {
|
||||
a.mu.Lock()
|
||||
delete(a.pending, id)
|
||||
a.mu.Unlock()
|
||||
}()
|
||||
|
||||
p, _ := json.Marshal(params)
|
||||
idRaw, _ := json.Marshal(id)
|
||||
if err := a.send(ctx, rpcMsg{JSONRPC: "2.0", ID: idRaw, Method: method, Params: p}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-a.done:
|
||||
return fmt.Errorf("connection to goose closed: %w", a.err)
|
||||
case msg := <-ch:
|
||||
if msg.Error != nil {
|
||||
return msg.Error
|
||||
}
|
||||
if result != nil && len(msg.Result) > 0 && string(msg.Result) != "null" {
|
||||
return json.Unmarshal(msg.Result, result)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (a *ACP) notify(method string, params any) {
|
||||
p, _ := json.Marshal(params)
|
||||
_ = a.send(context.Background(), rpcMsg{JSONRPC: "2.0", Method: method, Params: p})
|
||||
}
|
||||
|
||||
func (a *ACP) send(ctx context.Context, msg rpcMsg) error {
|
||||
b, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.writeMu.Lock()
|
||||
defer a.writeMu.Unlock()
|
||||
return a.conn.Write(ctx, b)
|
||||
}
|
||||
|
||||
func (a *ACP) readLoop() {
|
||||
defer close(a.done)
|
||||
for {
|
||||
data, err := a.conn.Read(context.Background())
|
||||
if err != nil {
|
||||
a.err = err
|
||||
return
|
||||
}
|
||||
var msg rpcMsg
|
||||
if json.Unmarshal(data, &msg) != nil {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case msg.Method != "" && len(msg.ID) > 0:
|
||||
go a.handleRequest(msg)
|
||||
case msg.Method == "session/update":
|
||||
a.handleUpdate(msg.Params)
|
||||
case msg.Method == "" && len(msg.ID) > 0:
|
||||
var id int64
|
||||
if json.Unmarshal(msg.ID, &id) == nil {
|
||||
a.mu.Lock()
|
||||
ch := a.pending[id]
|
||||
a.mu.Unlock()
|
||||
if ch != nil {
|
||||
ch <- msg
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *ACP) handleUpdate(params json.RawMessage) {
|
||||
var p struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
Update struct {
|
||||
Kind string `json:"sessionUpdate"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
Title string `json:"title"`
|
||||
Status string `json:"status"`
|
||||
} `json:"update"`
|
||||
}
|
||||
if json.Unmarshal(params, &p) != nil {
|
||||
return
|
||||
}
|
||||
u := Update{Kind: p.Update.Kind, Title: p.Update.Title, Status: p.Update.Status}
|
||||
// Message chunks carry one content block; tool updates carry a list.
|
||||
var block struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if json.Unmarshal(p.Update.Content, &block) == nil && block.Type == "text" {
|
||||
u.Text = block.Text
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
h := a.handlers[p.SessionID]
|
||||
a.mu.Unlock()
|
||||
if h != nil {
|
||||
h(u)
|
||||
}
|
||||
}
|
||||
|
||||
// Tool permission requests are auto-approved: the container is the sandbox.
|
||||
func (a *ACP) handleRequest(msg rpcMsg) {
|
||||
reply := rpcMsg{JSONRPC: "2.0", ID: msg.ID}
|
||||
|
||||
switch msg.Method {
|
||||
case "session/request_permission":
|
||||
var p struct {
|
||||
Options []struct {
|
||||
OptionID string `json:"optionId"`
|
||||
Kind string `json:"kind"`
|
||||
} `json:"options"`
|
||||
}
|
||||
_ = json.Unmarshal(msg.Params, &p)
|
||||
choice := ""
|
||||
for _, want := range []string{"allow_always", "allow_once"} {
|
||||
for _, o := range p.Options {
|
||||
if o.Kind == want && choice == "" {
|
||||
choice = o.OptionID
|
||||
}
|
||||
}
|
||||
}
|
||||
var outcome any = map[string]string{"outcome": "cancelled"}
|
||||
if choice != "" {
|
||||
outcome = map[string]string{"outcome": "selected", "optionId": choice}
|
||||
}
|
||||
reply.Result, _ = json.Marshal(map[string]any{"outcome": outcome})
|
||||
default:
|
||||
reply.Error = &rpcError{Code: -32601, Message: "method not supported by this client: " + msg.Method}
|
||||
}
|
||||
_ = a.send(context.Background(), reply)
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package acp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
func fakeGoose(t *testing.T) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/acp" || r.Header.Get("X-Secret-Key") != "s3cret" {
|
||||
http.Error(w, "nope", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
c, err := websocket.Accept(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ctx := context.Background()
|
||||
send := func(v any) { b, _ := json.Marshal(v); _ = c.Write(ctx, websocket.MessageText, b) }
|
||||
var promptID json.RawMessage
|
||||
|
||||
for {
|
||||
_, data, err := c.Read(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var m rpcMsg
|
||||
_ = json.Unmarshal(data, &m)
|
||||
switch m.Method {
|
||||
case "initialize":
|
||||
send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": map[string]any{
|
||||
"protocolVersion": 1, "agentCapabilities": map[string]any{"loadSession": true}}})
|
||||
case "session/new":
|
||||
send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": map[string]any{"sessionId": "sess-1"}})
|
||||
case "session/load":
|
||||
send(map[string]any{"jsonrpc": "2.0", "method": "session/update", "params": map[string]any{
|
||||
"sessionId": "sess-old", "update": map[string]any{"sessionUpdate": "agent_message_chunk",
|
||||
"content": map[string]any{"type": "text", "text": "old history"}}}})
|
||||
send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": nil})
|
||||
case "session/prompt":
|
||||
promptID = m.ID
|
||||
upd := func(u map[string]any) {
|
||||
send(map[string]any{"jsonrpc": "2.0", "method": "session/update",
|
||||
"params": map[string]any{"sessionId": "sess-1", "update": u}})
|
||||
}
|
||||
upd(map[string]any{"sessionUpdate": "agent_message_chunk", "content": map[string]any{"type": "text", "text": "Hello "}})
|
||||
upd(map[string]any{"sessionUpdate": "tool_call", "toolCallId": "t1", "title": "ls /root", "status": "pending"})
|
||||
send(map[string]any{"jsonrpc": "2.0", "id": "perm-1", "method": "session/request_permission",
|
||||
"params": map[string]any{"sessionId": "sess-1", "options": []map[string]any{
|
||||
{"optionId": "no", "name": "Reject", "kind": "reject_once"},
|
||||
{"optionId": "yes", "name": "Allow", "kind": "allow_once"}}}})
|
||||
case "":
|
||||
var res struct {
|
||||
Outcome struct{ Outcome, OptionID string } `json:"outcome"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Result, &res)
|
||||
text := "world"
|
||||
if res.Outcome.OptionID != "yes" {
|
||||
text = "PERMISSION WRONG"
|
||||
}
|
||||
send(map[string]any{"jsonrpc": "2.0", "method": "session/update", "params": map[string]any{
|
||||
"sessionId": "sess-1", "update": map[string]any{"sessionUpdate": "agent_message_chunk",
|
||||
"content": map[string]any{"type": "text", "text": text}}}})
|
||||
send(map[string]any{"jsonrpc": "2.0", "id": promptID, "result": map[string]any{"stopReason": "end_turn"}})
|
||||
}
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
func TestACP(t *testing.T) {
|
||||
srv := fakeGoose(t)
|
||||
defer srv.Close()
|
||||
addr := strings.TrimPrefix(srv.URL, "http://")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if _, err := DialACP(ctx, addr, "wrong"); err == nil {
|
||||
t.Fatal("wrong secret should fail")
|
||||
}
|
||||
a, err := DialACP(ctx, addr, "s3cret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.Close()
|
||||
if !a.CanLoad {
|
||||
t.Fatal("loadSession capability not read")
|
||||
}
|
||||
id, err := a.NewSession(ctx, "/root")
|
||||
if err != nil || id != "sess-1" {
|
||||
t.Fatalf("new session: %q %v", id, err)
|
||||
}
|
||||
if err := a.LoadSession(ctx, "sess-old", "/root"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var text strings.Builder
|
||||
var tools []string
|
||||
stop, err := a.Prompt(ctx, id, "hi", func(u Update) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
text.WriteString(u.Text)
|
||||
if u.Kind == "tool_call" {
|
||||
tools = append(tools, u.Title)
|
||||
}
|
||||
})
|
||||
if err != nil || stop != "end_turn" {
|
||||
t.Fatalf("prompt: %q %v", stop, err)
|
||||
}
|
||||
if text.String() != "Hello world" || len(tools) != 1 || tools[0] != "ls /root" {
|
||||
t.Fatalf("got text %q tools %v", text.String(), tools)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"sort"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/storage"
|
||||
)
|
||||
|
||||
var ErrMaxedOut = errors.New("no node has room for this container")
|
||||
|
||||
type NodeView struct {
|
||||
Name string
|
||||
Usable model.Size
|
||||
Used model.Size
|
||||
Drives []storage.DriveReport
|
||||
Running int
|
||||
}
|
||||
|
||||
func (v NodeView) Storage() (usableMB, committedMB, bookedMB int64) {
|
||||
for _, d := range v.Drives {
|
||||
usableMB += d.UsableMB
|
||||
committedMB += d.CommittedMB
|
||||
bookedMB += d.BookedMB
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func ratio(used, total float64) float64 {
|
||||
if total <= 0 {
|
||||
return math.Inf(1)
|
||||
}
|
||||
return used / total
|
||||
}
|
||||
|
||||
func (v NodeView) Homes() int {
|
||||
n := 0
|
||||
for _, d := range v.Drives {
|
||||
n += d.Homes
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (v NodeView) Load() (cpu, ram, storage float64) {
|
||||
us, cm, _ := v.Storage()
|
||||
return ratio(v.Used.CPUs, v.Usable.CPUs),
|
||||
ratio(float64(v.Used.MemoryMB), float64(v.Usable.MemoryMB)),
|
||||
ratio(float64(cm), float64(us))
|
||||
}
|
||||
|
||||
// The fullest of CPU, RAM and storage after adding the container. Placing
|
||||
// where this stays lowest keeps all three level across nodes.
|
||||
func (v NodeView) loadAfter(want model.Size, storageMB int64) float64 {
|
||||
us, cm, _ := v.Storage()
|
||||
return max(
|
||||
ratio(v.Used.CPUs+want.CPUs, v.Usable.CPUs),
|
||||
ratio(float64(v.Used.MemoryMB+want.MemoryMB), float64(v.Usable.MemoryMB)),
|
||||
ratio(float64(cm+storageMB), float64(us)),
|
||||
)
|
||||
}
|
||||
|
||||
func ChoosePlacement(views []NodeView, want model.Size, storageMB int64) (node, drive string, err error) {
|
||||
sorted := append([]NodeView(nil), views...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
|
||||
|
||||
best := math.Inf(1)
|
||||
roomForCompute := false
|
||||
for _, v := range sorted {
|
||||
if !v.Used.Add(want).Fits(v.Usable) {
|
||||
continue
|
||||
}
|
||||
roomForCompute = true
|
||||
d, err := storage.PickDrive(v.Drives, storageMB)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if score := v.loadAfter(want, storageMB); score < best {
|
||||
best, node, drive = score, v.Name, d
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case node != "":
|
||||
return node, drive, nil
|
||||
case roomForCompute:
|
||||
return "", "", storage.ErrStorageFull
|
||||
}
|
||||
return "", "", ErrMaxedOut
|
||||
}
|
||||
|
||||
func BuildView(info node.NodeInfo, owners []model.Owner, defaultSize model.Size) NodeView {
|
||||
v := NodeView{Name: info.Name, Usable: info.Usable}
|
||||
|
||||
byMachine := map[string]*model.Owner{}
|
||||
for i := range owners {
|
||||
if owners[i].Node == info.Name {
|
||||
byMachine[owners[i].Machine] = &owners[i]
|
||||
}
|
||||
}
|
||||
v.Running = len(info.Running)
|
||||
for _, r := range info.Running {
|
||||
if u := byMachine[r.Name]; u != nil {
|
||||
v.Used = v.Used.Add(u.Size)
|
||||
} else {
|
||||
v.Used = v.Used.Add(defaultSize)
|
||||
}
|
||||
}
|
||||
|
||||
for _, d := range info.Drives {
|
||||
r := storage.DriveReport{Path: d.Path}
|
||||
var homesUsed int64
|
||||
for machine, used := range d.HomesMB {
|
||||
homesUsed += used
|
||||
if u := byMachine[machine]; u != nil && u.Drive == d.Path {
|
||||
continue
|
||||
}
|
||||
r.CommittedMB += used
|
||||
r.BookedMB += used
|
||||
r.Homes++
|
||||
}
|
||||
for _, u := range byMachine {
|
||||
if u.Drive != d.Path {
|
||||
continue
|
||||
}
|
||||
used := d.HomesMB[u.Machine]
|
||||
r.CommittedMB += storage.Committed(u.StorageMinMB, used)
|
||||
r.BookedMB += u.StorageMaxMB
|
||||
r.Homes++
|
||||
}
|
||||
r.UsableMB = d.FreeMB + homesUsed - d.ReserveMB
|
||||
v.Drives = append(v.Drives, r)
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/storage"
|
||||
)
|
||||
|
||||
func TestPlacementBalancesAllThree(t *testing.T) {
|
||||
drive := func(usable, committed int64) []storage.DriveReport {
|
||||
return []storage.DriveReport{{Path: "/d", UsableMB: usable, CommittedMB: committed}}
|
||||
}
|
||||
want := model.Size{MemoryMB: 2048, CPUs: 1}
|
||||
views := []NodeView{
|
||||
{Name: "cpu-busy", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 2048, CPUs: 6}, Drives: drive(100_000, 10_000)},
|
||||
{Name: "ram-busy", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 12288, CPUs: 1}, Drives: drive(100_000, 10_000)},
|
||||
{Name: "disk-busy", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 2048, CPUs: 1}, Drives: drive(100_000, 80_000)},
|
||||
{Name: "calm", Usable: model.Size{MemoryMB: 16384, CPUs: 8}, Used: model.Size{MemoryMB: 4096, CPUs: 2}, Drives: drive(100_000, 20_000)},
|
||||
}
|
||||
node, d, err := ChoosePlacement(views, want, 5120)
|
||||
if err != nil || node != "calm" || d != "/d" {
|
||||
t.Fatalf("got %s %s %v, want calm", node, d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlacementSpreadsAcrossEqualNodes(t *testing.T) {
|
||||
views := []NodeView{
|
||||
{Name: "a", Usable: model.Size{MemoryMB: 8192, CPUs: 4}, Drives: []storage.DriveReport{{Path: "/a", UsableMB: 50_000}}},
|
||||
{Name: "b", Usable: model.Size{MemoryMB: 8192, CPUs: 4}, Drives: []storage.DriveReport{{Path: "/b", UsableMB: 50_000}}},
|
||||
}
|
||||
want := model.Size{MemoryMB: 2048, CPUs: 1}
|
||||
counts := map[string]int{}
|
||||
for i := 0; i < 4; i++ {
|
||||
node, _, err := ChoosePlacement(views, want, 5120)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counts[node]++
|
||||
for j := range views {
|
||||
if views[j].Name == node {
|
||||
views[j].Used = views[j].Used.Add(want)
|
||||
views[j].Drives[0].CommittedMB += 5120
|
||||
}
|
||||
}
|
||||
}
|
||||
if counts["a"] != 2 || counts["b"] != 2 {
|
||||
t.Fatalf("uneven spread: %v", counts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlacementWhenFull(t *testing.T) {
|
||||
full := []NodeView{{Name: "a", Usable: model.Size{MemoryMB: 4096, CPUs: 2}, Used: model.Size{MemoryMB: 4096, CPUs: 2}, Drives: []storage.DriveReport{{Path: "/a", UsableMB: 50_000}}}}
|
||||
if _, _, err := ChoosePlacement(full, model.Size{MemoryMB: 1024, CPUs: 1}, 1024); !errors.Is(err, ErrMaxedOut) {
|
||||
t.Fatalf("want maxed out, got %v", err)
|
||||
}
|
||||
noDisk := []NodeView{{Name: "a", Usable: model.Size{MemoryMB: 4096, CPUs: 2}, Drives: []storage.DriveReport{{Path: "/a", UsableMB: 1000}}}}
|
||||
if _, _, err := ChoosePlacement(noDisk, model.Size{MemoryMB: 1024, CPUs: 1}, 5120); !errors.Is(err, storage.ErrStorageFull) {
|
||||
t.Fatalf("want storage full, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildView(t *testing.T) {
|
||||
info := node.NodeInfo{
|
||||
Name: "n1",
|
||||
Usable: model.Size{MemoryMB: 16384, CPUs: 8},
|
||||
Running: []node.RunningMachine{{Name: "c-a"}, {Name: "u-stranger"}},
|
||||
Drives: []node.DriveInfo{{Path: "/d", FreeMB: 90_000, ReserveMB: 10_000,
|
||||
HomesMB: map[string]int64{"c-a": 8000, "c-b": 100, "u-stranger": 3000}}},
|
||||
}
|
||||
owners := []model.Owner{
|
||||
{ID: "A", Machine: "c-a", Node: "n1", Drive: "/d", Size: model.Size{MemoryMB: 4096, CPUs: 2}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
||||
{ID: "B", Machine: "c-b", Node: "n1", Drive: "/d", Size: model.Size{MemoryMB: 2048, CPUs: 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
||||
{ID: "C", Machine: "c-c", Node: "n1", Drive: "/d", Size: model.Size{MemoryMB: 2048, CPUs: 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
||||
{ID: "D", Machine: "c-d", Node: "other", Drive: "/d", Size: model.Size{MemoryMB: 2048, CPUs: 1}, StorageMinMB: 5120, StorageMaxMB: 10240},
|
||||
}
|
||||
v := BuildView(info, owners, model.Size{MemoryMB: 1024, CPUs: 1})
|
||||
if v.Used != (model.Size{MemoryMB: 4096 + 1024, CPUs: 3}) {
|
||||
t.Fatalf("used = %v", v.Used)
|
||||
}
|
||||
d := v.Drives[0]
|
||||
if d.UsableMB != 90_000+11_100-10_000 {
|
||||
t.Fatalf("usable = %d", d.UsableMB)
|
||||
}
|
||||
if d.CommittedMB != 8000+5120+5120+3000 || d.BookedMB != 3*10240+3000 || d.Homes != 4 {
|
||||
t.Fatalf("committed %d booked %d homes %d", d.CommittedMB, d.BookedMB, d.Homes)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,249 @@
|
||||
package bot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/slack-go/slack"
|
||||
|
||||
"slackerbot/internal/acp"
|
||||
"slackerbot/internal/manager"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/state"
|
||||
"slackerbot/internal/testutil"
|
||||
"slackerbot/internal/testutil/clustertest"
|
||||
)
|
||||
|
||||
func TestOwnership(t *testing.T) {
|
||||
dm := incoming{user: "U42", channel: "D9", dm: true}
|
||||
if id, kind := dm.owner(); id != "U42" || kind != model.OwnerUser {
|
||||
t.Fatalf("DM should be owned by the user, got %s %s", id, kind)
|
||||
}
|
||||
ch := incoming{user: "U42", channel: "C7"}
|
||||
if id, kind := ch.owner(); id != "C7" || kind != model.OwnerChannel {
|
||||
t.Fatalf("channel message should be owned by the channel, got %s %s", id, kind)
|
||||
}
|
||||
other := incoming{user: "U99", channel: "C7"}
|
||||
a, ka := ch.owner()
|
||||
b, kb := other.owner()
|
||||
if model.MachineName(a, ka) != model.MachineName(b, kb) || model.MachineName(a, ka) != "c-c7" {
|
||||
t.Fatal("two people in one channel should share a container")
|
||||
}
|
||||
if model.MachineName("U42", model.OwnerUser) != "u-u42" || !model.IsOurs("u-u42") || !model.IsOurs("c-c7") || model.IsOurs("web") {
|
||||
t.Fatal("machine naming is off")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTitle(t *testing.T) {
|
||||
if got := title(" fix the build\nplease"); got != "fix the build" {
|
||||
t.Fatal(got)
|
||||
}
|
||||
if got := title(strings.Repeat("é", 80)); !strings.HasSuffix(got, "…") || len([]rune(got)) != 61 {
|
||||
t.Fatal(got)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeSlack struct {
|
||||
mu sync.Mutex
|
||||
msgs map[string]string
|
||||
order []string
|
||||
n int
|
||||
}
|
||||
|
||||
func (f *fakeSlack) handler(w http.ResponseWriter, r *http.Request) {
|
||||
_ = r.ParseForm()
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
ts := r.Form.Get("ts")
|
||||
if strings.HasSuffix(r.URL.Path, "chat.postMessage") {
|
||||
f.n++
|
||||
ts = "ts" + string(rune('0'+f.n))
|
||||
f.order = append(f.order, ts)
|
||||
}
|
||||
f.msgs[ts] = r.Form.Get("text")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "D1", "ts": ts, "text": f.msgs[ts]})
|
||||
}
|
||||
|
||||
func TestStreamerSplitsLongReplies(t *testing.T) {
|
||||
f := &fakeSlack{msgs: map[string]string{"ph": ""}, order: []string{"ph"}}
|
||||
srv := httptest.NewServer(http.HandlerFunc(f.handler))
|
||||
defer srv.Close()
|
||||
api := slack.New("xoxb-test", slack.OptionAPIURL(srv.URL+"/"))
|
||||
|
||||
st := NewStreamer(api, "D1", "root", "ph")
|
||||
line := strings.Repeat("x", 99) + "\n"
|
||||
for i := 0; i < 80; i++ {
|
||||
st.Update(acp.Update{Kind: "agent_message_chunk", Text: line})
|
||||
}
|
||||
st.Finish("end_turn", nil)
|
||||
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if len(f.order) != 3 {
|
||||
t.Fatalf("want 3 messages, got %d", len(f.order))
|
||||
}
|
||||
var all strings.Builder
|
||||
for _, ts := range f.order {
|
||||
if len(f.msgs[ts]) > slackMsgLimit {
|
||||
t.Fatalf("message %s too long: %d", ts, len(f.msgs[ts]))
|
||||
}
|
||||
all.WriteString(f.msgs[ts])
|
||||
}
|
||||
if all.Len() != 8000 {
|
||||
t.Fatalf("text lost or duplicated: %d chars", all.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveTarget(t *testing.T) {
|
||||
b := &Bot{}
|
||||
for in, want := range map[string]string{"": "", "auto": "", "Slave-1": "slave-1", "to node-b": "node-b"} {
|
||||
if got, ok := b.moveTarget(in); !ok || got != want {
|
||||
t.Fatalf("move %q = %q %v, want %q", in, got, ok, want)
|
||||
}
|
||||
}
|
||||
if _, ok := b.moveTarget("the files into the build folder"); ok {
|
||||
t.Fatal("a normal sentence should go to the agent")
|
||||
}
|
||||
}
|
||||
|
||||
type slackLog struct {
|
||||
mu sync.Mutex
|
||||
n int
|
||||
text map[string]string
|
||||
under map[string]string
|
||||
order []string
|
||||
}
|
||||
|
||||
func (s *slackLog) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
_ = r.ParseForm()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ts := r.Form.Get("ts")
|
||||
if strings.HasSuffix(r.URL.Path, "chat.postMessage") {
|
||||
s.n++
|
||||
ts = fmt.Sprintf("%d.0", s.n)
|
||||
s.order = append(s.order, ts)
|
||||
s.under[ts] = r.Form.Get("thread_ts")
|
||||
}
|
||||
if !strings.HasSuffix(r.URL.Path, "chat.getPermalink") {
|
||||
s.text[ts] = r.Form.Get("text")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "channel": "C1", "ts": ts, "permalink": "https://slack.example/" + ts})
|
||||
}
|
||||
|
||||
func TestEveryMoveStartsAnAnnouncedThread(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), false)
|
||||
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
|
||||
srv := httptest.NewServer(log)
|
||||
defer srv.Close()
|
||||
b := &Bot{cfg: c.Cfg, store: c.Store, mgr: c.Mgr, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/")),
|
||||
acps: map[string]*acp.ACP{}, busy: map[string]int{}}
|
||||
|
||||
b.moveFor(c.Ctx, "C1", "C1", "slave-1", "<@U1>", "100.0")
|
||||
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "master"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.runMove(c.Ctx, "C1", "C1", manager.MoveOptions{}, "auto process (master was full), for <@U2>'s message", nil)
|
||||
|
||||
log.mu.Lock()
|
||||
defer log.mu.Unlock()
|
||||
var roots []string
|
||||
for _, ts := range log.order {
|
||||
if log.under[ts] == "" {
|
||||
roots = append(roots, ts)
|
||||
}
|
||||
}
|
||||
linked := 0
|
||||
for ts, parent := range log.under {
|
||||
if parent == "100.0" && strings.Contains(log.text[ts], "Follow the move here") {
|
||||
linked++
|
||||
}
|
||||
}
|
||||
if linked != 1 {
|
||||
t.Fatalf("the thread where the move started should get one link to the move thread, got %d", linked)
|
||||
}
|
||||
if len(roots) != 2 {
|
||||
t.Fatalf("want one new top-level message per move, got %d", len(roots))
|
||||
}
|
||||
checks := []string{"started by <@U1>: master → slave-1", "started by auto process (master was full), for <@U2>'s message"}
|
||||
for i, root := range roots {
|
||||
if !strings.Contains(log.text[root], checks[i]) || !strings.Contains(log.text[root], ":white_check_mark: done") {
|
||||
t.Fatalf("root %d = %q", i, log.text[root])
|
||||
}
|
||||
replies := 0
|
||||
for ts, parent := range log.under {
|
||||
if parent == root {
|
||||
replies++
|
||||
if !strings.Contains(log.text[ts], "Moved from") {
|
||||
t.Fatalf("progress reply = %q", log.text[ts])
|
||||
}
|
||||
}
|
||||
}
|
||||
if replies != 1 {
|
||||
t.Fatalf("want progress in one thread reply, got %d", replies)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManageCommandsInSlack(t *testing.T) {
|
||||
log := &slackLog{text: map[string]string{}, under: map[string]string{}}
|
||||
srv := httptest.NewServer(log)
|
||||
defer srv.Close()
|
||||
store, _ := state.Open(filepath.Join(t.TempDir(), "state.json"))
|
||||
store.SeedPermissions(model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UBOSS": {"all"}}})
|
||||
b := &Bot{store: store, api: slack.New("xoxb", slack.OptionAPIURL(srv.URL+"/"))}
|
||||
ctx := context.Background()
|
||||
say := func(user, text string) string {
|
||||
m := incoming{user: user, channel: "C1", ts: "1.0", text: text}
|
||||
if !b.command(ctx, m) {
|
||||
return "(went to the agent)"
|
||||
}
|
||||
log.mu.Lock()
|
||||
defer log.mu.Unlock()
|
||||
return log.text[log.order[len(log.order)-1]]
|
||||
}
|
||||
|
||||
if got := say("UANA", "grant <@UANA> all"); !strings.Contains(got, ":lock:") {
|
||||
t.Fatalf("non-managers can't grant: %q", got)
|
||||
}
|
||||
if got := say("UBOSS", "grant <@UANA> move manage"); !strings.Contains(got, "<@UANA> can now: move, manage") {
|
||||
t.Fatalf("manager grant: %q", got)
|
||||
}
|
||||
if got := say("UANA", "grant <@UCAL> scale"); !strings.Contains(got, "<@UCAL> can now: scale") {
|
||||
t.Fatalf("channel manager grant: %q", got)
|
||||
}
|
||||
if got := say("UANA", "grant default <@UCAL> all"); !strings.Contains(got, ":lock:") {
|
||||
t.Fatalf("channel managers can't touch the default list: %q", got)
|
||||
}
|
||||
if got := say("UANA", "managers add <@UCAL>"); !strings.Contains(got, ":lock:") {
|
||||
t.Fatalf("only user managers add managers: %q", got)
|
||||
}
|
||||
if got := say("UBOSS", "managers remove <@UBOSS>"); !strings.Contains(got, "nobody able to manage") {
|
||||
t.Fatalf("last manager guard: %q", got)
|
||||
}
|
||||
if got := say("UBOSS", "dm storage off"); !strings.Contains(got, "now turned off") {
|
||||
t.Fatalf("dm toggle: %q", got)
|
||||
}
|
||||
if store.Permissions().Allowed(model.OwnerUser, "D1", "UX", model.ActStorage) {
|
||||
t.Fatal("dm toggle not saved")
|
||||
}
|
||||
if got := say("UANA", "grant me access to the repo"); got != "(went to the agent)" {
|
||||
t.Fatalf("plain sentence: %q", got)
|
||||
}
|
||||
if got := say("UANA", "permissions reset"); !strings.Contains(got, "default super-user list again") {
|
||||
t.Fatalf("reset: %q", got)
|
||||
}
|
||||
if store.Permissions().Allowed(model.OwnerChannel, "C1", "UCAL", model.ActScale) {
|
||||
t.Fatal("reset should drop the channel's own list")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package bot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"slackerbot/internal/commands"
|
||||
"slackerbot/internal/model"
|
||||
)
|
||||
|
||||
func mentions(users []string) string {
|
||||
out := make([]string, len(users))
|
||||
for i, u := range users {
|
||||
out[i] = "<@" + u + ">"
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
func actionList(actions []model.Action) string {
|
||||
if len(actions) == 0 {
|
||||
return "nothing"
|
||||
}
|
||||
names := make([]string, len(actions))
|
||||
for i, a := range actions {
|
||||
names[i] = string(a)
|
||||
}
|
||||
return strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// manageCommand handles permission changes. It returns false if the message
|
||||
// isn't one of these commands, so it goes to the agent.
|
||||
func (b *Bot) manageCommand(ctx context.Context, m incoming, verb, args string) bool {
|
||||
_, kind := m.owner()
|
||||
p := b.store.Permissions()
|
||||
reply := func(text string) bool {
|
||||
b.say(ctx, m.channel, "", text)
|
||||
return true
|
||||
}
|
||||
denied := func(global bool) bool {
|
||||
who := "user managers"
|
||||
if !global {
|
||||
who = "user managers and people with *manage* permission here"
|
||||
}
|
||||
names := make([]string, len(p.Managers))
|
||||
for i, u := range p.Managers {
|
||||
names[i] = b.displayName(ctx, u)
|
||||
}
|
||||
return reply(fmt.Sprintf(":lock: Only %s can do that. User managers: %s.", who, strings.Join(names, ", ")))
|
||||
}
|
||||
update := func(change func(p *model.PermissionsConfig) error) error {
|
||||
return b.store.UpdatePermissions(change)
|
||||
}
|
||||
|
||||
switch verb {
|
||||
case "grant", "revoke":
|
||||
g, err := commands.ParseGrant(args)
|
||||
if errors.Is(err, commands.ErrNotCommand) {
|
||||
return false
|
||||
}
|
||||
if err != nil {
|
||||
return reply(":warning: " + err.Error() + ". Try `grant @alice move scale` or `revoke @alice`.")
|
||||
}
|
||||
channel := m.channel
|
||||
if g.ToDefault {
|
||||
channel = ""
|
||||
if !p.IsManager(m.user) {
|
||||
return denied(true)
|
||||
}
|
||||
} else {
|
||||
if kind == model.OwnerUser {
|
||||
return reply("DMs don't have a super-user list. User managers can switch DM permissions with `dm <permission> on|off`.")
|
||||
}
|
||||
if !p.CanManage(channel, m.user) {
|
||||
return denied(false)
|
||||
}
|
||||
}
|
||||
if verb == "grant" && len(g.Actions) == 0 {
|
||||
return reply(":warning: Say what to grant, like `grant @alice move scale` (or `all`, or `manage`).")
|
||||
}
|
||||
if slices.Contains(g.Actions, model.ActManage) && !g.ToDefault && !p.CanManage(channel, m.user) {
|
||||
return denied(false)
|
||||
}
|
||||
err = update(func(p *model.PermissionsConfig) error {
|
||||
if verb == "grant" {
|
||||
p.Grant(channel, g.Users, g.Actions)
|
||||
} else {
|
||||
p.Revoke(channel, g.Users, g.Actions)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return reply(":warning: " + err.Error())
|
||||
}
|
||||
where := "in this channel"
|
||||
if g.ToDefault {
|
||||
where = "in every channel that uses the default list"
|
||||
}
|
||||
after := b.store.Permissions()
|
||||
var lines []string
|
||||
for _, u := range g.Users {
|
||||
var actions []model.Action
|
||||
if g.ToDefault {
|
||||
list := after.Default
|
||||
for _, a := range model.GrantActions {
|
||||
if model.Grants(list[u], a) {
|
||||
actions = append(actions, a)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
actions = after.ActionsFor(channel, u)
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("• <@%s> can now: %s", u, actionList(actions)))
|
||||
}
|
||||
return reply(fmt.Sprintf("Done, %s:\n%s", where, strings.Join(lines, "\n")))
|
||||
|
||||
case "dm":
|
||||
words := strings.Fields(strings.ToLower(args))
|
||||
if len(words) != 2 || !slices.Contains(model.AllActions, model.Action(words[0])) {
|
||||
return false
|
||||
}
|
||||
on, ok := commands.ParseToggle(words[1])
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if !p.IsManager(m.user) {
|
||||
return denied(true)
|
||||
}
|
||||
if err := update(func(p *model.PermissionsConfig) error { p.DM[words[0]] = on; return nil }); err != nil {
|
||||
return reply(":warning: " + err.Error())
|
||||
}
|
||||
state := "turned off"
|
||||
if on {
|
||||
state = "allowed"
|
||||
}
|
||||
return reply(fmt.Sprintf("Done. Changing *%s* on your own DM container is now %s for everyone.", words[0], state))
|
||||
|
||||
case "managers":
|
||||
words := strings.Fields(args)
|
||||
if len(words) == 0 {
|
||||
return reply("*User managers:* " + mentions(p.Managers))
|
||||
}
|
||||
op := strings.ToLower(words[0])
|
||||
g, err := commands.ParseGrant(strings.Join(words[1:], " "))
|
||||
if (op != "add" && op != "remove") || err != nil || len(g.Actions) > 0 {
|
||||
return false
|
||||
}
|
||||
if !p.IsManager(m.user) {
|
||||
return denied(true)
|
||||
}
|
||||
err = update(func(p *model.PermissionsConfig) error {
|
||||
if op == "add" {
|
||||
p.AddManagers(g.Users)
|
||||
return nil
|
||||
}
|
||||
return p.RemoveManagers(g.Users)
|
||||
})
|
||||
if errors.Is(err, model.ErrLastManager) {
|
||||
return reply(":warning: That would leave nobody able to manage permissions. Add someone else first.")
|
||||
}
|
||||
if err != nil {
|
||||
return reply(":warning: " + err.Error())
|
||||
}
|
||||
verbText := "are now user managers"
|
||||
if op == "remove" {
|
||||
verbText = "are no longer user managers"
|
||||
}
|
||||
return reply(fmt.Sprintf("Done. %s %s.", mentions(g.Users), verbText))
|
||||
|
||||
case "permissions":
|
||||
if strings.ToLower(args) != "reset" {
|
||||
return false
|
||||
}
|
||||
if kind == model.OwnerUser {
|
||||
return reply("DMs don't have a super-user list to reset.")
|
||||
}
|
||||
if !p.CanManage(m.channel, m.user) {
|
||||
return denied(false)
|
||||
}
|
||||
if err := update(func(p *model.PermissionsConfig) error { delete(p.Channels, m.channel); return nil }); err != nil {
|
||||
return reply(":warning: " + err.Error())
|
||||
}
|
||||
return reply("Done. This channel uses the default super-user list again.")
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package bot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/slack-go/slack"
|
||||
|
||||
"slackerbot/internal/acp"
|
||||
)
|
||||
|
||||
const (
|
||||
slackMsgLimit = 3500
|
||||
flushEvery = 1500 * time.Millisecond // under Slack's edit rate limit
|
||||
)
|
||||
|
||||
type Streamer struct {
|
||||
api *slack.Client
|
||||
channel string
|
||||
thread string
|
||||
|
||||
mu sync.Mutex
|
||||
full strings.Builder
|
||||
dirty bool
|
||||
|
||||
// Only the flush loop touches these.
|
||||
ts string
|
||||
offset int
|
||||
|
||||
quit chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func NewStreamer(api *slack.Client, channel, thread, placeholderTS string) *Streamer {
|
||||
s := &Streamer{api: api, channel: channel, thread: thread, ts: placeholderTS,
|
||||
quit: make(chan struct{}), done: make(chan struct{})}
|
||||
go s.loop()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *Streamer) Update(u acp.Update) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
switch u.Kind {
|
||||
case "agent_message_chunk":
|
||||
s.full.WriteString(u.Text)
|
||||
case "tool_call":
|
||||
if u.Title == "" {
|
||||
return
|
||||
}
|
||||
if t := s.full.String(); t != "" && !strings.HasSuffix(t, "\n") {
|
||||
s.full.WriteString("\n")
|
||||
}
|
||||
s.full.WriteString("> :wrench: " + u.Title + "\n")
|
||||
default:
|
||||
return
|
||||
}
|
||||
s.dirty = true
|
||||
}
|
||||
|
||||
func (s *Streamer) Finish(stopReason string, err error) {
|
||||
close(s.quit)
|
||||
<-s.done
|
||||
|
||||
s.mu.Lock()
|
||||
note := ""
|
||||
switch {
|
||||
case err != nil:
|
||||
note = ":warning: " + err.Error()
|
||||
case stopReason == "max_tokens", stopReason == "max_turn_requests":
|
||||
note = "_(stopped: hit the agent's length limit. Reply to keep going.)_"
|
||||
case stopReason == "refusal":
|
||||
note = "_(the agent declined this one)_"
|
||||
case stopReason == "cancelled":
|
||||
note = "_(cancelled)_"
|
||||
}
|
||||
if note != "" {
|
||||
if s.full.Len() > 0 {
|
||||
s.full.WriteString("\n\n")
|
||||
}
|
||||
s.full.WriteString(note)
|
||||
}
|
||||
if s.full.Len() == 0 {
|
||||
s.full.WriteString("_(no reply)_")
|
||||
}
|
||||
s.dirty = true
|
||||
s.mu.Unlock()
|
||||
|
||||
s.flush()
|
||||
}
|
||||
|
||||
func (s *Streamer) loop() {
|
||||
defer close(s.done)
|
||||
t := time.NewTicker(flushEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-s.quit:
|
||||
return
|
||||
case <-t.C:
|
||||
s.flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Streamer) flush() {
|
||||
s.mu.Lock()
|
||||
if !s.dirty {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.dirty = false
|
||||
text := s.full.String()[s.offset:]
|
||||
s.mu.Unlock()
|
||||
|
||||
ctx := context.Background()
|
||||
for len(text) > slackMsgLimit {
|
||||
cut := strings.LastIndex(text[:slackMsgLimit], "\n")
|
||||
if cut < slackMsgLimit/2 {
|
||||
cut = slackMsgLimit
|
||||
for cut > 0 && !utf8.RuneStart(text[cut]) {
|
||||
cut--
|
||||
}
|
||||
}
|
||||
s.edit(ctx, text[:cut])
|
||||
_, ts, err := s.api.PostMessageContext(ctx, s.channel,
|
||||
slack.MsgOptionText("…", false), slack.MsgOptionTS(s.thread))
|
||||
if err != nil {
|
||||
log.Printf("slack post: %v", err)
|
||||
return
|
||||
}
|
||||
s.ts = ts
|
||||
s.offset += cut
|
||||
text = text[cut:]
|
||||
}
|
||||
s.edit(ctx, text)
|
||||
}
|
||||
|
||||
func (s *Streamer) edit(ctx context.Context, text string) {
|
||||
if strings.TrimSpace(text) == "" {
|
||||
text = "…"
|
||||
}
|
||||
if _, _, _, err := s.api.UpdateMessageContext(ctx, s.channel, s.ts, slack.MsgOptionText(text, false)); err != nil {
|
||||
log.Printf("slack update: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/units"
|
||||
)
|
||||
|
||||
var ErrNotCommand = errors.New("not a command")
|
||||
|
||||
var unitSpace = regexp.MustCompile(`(\d)\s+(tib|tb|t|gib|gb|g|mib|mb|m|cpus|cpu|cores|core)\b`)
|
||||
var sizeToken = regexp.MustCompile(`^([+-]?)(\d+(?:\.\d+)?)(g|gb|gib|m|mb|mib|cpu|cpus|core|cores)$`)
|
||||
|
||||
func ParseScale(args string, cur model.Size) (model.Size, error) {
|
||||
args = unitSpace.ReplaceAllString(strings.ToLower(args), "$1$2")
|
||||
words := strings.Fields(args)
|
||||
if len(words) == 0 {
|
||||
return model.Size{MemoryMB: cur.MemoryMB * 2, CPUs: cur.CPUs * 2}, nil
|
||||
}
|
||||
if len(words) == 1 {
|
||||
switch words[0] {
|
||||
case "up", "out":
|
||||
return model.Size{MemoryMB: cur.MemoryMB * 2, CPUs: cur.CPUs * 2}, nil
|
||||
case "down", "in":
|
||||
return model.Size{MemoryMB: cur.MemoryMB / 2, CPUs: cur.CPUs / 2}, nil
|
||||
}
|
||||
}
|
||||
|
||||
next := cur
|
||||
var memDone, cpuDone bool
|
||||
for i, w := range words {
|
||||
parts := sizeToken.FindStringSubmatch(w)
|
||||
if parts == nil {
|
||||
if i == 0 {
|
||||
return cur, ErrNotCommand
|
||||
}
|
||||
return cur, fmt.Errorf("I didn't understand `%s`", w)
|
||||
}
|
||||
sign, num, unit := parts[1], parts[2], parts[3]
|
||||
n, _ := strconv.ParseFloat(num, 64)
|
||||
|
||||
if unit == "cpu" || unit == "cpus" || unit == "core" || unit == "cores" {
|
||||
if cpuDone {
|
||||
return cur, errors.New("CPU was given twice")
|
||||
}
|
||||
cpuDone = true
|
||||
next.CPUs = apply(sign, cur.CPUs, n)
|
||||
continue
|
||||
}
|
||||
if memDone {
|
||||
return cur, errors.New("memory was given twice")
|
||||
}
|
||||
memDone = true
|
||||
mb := n
|
||||
if strings.HasPrefix(unit, "g") {
|
||||
mb = n * 1024
|
||||
}
|
||||
next.MemoryMB = int(math.Round(apply(sign, float64(cur.MemoryMB), mb)))
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func apply(sign string, cur, n float64) float64 {
|
||||
switch sign {
|
||||
case "+":
|
||||
return cur + n
|
||||
case "-":
|
||||
return cur - n
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
type KeepSetting struct {
|
||||
Show bool
|
||||
Reset bool
|
||||
Indefinite bool
|
||||
Limit time.Duration
|
||||
}
|
||||
|
||||
var dayToken = regexp.MustCompile(`(\d+(?:\.\d+)?)([dw])`)
|
||||
|
||||
var wordUnit = regexp.MustCompile(`(\d+(?:\.\d+)?)\s*(weeks?|wks?|days?|hours?|hrs?|minutes?|mins?)\b`)
|
||||
|
||||
func ParseKeep(args string) (KeepSetting, error) {
|
||||
a := strings.ToLower(strings.TrimSpace(args))
|
||||
switch a {
|
||||
case "":
|
||||
return KeepSetting{Show: true}, nil
|
||||
case "default", "reset", "normal":
|
||||
return KeepSetting{Reset: true}, nil
|
||||
case "indefinite", "indefinitely", "forever", "alive", "running", "on":
|
||||
return KeepSetting{Indefinite: true}, nil
|
||||
}
|
||||
d, err := parseDuration(a)
|
||||
if err != nil {
|
||||
return KeepSetting{}, ErrNotCommand
|
||||
}
|
||||
if d <= 0 {
|
||||
return KeepSetting{}, errors.New("the time has to be more than zero")
|
||||
}
|
||||
return KeepSetting{Limit: d}, nil
|
||||
}
|
||||
|
||||
func parseDuration(s string) (time.Duration, error) {
|
||||
s = wordUnit.ReplaceAllStringFunc(s, func(m string) string {
|
||||
p := wordUnit.FindStringSubmatch(m)
|
||||
return p[1] + map[byte]string{'w': "w", 'd': "d", 'h': "h", 'm': "m"}[p[2][0]]
|
||||
})
|
||||
s = strings.ReplaceAll(s, " ", "")
|
||||
s = dayToken.ReplaceAllStringFunc(s, func(m string) string {
|
||||
p := dayToken.FindStringSubmatch(m)
|
||||
n, _ := strconv.ParseFloat(p[1], 64)
|
||||
hours := n * 24
|
||||
if p[2] == "w" {
|
||||
hours = n * 24 * 7
|
||||
}
|
||||
return strconv.FormatFloat(hours, 'f', -1, 64) + "h"
|
||||
})
|
||||
return time.ParseDuration(s)
|
||||
}
|
||||
|
||||
func Human(d time.Duration) string {
|
||||
d = d.Round(time.Minute)
|
||||
days := d / (24 * time.Hour)
|
||||
d -= days * 24 * time.Hour
|
||||
out := ""
|
||||
if days > 0 {
|
||||
out = fmt.Sprintf("%dd", days)
|
||||
}
|
||||
if d > 0 {
|
||||
r := strings.TrimSuffix(d.String(), "0s")
|
||||
if strings.HasSuffix(r, "h0m") {
|
||||
r = strings.TrimSuffix(r, "0m")
|
||||
}
|
||||
out += r
|
||||
}
|
||||
if out == "" {
|
||||
out = "0m"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func ParseStorage(args string, curMin, curMax int64) (min, max int64, err error) {
|
||||
args = unitSpace.ReplaceAllString(strings.ToLower(args), "$1$2")
|
||||
words := strings.Fields(args)
|
||||
min, max = curMin, curMax
|
||||
var minDone, maxDone bool
|
||||
|
||||
for i := 0; i < len(words); i++ {
|
||||
target := "max"
|
||||
switch words[i] {
|
||||
case "min", "guarantee", "guaranteed", "reserve":
|
||||
target = "min"
|
||||
i++
|
||||
case "max", "cap", "limit":
|
||||
i++
|
||||
}
|
||||
if i >= len(words) {
|
||||
return curMin, curMax, fmt.Errorf("%s needs a size after it, like `%s 10G`", target, target)
|
||||
}
|
||||
p := units.StorageToken.FindStringSubmatch(words[i])
|
||||
if p == nil {
|
||||
if i == 0 {
|
||||
return curMin, curMax, ErrNotCommand
|
||||
}
|
||||
return curMin, curMax, fmt.Errorf("I didn't understand `%s`", words[i])
|
||||
}
|
||||
n := units.ToMB(p[2], p[3])
|
||||
if target == "min" {
|
||||
if minDone {
|
||||
return curMin, curMax, errors.New("min was given twice")
|
||||
}
|
||||
minDone = true
|
||||
min = int64(apply(p[1], float64(curMin), float64(n)))
|
||||
} else {
|
||||
if maxDone {
|
||||
return curMin, curMax, errors.New("max was given twice")
|
||||
}
|
||||
maxDone = true
|
||||
max = int64(apply(p[1], float64(curMax), float64(n)))
|
||||
}
|
||||
}
|
||||
return min, max, nil
|
||||
}
|
||||
|
||||
var mentionRe = regexp.MustCompile(`^<@([UW][A-Z0-9]+)(?:\|[^>]*)?>[,.]?$`)
|
||||
|
||||
type GrantCmd struct {
|
||||
ToDefault bool
|
||||
Users []string
|
||||
Actions []model.Action
|
||||
}
|
||||
|
||||
func ParseGrant(args string) (GrantCmd, error) {
|
||||
var g GrantCmd
|
||||
words := strings.Fields(args)
|
||||
i := 0
|
||||
if len(words) > 0 && strings.ToLower(words[0]) == "default" {
|
||||
g.ToDefault, i = true, 1
|
||||
}
|
||||
for ; i < len(words); i++ {
|
||||
if m := mentionRe.FindStringSubmatch(words[i]); m != nil {
|
||||
g.Users = append(g.Users, m[1])
|
||||
continue
|
||||
}
|
||||
w := strings.ToLower(strings.Trim(words[i], ",."))
|
||||
if w == "all" || slices.Contains(model.GrantActions, model.Action(w)) {
|
||||
g.Actions = append(g.Actions, model.Action(w))
|
||||
continue
|
||||
}
|
||||
if len(g.Users) == 0 && !g.ToDefault {
|
||||
return g, ErrNotCommand
|
||||
}
|
||||
return g, fmt.Errorf("I didn't understand `%s`", words[i])
|
||||
}
|
||||
switch {
|
||||
case len(g.Users) == 0 && !g.ToDefault:
|
||||
return g, ErrNotCommand
|
||||
case len(g.Users) == 0:
|
||||
return g, errors.New("name at least one person, like `grant default @alice all`")
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
func ParseToggle(word string) (on, ok bool) {
|
||||
switch strings.ToLower(word) {
|
||||
case "on", "yes", "true", "allow", "allowed", "enable":
|
||||
return true, true
|
||||
case "off", "no", "false", "deny", "denied", "disable":
|
||||
return false, true
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
func NeededPermission(verb, args string) model.Action {
|
||||
switch verb {
|
||||
case "scale":
|
||||
if _, err := ParseScale(args, model.Size{MemoryMB: 1024, CPUs: 1}); !errors.Is(err, ErrNotCommand) {
|
||||
return model.ActScale
|
||||
}
|
||||
case "keep":
|
||||
if k, err := ParseKeep(args); !errors.Is(err, ErrNotCommand) && !k.Show {
|
||||
return model.ActKeep
|
||||
}
|
||||
case "storage", "disk":
|
||||
if args == "" {
|
||||
return ""
|
||||
}
|
||||
if _, _, err := ParseStorage(args, 1024, 1024); !errors.Is(err, ErrNotCommand) {
|
||||
return model.ActStorage
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
)
|
||||
|
||||
func TestParseScale(t *testing.T) {
|
||||
cur := model.Size{MemoryMB: 2048, CPUs: 1}
|
||||
cases := []struct {
|
||||
in string
|
||||
want model.Size
|
||||
}{
|
||||
{"", model.Size{MemoryMB: 4096, CPUs: 2}},
|
||||
{"up", model.Size{MemoryMB: 4096, CPUs: 2}},
|
||||
{"down", model.Size{MemoryMB: 1024, CPUs: 0.5}},
|
||||
{"8G 4cpu", model.Size{MemoryMB: 8192, CPUs: 4}},
|
||||
{"8gb", model.Size{MemoryMB: 8192, CPUs: 1}},
|
||||
{"+2G", model.Size{MemoryMB: 4096, CPUs: 1}},
|
||||
{"-1cpu", model.Size{MemoryMB: 2048, CPUs: 0}},
|
||||
{"512M +0.5cpu", model.Size{MemoryMB: 512, CPUs: 1.5}},
|
||||
{"1.5G 3 cores", model.Size{MemoryMB: 1536, CPUs: 3}},
|
||||
{"+2 GB", model.Size{MemoryMB: 4096, CPUs: 1}},
|
||||
{"+1cpus -512mb", model.Size{MemoryMB: 1536, CPUs: 2}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := ParseScale(c.in, cur)
|
||||
if err != nil || got != c.want {
|
||||
t.Fatalf("scale %q = %v, %v; want %v", c.in, got, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := ParseScale("the database to 3 replicas", cur); !errors.Is(err, ErrNotCommand) {
|
||||
t.Fatal("a normal sentence should go to the agent")
|
||||
}
|
||||
if _, err := ParseScale("4G lots", cur); err == nil || errors.Is(err, ErrNotCommand) {
|
||||
t.Fatal("a bad second word should be a usage error")
|
||||
}
|
||||
if _, err := ParseScale("4G 8G", cur); err == nil {
|
||||
t.Fatal("memory twice should fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseKeep(t *testing.T) {
|
||||
for in, want := range map[string]time.Duration{
|
||||
"24h": 24 * time.Hour, "3d": 72 * time.Hour, "1d12h": 36 * time.Hour, "2w": 336 * time.Hour,
|
||||
"5 hours": 5 * time.Hour, "2 days": 48 * time.Hour, "1d 6h": 30 * time.Hour, "90 mins": 90 * time.Minute,
|
||||
} {
|
||||
k, err := ParseKeep(in)
|
||||
if err != nil || k.Limit != want {
|
||||
t.Fatalf("keep %q = %+v, %v; want %s", in, k, err, want)
|
||||
}
|
||||
}
|
||||
for _, in := range []string{"indefinite", "Indefinitely", "forever"} {
|
||||
if k, _ := ParseKeep(in); !k.Indefinite {
|
||||
t.Fatalf("keep %q should be indefinite", in)
|
||||
}
|
||||
}
|
||||
if k, _ := ParseKeep("default"); !k.Reset {
|
||||
t.Fatal("keep default should reset")
|
||||
}
|
||||
if k, _ := ParseKeep(""); !k.Show {
|
||||
t.Fatal("bare keep should show")
|
||||
}
|
||||
if _, err := ParseKeep("the tests passing"); !errors.Is(err, ErrNotCommand) {
|
||||
t.Fatal("a normal sentence should go to the agent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHuman(t *testing.T) {
|
||||
for d, want := range map[time.Duration]string{
|
||||
6 * time.Hour: "6h", 36 * time.Hour: "1d12h", 90 * time.Minute: "1h30m",
|
||||
45 * time.Minute: "45m", 168 * time.Hour: "7d", 0: "0m",
|
||||
} {
|
||||
if got := Human(d); got != want {
|
||||
t.Fatalf("Human(%s) = %q, want %q", d, got, want)
|
||||
}
|
||||
}
|
||||
if got := (model.Size{MemoryMB: 8192, CPUs: 4}).String(); got != "8 GB RAM, 4 CPUs" {
|
||||
t.Fatal(got)
|
||||
}
|
||||
if got := (model.Size{MemoryMB: 512, CPUs: 0.25}).String(); got != "512 MB RAM, 0.25 CPUs" {
|
||||
t.Fatal(got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStorage(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
min, max int64
|
||||
}{
|
||||
{"20G", 5120, 20480},
|
||||
{"+5G", 5120, 15360},
|
||||
{"-2G", 5120, 8192},
|
||||
{"max 30G", 5120, 30720},
|
||||
{"min 8G", 8192, 10240},
|
||||
{"min +1G max 30 GB", 6144, 30720},
|
||||
{"cap 1T", 5120, 1048576},
|
||||
}
|
||||
for _, c := range cases {
|
||||
min, max, err := ParseStorage(c.in, 5120, 10240)
|
||||
if err != nil || min != c.min || max != c.max {
|
||||
t.Fatalf("storage %q = %d/%d, %v; want %d/%d", c.in, min, max, err, c.min, c.max)
|
||||
}
|
||||
}
|
||||
if _, _, err := ParseStorage("is full, clean up the logs", 5120, 10240); !errors.Is(err, ErrNotCommand) {
|
||||
t.Fatal("a normal sentence should go to the agent")
|
||||
}
|
||||
for _, in := range []string{"min", "20G 30G", "min 5G min 6G", "20G lots"} {
|
||||
if _, _, err := ParseStorage(in, 5120, 10240); err == nil || errors.Is(err, ErrNotCommand) {
|
||||
t.Fatalf("%q should be a usage error, got %v", in, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeededPermission(t *testing.T) {
|
||||
for in, want := range map[[2]string]model.Action{
|
||||
{"scale", "up"}: model.ActScale,
|
||||
{"scale", "8G 4cpu"}: model.ActScale,
|
||||
{"scale", "the db to 3 replicas"}: "",
|
||||
{"keep", ""}: "",
|
||||
{"keep", "24h"}: model.ActKeep,
|
||||
{"keep", "indefinite"}: model.ActKeep,
|
||||
{"keep", "the tests green"}: "",
|
||||
{"storage", ""}: "",
|
||||
{"storage", "20G"}: model.ActStorage,
|
||||
{"storage", "is full, clean it up"}: "",
|
||||
{"status", ""}: "",
|
||||
} {
|
||||
if got := NeededPermission(in[0], in[1]); got != want {
|
||||
t.Errorf("%s %q needs %q, want %q", in[0], in[1], got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGrant(t *testing.T) {
|
||||
g, err := ParseGrant("<@U1> <@U2|bob>, move scale")
|
||||
if err != nil || len(g.Users) != 2 || g.Users[1] != "U2" || len(g.Actions) != 2 {
|
||||
t.Fatalf("%+v %v", g, err)
|
||||
}
|
||||
if g, _ := ParseGrant("default <@U1> all"); !g.ToDefault || g.Actions[0] != "all" {
|
||||
t.Fatalf("%+v", g)
|
||||
}
|
||||
for _, in := range []string{"me access to the repo", "all the things", ""} {
|
||||
if _, err := ParseGrant(in); !errors.Is(err, ErrNotCommand) {
|
||||
t.Errorf("%q should go to the agent, got %v", in, err)
|
||||
}
|
||||
}
|
||||
if _, err := ParseGrant("<@U1> teleport"); err == nil || errors.Is(err, ErrNotCommand) {
|
||||
t.Error("an unknown permission after a mention should be a usage error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/units"
|
||||
)
|
||||
|
||||
const MinStorageMB = 512
|
||||
|
||||
type StorageConfig struct {
|
||||
Min string `toml:"min"`
|
||||
Max string `toml:"max"`
|
||||
Limit string `toml:"limit"`
|
||||
|
||||
MinMB, MaxMB, LimitMB int64 `toml:"-"`
|
||||
}
|
||||
|
||||
type DriveConfig struct {
|
||||
Path string `toml:"path"`
|
||||
Reserve string `toml:"reserve"`
|
||||
|
||||
ReserveMB int64 `toml:"-"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Slack struct {
|
||||
AppToken string `toml:"app_token"`
|
||||
BotToken string `toml:"bot_token"`
|
||||
} `toml:"slack"`
|
||||
|
||||
System string `toml:"system"`
|
||||
|
||||
MachinesDir string `toml:"machines_dir"`
|
||||
StateDir string `toml:"state_dir"`
|
||||
|
||||
IdleTimeout time.Duration `toml:"idle_timeout"`
|
||||
MaxIdleTimeout time.Duration `toml:"max_idle_timeout"`
|
||||
AllowIndefinite bool `toml:"allow_indefinite"`
|
||||
BootTimeout time.Duration `toml:"boot_timeout"`
|
||||
|
||||
AgentCwd string `toml:"agent_cwd"`
|
||||
AgentEnv map[string]string `toml:"agent_env"`
|
||||
|
||||
ROBinds []string `toml:"ro_binds"`
|
||||
RWBinds []string `toml:"rw_binds"`
|
||||
|
||||
Container model.Size `toml:"container"`
|
||||
MaxContainer model.Size `toml:"max_container"`
|
||||
HostReserve model.Size `toml:"host_reserve"`
|
||||
|
||||
Storage StorageConfig `toml:"storage"`
|
||||
Drives []DriveConfig `toml:"drives"`
|
||||
|
||||
Cluster ClusterConfig `toml:"cluster"`
|
||||
|
||||
Permissions model.PermissionsConfig `toml:"permissions"`
|
||||
}
|
||||
|
||||
type ClusterConfig struct {
|
||||
Name string `toml:"name"`
|
||||
Secret string `toml:"secret"`
|
||||
Listen string `toml:"listen"`
|
||||
Master string `toml:"master"`
|
||||
MasterFingerprint string `toml:"master_fingerprint"`
|
||||
}
|
||||
|
||||
const (
|
||||
ModeMaster = "master"
|
||||
ModeSlave = "slave"
|
||||
)
|
||||
|
||||
func LoadConfig(path, mode string) (*Config, error) {
|
||||
c := &Config{
|
||||
MachinesDir: "/var/lib/machines",
|
||||
StateDir: "/var/lib/slackerbot",
|
||||
IdleTimeout: 6 * time.Hour,
|
||||
AllowIndefinite: true,
|
||||
BootTimeout: 2 * time.Minute,
|
||||
AgentCwd: "/root",
|
||||
Container: model.Size{MemoryMB: 2048, CPUs: 1},
|
||||
MaxContainer: model.Size{MemoryMB: 8192, CPUs: 4},
|
||||
HostReserve: model.Size{MemoryMB: 2048, CPUs: 1},
|
||||
Storage: StorageConfig{Min: "5G", Max: "10G", Limit: "50G"},
|
||||
}
|
||||
if _, err := toml.DecodeFile(path, c); err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", path, err)
|
||||
}
|
||||
|
||||
if v := os.Getenv("SLACK_APP_TOKEN"); v != "" {
|
||||
c.Slack.AppToken = v
|
||||
}
|
||||
if v := os.Getenv("SLACK_BOT_TOKEN"); v != "" {
|
||||
c.Slack.BotToken = v
|
||||
}
|
||||
for k, v := range c.AgentEnv {
|
||||
c.AgentEnv[k] = os.ExpandEnv(v)
|
||||
}
|
||||
c.Cluster.Secret = os.ExpandEnv(c.Cluster.Secret)
|
||||
if v := os.Getenv("CLUSTER_SECRET"); v != "" {
|
||||
c.Cluster.Secret = v
|
||||
}
|
||||
if c.Cluster.Name == "" {
|
||||
c.Cluster.Name, _ = os.Hostname()
|
||||
}
|
||||
|
||||
if err := c.loadStorage(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clustered := (mode == ModeMaster && c.Cluster.Listen != "") || mode == ModeSlave
|
||||
switch {
|
||||
case mode != ModeMaster && mode != ModeSlave:
|
||||
return nil, fmt.Errorf("mode must be %q or %q", ModeMaster, ModeSlave)
|
||||
case mode == ModeMaster && (c.Slack.AppToken == "" || c.Slack.BotToken == ""):
|
||||
return nil, fmt.Errorf("slack app_token and bot_token are required")
|
||||
case mode == ModeMaster && c.Permissions.Validate() != nil:
|
||||
return nil, c.Permissions.Validate()
|
||||
case mode == ModeSlave && !strings.HasPrefix(c.Cluster.Master, "wss://"):
|
||||
return nil, fmt.Errorf("cluster.master must be the master's address, like wss://master.example:7946")
|
||||
case clustered && len(c.Cluster.Secret) < 16:
|
||||
return nil, fmt.Errorf("cluster.secret must be at least 16 characters, and the same on every node")
|
||||
case c.Cluster.Name == "":
|
||||
return nil, fmt.Errorf("cluster.name is required")
|
||||
case c.System == "":
|
||||
return nil, fmt.Errorf("system (the NixOS build) is required")
|
||||
case c.Container.MemoryMB <= 0 || c.Container.CPUs <= 0:
|
||||
return nil, fmt.Errorf("container memory_mb and cpus must be above zero")
|
||||
case c.IdleTimeout <= 0:
|
||||
return nil, fmt.Errorf("idle_timeout must be above zero")
|
||||
case !c.Container.Fits(c.MaxContainer):
|
||||
return nil, fmt.Errorf("max_container must be at least as big as container")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Config) loadStorage() error {
|
||||
var err error
|
||||
s := &c.Storage
|
||||
if s.MinMB, err = units.ParseSizeMB(s.Min); err != nil {
|
||||
return fmt.Errorf("storage.min: %w", err)
|
||||
}
|
||||
if s.MaxMB, err = units.ParseSizeMB(s.Max); err != nil {
|
||||
return fmt.Errorf("storage.max: %w", err)
|
||||
}
|
||||
if s.LimitMB, err = units.ParseSizeMB(s.Limit); err != nil {
|
||||
return fmt.Errorf("storage.limit: %w", err)
|
||||
}
|
||||
switch {
|
||||
case s.MinMB < MinStorageMB:
|
||||
return fmt.Errorf("storage.min must be at least %s", units.FormatMB(MinStorageMB))
|
||||
case s.MinMB > s.MaxMB:
|
||||
return fmt.Errorf("storage.min (%s) can't be more than storage.max (%s)", s.Min, s.Max)
|
||||
case s.MaxMB > s.LimitMB:
|
||||
return fmt.Errorf("storage.max (%s) can't be more than storage.limit (%s)", s.Max, s.Limit)
|
||||
case len(c.Drives) == 0:
|
||||
return fmt.Errorf("add at least one [[drives]] entry (a folder on a btrfs drive)")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for i := range c.Drives {
|
||||
d := &c.Drives[i]
|
||||
if d.Path == "" || !filepath.IsAbs(d.Path) {
|
||||
return fmt.Errorf("drives[%d].path must be a full path", i)
|
||||
}
|
||||
d.Path = filepath.Clean(d.Path)
|
||||
if seen[d.Path] {
|
||||
return fmt.Errorf("drive %s is listed twice", d.Path)
|
||||
}
|
||||
seen[d.Path] = true
|
||||
if d.Reserve == "" {
|
||||
d.Reserve = "0G"
|
||||
}
|
||||
if d.ReserveMB, err = units.ParseSizeMB(d.Reserve); err != nil {
|
||||
return fmt.Errorf("drive %s reserve: %w", d.Path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestConfigExample(t *testing.T) {
|
||||
t.Setenv("SLACK_APP_TOKEN", "xapp-test")
|
||||
t.Setenv("SLACK_BOT_TOKEN", "xoxb-test")
|
||||
t.Setenv("ANTHROPIC_API_KEY", "sk-test")
|
||||
t.Setenv("CLUSTER_SECRET", "cluster-secret-1234567890")
|
||||
c, err := LoadConfig("../../../config.example.toml", ModeMaster)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.IdleTimeout != 6*time.Hour || c.Container.MemoryMB != 2048 || c.MaxContainer.CPUs != 4 {
|
||||
t.Fatalf("unexpected config: %+v", c)
|
||||
}
|
||||
if c.Storage.MinMB != 5120 || c.Storage.MaxMB != 10240 || len(c.Drives) != 2 || c.Drives[0].ReserveMB != 20480 {
|
||||
t.Fatalf("unexpected storage config: %+v %+v", c.Storage, c.Drives)
|
||||
}
|
||||
if c.AgentEnv["ANTHROPIC_API_KEY"] != "sk-test" {
|
||||
t.Fatal("env var not expanded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
)
|
||||
|
||||
type Capacity struct {
|
||||
Total model.Size
|
||||
Usable model.Size
|
||||
}
|
||||
|
||||
func DetectCapacity(reserve model.Size) (Capacity, error) {
|
||||
memMB, err := totalMemoryMB()
|
||||
if err != nil {
|
||||
return Capacity{}, err
|
||||
}
|
||||
total := model.Size{MemoryMB: memMB, CPUs: float64(runtime.NumCPU())}
|
||||
usable := total.Sub(reserve)
|
||||
if usable.MemoryMB <= 0 || usable.CPUs <= 0 {
|
||||
return Capacity{}, fmt.Errorf("host_reserve (%s) leaves nothing on this node (%s)", reserve, total)
|
||||
}
|
||||
return Capacity{Total: total, Usable: usable}, nil
|
||||
}
|
||||
|
||||
func (c Capacity) Room(used, want model.Size) bool {
|
||||
return used.Add(want).Fits(c.Usable)
|
||||
}
|
||||
|
||||
func (c Capacity) Slots(size model.Size) int {
|
||||
byMem := c.Usable.MemoryMB / size.MemoryMB
|
||||
byCPU := int(c.Usable.CPUs/size.CPUs + 1e-9)
|
||||
return min(byMem, byCPU)
|
||||
}
|
||||
|
||||
func totalMemoryMB() (int, error) {
|
||||
f, err := os.Open("/proc/meminfo")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
fields := strings.Fields(sc.Text())
|
||||
if len(fields) >= 2 && fields[0] == "MemTotal:" {
|
||||
kb, err := strconv.Atoi(fields[1])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return kb / 1024, nil
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("MemTotal not found in /proc/meminfo")
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
)
|
||||
|
||||
func TestCapacity(t *testing.T) {
|
||||
c := Capacity{Usable: model.Size{MemoryMB: 6144, CPUs: 3}}
|
||||
def := model.Size{MemoryMB: 2048, CPUs: 1}
|
||||
if got := c.Slots(def); got != 3 {
|
||||
t.Fatalf("slots = %d, want 3", got)
|
||||
}
|
||||
if !c.Room(model.Size{MemoryMB: 4096, CPUs: 2}, def) {
|
||||
t.Fatal("third container should fit")
|
||||
}
|
||||
if c.Room(model.Size{MemoryMB: 6144, CPUs: 3}, def) {
|
||||
t.Fatal("fourth container should not fit")
|
||||
}
|
||||
if c.Room(model.Size{MemoryMB: 2048, CPUs: 2.5}, def) {
|
||||
t.Fatal("should be blocked by CPU")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/godbus/dbus/v5"
|
||||
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
ServicePort = 1984
|
||||
|
||||
machinedName = "org.freedesktop.machine1"
|
||||
machinedPath = "/org/freedesktop/machine1"
|
||||
machinedMgr = "org.freedesktop.machine1.Manager"
|
||||
sigPoweroff = 38 // SIGRTMIN+4, what `machinectl poweroff` sends
|
||||
|
||||
agentEnvInside = "/var/lib/slackerbot/agent.env"
|
||||
)
|
||||
|
||||
type Local struct {
|
||||
cfg *config.Config
|
||||
name string
|
||||
cap Capacity
|
||||
bus *dbus.Conn
|
||||
system string
|
||||
|
||||
exports transfers
|
||||
imports transfers
|
||||
}
|
||||
|
||||
func New(cfg *config.Config) (*Local, error) {
|
||||
system, err := filepath.EvalSymlinks(cfg.System)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(system, "init")); err != nil {
|
||||
return nil, fmt.Errorf("%s has no init. Is it a NixOS build with boot.isContainer = true?", system)
|
||||
}
|
||||
for _, d := range cfg.Drives {
|
||||
if err := storage.PrepareDrive(d.Path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
capacity, err := DetectCapacity(cfg.HostReserve)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bus, err := dbus.ConnectSystemBus()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connecting to the system bus: %w", err)
|
||||
}
|
||||
return &Local{cfg: cfg, name: cfg.Cluster.Name, cap: capacity, bus: bus, system: system}, nil
|
||||
}
|
||||
|
||||
func unitName(machine string) string { return "slackerbot-" + machine + ".service" }
|
||||
|
||||
func (l *Local) Name() string { return l.name }
|
||||
|
||||
func (l *Local) machined() dbus.BusObject { return l.bus.Object(machinedName, machinedPath) }
|
||||
|
||||
func (l *Local) Info(ctx context.Context) (node.NodeInfo, error) {
|
||||
info := node.NodeInfo{Name: l.name, Total: l.cap.Total, Usable: l.cap.Usable}
|
||||
for _, d := range l.cfg.Drives {
|
||||
free, err := storage.FreeMB(d.Path)
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
di := node.DriveInfo{Path: d.Path, FreeMB: free, ReserveMB: d.ReserveMB, HomesMB: map[string]int64{}}
|
||||
entries, err := os.ReadDir(d.Path)
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() && model.IsOurs(e.Name()) {
|
||||
used, _ := storage.UsedMB(filepath.Join(d.Path, e.Name()))
|
||||
di.HomesMB[e.Name()] = used
|
||||
}
|
||||
}
|
||||
info.Drives = append(info.Drives, di)
|
||||
}
|
||||
running, err := l.Running(ctx)
|
||||
info.Running = running
|
||||
return info, err
|
||||
}
|
||||
|
||||
func (l *Local) Running(ctx context.Context) ([]node.RunningMachine, error) {
|
||||
var list []struct {
|
||||
Name, Class, Service string
|
||||
Path dbus.ObjectPath
|
||||
}
|
||||
if err := l.machined().Call(machinedMgr+".ListMachines", 0).Store(&list); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []node.RunningMachine
|
||||
for _, x := range list {
|
||||
if !model.IsOurs(x.Name) {
|
||||
continue
|
||||
}
|
||||
r := node.RunningMachine{Name: x.Name}
|
||||
if v, err := l.bus.Object(machinedName, x.Path).GetProperty("org.freedesktop.machine1.Machine.Timestamp"); err == nil {
|
||||
if us, ok := v.Value().(uint64); ok {
|
||||
r.Since = time.UnixMicro(int64(us))
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (l *Local) IsRunning(machine string) bool {
|
||||
var p dbus.ObjectPath
|
||||
return l.machined().Call(machinedMgr+".GetMachine", 0, machine).Store(&p) == nil
|
||||
}
|
||||
|
||||
func (l *Local) drive(path string) (config.DriveConfig, error) {
|
||||
for _, d := range l.cfg.Drives {
|
||||
if d.Path == path {
|
||||
return d, nil
|
||||
}
|
||||
}
|
||||
return config.DriveConfig{}, fmt.Errorf("drive %s isn't configured on node %s", path, l.name)
|
||||
}
|
||||
|
||||
func (l *Local) Start(ctx context.Context, spec node.StartSpec) error {
|
||||
if !model.IsOurs(spec.Machine) {
|
||||
return fmt.Errorf("unexpected machine name %q", spec.Machine)
|
||||
}
|
||||
if _, err := l.drive(spec.Drive); err != nil {
|
||||
return err
|
||||
}
|
||||
home := filepath.Join(spec.Drive, spec.Machine)
|
||||
if _, err := os.Stat(home); errors.Is(err, os.ErrNotExist) {
|
||||
if err := storage.CreateSubvolume(home); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := storage.SetQuota(home, spec.StorageMaxMB); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Only the home folder persists. NixOS rebuilds the rest on boot.
|
||||
root := filepath.Join(l.cfg.MachinesDir, spec.Machine)
|
||||
if err := os.RemoveAll(root); err != nil {
|
||||
return fmt.Errorf("clearing old container files: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if f, err := os.OpenFile(filepath.Join(root, "etc/os-release"), os.O_CREATE, 0o644); err == nil {
|
||||
f.Close()
|
||||
}
|
||||
|
||||
envFile, err := l.writeEnvFile(spec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"--unit=" + unitName(spec.Machine),
|
||||
"--collect",
|
||||
"--quiet",
|
||||
"--description=" + spec.Description,
|
||||
// Same as systemd-nspawn@.service.
|
||||
"--property=Type=notify",
|
||||
"--property=KillMode=mixed",
|
||||
"--property=Delegate=yes",
|
||||
fmt.Sprintf("--property=MemoryMax=%dM", spec.Size.MemoryMB),
|
||||
fmt.Sprintf("--property=CPUQuota=%d%%", int(math.Round(spec.Size.CPUs*100))),
|
||||
|
||||
"systemd-nspawn",
|
||||
"--machine=" + spec.Machine,
|
||||
"--directory=" + root,
|
||||
"--keep-unit",
|
||||
"--network-veth", // own network, so every container can use port 1984
|
||||
"--bind-ro=/nix/store",
|
||||
"--bind-ro=" + envFile + ":" + agentEnvInside,
|
||||
"--bind=" + home + ":/root",
|
||||
}
|
||||
for _, b := range l.cfg.ROBinds {
|
||||
args = append(args, "--bind-ro="+strings.ReplaceAll(b, "{machine}", spec.Machine))
|
||||
}
|
||||
for _, b := range l.cfg.RWBinds {
|
||||
b = strings.ReplaceAll(b, "{machine}", spec.Machine)
|
||||
src, _, _ := strings.Cut(b, ":")
|
||||
if err := os.MkdirAll(src, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
args = append(args, "--bind="+b)
|
||||
}
|
||||
args = append(args, filepath.Join(l.system, "init"))
|
||||
|
||||
if out, err := exec.CommandContext(ctx, "systemd-run", args...).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("systemd-run: %v: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
|
||||
// Wait until machined lists it, so the next capacity check counts it.
|
||||
for i := 0; i < 50; i++ {
|
||||
if l.IsRunning(spec.Machine) {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("%s started but never registered with machined", spec.Machine)
|
||||
}
|
||||
|
||||
func (l *Local) writeEnvFile(spec node.StartSpec) (string, error) {
|
||||
dir := filepath.Join(l.cfg.StateDir, "env")
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var b strings.Builder
|
||||
line := func(k, v string) {
|
||||
v = strings.ReplaceAll(v, `\`, `\\`)
|
||||
v = strings.ReplaceAll(v, `"`, `\"`)
|
||||
fmt.Fprintf(&b, "%s=\"%s\"\n", k, v)
|
||||
}
|
||||
line("GOOSE_SERVER__SECRET_KEY", spec.Secret)
|
||||
for k, v := range spec.Env {
|
||||
line(k, v)
|
||||
}
|
||||
path := filepath.Join(dir, spec.Machine+".env")
|
||||
return path, os.WriteFile(path, []byte(b.String()), 0o600)
|
||||
}
|
||||
|
||||
func (l *Local) Poweroff(ctx context.Context, machine string) error {
|
||||
return l.machined().CallWithContext(ctx, machinedMgr+".KillMachine", 0, machine, "leader", int32(sigPoweroff)).Err
|
||||
}
|
||||
|
||||
func (l *Local) SetLimits(ctx context.Context, machine string, s model.Size) error {
|
||||
type prop struct {
|
||||
Name string
|
||||
Value dbus.Variant
|
||||
}
|
||||
props := []prop{
|
||||
{"MemoryMax", dbus.MakeVariant(uint64(s.MemoryMB) << 20)},
|
||||
{"CPUQuotaPerSecUSec", dbus.MakeVariant(uint64(s.CPUs * 1e6))},
|
||||
}
|
||||
return l.bus.Object("org.freedesktop.systemd1", "/org/freedesktop/systemd1").
|
||||
CallWithContext(ctx, "org.freedesktop.systemd1.Manager.SetUnitProperties", 0, unitName(machine), true, props).Err
|
||||
}
|
||||
|
||||
func (l *Local) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
|
||||
if _, err := l.drive(drive); err != nil {
|
||||
return err
|
||||
}
|
||||
if !model.IsOurs(machine) {
|
||||
return fmt.Errorf("unexpected machine name %q", machine)
|
||||
}
|
||||
return storage.SetQuota(filepath.Join(drive, machine), mb)
|
||||
}
|
||||
|
||||
func (l *Local) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
|
||||
addr, err := l.waitReady(ctx, machine)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return node.DialAgentWS(ctx, addr, secret)
|
||||
}
|
||||
|
||||
func (l *Local) address(machine string) (string, error) {
|
||||
var addrs []struct {
|
||||
Family int32
|
||||
Addr []byte
|
||||
}
|
||||
if err := l.machined().Call(machinedMgr+".GetMachineAddresses", 0, machine).Store(&addrs); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, a := range addrs {
|
||||
ip := net.IP(a.Addr)
|
||||
if a.Family == syscall.AF_INET && !ip.IsLinkLocalUnicast() {
|
||||
return net.JoinHostPort(ip.String(), strconv.Itoa(ServicePort)), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("%s has no IPv4 address yet", machine)
|
||||
}
|
||||
|
||||
func (l *Local) waitReady(ctx context.Context, machine string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, l.cfg.BootTimeout)
|
||||
defer cancel()
|
||||
client := http.Client{Timeout: 3 * time.Second}
|
||||
|
||||
var lastErr error
|
||||
for {
|
||||
addr, err := l.address(machine)
|
||||
if err == nil {
|
||||
var resp *http.Response
|
||||
resp, err = client.Get("http://" + addr + "/status")
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return addr, nil
|
||||
}
|
||||
err = fmt.Errorf("/status returned %s", resp.Status)
|
||||
}
|
||||
}
|
||||
lastErr = err
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", fmt.Errorf("goose didn't answer on port %d within %s (last error: %v)", ServicePort, l.cfg.BootTimeout, lastErr)
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Local) Capacity() Capacity { return l.cap }
|
||||
@@ -0,0 +1,265 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/storage"
|
||||
"slackerbot/internal/util"
|
||||
)
|
||||
|
||||
type transfer struct {
|
||||
cmd *exec.Cmd
|
||||
out io.ReadCloser
|
||||
in io.WriteCloser
|
||||
stderr bytes.Buffer
|
||||
|
||||
snapshot string
|
||||
tmpDir string
|
||||
drive string
|
||||
machine string
|
||||
seen map[string]bool
|
||||
latest string
|
||||
}
|
||||
|
||||
type transfers struct {
|
||||
mu sync.Mutex
|
||||
m map[string]*transfer
|
||||
}
|
||||
|
||||
func (t *transfers) add(x *transfer) string {
|
||||
id := util.RandomHex(8)
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.m == nil {
|
||||
t.m = map[string]*transfer{}
|
||||
}
|
||||
t.m[id] = x
|
||||
return id
|
||||
}
|
||||
|
||||
func (t *transfers) get(id string) (*transfer, error) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if x := t.m[id]; x != nil {
|
||||
return x, nil
|
||||
}
|
||||
return nil, fmt.Errorf("no transfer %s", id)
|
||||
}
|
||||
|
||||
func (t *transfers) take(id string) *transfer {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
x := t.m[id]
|
||||
delete(t.m, id)
|
||||
return x
|
||||
}
|
||||
|
||||
func (l *Local) home(drive, machine string) (string, error) {
|
||||
if _, err := l.drive(drive); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !model.IsOurs(machine) {
|
||||
return "", fmt.Errorf("unexpected machine name %q", machine)
|
||||
}
|
||||
return filepath.Join(drive, machine), nil
|
||||
}
|
||||
|
||||
func (l *Local) WaitStopped(ctx context.Context, machine string) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
for l.IsRunning(machine) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("%s didn't stop in time", machine)
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// With a parent export, only the changes since the parent's snapshot are sent.
|
||||
func (l *Local) ExportHome(ctx context.Context, drive, machine, parent string) (string, error) {
|
||||
home, err := l.home(drive, machine)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
args := []string{"send", "-q"}
|
||||
if parent != "" {
|
||||
p, err := l.exports.get(parent)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
args = append(args, "-p", p.snapshot)
|
||||
}
|
||||
snapshot := filepath.Join(drive, ".move-"+machine+"-"+util.RandomHex(4))
|
||||
if out, err := exec.Command("btrfs", "subvolume", "snapshot", "-r", home, snapshot).CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("snapshot: %v: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
x := &transfer{snapshot: snapshot, drive: drive, machine: machine}
|
||||
x.cmd = exec.Command("btrfs", append(args, snapshot)...)
|
||||
x.cmd.Stderr = &x.stderr
|
||||
if x.out, err = x.cmd.StdoutPipe(); err == nil {
|
||||
err = x.cmd.Start()
|
||||
}
|
||||
if err != nil {
|
||||
storage.DeleteSubvolume(snapshot)
|
||||
return "", err
|
||||
}
|
||||
return l.exports.add(x), nil
|
||||
}
|
||||
|
||||
func (l *Local) ReadExport(ctx context.Context, id string, max int) ([]byte, bool, error) {
|
||||
x, err := l.exports.get(id)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
buf := make([]byte, max)
|
||||
n, err := io.ReadFull(x.out, buf)
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
if werr := x.cmd.Wait(); werr != nil {
|
||||
return nil, false, fmt.Errorf("btrfs send: %v: %s", werr, strings.TrimSpace(x.stderr.String()))
|
||||
}
|
||||
return buf[:n], true, nil
|
||||
}
|
||||
return buf[:n], false, err
|
||||
}
|
||||
|
||||
func (l *Local) CloseExport(ctx context.Context, id string) error {
|
||||
x := l.exports.take(id)
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
if x.cmd.ProcessState == nil {
|
||||
_ = x.cmd.Process.Kill()
|
||||
_ = x.cmd.Wait()
|
||||
}
|
||||
return storage.DeleteSubvolume(x.snapshot)
|
||||
}
|
||||
|
||||
func (l *Local) ImportHome(ctx context.Context, drive, machine string) (string, error) {
|
||||
home, err := l.home(drive, machine)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := os.Stat(home); err == nil {
|
||||
return "", fmt.Errorf("%s already has a home folder for %s; remove it first", l.name, machine)
|
||||
}
|
||||
x := &transfer{drive: drive, machine: machine, seen: map[string]bool{},
|
||||
tmpDir: filepath.Join(drive, ".incoming-"+machine+"-"+util.RandomHex(4))}
|
||||
if err := os.Mkdir(x.tmpDir, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return l.imports.add(x), nil
|
||||
}
|
||||
|
||||
func (l *Local) WriteImport(ctx context.Context, id string, data []byte) error {
|
||||
x, err := l.imports.get(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if x.cmd == nil {
|
||||
x.stderr.Reset()
|
||||
x.cmd = exec.Command("btrfs", "receive", "-q", x.tmpDir)
|
||||
x.cmd.Stderr = &x.stderr
|
||||
if x.in, err = x.cmd.StdinPipe(); err == nil {
|
||||
err = x.cmd.Start()
|
||||
}
|
||||
if err != nil {
|
||||
x.cmd = nil
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := x.in.Write(data); err != nil {
|
||||
return fmt.Errorf("btrfs receive: %v: %s", err, strings.TrimSpace(x.stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *Local) EndReceive(ctx context.Context, id string) error {
|
||||
x, err := l.imports.get(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if x.cmd == nil {
|
||||
return errors.New("nothing was received")
|
||||
}
|
||||
x.in.Close()
|
||||
err = x.cmd.Wait()
|
||||
x.cmd = nil
|
||||
if err != nil {
|
||||
return fmt.Errorf("btrfs receive: %v: %s", err, strings.TrimSpace(x.stderr.String()))
|
||||
}
|
||||
entries, err := os.ReadDir(x.tmpDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !x.seen[e.Name()] {
|
||||
x.seen[e.Name()] = true
|
||||
x.latest = e.Name()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Received subvolumes are read-only; a writable snapshot of the latest one
|
||||
// becomes the new home.
|
||||
func (l *Local) FinishImport(ctx context.Context, id string, quotaMB int64) error {
|
||||
x := l.imports.take(id)
|
||||
if x == nil {
|
||||
return fmt.Errorf("no transfer %s", id)
|
||||
}
|
||||
defer cleanupIncoming(x.tmpDir)
|
||||
if x.latest == "" {
|
||||
return errors.New("nothing was received")
|
||||
}
|
||||
home := filepath.Join(x.drive, x.machine)
|
||||
if out, err := exec.Command("btrfs", "subvolume", "snapshot", filepath.Join(x.tmpDir, x.latest), home).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("snapshot: %v: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return storage.SetQuota(home, quotaMB)
|
||||
}
|
||||
|
||||
func (l *Local) AbortImport(ctx context.Context, id string) error {
|
||||
x := l.imports.take(id)
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
if x.cmd != nil {
|
||||
x.in.Close()
|
||||
_ = x.cmd.Process.Kill()
|
||||
_ = x.cmd.Wait()
|
||||
}
|
||||
cleanupIncoming(x.tmpDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *Local) DeleteHome(ctx context.Context, drive, machine string) error {
|
||||
home, err := l.home(drive, machine)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if l.IsRunning(machine) {
|
||||
return fmt.Errorf("%s is running", machine)
|
||||
}
|
||||
return storage.DeleteSubvolume(home)
|
||||
}
|
||||
|
||||
func cleanupIncoming(dir string) {
|
||||
entries, _ := os.ReadDir(dir)
|
||||
for _, e := range entries {
|
||||
_ = storage.DeleteSubvolume(filepath.Join(dir, e.Name()))
|
||||
}
|
||||
_ = os.RemoveAll(dir)
|
||||
}
|
||||
@@ -0,0 +1,787 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
)
|
||||
|
||||
const (
|
||||
reportEvery = 5 * time.Second
|
||||
offlineAfter = 20 * time.Second
|
||||
callTimeout = 3 * time.Minute
|
||||
pendingTimeout = 30 * time.Second
|
||||
streamBuffer = 4096
|
||||
)
|
||||
|
||||
type frame struct {
|
||||
Type string `json:"t"`
|
||||
ID int64 `json:"id,omitempty"`
|
||||
Method string `json:"m,omitempty"`
|
||||
Params json.RawMessage `json:"p,omitempty"`
|
||||
Result json.RawMessage `json:"r,omitempty"`
|
||||
Error string `json:"e,omitempty"`
|
||||
Data string `json:"d,omitempty"`
|
||||
}
|
||||
|
||||
type link struct {
|
||||
conn *websocket.Conn
|
||||
writeMu sync.Mutex
|
||||
nextID atomic.Int64
|
||||
|
||||
mu sync.Mutex
|
||||
pending map[int64]chan frame
|
||||
streams map[int64]*stream
|
||||
|
||||
onCall func(f frame)
|
||||
onReport func(info node.NodeInfo)
|
||||
|
||||
done chan struct{}
|
||||
err error
|
||||
}
|
||||
|
||||
func newLink(conn *websocket.Conn) *link {
|
||||
conn.SetReadLimit(64 << 20)
|
||||
return &link{
|
||||
conn: conn,
|
||||
pending: map[int64]chan frame{},
|
||||
streams: map[int64]*stream{},
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *link) Done() <-chan struct{} { return l.done }
|
||||
|
||||
func (l *link) Close() { l.conn.Close(websocket.StatusNormalClosure, "") }
|
||||
|
||||
func (l *link) send(ctx context.Context, f frame) error {
|
||||
b, err := json.Marshal(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l.writeMu.Lock()
|
||||
defer l.writeMu.Unlock()
|
||||
return l.conn.Write(ctx, websocket.MessageText, b)
|
||||
}
|
||||
|
||||
func (l *link) run() {
|
||||
defer func() {
|
||||
l.mu.Lock()
|
||||
streams := l.streams
|
||||
l.streams = map[int64]*stream{}
|
||||
l.mu.Unlock()
|
||||
for _, s := range streams {
|
||||
s.closeLocal()
|
||||
}
|
||||
close(l.done)
|
||||
}()
|
||||
for {
|
||||
_, data, err := l.conn.Read(context.Background())
|
||||
if err != nil {
|
||||
l.err = err
|
||||
return
|
||||
}
|
||||
var f frame
|
||||
if json.Unmarshal(data, &f) != nil {
|
||||
continue
|
||||
}
|
||||
switch f.Type {
|
||||
case "call":
|
||||
if l.onCall != nil {
|
||||
go l.onCall(f)
|
||||
}
|
||||
case "reply":
|
||||
l.mu.Lock()
|
||||
ch := l.pending[f.ID]
|
||||
l.mu.Unlock()
|
||||
if ch != nil {
|
||||
ch <- f
|
||||
}
|
||||
case "report":
|
||||
var info node.NodeInfo
|
||||
if l.onReport != nil && json.Unmarshal(f.Params, &info) == nil {
|
||||
l.onReport(info)
|
||||
}
|
||||
case "data":
|
||||
if s := l.stream(f.ID); s != nil {
|
||||
select {
|
||||
case s.in <- []byte(f.Data):
|
||||
default:
|
||||
s.Close()
|
||||
}
|
||||
}
|
||||
case "close":
|
||||
if s := l.stream(f.ID); s != nil {
|
||||
s.closeLocal()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *link) call(ctx context.Context, method string, params, result any) error {
|
||||
return l.callWithID(ctx, l.nextID.Add(1), method, params, result)
|
||||
}
|
||||
|
||||
func (l *link) callWithID(ctx context.Context, id int64, method string, params, result any) error {
|
||||
ch := make(chan frame, 1)
|
||||
l.mu.Lock()
|
||||
l.pending[id] = ch
|
||||
l.mu.Unlock()
|
||||
defer func() {
|
||||
l.mu.Lock()
|
||||
delete(l.pending, id)
|
||||
l.mu.Unlock()
|
||||
}()
|
||||
|
||||
p, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := l.send(ctx, frame{Type: "call", ID: id, Method: method, Params: p}); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-l.done:
|
||||
return fmt.Errorf("link closed: %v", l.err)
|
||||
case f := <-ch:
|
||||
if f.Error != "" {
|
||||
return errors.New(f.Error)
|
||||
}
|
||||
if result != nil && len(f.Result) > 0 {
|
||||
return json.Unmarshal(f.Result, result)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (l *link) reply(id int64, result any, err error) {
|
||||
f := frame{Type: "reply", ID: id}
|
||||
if err != nil {
|
||||
f.Error = err.Error()
|
||||
} else if result != nil {
|
||||
f.Result, _ = json.Marshal(result)
|
||||
}
|
||||
_ = l.send(context.Background(), f)
|
||||
}
|
||||
|
||||
func (l *link) report(ctx context.Context, info node.NodeInfo) error {
|
||||
p, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return l.send(ctx, frame{Type: "report", Params: p})
|
||||
}
|
||||
|
||||
func (l *link) stream(id int64) *stream {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.streams[id]
|
||||
}
|
||||
|
||||
func (l *link) addStream(id int64) *stream {
|
||||
s := &stream{l: l, id: id, in: make(chan []byte, streamBuffer), closed: make(chan struct{})}
|
||||
l.mu.Lock()
|
||||
l.streams[id] = s
|
||||
l.mu.Unlock()
|
||||
return s
|
||||
}
|
||||
|
||||
func (l *link) dropStream(id int64) {
|
||||
l.mu.Lock()
|
||||
delete(l.streams, id)
|
||||
l.mu.Unlock()
|
||||
}
|
||||
|
||||
// The stream shares its ID with the "open_agent" call, so it's registered
|
||||
// before the call goes out and no early data is lost.
|
||||
func (l *link) openAgent(ctx context.Context, machine, secret string) (*stream, error) {
|
||||
id := l.nextID.Add(1)
|
||||
s := l.addStream(id)
|
||||
err := l.callWithID(ctx, id, "open_agent", map[string]string{"machine": machine, "secret": secret}, nil)
|
||||
if err != nil {
|
||||
l.dropStream(id)
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
l *link
|
||||
id int64
|
||||
in chan []byte
|
||||
closed chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (s *stream) Read(ctx context.Context) ([]byte, error) {
|
||||
select {
|
||||
case b := <-s.in:
|
||||
return b, nil
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case b := <-s.in:
|
||||
return b, nil
|
||||
case <-s.closed:
|
||||
return nil, errors.New("stream closed")
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *stream) Write(ctx context.Context, b []byte) error {
|
||||
select {
|
||||
case <-s.closed:
|
||||
return errors.New("stream closed")
|
||||
default:
|
||||
}
|
||||
return s.l.send(ctx, frame{Type: "data", ID: s.id, Data: string(b)})
|
||||
}
|
||||
|
||||
func (s *stream) Close() error {
|
||||
s.once.Do(func() {
|
||||
_ = s.l.send(context.Background(), frame{Type: "close", ID: s.id})
|
||||
s.l.dropStream(s.id)
|
||||
close(s.closed)
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stream) closeLocal() {
|
||||
s.once.Do(func() {
|
||||
s.l.dropStream(s.id)
|
||||
close(s.closed)
|
||||
})
|
||||
}
|
||||
|
||||
type Remote struct {
|
||||
name string
|
||||
|
||||
mu sync.Mutex
|
||||
link *link
|
||||
info node.NodeInfo
|
||||
infoAt time.Time
|
||||
starting map[string]time.Time
|
||||
stopping map[string]time.Time
|
||||
}
|
||||
|
||||
func (r *Remote) Name() string { return r.name }
|
||||
|
||||
func (r *Remote) Online() bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.link == nil || time.Since(r.infoAt) > offlineAfter {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case <-r.link.Done():
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Remote) currentLink() (*link, error) {
|
||||
if !r.Online() {
|
||||
return nil, &node.NodeOfflineError{Node: r.name}
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.link, nil
|
||||
}
|
||||
|
||||
func (r *Remote) setInfo(info node.NodeInfo) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
info.Name = r.name
|
||||
r.info, r.infoAt = info, time.Now()
|
||||
for _, m := range info.Running {
|
||||
delete(r.starting, m.Name)
|
||||
}
|
||||
for m := range r.stopping {
|
||||
if !containsMachine(info.Running, m) {
|
||||
delete(r.stopping, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func containsMachine(list []node.RunningMachine, name string) bool {
|
||||
for _, m := range list {
|
||||
if m.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Reports can lag behind a start or stop the master just asked for,
|
||||
// so recent requests override them for a short while.
|
||||
func (r *Remote) runningLocked() []node.RunningMachine {
|
||||
var out []node.RunningMachine
|
||||
for _, m := range r.info.Running {
|
||||
if t, ok := r.stopping[m.Name]; ok && time.Since(t) < pendingTimeout {
|
||||
continue
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
for m, t := range r.starting {
|
||||
if time.Since(t) < pendingTimeout && !containsMachine(out, m) {
|
||||
out = append(out, node.RunningMachine{Name: m, Since: t})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *Remote) Info(ctx context.Context) (node.NodeInfo, error) {
|
||||
if !r.Online() {
|
||||
return node.NodeInfo{}, &node.NodeOfflineError{Node: r.name}
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
info := r.info
|
||||
info.Running = r.runningLocked()
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (r *Remote) Running(ctx context.Context) ([]node.RunningMachine, error) {
|
||||
info, err := r.Info(ctx)
|
||||
return info.Running, err
|
||||
}
|
||||
|
||||
func (r *Remote) IsRunning(machine string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return containsMachine(r.runningLocked(), machine)
|
||||
}
|
||||
|
||||
func (r *Remote) callNode(ctx context.Context, method string, params, result any) error {
|
||||
l, err := r.currentLink()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
return l.call(ctx, method, params, result)
|
||||
}
|
||||
|
||||
func (r *Remote) Start(ctx context.Context, spec node.StartSpec) error {
|
||||
if err := r.callNode(ctx, "start", spec, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.starting[spec.Machine] = time.Now()
|
||||
delete(r.stopping, spec.Machine)
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Remote) Poweroff(ctx context.Context, machine string) error {
|
||||
if err := r.callNode(ctx, "poweroff", map[string]string{"machine": machine}, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.stopping[machine] = time.Now()
|
||||
delete(r.starting, machine)
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Remote) SetLimits(ctx context.Context, machine string, size model.Size) error {
|
||||
return r.callNode(ctx, "set_limits", callParams{Machine: machine, Size: size}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
|
||||
return r.callNode(ctx, "set_quota", callParams{Drive: drive, Machine: machine, MB: mb}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) WaitStopped(ctx context.Context, machine string) error {
|
||||
return r.callNode(ctx, "wait_stopped", callParams{Machine: machine}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) ExportHome(ctx context.Context, drive, machine, parent string) (string, error) {
|
||||
var res callParams
|
||||
err := r.callNode(ctx, "export_home", callParams{Drive: drive, Machine: machine, Parent: parent}, &res)
|
||||
return res.ID, err
|
||||
}
|
||||
|
||||
func (r *Remote) ReadExport(ctx context.Context, id string, max int) ([]byte, bool, error) {
|
||||
var res callParams
|
||||
err := r.callNode(ctx, "read_export", callParams{ID: id, Max: max}, &res)
|
||||
return res.Data, res.EOF, err
|
||||
}
|
||||
|
||||
func (r *Remote) CloseExport(ctx context.Context, id string) error {
|
||||
return r.callNode(ctx, "close_export", callParams{ID: id}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) ImportHome(ctx context.Context, drive, machine string) (string, error) {
|
||||
var res callParams
|
||||
err := r.callNode(ctx, "import_home", callParams{Drive: drive, Machine: machine}, &res)
|
||||
return res.ID, err
|
||||
}
|
||||
|
||||
func (r *Remote) WriteImport(ctx context.Context, id string, data []byte) error {
|
||||
return r.callNode(ctx, "write_import", callParams{ID: id, Data: data}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) EndReceive(ctx context.Context, id string) error {
|
||||
return r.callNode(ctx, "end_receive", callParams{ID: id}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) FinishImport(ctx context.Context, id string, quotaMB int64) error {
|
||||
return r.callNode(ctx, "finish_import", callParams{ID: id, MB: quotaMB}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) AbortImport(ctx context.Context, id string) error {
|
||||
return r.callNode(ctx, "abort_import", callParams{ID: id}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) DeleteHome(ctx context.Context, drive, machine string) error {
|
||||
return r.callNode(ctx, "delete_home", callParams{Drive: drive, Machine: machine}, nil)
|
||||
}
|
||||
|
||||
func (r *Remote) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
|
||||
l, err := r.currentLink()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
return l.openAgent(ctx, machine, secret)
|
||||
}
|
||||
|
||||
func (m *Manager) attach(name string, l *link) (*Remote, error) {
|
||||
if name == "" || name == m.local.Name() {
|
||||
return nil, fmt.Errorf("node name %q is empty or taken by the master", name)
|
||||
}
|
||||
m.mu.Lock()
|
||||
r := m.remotes[name]
|
||||
if r == nil {
|
||||
r = &Remote{name: name, starting: map[string]time.Time{}, stopping: map[string]time.Time{}}
|
||||
m.remotes[name] = r
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
r.mu.Lock()
|
||||
old := r.link
|
||||
r.link = l
|
||||
r.infoAt = time.Time{}
|
||||
r.mu.Unlock()
|
||||
if old != nil {
|
||||
old.Close()
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (m *Manager) ServeCluster(ctx context.Context) error {
|
||||
cc := m.cfg.Cluster
|
||||
cert, fingerprint, err := loadOrCreateCert(m.cfg.StateDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("cluster: listening on %s, certificate fingerprint %s", cc.Listen, fingerprint)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/cluster", func(w http.ResponseWriter, req *http.Request) {
|
||||
if subtle.ConstantTimeCompare([]byte(req.Header.Get("X-Cluster-Secret")), []byte(cc.Secret)) != 1 {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
name := req.Header.Get("X-Node-Name")
|
||||
conn, err := websocket.Accept(w, req, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
l := newLink(conn)
|
||||
r, err := m.attach(name, l)
|
||||
if err != nil {
|
||||
conn.Close(websocket.StatusPolicyViolation, err.Error())
|
||||
return
|
||||
}
|
||||
back := false
|
||||
l.onReport = func(info node.NodeInfo) {
|
||||
r.setInfo(info)
|
||||
if !back {
|
||||
back = true
|
||||
log.Printf("cluster: node %s connected", name)
|
||||
if m.onNodeBack != nil {
|
||||
go m.onNodeBack(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
l.onCall = func(f frame) { l.reply(f.ID, nil, errors.New("the master takes no calls")) }
|
||||
l.run()
|
||||
log.Printf("cluster: node %s disconnected: %v", name, l.err)
|
||||
})
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: cc.Listen,
|
||||
Handler: mux,
|
||||
TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13},
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
srv.Close()
|
||||
}()
|
||||
if err := srv.ListenAndServeTLS("", ""); !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadOrCreateCert(dir string) (tls.Certificate, string, error) {
|
||||
certPath, keyPath := filepath.Join(dir, "cluster.crt"), filepath.Join(dir, "cluster.key")
|
||||
if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
|
||||
return cert, fingerprintOf(cert.Certificate[0]), nil
|
||||
}
|
||||
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
serial, _ := rand.Int(rand.Reader, big.NewInt(1<<62))
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: "slackerbot master"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().AddDate(20, 0, 0),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(key)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
|
||||
return tls.Certificate{}, "", err
|
||||
}
|
||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
return cert, fingerprintOf(der), err
|
||||
}
|
||||
|
||||
func fingerprintOf(der []byte) string {
|
||||
sum := sha256.Sum256(der)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func normalizeFingerprint(s string) string {
|
||||
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), ":", ""))
|
||||
}
|
||||
|
||||
func RunSlave(ctx context.Context, cfg *config.Config, local node.Node) error {
|
||||
backoff := time.Second
|
||||
for {
|
||||
started := time.Now()
|
||||
err := slaveSession(ctx, cfg, local)
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if time.Since(started) > time.Minute {
|
||||
backoff = time.Second
|
||||
}
|
||||
log.Printf("cluster: link to master lost (%v), retrying in %s", err, backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
backoff = min(backoff*2, 30*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func pinnedTLS(cfg *config.Config) *tls.Config {
|
||||
pinned := normalizeFingerprint(cfg.Cluster.MasterFingerprint)
|
||||
savedPath := filepath.Join(cfg.StateDir, "master.fingerprint")
|
||||
if pinned == "" {
|
||||
if b, err := os.ReadFile(savedPath); err == nil {
|
||||
pinned = normalizeFingerprint(string(b))
|
||||
}
|
||||
}
|
||||
return &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
// Not verified against a CA: the master's self-signed certificate is pinned instead.
|
||||
InsecureSkipVerify: true,
|
||||
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(raw) == 0 {
|
||||
return errors.New("master sent no certificate")
|
||||
}
|
||||
got := fingerprintOf(raw[0])
|
||||
if pinned == "" {
|
||||
log.Printf("cluster: trusting master certificate %s from now on", got)
|
||||
pinned = got
|
||||
_ = os.MkdirAll(cfg.StateDir, 0o700)
|
||||
return os.WriteFile(savedPath, []byte(got+"\n"), 0o600)
|
||||
}
|
||||
if got != pinned {
|
||||
return fmt.Errorf("master certificate %s doesn't match the pinned %s", got, pinned)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func slaveSession(ctx context.Context, cfg *config.Config, local node.Node) error {
|
||||
client := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(cfg), Proxy: http.ProxyFromEnvironment}}
|
||||
dialCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
conn, _, err := websocket.Dial(dialCtx, strings.TrimSuffix(cfg.Cluster.Master, "/")+"/cluster", &websocket.DialOptions{
|
||||
HTTPClient: client,
|
||||
HTTPHeader: http.Header{
|
||||
"X-Cluster-Secret": []string{cfg.Cluster.Secret},
|
||||
"X-Node-Name": []string{cfg.Cluster.Name},
|
||||
},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l := newLink(conn)
|
||||
defer l.Close()
|
||||
l.onCall = func(f frame) { serveCall(ctx, l, local, f) }
|
||||
go l.run()
|
||||
log.Printf("cluster: connected to master %s as %s", cfg.Cluster.Master, cfg.Cluster.Name)
|
||||
|
||||
t := time.NewTicker(reportEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
info, err := local.Info(ctx)
|
||||
if err != nil {
|
||||
log.Printf("cluster: reading node info: %v", err)
|
||||
} else if err := l.report(ctx, info); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-l.Done():
|
||||
return l.err
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type callParams struct {
|
||||
Machine string `json:"machine,omitempty"`
|
||||
Secret string `json:"secret,omitempty"`
|
||||
Drive string `json:"drive,omitempty"`
|
||||
MB int64 `json:"mb,omitempty"`
|
||||
Size model.Size `json:"size"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Parent string `json:"parent,omitempty"`
|
||||
Max int `json:"max,omitempty"`
|
||||
Data []byte `json:"data,omitempty"`
|
||||
EOF bool `json:"eof,omitempty"`
|
||||
}
|
||||
|
||||
func serveCall(ctx context.Context, l *link, local node.Node, f frame) {
|
||||
var p callParams
|
||||
_ = json.Unmarshal(f.Params, &p)
|
||||
|
||||
switch f.Method {
|
||||
case "start":
|
||||
var spec node.StartSpec
|
||||
if err := json.Unmarshal(f.Params, &spec); err != nil {
|
||||
l.reply(f.ID, nil, err)
|
||||
return
|
||||
}
|
||||
l.reply(f.ID, nil, local.Start(ctx, spec))
|
||||
case "poweroff":
|
||||
l.reply(f.ID, nil, local.Poweroff(ctx, p.Machine))
|
||||
case "set_limits":
|
||||
l.reply(f.ID, nil, local.SetLimits(ctx, p.Machine, p.Size))
|
||||
case "set_quota":
|
||||
l.reply(f.ID, nil, local.SetQuota(ctx, p.Drive, p.Machine, p.MB))
|
||||
case "wait_stopped":
|
||||
l.reply(f.ID, nil, local.WaitStopped(ctx, p.Machine))
|
||||
case "export_home":
|
||||
id, err := local.ExportHome(ctx, p.Drive, p.Machine, p.Parent)
|
||||
l.reply(f.ID, callParams{ID: id}, err)
|
||||
case "read_export":
|
||||
data, eof, err := local.ReadExport(ctx, p.ID, min(p.Max, transferChunk))
|
||||
l.reply(f.ID, callParams{Data: data, EOF: eof}, err)
|
||||
case "close_export":
|
||||
l.reply(f.ID, nil, local.CloseExport(ctx, p.ID))
|
||||
case "import_home":
|
||||
id, err := local.ImportHome(ctx, p.Drive, p.Machine)
|
||||
l.reply(f.ID, callParams{ID: id}, err)
|
||||
case "write_import":
|
||||
l.reply(f.ID, nil, local.WriteImport(ctx, p.ID, p.Data))
|
||||
case "end_receive":
|
||||
l.reply(f.ID, nil, local.EndReceive(ctx, p.ID))
|
||||
case "finish_import":
|
||||
l.reply(f.ID, nil, local.FinishImport(ctx, p.ID, p.MB))
|
||||
case "abort_import":
|
||||
l.reply(f.ID, nil, local.AbortImport(ctx, p.ID))
|
||||
case "delete_home":
|
||||
l.reply(f.ID, nil, local.DeleteHome(ctx, p.Drive, p.Machine))
|
||||
case "open_agent":
|
||||
agent, err := local.DialAgent(ctx, p.Machine, p.Secret)
|
||||
if err != nil {
|
||||
l.reply(f.ID, nil, err)
|
||||
return
|
||||
}
|
||||
s := l.addStream(f.ID)
|
||||
l.reply(f.ID, nil, nil)
|
||||
pipe(ctx, agent, s)
|
||||
default:
|
||||
l.reply(f.ID, nil, fmt.Errorf("unknown call %q", f.Method))
|
||||
}
|
||||
}
|
||||
|
||||
func pipe(ctx context.Context, a, b node.MsgConn) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
copyTo := func(dst, src node.MsgConn) {
|
||||
defer cancel()
|
||||
for {
|
||||
msg, err := src.Read(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := dst.Write(ctx, msg); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
go copyTo(a, b)
|
||||
copyTo(b, a)
|
||||
<-ctx.Done()
|
||||
a.Close()
|
||||
b.Close()
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/state"
|
||||
"slackerbot/internal/testutil"
|
||||
)
|
||||
|
||||
func TestMasterAndSlave(t *testing.T) {
|
||||
addr := testutil.FreePort(t)
|
||||
secret := "cluster-secret-1234567890"
|
||||
masterDir, slaveDir := t.TempDir(), t.TempDir()
|
||||
|
||||
masterCfg := &config.Config{StateDir: masterDir, Container: model.Size{MemoryMB: 1024, CPUs: 1}}
|
||||
masterCfg.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Listen: addr}
|
||||
store, err := state.Open(filepath.Join(masterDir, "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mgr := New(masterCfg, store, testutil.NewFakeNode("master"))
|
||||
backCh := make(chan string, 4)
|
||||
mgr.onNodeBack = func(n string) { backCh <- n }
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go mgr.ServeCluster(ctx)
|
||||
|
||||
slaveCfg := &config.Config{StateDir: slaveDir}
|
||||
slaveCfg.Cluster = config.ClusterConfig{Name: "slave-1", Secret: secret, Master: "wss://" + addr}
|
||||
slave := testutil.NewFakeNode("slave-1")
|
||||
slaveCtx, stopSlave := context.WithCancel(ctx)
|
||||
go RunSlave(slaveCtx, slaveCfg, slave)
|
||||
|
||||
testutil.WaitFor(t, "slave to come online", func() bool { _, err := mgr.nodeByName("slave-1"); return err == nil })
|
||||
if got := <-backCh; got != "slave-1" {
|
||||
t.Fatalf("back callback for %q", got)
|
||||
}
|
||||
pinned, err := os.ReadFile(filepath.Join(slaveDir, "master.fingerprint"))
|
||||
if err != nil || len(strings.TrimSpace(string(pinned))) != 64 {
|
||||
t.Fatalf("slave should pin the master's certificate, got %q %v", pinned, err)
|
||||
}
|
||||
|
||||
n, _ := mgr.nodeByName("slave-1")
|
||||
spec := node.StartSpec{Machine: "c-abc", Secret: "agent-secret", Size: model.Size{MemoryMB: 1024, CPUs: 1}, Drive: "/d"}
|
||||
if err := n.Start(ctx, spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slave.IsRunning("c-abc") || slave.Last.Drive != "/d" {
|
||||
t.Fatal("start didn't reach the slave")
|
||||
}
|
||||
if !n.IsRunning("c-abc") {
|
||||
t.Fatal("master should count the container as running right away")
|
||||
}
|
||||
|
||||
conn, err := n.DialAgent(ctx, "c-abc", "agent-secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.Write(ctx, []byte(`{"jsonrpc":"2.0"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
got, err := conn.Read(readCtx)
|
||||
readCancel()
|
||||
if err != nil || string(got) != `echo: {"jsonrpc":"2.0"}` {
|
||||
t.Fatalf("relay got %q %v", got, err)
|
||||
}
|
||||
conn.Close()
|
||||
if _, err := n.DialAgent(ctx, "c-abc", "wrong"); err == nil {
|
||||
t.Fatal("a bad agent secret should fail through the relay")
|
||||
}
|
||||
|
||||
if err := n.Poweroff(ctx, "c-abc"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if slave.IsRunning("c-abc") || n.IsRunning("c-abc") {
|
||||
t.Fatal("poweroff didn't take")
|
||||
}
|
||||
|
||||
stopSlave()
|
||||
testutil.WaitFor(t, "slave to go offline", func() bool {
|
||||
_, err := mgr.nodeByName("slave-1")
|
||||
var off *node.NodeOfflineError
|
||||
return errors.As(err, &off) && off.Node == "slave-1"
|
||||
})
|
||||
if off := mgr.OfflineNodes(); len(off) != 1 || off[0] != "slave-1" {
|
||||
t.Fatalf("offline nodes = %v", off)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlaveRejectsWrongMasterOrSecret(t *testing.T) {
|
||||
addr := testutil.FreePort(t)
|
||||
secret := "cluster-secret-1234567890"
|
||||
masterCfg := &config.Config{StateDir: t.TempDir()}
|
||||
masterCfg.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Listen: addr}
|
||||
store, _ := state.Open(filepath.Join(masterCfg.StateDir, "state.json"))
|
||||
mgr := New(masterCfg, store, testutil.NewFakeNode("master"))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go mgr.ServeCluster(ctx)
|
||||
testutil.WaitFor(t, "master to listen", func() bool {
|
||||
c, err := net.Dial("tcp", addr)
|
||||
if err == nil {
|
||||
c.Close()
|
||||
}
|
||||
return err == nil
|
||||
})
|
||||
|
||||
wrongPin := &config.Config{StateDir: t.TempDir()}
|
||||
wrongPin.Cluster = config.ClusterConfig{Name: "s", Secret: secret, Master: "wss://" + addr, MasterFingerprint: strings.Repeat("ab", 32)}
|
||||
if err := slaveSession(ctx, wrongPin, testutil.NewFakeNode("s")); err == nil {
|
||||
t.Fatal("a master with a different certificate should be refused")
|
||||
}
|
||||
|
||||
wrongSecret := &config.Config{StateDir: t.TempDir()}
|
||||
wrongSecret.Cluster = config.ClusterConfig{Name: "s", Secret: "not-the-secret-at-all", Master: "wss://" + addr}
|
||||
if err := slaveSession(ctx, wrongSecret, testutil.NewFakeNode("s")); err == nil {
|
||||
t.Fatal("a slave with the wrong secret should be refused")
|
||||
}
|
||||
|
||||
sameName := &config.Config{StateDir: t.TempDir()}
|
||||
sameName.Cluster = config.ClusterConfig{Name: "master", Secret: secret, Master: "wss://" + addr}
|
||||
if err := slaveSession(ctx, sameName, testutil.NewFakeNode("master")); err == nil {
|
||||
t.Fatal("a slave can't take the master's name")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"slackerbot/internal/balance"
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/state"
|
||||
"slackerbot/internal/storage"
|
||||
"slackerbot/internal/util"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrTooBig = errors.New("bigger than max_container allows")
|
||||
ErrTooSmall = errors.New("smaller than the minimum size")
|
||||
|
||||
MinContainer = model.Size{MemoryMB: 256, CPUs: 0.25}
|
||||
)
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateRunning State = "running"
|
||||
StateStopped State = "stopped"
|
||||
StateNone State = "not created"
|
||||
StateOffline State = "on an offline node"
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
cfg *config.Config
|
||||
store *state.Store
|
||||
local node.Node
|
||||
|
||||
mu sync.RWMutex
|
||||
remotes map[string]*Remote
|
||||
onNodeBack func(node string)
|
||||
|
||||
startMu sync.Mutex
|
||||
ownerMu util.KeyedMutex
|
||||
migrating sync.Map
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, store *state.Store, local node.Node) *Manager {
|
||||
return &Manager{cfg: cfg, store: store, local: local, remotes: map[string]*Remote{}}
|
||||
}
|
||||
|
||||
func (m *Manager) SetNodeBackHandler(fn func(node string)) { m.onNodeBack = fn }
|
||||
|
||||
func (m *Manager) nodeByName(name string) (node.Node, error) {
|
||||
if name == m.local.Name() {
|
||||
return m.local, nil
|
||||
}
|
||||
m.mu.RLock()
|
||||
r := m.remotes[name]
|
||||
m.mu.RUnlock()
|
||||
if r != nil && r.Online() {
|
||||
return r, nil
|
||||
}
|
||||
return nil, &node.NodeOfflineError{Node: name}
|
||||
}
|
||||
|
||||
func (m *Manager) onlineNodes() []node.Node {
|
||||
nodes := []node.Node{m.local}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
for _, r := range m.remotes {
|
||||
if r.Online() {
|
||||
nodes = append(nodes, r)
|
||||
}
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
func (m *Manager) OfflineNodes() []string {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
var out []string
|
||||
for name, r := range m.remotes {
|
||||
if !r.Online() {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Manager) Views(ctx context.Context) ([]balance.NodeView, error) {
|
||||
owners := m.store.Owners()
|
||||
var views []balance.NodeView
|
||||
for _, n := range m.onlineNodes() {
|
||||
info, err := n.Info(ctx)
|
||||
if err != nil {
|
||||
if n == m.local {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
views = append(views, balance.BuildView(info, owners, m.cfg.Container))
|
||||
}
|
||||
sort.Slice(views, func(i, j int) bool { return views[i].Name < views[j].Name })
|
||||
return views, nil
|
||||
}
|
||||
|
||||
func (m *Manager) view(ctx context.Context, n node.Node) (balance.NodeView, node.NodeInfo, error) {
|
||||
info, err := n.Info(ctx)
|
||||
if err != nil {
|
||||
return balance.NodeView{}, info, err
|
||||
}
|
||||
return balance.BuildView(info, m.store.Owners(), m.cfg.Container), info, nil
|
||||
}
|
||||
|
||||
func (m *Manager) Totals(ctx context.Context) (used, usable model.Size) {
|
||||
views, _ := m.Views(ctx)
|
||||
for _, v := range views {
|
||||
used, usable = used.Add(v.Used), usable.Add(v.Usable)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (m *Manager) ownerRecord(id string, kind model.OwnerKind) (*model.Owner, error) {
|
||||
if u := m.store.Owner(id); u != nil {
|
||||
return u, nil
|
||||
}
|
||||
u := &model.Owner{
|
||||
ID: id,
|
||||
Kind: kind,
|
||||
Machine: model.MachineName(id, kind),
|
||||
Secret: util.RandomHex(32),
|
||||
Size: m.cfg.Container,
|
||||
StorageMinMB: m.cfg.Storage.MinMB,
|
||||
StorageMaxMB: m.cfg.Storage.MaxMB,
|
||||
LastActive: time.Now(),
|
||||
}
|
||||
return u, m.store.PutOwner(u)
|
||||
}
|
||||
|
||||
func (m *Manager) Owner(id string, kind model.OwnerKind) (*model.Owner, error) {
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
return m.ownerRecord(id, kind)
|
||||
}
|
||||
|
||||
func (m *Manager) spec(u *model.Owner) node.StartSpec {
|
||||
return node.StartSpec{
|
||||
Machine: u.Machine,
|
||||
Description: "slackerbot container for " + string(u.Kind) + " " + u.ID,
|
||||
Secret: u.Secret,
|
||||
Size: u.Size,
|
||||
Drive: u.Drive,
|
||||
StorageMaxMB: u.StorageMaxMB,
|
||||
Env: m.cfg.AgentEnv,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) Ensure(ctx context.Context, id string, kind model.OwnerKind) (*model.Owner, error) {
|
||||
if m.IsMigrating(id) {
|
||||
return nil, ErrMigrating
|
||||
}
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
|
||||
u, err := m.ownerRecord(id, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if u.Node != "" {
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return u, err
|
||||
}
|
||||
if n.IsRunning(u.Machine) {
|
||||
return u, nil
|
||||
}
|
||||
}
|
||||
|
||||
m.startMu.Lock()
|
||||
defer m.startMu.Unlock()
|
||||
|
||||
if u.Node == "" {
|
||||
views, err := m.Views(ctx)
|
||||
if err != nil {
|
||||
return u, err
|
||||
}
|
||||
target, drive, err := balance.ChoosePlacement(views, u.Size, u.StorageMinMB)
|
||||
if err != nil {
|
||||
return u, err
|
||||
}
|
||||
u.Node, u.Drive = target, drive
|
||||
if err := m.store.SetPlacement(u.ID, target, drive); err != nil {
|
||||
return u, err
|
||||
}
|
||||
}
|
||||
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return u, err
|
||||
}
|
||||
v, _, err := m.view(ctx, n)
|
||||
if err != nil {
|
||||
return u, err
|
||||
}
|
||||
if !v.Used.Add(u.Size).Fits(v.Usable) {
|
||||
return u, &node.NodeFullError{Node: u.Node}
|
||||
}
|
||||
return u, n.Start(ctx, m.spec(u))
|
||||
}
|
||||
|
||||
// The old home folder stays on its node, untouched.
|
||||
func (m *Manager) Recreate(id string) error {
|
||||
if m.IsMigrating(id) {
|
||||
return ErrMigrating
|
||||
}
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
if err := m.store.SetPlacement(id, "", ""); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.store.DeleteSessions(id)
|
||||
}
|
||||
|
||||
func (m *Manager) DialAgent(ctx context.Context, u *model.Owner) (node.MsgConn, error) {
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return n.DialAgent(ctx, u.Machine, u.Secret)
|
||||
}
|
||||
|
||||
func (m *Manager) State(u *model.Owner) State {
|
||||
if u == nil || u.Node == "" {
|
||||
return StateNone
|
||||
}
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return StateOffline
|
||||
}
|
||||
if n.IsRunning(u.Machine) {
|
||||
return StateRunning
|
||||
}
|
||||
return StateStopped
|
||||
}
|
||||
|
||||
func (m *Manager) Poweroff(ctx context.Context, u *model.Owner) error {
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return n.Poweroff(ctx, u.Machine)
|
||||
}
|
||||
|
||||
func (m *Manager) RunningOwners(ctx context.Context) []model.Owner {
|
||||
byNodeMachine := map[string]model.Owner{}
|
||||
for _, u := range m.store.Owners() {
|
||||
byNodeMachine[u.Node+"/"+u.Machine] = u
|
||||
}
|
||||
var out []model.Owner
|
||||
for _, n := range m.onlineNodes() {
|
||||
running, err := n.Running(ctx)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, r := range running {
|
||||
if u, ok := byNodeMachine[n.Name()+"/"+r.Name]; ok {
|
||||
out = append(out, u)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Manager) Resize(ctx context.Context, id string, kind model.OwnerKind, change func(model.Size) (model.Size, error)) (old, next model.Size, err error) {
|
||||
if m.IsMigrating(id) {
|
||||
return old, old, ErrMigrating
|
||||
}
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
m.startMu.Lock()
|
||||
defer m.startMu.Unlock()
|
||||
|
||||
u, err := m.ownerRecord(id, kind)
|
||||
if err != nil {
|
||||
return model.Size{}, model.Size{}, err
|
||||
}
|
||||
old = u.Size
|
||||
if next, err = change(old); err != nil {
|
||||
return old, old, err
|
||||
}
|
||||
next.CPUs = math.Round(next.CPUs*100) / 100
|
||||
switch {
|
||||
case !MinContainer.Fits(next):
|
||||
return old, old, ErrTooSmall
|
||||
case !next.Fits(m.cfg.MaxContainer):
|
||||
return old, old, ErrTooBig
|
||||
case next == old:
|
||||
return old, old, nil
|
||||
}
|
||||
|
||||
if u.Node != "" {
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return old, old, err
|
||||
}
|
||||
if n.IsRunning(u.Machine) {
|
||||
v, _, err := m.view(ctx, n)
|
||||
if err != nil {
|
||||
return old, old, err
|
||||
}
|
||||
growth := model.Size{MemoryMB: max(0, next.MemoryMB-old.MemoryMB), CPUs: math.Max(0, next.CPUs-old.CPUs)}
|
||||
if !v.Used.Add(growth).Fits(v.Usable) {
|
||||
return old, old, &node.NodeFullError{Node: u.Node}
|
||||
}
|
||||
if err := n.SetLimits(ctx, u.Machine, next); err != nil {
|
||||
return old, old, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return old, next, m.store.SetSize(id, next)
|
||||
}
|
||||
|
||||
type StorageChange struct {
|
||||
OldMin, OldMax, NewMin, NewMax int64
|
||||
UsedMB int64
|
||||
Drive *storage.DriveReport
|
||||
}
|
||||
|
||||
func (m *Manager) SetStorage(ctx context.Context, id string, kind model.OwnerKind, change func(min, max int64) (int64, int64, error)) (*StorageChange, error) {
|
||||
if m.IsMigrating(id) {
|
||||
return &StorageChange{}, ErrMigrating
|
||||
}
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
m.startMu.Lock()
|
||||
defer m.startMu.Unlock()
|
||||
|
||||
u, err := m.ownerRecord(id, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := &StorageChange{OldMin: u.StorageMinMB, OldMax: u.StorageMaxMB}
|
||||
if c.NewMin, c.NewMax, err = change(c.OldMin, c.OldMax); err != nil {
|
||||
return c, err
|
||||
}
|
||||
switch {
|
||||
case c.NewMin < config.MinStorageMB || c.NewMax < config.MinStorageMB:
|
||||
return c, storage.ErrStorageTooSmall
|
||||
case c.NewMin > c.NewMax:
|
||||
return c, storage.ErrMinOverMax
|
||||
case c.NewMax > m.cfg.Storage.LimitMB:
|
||||
return c, storage.ErrStorageTooBig
|
||||
}
|
||||
|
||||
if u.Node == "" {
|
||||
views, err := m.Views(ctx)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
if _, _, err := balance.ChoosePlacement(views, u.Size, c.NewMin); err != nil {
|
||||
return c, err
|
||||
}
|
||||
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
|
||||
}
|
||||
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
v, info, err := m.view(ctx, n)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
for _, d := range info.Drives {
|
||||
if d.Path == u.Drive {
|
||||
c.UsedMB = d.HomesMB[u.Machine]
|
||||
}
|
||||
}
|
||||
if c.NewMax < c.UsedMB {
|
||||
return c, storage.ErrBelowUsage
|
||||
}
|
||||
var r *storage.DriveReport
|
||||
for i := range v.Drives {
|
||||
if v.Drives[i].Path == u.Drive {
|
||||
r = &v.Drives[i]
|
||||
}
|
||||
}
|
||||
if r == nil {
|
||||
return c, fmt.Errorf("drive %s is no longer configured on %s", u.Drive, u.Node)
|
||||
}
|
||||
extra := storage.Committed(c.NewMin, c.UsedMB) - storage.Committed(c.OldMin, c.UsedMB)
|
||||
if extra > 0 && extra > r.HeadroomMB() {
|
||||
return c, storage.ErrStorageFull
|
||||
}
|
||||
if err := n.SetQuota(ctx, u.Drive, u.Machine, c.NewMax); err != nil {
|
||||
return c, err
|
||||
}
|
||||
r.CommittedMB += extra
|
||||
r.BookedMB += c.NewMax - c.OldMax
|
||||
c.Drive = r
|
||||
return c, m.store.SetStorage(id, c.NewMin, c.NewMax)
|
||||
}
|
||||
|
||||
func (m *Manager) StorageUsed(ctx context.Context, u *model.Owner) (int64, bool) {
|
||||
if u == nil || u.Node == "" {
|
||||
return 0, false
|
||||
}
|
||||
n, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
info, err := n.Info(ctx)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
for _, d := range info.Drives {
|
||||
if used, ok := d.HomesMB[u.Machine]; ok && d.Path == u.Drive {
|
||||
return used, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"slackerbot/internal/balance"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/storage"
|
||||
)
|
||||
|
||||
const transferChunk = 4 << 20
|
||||
|
||||
var (
|
||||
ErrMigrating = errors.New("this container is being moved")
|
||||
ErrNoContainer = errors.New("there's no container here yet")
|
||||
ErrSameNode = errors.New("the container is already on that node")
|
||||
)
|
||||
|
||||
type UnknownNodeError struct{ Node string }
|
||||
|
||||
func (e *UnknownNodeError) Error() string { return fmt.Sprintf("there's no node called %s", e.Node) }
|
||||
|
||||
type MovePhase string
|
||||
|
||||
const (
|
||||
PhaseCopy MovePhase = "copy"
|
||||
PhaseFinal MovePhase = "final"
|
||||
PhaseStart MovePhase = "start"
|
||||
PhaseCleanup MovePhase = "cleanup"
|
||||
)
|
||||
|
||||
type MoveProgress struct {
|
||||
To string
|
||||
Phase MovePhase
|
||||
CopiedMB int64
|
||||
TotalMB int64
|
||||
}
|
||||
|
||||
type MoveOptions struct {
|
||||
Target string
|
||||
StartAfter bool
|
||||
WaitIdle func(ctx context.Context)
|
||||
Progress func(MoveProgress)
|
||||
}
|
||||
|
||||
type MoveResult struct {
|
||||
From, To string
|
||||
WasRunning bool
|
||||
Started bool
|
||||
CopiedMB int64
|
||||
LeftOld error
|
||||
}
|
||||
|
||||
func (m *Manager) IsMigrating(id string) bool {
|
||||
_, ok := m.migrating.Load(id)
|
||||
return ok
|
||||
}
|
||||
|
||||
// Migrate copies the owner's home folder to another node and switches the
|
||||
// container over. A running container keeps running through the bulk copy
|
||||
// and is only stopped for a final incremental sync.
|
||||
func (m *Manager) Migrate(ctx context.Context, id string, opts MoveOptions) (*MoveResult, error) {
|
||||
if _, busy := m.migrating.LoadOrStore(id, true); busy {
|
||||
return nil, ErrMigrating
|
||||
}
|
||||
defer m.migrating.Delete(id)
|
||||
unlock := m.ownerMu.Lock(id)
|
||||
defer unlock()
|
||||
|
||||
u := m.store.Owner(id)
|
||||
if u == nil || u.Node == "" {
|
||||
return nil, ErrNoContainer
|
||||
}
|
||||
src, err := m.nodeByName(u.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &MoveResult{From: u.Node, WasRunning: src.IsRunning(u.Machine)}
|
||||
needStart := res.WasRunning || opts.StartAfter
|
||||
|
||||
usedMB, _ := m.StorageUsed(ctx, u)
|
||||
target, drive, err := m.pickTarget(ctx, u, opts.Target, needStart, storage.Committed(u.StorageMinMB, usedMB))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.To = target
|
||||
dst, err := m.nodeByName(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
progress := func(p MovePhase, copied int64) {
|
||||
if opts.Progress != nil {
|
||||
opts.Progress(MoveProgress{To: target, Phase: p, CopiedMB: copied >> 20, TotalMB: usedMB})
|
||||
}
|
||||
}
|
||||
|
||||
imp, err := dst.ImportHome(ctx, drive, u.Machine)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var exports []string
|
||||
stopped := false
|
||||
rollback := func(cause error) (*MoveResult, error) {
|
||||
_ = dst.AbortImport(context.Background(), imp)
|
||||
for _, e := range exports {
|
||||
_ = src.CloseExport(context.Background(), e)
|
||||
}
|
||||
if stopped {
|
||||
if err := src.Start(context.Background(), m.spec(u)); err != nil {
|
||||
return res, fmt.Errorf("%w (and restarting it on %s failed: %v)", cause, u.Node, err)
|
||||
}
|
||||
}
|
||||
return res, cause
|
||||
}
|
||||
|
||||
var copied int64
|
||||
send := func(parent string, phase MovePhase) error {
|
||||
exp, err := src.ExportHome(ctx, u.Drive, u.Machine, parent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
exports = append(exports, exp)
|
||||
for {
|
||||
data, eof, err := src.ReadExport(ctx, exp, transferChunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(data) > 0 {
|
||||
if err := dst.WriteImport(ctx, imp, data); err != nil {
|
||||
return err
|
||||
}
|
||||
copied += int64(len(data))
|
||||
progress(phase, copied)
|
||||
}
|
||||
if eof {
|
||||
return dst.EndReceive(ctx, imp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
progress(PhaseCopy, 0)
|
||||
if err := send("", PhaseCopy); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
if res.WasRunning {
|
||||
progress(PhaseFinal, copied)
|
||||
if opts.WaitIdle != nil {
|
||||
opts.WaitIdle(ctx)
|
||||
}
|
||||
if err := src.Poweroff(ctx, u.Machine); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
stopped = true
|
||||
if err := src.WaitStopped(ctx, u.Machine); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
if err := send(exports[0], PhaseFinal); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
}
|
||||
if err := dst.FinishImport(ctx, imp, u.StorageMaxMB); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
res.CopiedMB = copied >> 20
|
||||
|
||||
oldNode, oldDrive := u.Node, u.Drive
|
||||
if err := m.store.SetPlacement(id, target, drive); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
u.Node, u.Drive = target, drive
|
||||
|
||||
progress(PhaseCleanup, copied)
|
||||
for _, e := range exports {
|
||||
_ = src.CloseExport(ctx, e)
|
||||
}
|
||||
if res.LeftOld = src.DeleteHome(ctx, oldDrive, u.Machine); res.LeftOld != nil {
|
||||
log.Printf("move: old home of %s left on %s: %v", u.Machine, oldNode, res.LeftOld)
|
||||
}
|
||||
|
||||
if !needStart {
|
||||
return res, nil
|
||||
}
|
||||
progress(PhaseStart, copied)
|
||||
if err := dst.Start(ctx, m.spec(u)); err != nil {
|
||||
return res, fmt.Errorf("moved to %s, but starting it there failed: %w", target, err)
|
||||
}
|
||||
res.Started = true
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (m *Manager) pickTarget(ctx context.Context, u *model.Owner, want string, needStart bool, storageMB int64) (target, drive string, err error) {
|
||||
if want == u.Node {
|
||||
return "", "", ErrSameNode
|
||||
}
|
||||
m.startMu.Lock()
|
||||
defer m.startMu.Unlock()
|
||||
|
||||
views, err := m.Views(ctx)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
compute := model.Size{}
|
||||
if needStart {
|
||||
compute = u.Size
|
||||
}
|
||||
var candidates []balance.NodeView
|
||||
for _, v := range views {
|
||||
if v.Name != u.Node && (want == "" || v.Name == want) {
|
||||
candidates = append(candidates, v)
|
||||
}
|
||||
}
|
||||
if want != "" && len(candidates) == 0 {
|
||||
if m.KnownNode(want) {
|
||||
return "", "", &node.NodeOfflineError{Node: want}
|
||||
}
|
||||
return "", "", &UnknownNodeError{Node: want}
|
||||
}
|
||||
target, drive, err = balance.ChoosePlacement(candidates, compute, storageMB)
|
||||
if err != nil && want != "" {
|
||||
return "", "", &node.NodeFullError{Node: want}
|
||||
}
|
||||
return target, drive, err
|
||||
}
|
||||
|
||||
func (m *Manager) KnownNode(name string) bool {
|
||||
if name == m.local.Name() {
|
||||
return true
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
_, ok := m.remotes[name]
|
||||
return ok
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package manager_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"slackerbot/internal/manager"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/testutil"
|
||||
"slackerbot/internal/testutil/clustertest"
|
||||
)
|
||||
|
||||
func TestMoveLiveContainer(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), true)
|
||||
|
||||
var phases []manager.MovePhase
|
||||
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{
|
||||
Target: "slave-1",
|
||||
WaitIdle: func(context.Context) {
|
||||
c.Master.Mu.Lock()
|
||||
c.Master.Homes["/d/c-c1"] = append(c.Master.Homes["/d/c-c1"], []byte("written during the bulk copy")...)
|
||||
c.Master.Mu.Unlock()
|
||||
},
|
||||
Progress: func(p manager.MoveProgress) {
|
||||
if len(phases) == 0 || phases[len(phases)-1] != p.Phase {
|
||||
phases = append(phases, p.Phase)
|
||||
}
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := append(testutil.HomeData(), []byte("written during the bulk copy")...)
|
||||
if !bytes.Equal(c.Slave.Homes["/d/c-c1"], want) {
|
||||
t.Fatal("the late write didn't make it to the new node")
|
||||
}
|
||||
if c.Slave.Passes != 2 {
|
||||
t.Fatalf("a live move should copy twice (bulk + final sync), got %d", c.Slave.Passes)
|
||||
}
|
||||
if _, still := c.Master.Homes["/d/c-c1"]; still {
|
||||
t.Fatal("old home should be deleted after a good move")
|
||||
}
|
||||
if !c.Slave.IsRunning("c-c1") || c.Master.IsRunning("c-c1") || !res.Started {
|
||||
t.Fatal("container should be running on the new node only")
|
||||
}
|
||||
if u := c.Store.Owner("C1"); u.Node != "slave-1" || u.Drive != "/d" {
|
||||
t.Fatalf("placement not switched: %+v", u)
|
||||
}
|
||||
if len(c.Master.Exports) != 0 {
|
||||
t.Fatal("snapshots on the old node should be cleaned up")
|
||||
}
|
||||
wantPhases := []manager.MovePhase{manager.PhaseCopy, manager.PhaseFinal, manager.PhaseCleanup, manager.PhaseStart}
|
||||
if len(phases) != len(wantPhases) {
|
||||
t.Fatalf("phases %v, want %v", phases, wantPhases)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveAutoStoppedContainer(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), false)
|
||||
|
||||
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.To != "slave-1" || res.Started || c.Slave.IsRunning("c-c1") {
|
||||
t.Fatalf("a stopped container should move without starting: %+v", res)
|
||||
}
|
||||
if c.Slave.Passes != 1 {
|
||||
t.Fatalf("a stopped container needs one copy, got %d", c.Slave.Passes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveFullNodeStartsElsewhere(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), false)
|
||||
c.Master.Mu.Lock()
|
||||
c.Master.Usable = model.Size{MemoryMB: 512, CPUs: 0.5}
|
||||
c.Master.Mu.Unlock()
|
||||
|
||||
_, err := c.Mgr.Ensure(c.Ctx, "C1", model.OwnerChannel)
|
||||
var full *node.NodeFullError
|
||||
if !errors.As(err, &full) || full.Node != "master" {
|
||||
t.Fatalf("want node full on master, got %v", err)
|
||||
}
|
||||
res, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{StartAfter: true})
|
||||
if err != nil || !res.Started || !c.Slave.IsRunning("c-c1") {
|
||||
t.Fatalf("auto-move should start it on the new node: %+v %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveFailureRollsBack(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), true)
|
||||
|
||||
_, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{
|
||||
Target: "slave-1",
|
||||
WaitIdle: func(context.Context) {
|
||||
c.Slave.Mu.Lock()
|
||||
c.Slave.FailWrites = true
|
||||
c.Slave.Mu.Unlock()
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected the final sync to fail")
|
||||
}
|
||||
if u := c.Store.Owner("C1"); u.Node != "master" {
|
||||
t.Fatalf("container should stay on its old node, got %s", u.Node)
|
||||
}
|
||||
if !c.Master.IsRunning("c-c1") {
|
||||
t.Fatal("container should be restarted on its old node")
|
||||
}
|
||||
if _, ok := c.Master.Homes["/d/c-c1"]; !ok {
|
||||
t.Fatal("old home must survive a failed move")
|
||||
}
|
||||
if _, ok := c.Slave.Homes["/d/c-c1"]; ok || len(c.Slave.Imports) != 0 {
|
||||
t.Fatal("half-copied home on the target should be cleaned up")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveRefusals(t *testing.T) {
|
||||
c := clustertest.New(t)
|
||||
c.AddOwner(t, testutil.HomeData(), false)
|
||||
|
||||
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "master"}); !errors.Is(err, manager.ErrSameNode) {
|
||||
t.Fatalf("same node: %v", err)
|
||||
}
|
||||
var unknown *manager.UnknownNodeError
|
||||
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "nope"}); !errors.As(err, &unknown) {
|
||||
t.Fatalf("unknown node: %v", err)
|
||||
}
|
||||
if _, err := c.Mgr.Migrate(c.Ctx, "nobody", manager.MoveOptions{}); !errors.Is(err, manager.ErrNoContainer) {
|
||||
t.Fatalf("no container: %v", err)
|
||||
}
|
||||
c.Slave.Mu.Lock()
|
||||
c.Slave.Usable = model.Size{MemoryMB: 512, CPUs: 0.5}
|
||||
c.Slave.Mu.Unlock()
|
||||
c.Master.Mu.Lock()
|
||||
c.Master.Runs["c-c1"] = true
|
||||
c.Master.Mu.Unlock()
|
||||
testutil.WaitFor(t, "slave report", func() bool {
|
||||
views, _ := c.Mgr.Views(c.Ctx)
|
||||
for _, v := range views {
|
||||
if v.Name == "slave-1" {
|
||||
return v.Usable.MemoryMB == 512
|
||||
}
|
||||
}
|
||||
return false
|
||||
})
|
||||
var full *node.NodeFullError
|
||||
if _, err := c.Mgr.Migrate(c.Ctx, "C1", manager.MoveOptions{Target: "slave-1"}); !errors.As(err, &full) {
|
||||
t.Fatalf("full target: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Size struct {
|
||||
MemoryMB int `toml:"memory_mb"`
|
||||
CPUs float64 `toml:"cpus"`
|
||||
}
|
||||
|
||||
func (s Size) Add(o Size) Size { return Size{s.MemoryMB + o.MemoryMB, s.CPUs + o.CPUs} }
|
||||
func (s Size) Sub(o Size) Size { return Size{s.MemoryMB - o.MemoryMB, s.CPUs - o.CPUs} }
|
||||
func (s Size) Fits(in Size) bool {
|
||||
return s.MemoryMB <= in.MemoryMB && s.CPUs <= in.CPUs+1e-9
|
||||
}
|
||||
func (s Size) String() string {
|
||||
mem := fmt.Sprintf("%d MB", s.MemoryMB)
|
||||
if s.MemoryMB >= 1024 && s.MemoryMB%1024 == 0 {
|
||||
mem = fmt.Sprintf("%d GB", s.MemoryMB/1024)
|
||||
}
|
||||
cpu := strconv.FormatFloat(s.CPUs, 'f', -1, 64) + " CPUs"
|
||||
if s.CPUs == 1 {
|
||||
cpu = "1 CPU"
|
||||
}
|
||||
return mem + " RAM, " + cpu
|
||||
}
|
||||
|
||||
type OwnerKind string
|
||||
|
||||
const (
|
||||
OwnerUser OwnerKind = "user"
|
||||
OwnerChannel OwnerKind = "channel"
|
||||
)
|
||||
|
||||
type Owner struct {
|
||||
ID string `json:"id"`
|
||||
Kind OwnerKind `json:"kind"`
|
||||
Machine string `json:"machine"`
|
||||
Secret string `json:"secret"`
|
||||
Size Size `json:"size"`
|
||||
LastActive time.Time `json:"last_active"`
|
||||
|
||||
Node string `json:"node,omitempty"`
|
||||
Drive string `json:"drive,omitempty"`
|
||||
StorageMinMB int64 `json:"storage_min_mb"`
|
||||
StorageMaxMB int64 `json:"storage_max_mb"`
|
||||
|
||||
// 0 means the config default.
|
||||
IdleTimeout time.Duration `json:"idle_timeout,omitempty"`
|
||||
}
|
||||
|
||||
const Indefinite time.Duration = -1
|
||||
|
||||
type Session struct {
|
||||
Channel string `json:"channel"`
|
||||
ThreadTS string `json:"thread_ts"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
ACPID string `json:"acp_id"`
|
||||
Title string `json:"title"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func MachineName(id string, kind OwnerKind) string {
|
||||
if kind == OwnerChannel {
|
||||
return "c-" + strings.ToLower(id)
|
||||
}
|
||||
return "u-" + strings.ToLower(id)
|
||||
}
|
||||
|
||||
func IsOurs(name string) bool { return strings.HasPrefix(name, "u-") || strings.HasPrefix(name, "c-") }
|
||||
@@ -0,0 +1,244 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Action string
|
||||
|
||||
const (
|
||||
ActScale Action = "scale"
|
||||
ActStorage Action = "storage"
|
||||
ActMove Action = "move"
|
||||
ActKeep Action = "keep"
|
||||
ActManage Action = "manage"
|
||||
)
|
||||
|
||||
var (
|
||||
AllActions = []Action{ActScale, ActStorage, ActMove, ActKeep}
|
||||
GrantActions = append(slices.Clone(AllActions), ActManage)
|
||||
|
||||
ErrLastManager = errors.New("that would leave nobody able to manage permissions")
|
||||
)
|
||||
|
||||
type PermissionsConfig struct {
|
||||
Managers []string `toml:"managers" json:"managers"`
|
||||
Default map[string][]string `toml:"default" json:"default"`
|
||||
Channels map[string]map[string][]string `toml:"channels" json:"channels"`
|
||||
DM map[string]bool `toml:"dm" json:"dm"`
|
||||
}
|
||||
|
||||
func (p PermissionsConfig) Validate() error {
|
||||
if len(p.Managers) == 0 {
|
||||
return fmt.Errorf("permissions.managers must name at least one user manager")
|
||||
}
|
||||
if len(p.Default) == 0 {
|
||||
return fmt.Errorf("[permissions.default] must name at least one super user, e.g. U0ADMIN = [\"all\"]")
|
||||
}
|
||||
check := func(where string, list map[string][]string) error {
|
||||
for user, actions := range list {
|
||||
for _, a := range actions {
|
||||
if a != "all" && !slices.Contains(GrantActions, Action(a)) {
|
||||
return fmt.Errorf("%s: unknown permission %q for %s (use %s, manage or all)", where, a, user, actionNames())
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := check("permissions.default", p.Default); err != nil {
|
||||
return err
|
||||
}
|
||||
for channel, list := range p.Channels {
|
||||
if err := check("permissions.channels."+channel, list); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for a := range p.DM {
|
||||
if !slices.Contains(AllActions, Action(a)) {
|
||||
return fmt.Errorf("permissions.dm: unknown permission %q (use %s)", a, actionNames())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p PermissionsConfig) Clone() PermissionsConfig {
|
||||
c := PermissionsConfig{
|
||||
Managers: slices.Clone(p.Managers),
|
||||
Default: cloneList(p.Default),
|
||||
Channels: map[string]map[string][]string{},
|
||||
DM: maps.Clone(p.DM),
|
||||
}
|
||||
for ch, list := range p.Channels {
|
||||
c.Channels[ch] = cloneList(list)
|
||||
}
|
||||
if c.DM == nil {
|
||||
c.DM = map[string]bool{}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func cloneList(list map[string][]string) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for user, actions := range list {
|
||||
out[user] = slices.Clone(actions)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func actionNames() string {
|
||||
names := make([]string, len(AllActions))
|
||||
for i, a := range AllActions {
|
||||
names[i] = string(a)
|
||||
}
|
||||
return strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
func (p PermissionsConfig) SuperUsers(channel string) (list map[string][]string, own bool) {
|
||||
if l := p.Channels[channel]; len(l) > 0 {
|
||||
return l, true
|
||||
}
|
||||
return p.Default, false
|
||||
}
|
||||
|
||||
// "all" covers the container actions only; managing permissions is always granted by name.
|
||||
func Grants(actions []string, a Action) bool {
|
||||
if a == ActManage {
|
||||
return slices.Contains(actions, string(ActManage))
|
||||
}
|
||||
return slices.Contains(actions, "all") || slices.Contains(actions, string(a))
|
||||
}
|
||||
|
||||
func (p PermissionsConfig) Allowed(kind OwnerKind, channel, user string, a Action) bool {
|
||||
if kind == OwnerUser {
|
||||
allowed, set := p.DM[string(a)]
|
||||
return !set || allowed
|
||||
}
|
||||
list, _ := p.SuperUsers(channel)
|
||||
return Grants(list[user], a)
|
||||
}
|
||||
|
||||
func (p PermissionsConfig) IsManager(user string) bool { return slices.Contains(p.Managers, user) }
|
||||
|
||||
func (p PermissionsConfig) CanManage(channel, user string) bool {
|
||||
return p.IsManager(user) || p.Allowed(OwnerChannel, channel, user, ActManage)
|
||||
}
|
||||
|
||||
func (p PermissionsConfig) Holders(channel string, a Action) []string {
|
||||
list, _ := p.SuperUsers(channel)
|
||||
var out []string
|
||||
for user, actions := range list {
|
||||
if Grants(actions, a) {
|
||||
out = append(out, user)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (p PermissionsConfig) ActionsFor(channel, user string) []Action {
|
||||
list, _ := p.SuperUsers(channel)
|
||||
var out []Action
|
||||
for _, a := range GrantActions {
|
||||
if Grants(list[user], a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func expand(actions []Action) []Action {
|
||||
if slices.Contains(actions, "all") {
|
||||
out := slices.Clone(AllActions)
|
||||
for _, a := range actions {
|
||||
if a != "all" && !slices.Contains(out, a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
return actions
|
||||
}
|
||||
|
||||
// An empty channel means the default list. The first grant in a channel that
|
||||
// still uses the default list copies it, so nobody loses access there.
|
||||
func (p *PermissionsConfig) Grant(channel string, users []string, actions []Action) {
|
||||
list := p.Default
|
||||
if channel != "" {
|
||||
if len(p.Channels[channel]) == 0 {
|
||||
p.Channels[channel] = cloneList(p.Default)
|
||||
}
|
||||
list = p.Channels[channel]
|
||||
}
|
||||
for _, user := range users {
|
||||
have := expandStrings(list[user])
|
||||
for _, a := range expand(actions) {
|
||||
if !slices.Contains(have, string(a)) {
|
||||
have = append(have, string(a))
|
||||
}
|
||||
}
|
||||
list[user] = have
|
||||
}
|
||||
}
|
||||
|
||||
// No actions means everything.
|
||||
func (p *PermissionsConfig) Revoke(channel string, users []string, actions []Action) {
|
||||
list := p.Default
|
||||
if channel != "" {
|
||||
if len(p.Channels[channel]) == 0 {
|
||||
p.Channels[channel] = cloneList(p.Default)
|
||||
}
|
||||
list = p.Channels[channel]
|
||||
}
|
||||
remove := expand(actions)
|
||||
for _, user := range users {
|
||||
if len(actions) == 0 {
|
||||
delete(list, user)
|
||||
continue
|
||||
}
|
||||
var keep []string
|
||||
for _, a := range expandStrings(list[user]) {
|
||||
if !slices.Contains(remove, Action(a)) {
|
||||
keep = append(keep, a)
|
||||
}
|
||||
}
|
||||
if len(keep) == 0 {
|
||||
delete(list, user)
|
||||
} else {
|
||||
list[user] = keep
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func expandStrings(actions []string) []string {
|
||||
as := make([]Action, len(actions))
|
||||
for i, a := range actions {
|
||||
as[i] = Action(a)
|
||||
}
|
||||
var out []string
|
||||
for _, a := range expand(as) {
|
||||
out = append(out, string(a))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *PermissionsConfig) AddManagers(users []string) {
|
||||
for _, u := range users {
|
||||
if !p.IsManager(u) {
|
||||
p.Managers = append(p.Managers, u)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PermissionsConfig) RemoveManagers(users []string) error {
|
||||
keep := slices.DeleteFunc(slices.Clone(p.Managers), func(u string) bool { return slices.Contains(users, u) })
|
||||
if len(keep) == 0 {
|
||||
return ErrLastManager
|
||||
}
|
||||
p.Managers = keep
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package model_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/model"
|
||||
)
|
||||
|
||||
func TestPermissions(t *testing.T) {
|
||||
t.Setenv("SLACK_APP_TOKEN", "xapp-test")
|
||||
t.Setenv("SLACK_BOT_TOKEN", "xoxb-test")
|
||||
t.Setenv("CLUSTER_SECRET", "cluster-secret-1234567890")
|
||||
c, err := config.LoadConfig("../../../config.example.toml", config.ModeMaster)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := c.Permissions
|
||||
|
||||
cases := []struct {
|
||||
kind model.OwnerKind
|
||||
channel, user string
|
||||
action model.Action
|
||||
want bool
|
||||
}{
|
||||
{model.OwnerChannel, "C07ABC123", "U0111", model.ActMove, true},
|
||||
{model.OwnerChannel, "C07ABC123", "U0111", model.ActScale, false},
|
||||
{model.OwnerChannel, "C07ABC123", "U0222", model.ActKeep, true},
|
||||
{model.OwnerChannel, "C07ABC123", "U0ADMIN", model.ActMove, false},
|
||||
{model.OwnerChannel, "C0OTHER", "U0ADMIN", model.ActStorage, true},
|
||||
{model.OwnerChannel, "C0OTHER", "U0111", model.ActMove, false},
|
||||
{model.OwnerUser, "D1", "U9", model.ActScale, true},
|
||||
{model.OwnerUser, "D1", "U9", model.ActStorage, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := p.Allowed(c.kind, c.channel, c.user, c.action); got != c.want {
|
||||
t.Errorf("%s %s %s %s = %v, want %v", c.kind, c.channel, c.user, c.action, got, c.want)
|
||||
}
|
||||
}
|
||||
if h := p.Holders("C07ABC123", model.ActMove); len(h) != 2 {
|
||||
t.Errorf("move holders = %v", h)
|
||||
}
|
||||
if a := p.ActionsFor("C0OTHER", "U0ADMIN"); len(a) != len(model.AllActions) {
|
||||
t.Errorf("all should expand to every action, got %v", a)
|
||||
}
|
||||
|
||||
open := model.PermissionsConfig{Default: map[string][]string{"U1": {"all"}}}
|
||||
if !open.Allowed(model.OwnerUser, "D1", "U9", model.ActMove) {
|
||||
t.Error("DM permissions should default to allowed")
|
||||
}
|
||||
if (model.PermissionsConfig{}).Validate() == nil {
|
||||
t.Error("a missing default super user should be refused")
|
||||
}
|
||||
bad := model.PermissionsConfig{Default: map[string][]string{"U1": {"delete-everything"}}}
|
||||
if bad.Validate() == nil {
|
||||
t.Error("unknown permission names should be refused")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrantRevoke(t *testing.T) {
|
||||
p := model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UDEF": {"all"}}}.Clone()
|
||||
|
||||
p.Grant("C1", []string{"UANA"}, []model.Action{model.ActMove})
|
||||
if !p.Allowed(model.OwnerChannel, "C1", "UDEF", model.ActScale) {
|
||||
t.Fatal("first grant in a channel should keep everyone from the default list")
|
||||
}
|
||||
if !p.Allowed(model.OwnerChannel, "C1", "UANA", model.ActMove) || p.Allowed(model.OwnerChannel, "C1", "UANA", model.ActScale) {
|
||||
t.Fatal("grant should add exactly what was asked")
|
||||
}
|
||||
if p.Allowed(model.OwnerChannel, "C2", "UANA", model.ActMove) {
|
||||
t.Fatal("grants stay in their channel")
|
||||
}
|
||||
|
||||
p.Grant("C1", []string{"UANA"}, []model.Action{"all"})
|
||||
if p.CanManage("C1", "UANA") {
|
||||
t.Fatal("all must not include manage")
|
||||
}
|
||||
p.Grant("C1", []string{"UANA"}, []model.Action{model.ActManage})
|
||||
if !p.CanManage("C1", "UANA") || p.CanManage("C2", "UANA") {
|
||||
t.Fatal("manage is per channel")
|
||||
}
|
||||
|
||||
p.Revoke("C1", []string{"UDEF"}, []model.Action{model.ActScale})
|
||||
if p.Allowed(model.OwnerChannel, "C1", "UDEF", model.ActScale) || !p.Allowed(model.OwnerChannel, "C1", "UDEF", model.ActMove) {
|
||||
t.Fatal("revoking one action out of all should keep the rest")
|
||||
}
|
||||
p.Revoke("C1", []string{"UANA"}, nil)
|
||||
if len(p.ActionsFor("C1", "UANA")) != 0 {
|
||||
t.Fatal("revoke with no actions removes the person")
|
||||
}
|
||||
|
||||
p.Grant("", []string{"UNEW"}, []model.Action{model.ActKeep})
|
||||
if !p.Allowed(model.OwnerChannel, "C9", "UNEW", model.ActKeep) {
|
||||
t.Fatal("default grants apply to channels without their own list")
|
||||
}
|
||||
|
||||
p.AddManagers([]string{"UTWO"})
|
||||
if err := p.RemoveManagers([]string{"UBOSS", "UTWO"}); !errors.Is(err, model.ErrLastManager) {
|
||||
t.Fatal("removing every manager must be refused")
|
||||
}
|
||||
if err := p.RemoveManagers([]string{"UBOSS"}); err != nil || p.IsManager("UBOSS") {
|
||||
t.Fatal("removing one of two managers should work")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package node
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
)
|
||||
|
||||
type Node interface {
|
||||
Name() string
|
||||
Info(ctx context.Context) (NodeInfo, error)
|
||||
Running(ctx context.Context) ([]RunningMachine, error)
|
||||
IsRunning(machine string) bool
|
||||
Start(ctx context.Context, spec StartSpec) error
|
||||
Poweroff(ctx context.Context, machine string) error
|
||||
SetLimits(ctx context.Context, machine string, size model.Size) error
|
||||
SetQuota(ctx context.Context, drive, machine string, mb int64) error
|
||||
DialAgent(ctx context.Context, machine, secret string) (MsgConn, error)
|
||||
|
||||
WaitStopped(ctx context.Context, machine string) error
|
||||
ExportHome(ctx context.Context, drive, machine, parent string) (id string, err error)
|
||||
ReadExport(ctx context.Context, id string, max int) (data []byte, eof bool, err error)
|
||||
CloseExport(ctx context.Context, id string) error
|
||||
ImportHome(ctx context.Context, drive, machine string) (id string, err error)
|
||||
WriteImport(ctx context.Context, id string, data []byte) error
|
||||
EndReceive(ctx context.Context, id string) error
|
||||
FinishImport(ctx context.Context, id string, quotaMB int64) error
|
||||
AbortImport(ctx context.Context, id string) error
|
||||
DeleteHome(ctx context.Context, drive, machine string) error
|
||||
}
|
||||
|
||||
type NodeInfo struct {
|
||||
Name string `json:"name"`
|
||||
Total model.Size `json:"total"`
|
||||
Usable model.Size `json:"usable"`
|
||||
Drives []DriveInfo `json:"drives"`
|
||||
Running []RunningMachine `json:"running"`
|
||||
}
|
||||
|
||||
type DriveInfo struct {
|
||||
Path string `json:"path"`
|
||||
FreeMB int64 `json:"free_mb"`
|
||||
ReserveMB int64 `json:"reserve_mb"`
|
||||
HomesMB map[string]int64 `json:"homes_mb"`
|
||||
}
|
||||
|
||||
type RunningMachine struct {
|
||||
Name string `json:"name"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
type StartSpec struct {
|
||||
Machine string `json:"machine"`
|
||||
Description string `json:"description"`
|
||||
Secret string `json:"secret"`
|
||||
Size model.Size `json:"size"`
|
||||
Drive string `json:"drive"`
|
||||
StorageMaxMB int64 `json:"storage_max_mb"`
|
||||
Env map[string]string `json:"env"`
|
||||
}
|
||||
|
||||
type NodeOfflineError struct{ Node string }
|
||||
|
||||
func (e *NodeOfflineError) Error() string { return fmt.Sprintf("node %s is offline", e.Node) }
|
||||
|
||||
type NodeFullError struct{ Node string }
|
||||
|
||||
func (e *NodeFullError) Error() string { return fmt.Sprintf("node %s is full", e.Node) }
|
||||
|
||||
type MsgConn interface {
|
||||
Read(ctx context.Context) ([]byte, error)
|
||||
Write(ctx context.Context, b []byte) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type wsConn struct{ c *websocket.Conn }
|
||||
|
||||
func (w wsConn) Read(ctx context.Context) ([]byte, error) {
|
||||
_, b, err := w.c.Read(ctx)
|
||||
return b, err
|
||||
}
|
||||
|
||||
func (w wsConn) Write(ctx context.Context, b []byte) error {
|
||||
return w.c.Write(ctx, websocket.MessageText, b)
|
||||
}
|
||||
|
||||
func (w wsConn) Close() error { return w.c.Close(websocket.StatusNormalClosure, "") }
|
||||
|
||||
func DialAgentWS(ctx context.Context, addr, secret string) (MsgConn, error) {
|
||||
conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/acp", &websocket.DialOptions{
|
||||
HTTPHeader: http.Header{"X-Secret-Key": []string{secret}},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connecting to goose: %w", err)
|
||||
}
|
||||
conn.SetReadLimit(32 << 20)
|
||||
return wsConn{conn}, nil
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
path string
|
||||
mu sync.Mutex
|
||||
data struct {
|
||||
Owners map[string]*model.Owner `json:"owners"`
|
||||
Sessions map[string]*model.Session `json:"sessions"`
|
||||
Permissions *model.PermissionsConfig `json:"permissions,omitempty"`
|
||||
}
|
||||
}
|
||||
|
||||
func Open(path string) (*Store, error) {
|
||||
s := &Store{path: path}
|
||||
s.data.Owners = map[string]*model.Owner{}
|
||||
s.data.Sessions = map[string]*model.Session{}
|
||||
|
||||
b, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return s, os.MkdirAll(filepath.Dir(path), 0o700)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(b, &s.data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.data.Owners == nil {
|
||||
s.data.Owners = map[string]*model.Owner{}
|
||||
}
|
||||
if s.data.Sessions == nil {
|
||||
s.data.Sessions = map[string]*model.Session{}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Store) save() error {
|
||||
b, err := json.MarshalIndent(&s.data, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := s.path + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, s.path)
|
||||
}
|
||||
|
||||
func (s *Store) Owner(id string) *model.Owner {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u, ok := s.data.Owners[id]; ok {
|
||||
c := *u
|
||||
return &c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) OwnerByMachine(machine string) *model.Owner {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, u := range s.data.Owners {
|
||||
if u.Machine == machine {
|
||||
c := *u
|
||||
return &c
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) PutOwner(u *model.Owner) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
c := *u
|
||||
s.data.Owners[u.ID] = &c
|
||||
return s.save()
|
||||
}
|
||||
|
||||
func (s *Store) Touch(id string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u, ok := s.data.Owners[id]; ok {
|
||||
u.LastActive = time.Now()
|
||||
return s.save()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) SetSize(id string, size model.Size) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u, ok := s.data.Owners[id]; ok {
|
||||
u.Size = size
|
||||
return s.save()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Owners() []model.Owner {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make([]model.Owner, 0, len(s.data.Owners))
|
||||
for _, u := range s.data.Owners {
|
||||
out = append(out, *u)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Store) SetStorage(id string, minMB, maxMB int64) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u, ok := s.data.Owners[id]; ok {
|
||||
u.StorageMinMB, u.StorageMaxMB = minMB, maxMB
|
||||
return s.save()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) SetPlacement(id, node, drive string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u, ok := s.data.Owners[id]; ok {
|
||||
u.Node, u.Drive = node, drive
|
||||
return s.save()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) DeleteSessions(ownerID string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for k, x := range s.data.Sessions {
|
||||
if x.OwnerID == ownerID {
|
||||
delete(s.data.Sessions, k)
|
||||
}
|
||||
}
|
||||
return s.save()
|
||||
}
|
||||
|
||||
func (s *Store) SetIdleTimeout(id string, d time.Duration) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u, ok := s.data.Owners[id]; ok {
|
||||
u.IdleTimeout = d
|
||||
return s.save()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// The config's permissions are only a starting point: once saved here,
|
||||
// changes made in Slack win over later config edits.
|
||||
func (s *Store) SeedPermissions(initial model.PermissionsConfig) (seeded bool, err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.data.Permissions != nil {
|
||||
return false, nil
|
||||
}
|
||||
p := initial.Clone()
|
||||
s.data.Permissions = &p
|
||||
return true, s.save()
|
||||
}
|
||||
|
||||
func (s *Store) Permissions() model.PermissionsConfig {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.data.Permissions == nil {
|
||||
return model.PermissionsConfig{}
|
||||
}
|
||||
return s.data.Permissions.Clone()
|
||||
}
|
||||
|
||||
func (s *Store) UpdatePermissions(change func(p *model.PermissionsConfig) error) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var p model.PermissionsConfig
|
||||
if s.data.Permissions != nil {
|
||||
p = s.data.Permissions.Clone()
|
||||
} else {
|
||||
p = model.PermissionsConfig{}.Clone()
|
||||
}
|
||||
if err := change(&p); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.data.Permissions = &p
|
||||
return s.save()
|
||||
}
|
||||
|
||||
func sessionKey(channel, threadTS string) string { return channel + "/" + threadTS }
|
||||
|
||||
func (s *Store) Session(channel, threadTS string) *model.Session {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if x, ok := s.data.Sessions[sessionKey(channel, threadTS)]; ok {
|
||||
c := *x
|
||||
return &c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) AddSession(x *model.Session) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
c := *x
|
||||
s.data.Sessions[sessionKey(x.Channel, x.ThreadTS)] = &c
|
||||
return s.save()
|
||||
}
|
||||
|
||||
func (s *Store) Sessions(ownerID string, limit int) []model.Session {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []model.Session
|
||||
for _, x := range s.data.Sessions {
|
||||
if x.OwnerID == ownerID {
|
||||
out = append(out, *x)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) })
|
||||
if limit > 0 && len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
)
|
||||
|
||||
func TestStore(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = s.PutOwner(&model.Owner{ID: "C1", Kind: model.OwnerChannel, Machine: "c-c1", Size: model.Size{MemoryMB: 1024, CPUs: 1}})
|
||||
_ = s.AddSession(&model.Session{Channel: "D1", ThreadTS: "1.0", OwnerID: "C1", ACPID: "a", CreatedAt: time.Now().Add(-time.Hour)})
|
||||
_ = s.AddSession(&model.Session{Channel: "D1", ThreadTS: "2.0", OwnerID: "C1", ACPID: "b", CreatedAt: time.Now()})
|
||||
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if u := s2.OwnerByMachine("c-c1"); u == nil || u.Size.MemoryMB != 1024 {
|
||||
t.Fatal("user not saved")
|
||||
}
|
||||
list := s2.Sessions("C1", 0)
|
||||
if len(list) != 2 || list[0].ACPID != "b" {
|
||||
t.Fatalf("sessions not newest-first: %+v", list)
|
||||
}
|
||||
if s2.Session("D1", "1.0").ACPID != "a" {
|
||||
t.Fatal("session lookup failed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPermissionsSavedAndSeededOnce(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
s, _ := Open(path)
|
||||
initial := model.PermissionsConfig{Managers: []string{"UBOSS"}, Default: map[string][]string{"UBOSS": {"all"}}}
|
||||
if seeded, err := s.SeedPermissions(initial); !seeded || err != nil {
|
||||
t.Fatalf("first start should seed: %v %v", seeded, err)
|
||||
}
|
||||
if err := s.UpdatePermissions(func(p *model.PermissionsConfig) error {
|
||||
p.Grant("C1", []string{"UANA"}, []model.Action{model.ActMove})
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s2, _ := Open(path)
|
||||
changed := model.PermissionsConfig{Managers: []string{"USOMEONE"}, Default: map[string][]string{"USOMEONE": {"all"}}}
|
||||
if seeded, _ := s2.SeedPermissions(changed); seeded {
|
||||
t.Fatal("config must not override saved permissions")
|
||||
}
|
||||
p := s2.Permissions()
|
||||
if !p.IsManager("UBOSS") || p.IsManager("USOMEONE") || !p.Allowed(model.OwnerChannel, "C1", "UANA", model.ActMove) {
|
||||
t.Fatalf("saved permissions lost: %+v", p)
|
||||
}
|
||||
if err := s2.UpdatePermissions(func(p *model.PermissionsConfig) error {
|
||||
p.Default["UX"] = []string{"teleport"}
|
||||
return nil
|
||||
}); err == nil {
|
||||
t.Fatal("invalid changes must be refused")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
const btrfsMagic = 0x9123683E
|
||||
|
||||
var (
|
||||
ErrStorageFull = errors.New("not enough storage left on the drives")
|
||||
ErrStorageTooBig = errors.New("above the storage limit")
|
||||
ErrMinOverMax = errors.New("the guaranteed amount can't be more than the max")
|
||||
ErrBelowUsage = errors.New("below what's already in use")
|
||||
ErrStorageTooSmall = errors.New("too small")
|
||||
)
|
||||
|
||||
type DriveReport struct {
|
||||
Path string
|
||||
UsableMB int64
|
||||
CommittedMB int64
|
||||
BookedMB int64
|
||||
Homes int
|
||||
}
|
||||
|
||||
func (d DriveReport) HeadroomMB() int64 { return d.UsableMB - d.CommittedMB }
|
||||
|
||||
func (d DriveReport) Overbooked() bool { return d.BookedMB > d.UsableMB }
|
||||
|
||||
func Committed(minMB, usedMB int64) int64 { return max(minMB, usedMB) }
|
||||
|
||||
func PickDrive(reports []DriveReport, needMB int64) (string, error) {
|
||||
best := -1
|
||||
for i, r := range reports {
|
||||
if r.HeadroomMB() >= needMB && (best < 0 || r.HeadroomMB() > reports[best].HeadroomMB()) {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return "", ErrStorageFull
|
||||
}
|
||||
return reports[best].Path, nil
|
||||
}
|
||||
|
||||
func PrepareDrive(path string) error {
|
||||
if err := os.MkdirAll(path, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
var st syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &st); err != nil {
|
||||
return err
|
||||
}
|
||||
if uint32(st.Type) != btrfsMagic {
|
||||
return fmt.Errorf("%s is not on a btrfs filesystem", path)
|
||||
}
|
||||
if out, err := exec.Command("btrfs", "quota", "enable", path).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("turning on btrfs quotas for %s: %v: %s", path, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func FreeMB(path string) (int64, error) {
|
||||
var st syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &st); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int64(st.Bavail) * int64(st.Bsize) / (1 << 20), nil
|
||||
}
|
||||
|
||||
func CreateSubvolume(path string) error {
|
||||
if out, err := exec.Command("btrfs", "subvolume", "create", path).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("creating %s: %v: %s", path, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func DeleteSubvolume(path string) error {
|
||||
if out, err := exec.Command("btrfs", "subvolume", "delete", path).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("deleting %s: %v: %s", path, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SetQuota(path string, mb int64) error {
|
||||
bytes := strconv.FormatInt(mb<<20, 10)
|
||||
if out, err := exec.Command("btrfs", "qgroup", "limit", bytes, path).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("setting quota on %s: %v: %s", path, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Quota numbers can lag a few seconds behind writes.
|
||||
func UsedMB(path string) (int64, error) {
|
||||
out, err := exec.Command("btrfs", "qgroup", "show", "-f", "--raw", path).CombinedOutput()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("reading usage of %s: %v: %s", path, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return parseQgroupShow(string(out))
|
||||
}
|
||||
|
||||
// Parses `btrfs qgroup show --raw`: "0/257 <referenced bytes> <exclusive bytes> ...".
|
||||
func parseQgroupShow(out string) (int64, error) {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) >= 2 && strings.HasPrefix(f[0], "0/") {
|
||||
b, err := strconv.ParseInt(f[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return b >> 20, nil
|
||||
}
|
||||
}
|
||||
return 0, errors.New("no quota group found")
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDriveMath(t *testing.T) {
|
||||
d := DriveReport{Path: "/a", UsableMB: 100_000, CommittedMB: 90_000, BookedMB: 150_000}
|
||||
if d.HeadroomMB() != 10_000 || !d.Overbooked() {
|
||||
t.Fatalf("headroom %d overbooked %v", d.HeadroomMB(), d.Overbooked())
|
||||
}
|
||||
if Committed(5120, 8000) != 8000 || Committed(5120, 100) != 5120 {
|
||||
t.Fatal("a home holds its guarantee or its real use, whichever is bigger")
|
||||
}
|
||||
drives := []DriveReport{d, {Path: "/b", UsableMB: 50_000, CommittedMB: 20_000}}
|
||||
if got, _ := PickDrive(drives, 5120); got != "/b" {
|
||||
t.Fatalf("should pick the drive with the most room, got %s", got)
|
||||
}
|
||||
if _, err := PickDrive(drives, 40_000); !errors.Is(err, ErrStorageFull) {
|
||||
t.Fatal("a guarantee that fits nowhere should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseQgroupShow(t *testing.T) {
|
||||
out := "Qgroupid Referenced Exclusive Max referenced Max exclusive Path\n" +
|
||||
"-------- ---------- --------- -------------- ------------- ----\n" +
|
||||
"0/257 3221225472 3221225472 10737418240 none c-c07abc\n"
|
||||
if mb, err := parseQgroupShow(out); err != nil || mb != 3072 {
|
||||
t.Fatalf("got %d, %v", mb, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package clustertest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/manager"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/state"
|
||||
"slackerbot/internal/testutil"
|
||||
)
|
||||
|
||||
const ClusterSecret = "cluster-secret-1234567890"
|
||||
|
||||
type Cluster struct {
|
||||
Cfg *config.Config
|
||||
Mgr *manager.Manager
|
||||
Store *state.Store
|
||||
Master, Slave *testutil.FakeNode
|
||||
Ctx context.Context
|
||||
}
|
||||
|
||||
func New(t testing.TB) *Cluster {
|
||||
addr := testutil.FreePort(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
cfg := &config.Config{StateDir: t.TempDir(), Container: model.Size{MemoryMB: 1024, CPUs: 1}}
|
||||
cfg.Cluster = config.ClusterConfig{Name: "master", Secret: ClusterSecret, Listen: addr}
|
||||
store, err := state.Open(filepath.Join(cfg.StateDir, "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := &Cluster{Cfg: cfg, Store: store, Master: testutil.NewFakeNode("master"), Slave: testutil.NewFakeNode("slave-1"), Ctx: ctx}
|
||||
c.Mgr = manager.New(cfg, store, c.Master)
|
||||
go c.Mgr.ServeCluster(ctx)
|
||||
|
||||
slaveCfg := &config.Config{StateDir: t.TempDir()}
|
||||
slaveCfg.Cluster = config.ClusterConfig{Name: "slave-1", Secret: ClusterSecret, Master: "wss://" + addr}
|
||||
go manager.RunSlave(ctx, slaveCfg, c.Slave)
|
||||
testutil.WaitFor(t, "slave to come online", func() bool { return c.Mgr.State(&model.Owner{Node: "slave-1"}) != manager.StateOffline })
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Cluster) AddOwner(t testing.TB, data []byte, running bool) {
|
||||
u := &model.Owner{ID: "C1", Kind: model.OwnerChannel, Machine: "c-c1", Secret: "s", Size: model.Size{MemoryMB: 1024, CPUs: 1},
|
||||
Node: "master", Drive: "/d", StorageMinMB: 1024, StorageMaxMB: 2048}
|
||||
if err := c.Store.PutOwner(u); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Master.Mu.Lock()
|
||||
defer c.Master.Mu.Unlock()
|
||||
c.Master.Homes["/d/c-c1"] = data
|
||||
if running {
|
||||
c.Master.Runs["c-c1"] = true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/util"
|
||||
)
|
||||
|
||||
type FakeNode struct {
|
||||
name string
|
||||
Mu sync.Mutex
|
||||
Runs map[string]bool
|
||||
Last node.StartSpec
|
||||
Homes map[string][]byte
|
||||
Exports map[string]*fakeExport
|
||||
Imports map[string]*fakeImport
|
||||
Passes int
|
||||
FailWrites bool
|
||||
Usable model.Size
|
||||
}
|
||||
|
||||
func NewFakeNode(name string) *FakeNode {
|
||||
return &FakeNode{name: name, Runs: map[string]bool{}, Homes: map[string][]byte{},
|
||||
Exports: map[string]*fakeExport{}, Imports: map[string]*fakeImport{}, Usable: model.Size{MemoryMB: 8192, CPUs: 4}}
|
||||
}
|
||||
|
||||
func (f *FakeNode) Name() string { return f.name }
|
||||
func (f *FakeNode) Info(ctx context.Context) (node.NodeInfo, error) {
|
||||
running, _ := f.Running(ctx)
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
homes := map[string]int64{}
|
||||
for k, v := range f.Homes {
|
||||
homes[strings.TrimPrefix(k, "/d/")] = int64(len(v)) >> 20
|
||||
}
|
||||
return node.NodeInfo{Name: f.name, Usable: f.Usable, Running: running,
|
||||
Drives: []node.DriveInfo{{Path: "/d", FreeMB: 50_000, HomesMB: homes}}}, nil
|
||||
}
|
||||
func (f *FakeNode) Running(context.Context) ([]node.RunningMachine, error) {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
var out []node.RunningMachine
|
||||
for m := range f.Runs {
|
||||
out = append(out, node.RunningMachine{Name: m})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (f *FakeNode) IsRunning(machine string) bool {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
return f.Runs[machine]
|
||||
}
|
||||
func (f *FakeNode) Start(_ context.Context, spec node.StartSpec) error {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
f.Last, f.Runs[spec.Machine] = spec, true
|
||||
return nil
|
||||
}
|
||||
func (f *FakeNode) Poweroff(_ context.Context, machine string) error {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
delete(f.Runs, machine)
|
||||
return nil
|
||||
}
|
||||
func (f *FakeNode) SetLimits(context.Context, string, model.Size) error { return nil }
|
||||
func (f *FakeNode) SetQuota(context.Context, string, string, int64) error { return nil }
|
||||
func (f *FakeNode) DialAgent(_ context.Context, machine, secret string) (node.MsgConn, error) {
|
||||
if secret != "agent-secret" {
|
||||
return nil, errors.New("bad agent secret")
|
||||
}
|
||||
return newEcho(), nil
|
||||
}
|
||||
|
||||
func (f *FakeNode) WaitStopped(_ context.Context, machine string) error {
|
||||
if f.IsRunning(machine) {
|
||||
return errors.New("still running")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeNode) ExportHome(_ context.Context, drive, machine, parent string) (string, error) {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
data, ok := f.Homes[drive+"/"+machine]
|
||||
if !ok {
|
||||
return "", errors.New("no such home")
|
||||
}
|
||||
kind := "FULL:"
|
||||
if parent != "" {
|
||||
kind = "INCR:"
|
||||
}
|
||||
id := util.RandomHex(4)
|
||||
f.Exports[id] = &fakeExport{data: append([]byte(kind), data...)}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (f *FakeNode) ReadExport(_ context.Context, id string, max int) ([]byte, bool, error) {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
x := f.Exports[id]
|
||||
if x == nil {
|
||||
return nil, false, errors.New("no export")
|
||||
}
|
||||
n := min(max, len(x.data)-x.pos)
|
||||
chunk := x.data[x.pos : x.pos+n]
|
||||
x.pos += n
|
||||
return chunk, x.pos == len(x.data), nil
|
||||
}
|
||||
|
||||
func (f *FakeNode) CloseExport(_ context.Context, id string) error {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
delete(f.Exports, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeNode) ImportHome(_ context.Context, drive, machine string) (string, error) {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
if _, exists := f.Homes[drive+"/"+machine]; exists {
|
||||
return "", errors.New("home already exists")
|
||||
}
|
||||
id := util.RandomHex(4)
|
||||
f.Imports[id] = &fakeImport{key: drive + "/" + machine}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (f *FakeNode) WriteImport(_ context.Context, id string, data []byte) error {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
if f.FailWrites {
|
||||
return errors.New("disk on fire")
|
||||
}
|
||||
x := f.Imports[id]
|
||||
if x == nil {
|
||||
return errors.New("no import")
|
||||
}
|
||||
x.cur = append(x.cur, data...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeNode) EndReceive(_ context.Context, id string) error {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
x := f.Imports[id]
|
||||
if x == nil || len(x.cur) < 5 {
|
||||
return errors.New("nothing received")
|
||||
}
|
||||
if string(x.cur[:5]) == "INCR:" && x.latest == nil {
|
||||
return errors.New("incremental send without a parent")
|
||||
}
|
||||
x.latest, x.cur = x.cur[5:], nil
|
||||
x.passes++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeNode) FinishImport(_ context.Context, id string, quotaMB int64) error {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
x := f.Imports[id]
|
||||
delete(f.Imports, id)
|
||||
f.Homes[x.key] = x.latest
|
||||
f.Passes = x.passes
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeNode) AbortImport(_ context.Context, id string) error {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
delete(f.Imports, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeNode) DeleteHome(_ context.Context, drive, machine string) error {
|
||||
f.Mu.Lock()
|
||||
defer f.Mu.Unlock()
|
||||
delete(f.Homes, drive+"/"+machine)
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeExport struct {
|
||||
data []byte
|
||||
pos int
|
||||
}
|
||||
|
||||
type fakeImport struct {
|
||||
key string
|
||||
cur, latest []byte
|
||||
passes int
|
||||
}
|
||||
|
||||
type echo struct {
|
||||
ch chan []byte
|
||||
closed chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func newEcho() *echo { return &echo{ch: make(chan []byte, 16), closed: make(chan struct{})} }
|
||||
func (e *echo) Read(ctx context.Context) ([]byte, error) {
|
||||
select {
|
||||
case b := <-e.ch:
|
||||
return b, nil
|
||||
case <-e.closed:
|
||||
return nil, errors.New("closed")
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
func (e *echo) Write(_ context.Context, b []byte) error {
|
||||
e.ch <- append([]byte("echo: "), b...)
|
||||
return nil
|
||||
}
|
||||
func (e *echo) Close() error { e.once.Do(func() { close(e.closed) }); return nil }
|
||||
|
||||
func FreePort(t testing.TB) string {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer l.Close()
|
||||
return l.Addr().String()
|
||||
}
|
||||
|
||||
func WaitFor(t testing.TB, what string, cond func() bool) {
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for !cond() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for %s", what)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func HomeData() []byte { return bytes.Repeat([]byte("0123456789abcdef"), 600_000) }
|
||||
@@ -0,0 +1,42 @@
|
||||
package units
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var StorageToken = regexp.MustCompile(`^([+-]?)(\d+(?:\.\d+)?)(tib|tb|t|gib|gb|g|mib|mb|m)$`)
|
||||
|
||||
func ParseSizeMB(s string) (int64, error) {
|
||||
t := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(s), " ", ""))
|
||||
p := StorageToken.FindStringSubmatch(t)
|
||||
if p == nil || p[1] != "" {
|
||||
return 0, fmt.Errorf("%q isn't a size like 5G or 500M", s)
|
||||
}
|
||||
return ToMB(p[2], p[3]), nil
|
||||
}
|
||||
|
||||
func ToMB(num, unit string) int64 {
|
||||
n, _ := strconv.ParseFloat(num, 64)
|
||||
switch unit[0] {
|
||||
case 't':
|
||||
n *= 1024 * 1024
|
||||
case 'g':
|
||||
n *= 1024
|
||||
}
|
||||
return int64(math.Round(n))
|
||||
}
|
||||
|
||||
func FormatMB(mb int64) string {
|
||||
trim := func(f float64) string { return strconv.FormatFloat(math.Round(f*10)/10, 'f', -1, 64) }
|
||||
switch {
|
||||
case mb >= 1024*1024:
|
||||
return trim(float64(mb)/(1024*1024)) + " TB"
|
||||
case mb >= 1024:
|
||||
return trim(float64(mb)/1024) + " GB"
|
||||
}
|
||||
return fmt.Sprintf("%d MB", mb)
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package units
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSizes(t *testing.T) {
|
||||
for in, want := range map[string]int64{"5G": 5120, "500M": 500, "1.5gb": 1536, "1T": 1048576, "20 GB": 20480} {
|
||||
if got, err := ParseSizeMB(in); err != nil || got != want {
|
||||
t.Fatalf("ParseSizeMB(%q) = %d, %v; want %d", in, got, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := ParseSizeMB("+5G"); err == nil {
|
||||
t.Fatal("a signed size isn't a config size")
|
||||
}
|
||||
for mb, want := range map[int64]string{512: "512 MB", 5120: "5 GB", 2560: "2.5 GB", 1258291: "1.2 TB"} {
|
||||
if got := FormatMB(mb); got != want {
|
||||
t.Fatalf("FormatMB(%d) = %q, want %q", mb, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type KeyedMutex struct{ m sync.Map }
|
||||
|
||||
func (k *KeyedMutex) Lock(key string) (unlock func()) {
|
||||
v, _ := k.m.LoadOrStore(key, &sync.Mutex{})
|
||||
mu := v.(*sync.Mutex)
|
||||
mu.Lock()
|
||||
return mu.Unlock
|
||||
}
|
||||
|
||||
func RandomHex(n int) string {
|
||||
b := make([]byte, n)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
Reference in new issue
Block a user