version mismatch catch

This commit is contained in:
ari committed 2026-11-01 14:34:23 -05:00
1 parent 3ba96528bb
commit a90653260a
11 files changed
+181 -34

No files matched your search

+1
View File
@@ -46,6 +46,7 @@ flowchart LR
- The slave connects to `wss://master/cluster` with `X-Cluster-Secret` and `X-Node-Name`, and pins the master's self-signed certificate by fingerprint, either from the config or trusted on first use.
- Frames are JSON: `call`/`reply` for node operations, `report` for node info every 5s, and `data`/`close` for byte streams. Agent connections are tunnelled as streams.
- Each `report` carries the slave's build version. The bot compares it with the master's and flags any difference at the top of `help`, `nodes` and `status`. A slave that reports no version (an older build) counts as a mismatch.
- A slave that hasn't reported for 20s, or whose link has dropped, counts as offline. The master doesn't trust reports for a short while after it asks a slave to start or stop something, so stale reports don't undo the change.
## Migration
+20 -20
View File
@@ -48,26 +48,26 @@ Anyone in the channel can change these. In an `all-messages` thread, `move` stil
Send these as a **new message**, not in a thread. `move` and `listen` are the exceptions, and also work in a thread. In the assistant pane, the first message of a chat counts as new.
| Command | What it does |
| --------------------------------------------------- | ------------------------------------------------------------------------------- |
| `help` | Shows the command list |
| `status` | State, size, time unused and idle limit, with **Stop** and **Scale up** buttons |
| `sessions` | Past sessions with links. Open one and reply in its thread to continue |
| `stop` | Shuts the container down. Files are kept |
| `scale up` / `scale down` | Doubles or halves RAM and CPU |
| `scale 8G 4cpu` | Sets an exact size |
| `scale +2G` / `scale -1cpu` / `scale 512M +0.5cpu` | `+` or `-` changes by that much. Memory and CPU can be mixed or left out |
| `keep 24h` (or `3d`, `1d12h`, `2w`, `5 hours`) | Stops the container after that long unused |
| `keep indefinite` (or `forever`) | Never stops it for being unused. Use `stop` when done |
| `keep default` | Back to the server's default |
| `keep` | Shows the current idle limit |
| `storage` | Shows home folder usage, guaranteed space and max |
| `storage 20G` / `storage +5G` / `storage -2G` | Sets or changes the max (the cap) |
| `storage min 8G` / `storage min 8G max 20G` | Sets the guaranteed space, or both |
| `nodes` | Every node with what's free on it, and where this container runs |
| `move <node>` / `move to <node>` / `migrate <node>` | Moves this container to that node |
| `move` / `move auto` | Moves it to whichever other node keeps resources most even |
| `permissions` | Who may change what here |
| Command | What it does |
| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
| `help` | Shows the command list |
| `status` | State, size, time unused and idle limit, with **Stop** and **Scale up** buttons |
| `sessions` | Past sessions with links. Open one and reply in its thread to continue |
| `stop` | Shuts the container down. Files are kept |
| `scale up` / `scale down` | Doubles or halves RAM and CPU |
| `scale 8G 4cpu` | Sets an exact size |
| `scale +2G` / `scale -1cpu` / `scale 512M +0.5cpu` | `+` or `-` changes by that much. Memory and CPU can be mixed or left out |
| `keep 24h` (or `3d`, `1d12h`, `2w`, `5 hours`) | Stops the container after that long unused |
| `keep indefinite` (or `forever`) | Never stops it for being unused. Use `stop` when done |
| `keep default` | Back to the server's default |
| `keep` | Shows the current idle limit |
| `storage` | Shows home folder usage, guaranteed space and max |
| `storage 20G` / `storage +5G` / `storage -2G` | Sets or changes the max (the cap) |
| `storage min 8G` / `storage min 8G max 20G` | Sets the guaranteed space, or both |
| `nodes` | Every node with what's free on it, and where this container runs. Flags nodes running a different version than the master |
| `move <node>` / `move to <node>` / `migrate <node>` | Moves this container to that node |
| `move` / `move auto` | Moves it to whichever other node keeps resources most even |
| `permissions` | Who may change what here |
If what follows `scale`, `keep`, `storage` or `move` doesn't look like a size, a time or a node (e.g. "keep the tests passing"), the message goes to the agent instead.
+2 -1
View File
@@ -14,6 +14,7 @@ var ErrMaxedOut = errors.New("no node has room for this container")
type NodeView struct {
Name string
Version string
Usable model.Size
Used model.Size
Drives []storage.DriveReport
@@ -91,7 +92,7 @@ func ChoosePlacement(views []NodeView, want model.Size, storageMB int64) (node,
}
func BuildView(info node.NodeInfo, owners []model.Owner, defaultSize model.Size) NodeView {
v := NodeView{Name: info.Name, Usable: info.Usable}
v := NodeView{Name: info.Name, Version: info.Version, Usable: info.Usable}
byMachine := map[string]*model.Owner{}
for i := range owners {
+15 -3
View File
@@ -350,6 +350,9 @@ func (b *Bot) command(ctx context.Context, m incoming) bool {
switch {
case verb == "" || verb == "help" && args == "":
reply = helpText
if warning := b.versionWarning(ctx); warning != "" {
reply = warning + "\n\n" + reply
}
case verb == "sessions" && args == "":
reply = b.sessionList(ctx, ownerID)
case verb == "status" && args == "":
@@ -478,10 +481,14 @@ func (b *Bot) postStatus(ctx context.Context, channel, thread, ownerID string, k
}
fmt.Fprintf(&sb, "*Stops when unused for:* %s\n", b.describeIdle(u))
fmt.Fprintf(&sb, "*Home folder:* %s\n", b.describeStorage(ctx, u))
sb.WriteString("*Nodes:*")
views, _ := b.mgr.Views(ctx)
if warning := versionWarning(b.mgr.LocalName(), views); warning != "" {
sb.WriteString(warning + "\n")
}
sb.WriteString("*Nodes:*")
masterVersion, _ := versionMismatches(b.mgr.LocalName(), views)
for _, v := range views {
sb.WriteString("\n• " + describeNode(v))
sb.WriteString("\n• " + describeNode(v) + versionNote(b.mgr.LocalName(), masterVersion, v))
}
for _, name := range b.mgr.OfflineNodes() {
fmt.Fprintf(&sb, "\n• %s: offline", name)
@@ -917,9 +924,14 @@ func (b *Bot) nodesList(ctx context.Context, ownerID string) string {
here = u.Node
}
var sb strings.Builder
master := b.mgr.LocalName()
if warning := versionWarning(master, views); warning != "" {
sb.WriteString(warning + "\n\n")
}
sb.WriteString("*Nodes* (what's free on each):")
masterVersion, _ := versionMismatches(master, views)
for _, v := range views {
sb.WriteString("\n• " + describeNode(v))
sb.WriteString("\n• " + describeNode(v) + versionNote(master, masterVersion, v))
if v.Name == here {
sb.WriteString(" ← this container")
}
+67
View File
@@ -0,0 +1,67 @@
package bot
import (
"context"
"fmt"
"strings"
"slackerbot/internal/balance"
)
const unknownVersion = "an older version that doesn't report one"
func versionLabel(version string) string {
if version == "" {
return unknownVersion
}
return "`" + version + "`"
}
func versionMismatches(master string, views []balance.NodeView) (masterVersion string, odd []balance.NodeView) {
for _, v := range views {
if v.Name == master {
masterVersion = v.Version
}
}
for _, v := range views {
if v.Name != master && v.Version != masterVersion {
odd = append(odd, v)
}
}
return masterVersion, odd
}
func versionWarning(master string, views []balance.NodeView) string {
masterVersion, odd := versionMismatches(master, views)
if len(odd) == 0 {
return ""
}
var sb strings.Builder
fmt.Fprintf(&sb, ":warning: *Version mismatch.* The master (%s) runs %s, but", master, versionLabel(masterVersion))
for i, v := range odd {
if i > 0 {
sb.WriteString(",")
}
fmt.Fprintf(&sb, " %s runs %s", v.Name, versionLabel(v.Version))
}
sb.WriteString(". Update the nodes so they all run the same version.")
return sb.String()
}
func (b *Bot) versionWarning(ctx context.Context) string {
if b.mgr == nil {
return ""
}
views, err := b.mgr.Views(ctx)
if err != nil {
return ""
}
return versionWarning(b.mgr.LocalName(), views)
}
func versionNote(master string, masterVersion string, v balance.NodeView) string {
if v.Name == master || v.Version == masterVersion {
return ""
}
return " · :warning: version " + versionLabel(v.Version)
}
+61
View File
@@ -0,0 +1,61 @@
package bot
import (
"strings"
"testing"
"slackerbot/internal/balance"
"slackerbot/internal/testutil/clustertest"
)
func TestVersionWarning(t *testing.T) {
views := func(versions ...string) []balance.NodeView {
names := []string{"master", "a", "b"}
var out []balance.NodeView
for i, v := range versions {
out = append(out, balance.NodeView{Name: names[i], Version: v})
}
return out
}
if got := versionWarning("master", views("v1", "v1", "v1")); got != "" {
t.Fatalf("matching versions should not warn: %q", got)
}
if got := versionWarning("master", views("v1")); got != "" {
t.Fatalf("a lone master should not warn: %q", got)
}
got := versionWarning("master", views("v1", "v2", "v1"))
if !strings.Contains(got, "Version mismatch") || !strings.Contains(got, "a runs `v2`") || strings.Contains(got, "b runs") {
t.Fatalf("only the odd node should be named: %q", got)
}
got = versionWarning("master", views("v1", ""))
if !strings.Contains(got, "a runs an older version that doesn't report one") {
t.Fatalf("a slave reporting no version is a mismatch: %q", got)
}
}
func TestVersionMismatchShownInNodesAndHelp(t *testing.T) {
c := clustertest.New(t)
b := &Bot{cfg: c.Cfg, store: c.Store, mgr: c.Mgr}
if got := b.nodesList(c.Ctx, "C1"); strings.Contains(got, "Version mismatch") {
t.Fatalf("unexpected warning when no node reports a version: %s", got)
}
c.Master.Mu.Lock()
c.Master.Version = "2026.11.01-abc"
c.Master.Mu.Unlock()
got := b.nodesList(c.Ctx, "C1")
if !strings.Contains(got, "Version mismatch") || !strings.Contains(got, "slave-1 runs an older version") {
t.Fatalf("nodes should flag the mismatch: %s", got)
}
if !strings.Contains(got, ":warning: version") {
t.Fatalf("the odd node's line should be flagged: %s", got)
}
if help := b.versionWarning(c.Ctx); !strings.Contains(help, "Version mismatch") {
t.Fatalf("help should flag the mismatch: %s", help)
}
}
+9 -8
View File
@@ -35,17 +35,18 @@ const (
)
type Local struct {
cfg *config.Config
name string
cap Capacity
bus *dbus.Conn
system string
cfg *config.Config
name string
version string
cap Capacity
bus *dbus.Conn
system string
exports transfers
imports transfers
}
func New(cfg *config.Config) (*Local, error) {
func New(cfg *config.Config, version string) (*Local, error) {
system, err := filepath.EvalSymlinks(cfg.System)
if err != nil {
return nil, err
@@ -66,7 +67,7 @@ func New(cfg *config.Config) (*Local, error) {
if err != nil {
return nil, fmt.Errorf("connecting to the system bus: %w", err)
}
return &Local{cfg: cfg, name: cfg.Cluster.Name, cap: capacity, bus: bus, system: system}, nil
return &Local{cfg: cfg, name: cfg.Cluster.Name, version: version, cap: capacity, bus: bus, system: system}, nil
}
func unitName(machine string) string { return "slackerbot-" + machine + ".service" }
@@ -76,7 +77,7 @@ func (l *Local) Name() string { return l.name }
func (l *Local) machined() dbus.BusObject { return l.bus.Object(machinedName, machinedPath) }
func (l *Local) Info(ctx context.Context) (node.NodeInfo, error) {
info := node.NodeInfo{Name: l.name, Total: l.cap.Total, Usable: l.cap.Usable}
info := node.NodeInfo{Name: l.name, Version: l.version, Total: l.cap.Total, Usable: l.cap.Usable}
for _, d := range l.cfg.Drives {
free, err := storage.FreeMB(d.Path)
if err != nil {
+2
View File
@@ -71,6 +71,8 @@ func New(cfg *config.Config, store *state.Store, local node.Node) *Manager {
func (m *Manager) SetNodeBackHandler(fn func(node string)) { m.onNodeBack = fn }
func (m *Manager) LocalName() string { return m.local.Name() }
func (m *Manager) nodeByName(name string) (node.Node, error) {
if name == m.local.Name() {
return m.local, nil
+1
View File
@@ -36,6 +36,7 @@ type Node interface {
type NodeInfo struct {
Name string `json:"name"`
Version string `json:"version,omitempty"`
Total model.Size `json:"total"`
Usable model.Size `json:"usable"`
Drives []DriveInfo `json:"drives"`
+2 -1
View File
@@ -26,6 +26,7 @@ type FakeNode struct {
Passes int
FailWrites bool
Usable model.Size
Version string
}
func NewFakeNode(name string) *FakeNode {
@@ -42,7 +43,7 @@ func (f *FakeNode) Info(ctx context.Context) (node.NodeInfo, error) {
for k, v := range f.Homes {
homes[strings.TrimPrefix(k, "/d/")] = int64(len(v)) >> 20
}
return node.NodeInfo{Name: f.name, Usable: f.Usable, Running: running,
return node.NodeInfo{Name: f.name, Version: f.Version, Usable: f.Usable, Running: running,
Drives: []node.DriveInfo{{Path: "/d", FreeMB: 50_000, HomesMB: homes}}}, nil
}
func (f *FakeNode) Running(context.Context) ([]node.RunningMachine, error) {
+1 -1
View File
@@ -39,7 +39,7 @@ func main() {
log.Print("warning: not running as root; starting containers will fail")
}
node, err := local.New(cfg)
node, err := local.New(cfg, version)
if err != nil {
log.Fatal(err)
}