clean up of the codebase
This commit is contained in:
1 parent
7822529a38
commit
0ac61b4ece
52 files changed
+2545
-2274
No files matched your search
@@ -0,0 +1,325 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/godbus/dbus/v5"
|
||||
|
||||
"slackerbot/internal/config"
|
||||
"slackerbot/internal/model"
|
||||
"slackerbot/internal/node"
|
||||
"slackerbot/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
ServicePort = 1984
|
||||
|
||||
machinedName = "org.freedesktop.machine1"
|
||||
machinedPath = "/org/freedesktop/machine1"
|
||||
machinedMgr = "org.freedesktop.machine1.Manager"
|
||||
sigPoweroff = 38 // SIGRTMIN+4, what `machinectl poweroff` sends
|
||||
|
||||
agentEnvInside = "/var/lib/slackerbot/agent.env"
|
||||
)
|
||||
|
||||
type Local struct {
|
||||
cfg *config.Config
|
||||
name string
|
||||
cap Capacity
|
||||
bus *dbus.Conn
|
||||
system string
|
||||
|
||||
exports transfers
|
||||
imports transfers
|
||||
}
|
||||
|
||||
func New(cfg *config.Config) (*Local, error) {
|
||||
system, err := filepath.EvalSymlinks(cfg.System)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(system, "init")); err != nil {
|
||||
return nil, fmt.Errorf("%s has no init. Is it a NixOS build with boot.isContainer = true?", system)
|
||||
}
|
||||
for _, d := range cfg.Drives {
|
||||
if err := storage.PrepareDrive(d.Path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
capacity, err := DetectCapacity(cfg.HostReserve)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bus, err := dbus.ConnectSystemBus()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connecting to the system bus: %w", err)
|
||||
}
|
||||
return &Local{cfg: cfg, name: cfg.Cluster.Name, cap: capacity, bus: bus, system: system}, nil
|
||||
}
|
||||
|
||||
func unitName(machine string) string { return "slackerbot-" + machine + ".service" }
|
||||
|
||||
func (l *Local) Name() string { return l.name }
|
||||
|
||||
func (l *Local) machined() dbus.BusObject { return l.bus.Object(machinedName, machinedPath) }
|
||||
|
||||
func (l *Local) Info(ctx context.Context) (node.NodeInfo, error) {
|
||||
info := node.NodeInfo{Name: l.name, Total: l.cap.Total, Usable: l.cap.Usable}
|
||||
for _, d := range l.cfg.Drives {
|
||||
free, err := storage.FreeMB(d.Path)
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
di := node.DriveInfo{Path: d.Path, FreeMB: free, ReserveMB: d.ReserveMB, HomesMB: map[string]int64{}}
|
||||
entries, err := os.ReadDir(d.Path)
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() && model.IsOurs(e.Name()) {
|
||||
used, _ := storage.UsedMB(filepath.Join(d.Path, e.Name()))
|
||||
di.HomesMB[e.Name()] = used
|
||||
}
|
||||
}
|
||||
info.Drives = append(info.Drives, di)
|
||||
}
|
||||
running, err := l.Running(ctx)
|
||||
info.Running = running
|
||||
return info, err
|
||||
}
|
||||
|
||||
func (l *Local) Running(ctx context.Context) ([]node.RunningMachine, error) {
|
||||
var list []struct {
|
||||
Name, Class, Service string
|
||||
Path dbus.ObjectPath
|
||||
}
|
||||
if err := l.machined().Call(machinedMgr+".ListMachines", 0).Store(&list); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []node.RunningMachine
|
||||
for _, x := range list {
|
||||
if !model.IsOurs(x.Name) {
|
||||
continue
|
||||
}
|
||||
r := node.RunningMachine{Name: x.Name}
|
||||
if v, err := l.bus.Object(machinedName, x.Path).GetProperty("org.freedesktop.machine1.Machine.Timestamp"); err == nil {
|
||||
if us, ok := v.Value().(uint64); ok {
|
||||
r.Since = time.UnixMicro(int64(us))
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (l *Local) IsRunning(machine string) bool {
|
||||
var p dbus.ObjectPath
|
||||
return l.machined().Call(machinedMgr+".GetMachine", 0, machine).Store(&p) == nil
|
||||
}
|
||||
|
||||
func (l *Local) drive(path string) (config.DriveConfig, error) {
|
||||
for _, d := range l.cfg.Drives {
|
||||
if d.Path == path {
|
||||
return d, nil
|
||||
}
|
||||
}
|
||||
return config.DriveConfig{}, fmt.Errorf("drive %s isn't configured on node %s", path, l.name)
|
||||
}
|
||||
|
||||
func (l *Local) Start(ctx context.Context, spec node.StartSpec) error {
|
||||
if !model.IsOurs(spec.Machine) {
|
||||
return fmt.Errorf("unexpected machine name %q", spec.Machine)
|
||||
}
|
||||
if _, err := l.drive(spec.Drive); err != nil {
|
||||
return err
|
||||
}
|
||||
home := filepath.Join(spec.Drive, spec.Machine)
|
||||
if _, err := os.Stat(home); errors.Is(err, os.ErrNotExist) {
|
||||
if err := storage.CreateSubvolume(home); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := storage.SetQuota(home, spec.StorageMaxMB); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Only the home folder persists. NixOS rebuilds the rest on boot.
|
||||
root := filepath.Join(l.cfg.MachinesDir, spec.Machine)
|
||||
if err := os.RemoveAll(root); err != nil {
|
||||
return fmt.Errorf("clearing old container files: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if f, err := os.OpenFile(filepath.Join(root, "etc/os-release"), os.O_CREATE, 0o644); err == nil {
|
||||
f.Close()
|
||||
}
|
||||
|
||||
envFile, err := l.writeEnvFile(spec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"--unit=" + unitName(spec.Machine),
|
||||
"--collect",
|
||||
"--quiet",
|
||||
"--description=" + spec.Description,
|
||||
// Same as systemd-nspawn@.service.
|
||||
"--property=Type=notify",
|
||||
"--property=KillMode=mixed",
|
||||
"--property=Delegate=yes",
|
||||
fmt.Sprintf("--property=MemoryMax=%dM", spec.Size.MemoryMB),
|
||||
fmt.Sprintf("--property=CPUQuota=%d%%", int(math.Round(spec.Size.CPUs*100))),
|
||||
|
||||
"systemd-nspawn",
|
||||
"--machine=" + spec.Machine,
|
||||
"--directory=" + root,
|
||||
"--keep-unit",
|
||||
"--network-veth", // own network, so every container can use port 1984
|
||||
"--bind-ro=/nix/store",
|
||||
"--bind-ro=" + envFile + ":" + agentEnvInside,
|
||||
"--bind=" + home + ":/root",
|
||||
}
|
||||
for _, b := range l.cfg.ROBinds {
|
||||
args = append(args, "--bind-ro="+strings.ReplaceAll(b, "{machine}", spec.Machine))
|
||||
}
|
||||
for _, b := range l.cfg.RWBinds {
|
||||
b = strings.ReplaceAll(b, "{machine}", spec.Machine)
|
||||
src, _, _ := strings.Cut(b, ":")
|
||||
if err := os.MkdirAll(src, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
args = append(args, "--bind="+b)
|
||||
}
|
||||
args = append(args, filepath.Join(l.system, "init"))
|
||||
|
||||
if out, err := exec.CommandContext(ctx, "systemd-run", args...).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("systemd-run: %v: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
|
||||
// Wait until machined lists it, so the next capacity check counts it.
|
||||
for i := 0; i < 50; i++ {
|
||||
if l.IsRunning(spec.Machine) {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("%s started but never registered with machined", spec.Machine)
|
||||
}
|
||||
|
||||
func (l *Local) writeEnvFile(spec node.StartSpec) (string, error) {
|
||||
dir := filepath.Join(l.cfg.StateDir, "env")
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var b strings.Builder
|
||||
line := func(k, v string) {
|
||||
v = strings.ReplaceAll(v, `\`, `\\`)
|
||||
v = strings.ReplaceAll(v, `"`, `\"`)
|
||||
fmt.Fprintf(&b, "%s=\"%s\"\n", k, v)
|
||||
}
|
||||
line("GOOSE_SERVER__SECRET_KEY", spec.Secret)
|
||||
for k, v := range spec.Env {
|
||||
line(k, v)
|
||||
}
|
||||
path := filepath.Join(dir, spec.Machine+".env")
|
||||
return path, os.WriteFile(path, []byte(b.String()), 0o600)
|
||||
}
|
||||
|
||||
func (l *Local) Poweroff(ctx context.Context, machine string) error {
|
||||
return l.machined().CallWithContext(ctx, machinedMgr+".KillMachine", 0, machine, "leader", int32(sigPoweroff)).Err
|
||||
}
|
||||
|
||||
func (l *Local) SetLimits(ctx context.Context, machine string, s model.Size) error {
|
||||
type prop struct {
|
||||
Name string
|
||||
Value dbus.Variant
|
||||
}
|
||||
props := []prop{
|
||||
{"MemoryMax", dbus.MakeVariant(uint64(s.MemoryMB) << 20)},
|
||||
{"CPUQuotaPerSecUSec", dbus.MakeVariant(uint64(s.CPUs * 1e6))},
|
||||
}
|
||||
return l.bus.Object("org.freedesktop.systemd1", "/org/freedesktop/systemd1").
|
||||
CallWithContext(ctx, "org.freedesktop.systemd1.Manager.SetUnitProperties", 0, unitName(machine), true, props).Err
|
||||
}
|
||||
|
||||
func (l *Local) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
|
||||
if _, err := l.drive(drive); err != nil {
|
||||
return err
|
||||
}
|
||||
if !model.IsOurs(machine) {
|
||||
return fmt.Errorf("unexpected machine name %q", machine)
|
||||
}
|
||||
return storage.SetQuota(filepath.Join(drive, machine), mb)
|
||||
}
|
||||
|
||||
func (l *Local) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
|
||||
addr, err := l.waitReady(ctx, machine)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return node.DialAgentWS(ctx, addr, secret)
|
||||
}
|
||||
|
||||
func (l *Local) address(machine string) (string, error) {
|
||||
var addrs []struct {
|
||||
Family int32
|
||||
Addr []byte
|
||||
}
|
||||
if err := l.machined().Call(machinedMgr+".GetMachineAddresses", 0, machine).Store(&addrs); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, a := range addrs {
|
||||
ip := net.IP(a.Addr)
|
||||
if a.Family == syscall.AF_INET && !ip.IsLinkLocalUnicast() {
|
||||
return net.JoinHostPort(ip.String(), strconv.Itoa(ServicePort)), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("%s has no IPv4 address yet", machine)
|
||||
}
|
||||
|
||||
func (l *Local) waitReady(ctx context.Context, machine string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, l.cfg.BootTimeout)
|
||||
defer cancel()
|
||||
client := http.Client{Timeout: 3 * time.Second}
|
||||
|
||||
var lastErr error
|
||||
for {
|
||||
addr, err := l.address(machine)
|
||||
if err == nil {
|
||||
var resp *http.Response
|
||||
resp, err = client.Get("http://" + addr + "/status")
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return addr, nil
|
||||
}
|
||||
err = fmt.Errorf("/status returned %s", resp.Status)
|
||||
}
|
||||
}
|
||||
lastErr = err
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", fmt.Errorf("goose didn't answer on port %d within %s (last error: %v)", ServicePort, l.cfg.BootTimeout, lastErr)
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Local) Capacity() Capacity { return l.cap }
|
||||
Reference in new issue
Block a user