378 lines
11 KiB
Go
378 lines
11 KiB
Go
package local
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/godbus/dbus/v5"
|
|
|
|
"slackerbot/internal/config"
|
|
"slackerbot/internal/model"
|
|
"slackerbot/internal/node"
|
|
"slackerbot/internal/storage"
|
|
)
|
|
|
|
const (
|
|
ServicePort = 1984
|
|
|
|
machinedName = "org.freedesktop.machine1"
|
|
machinedPath = "/org/freedesktop/machine1"
|
|
machinedMgr = "org.freedesktop.machine1.Manager"
|
|
systemdName = "org.freedesktop.systemd1"
|
|
systemdPath = "/org/freedesktop/systemd1"
|
|
systemdMgr = "org.freedesktop.systemd1.Manager"
|
|
sigPoweroff = 38 // SIGRTMIN+4, what `machinectl poweroff` sends
|
|
|
|
agentEnvInside = "/var/lib/slackerbot/agent.env"
|
|
)
|
|
|
|
type Local struct {
|
|
cfg *config.Config
|
|
name string
|
|
version string
|
|
cap Capacity
|
|
bus *dbus.Conn
|
|
system string
|
|
|
|
exports transfers
|
|
imports transfers
|
|
}
|
|
|
|
func New(cfg *config.Config, version string) (*Local, error) {
|
|
system, err := filepath.EvalSymlinks(cfg.System)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := os.Stat(filepath.Join(system, "init")); err != nil {
|
|
return nil, fmt.Errorf("%s has no init. Is it a NixOS build with boot.isContainer = true?", system)
|
|
}
|
|
for _, d := range cfg.Drives {
|
|
if err := storage.PrepareDrive(d.Path); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
capacity, err := DetectCapacity(cfg.HostReserve)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
bus, err := dbus.ConnectSystemBus()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("connecting to the system bus: %w", err)
|
|
}
|
|
return &Local{cfg: cfg, name: cfg.Cluster.Name, version: version, cap: capacity, bus: bus, system: system}, nil
|
|
}
|
|
|
|
func unitName(machine string) string { return "slackerbot-" + machine + ".service" }
|
|
|
|
func (l *Local) Name() string { return l.name }
|
|
|
|
func (l *Local) machined() dbus.BusObject { return l.bus.Object(machinedName, machinedPath) }
|
|
|
|
func (l *Local) Info(ctx context.Context) (node.NodeInfo, error) {
|
|
info := node.NodeInfo{Name: l.name, Version: l.version, Total: l.cap.Total, Usable: l.cap.Usable}
|
|
for _, d := range l.cfg.Drives {
|
|
free, err := storage.FreeMB(d.Path)
|
|
if err != nil {
|
|
return info, err
|
|
}
|
|
di := node.DriveInfo{Path: d.Path, FreeMB: free, ReserveMB: d.ReserveMB, HomesMB: map[string]int64{}}
|
|
entries, err := os.ReadDir(d.Path)
|
|
if err != nil {
|
|
return info, err
|
|
}
|
|
for _, e := range entries {
|
|
if e.IsDir() && model.IsOurs(e.Name()) {
|
|
used, _ := storage.UsedMB(filepath.Join(d.Path, e.Name()))
|
|
di.HomesMB[e.Name()] = used
|
|
}
|
|
}
|
|
info.Drives = append(info.Drives, di)
|
|
}
|
|
running, err := l.Running(ctx)
|
|
info.Running = running
|
|
return info, err
|
|
}
|
|
|
|
func (l *Local) Running(ctx context.Context) ([]node.RunningMachine, error) {
|
|
var list []struct {
|
|
Name, Class, Service string
|
|
Path dbus.ObjectPath
|
|
}
|
|
if err := l.machined().Call(machinedMgr+".ListMachines", 0).Store(&list); err != nil {
|
|
return nil, err
|
|
}
|
|
var out []node.RunningMachine
|
|
for _, x := range list {
|
|
if !model.IsOurs(x.Name) {
|
|
continue
|
|
}
|
|
r := node.RunningMachine{Name: x.Name, Outdated: l.outdated(x.Name)}
|
|
if v, err := l.bus.Object(machinedName, x.Path).GetProperty("org.freedesktop.machine1.Machine.Timestamp"); err == nil {
|
|
if us, ok := v.Value().(uint64); ok {
|
|
r.Since = time.UnixMicro(int64(us))
|
|
}
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// The unit's command ends with the init of the system the container booted.
|
|
// When it can't be read, the container counts as up to date.
|
|
func (l *Local) outdated(machine string) bool {
|
|
args := l.execStart(unitName(machine))
|
|
if len(args) == 0 {
|
|
return false
|
|
}
|
|
init := args[len(args)-1]
|
|
return filepath.Base(init) == "init" && filepath.Dir(init) != l.system
|
|
}
|
|
|
|
func (l *Local) execStart(unit string) []string {
|
|
var path dbus.ObjectPath
|
|
if err := l.bus.Object(systemdName, systemdPath).Call(systemdMgr+".GetUnit", 0, unit).Store(&path); err != nil {
|
|
return nil
|
|
}
|
|
v, err := l.bus.Object(systemdName, path).GetProperty(systemdName + ".Service.ExecStart")
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var cmds []struct {
|
|
Path string
|
|
Args []string
|
|
IgnoreErrors bool
|
|
StartTime, StartTimeMono, ExitTime, ExitTimeMono uint64
|
|
PID uint32
|
|
Code, Status int32
|
|
}
|
|
if err := v.Store(&cmds); err != nil || len(cmds) == 0 {
|
|
return nil
|
|
}
|
|
return cmds[0].Args
|
|
}
|
|
|
|
func (l *Local) IsRunning(machine string) bool {
|
|
var p dbus.ObjectPath
|
|
return l.machined().Call(machinedMgr+".GetMachine", 0, machine).Store(&p) == nil
|
|
}
|
|
|
|
func (l *Local) drive(path string) (config.DriveConfig, error) {
|
|
for _, d := range l.cfg.Drives {
|
|
if d.Path == path {
|
|
return d, nil
|
|
}
|
|
}
|
|
return config.DriveConfig{}, fmt.Errorf("drive %s isn't configured on node %s", path, l.name)
|
|
}
|
|
|
|
func (l *Local) Start(ctx context.Context, spec node.StartSpec) error {
|
|
if !model.IsOurs(spec.Machine) {
|
|
return fmt.Errorf("unexpected machine name %q", spec.Machine)
|
|
}
|
|
if _, err := l.drive(spec.Drive); err != nil {
|
|
return err
|
|
}
|
|
home := filepath.Join(spec.Drive, spec.Machine)
|
|
if _, err := os.Stat(home); errors.Is(err, os.ErrNotExist) {
|
|
if err := storage.CreateSubvolume(home); err != nil {
|
|
return err
|
|
}
|
|
} else if err != nil {
|
|
return err
|
|
}
|
|
if err := storage.SetQuota(home, spec.StorageMaxMB); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Only the home folder persists. NixOS rebuilds the rest on boot.
|
|
root := filepath.Join(l.cfg.MachinesDir, spec.Machine)
|
|
// NixOS marks /var/empty immutable, which blocks RemoveAll.
|
|
_ = exec.Command("chattr", "-i", filepath.Join(root, "var/empty")).Run()
|
|
if err := os.RemoveAll(root); err != nil {
|
|
return fmt.Errorf("clearing old container files: %w", err)
|
|
}
|
|
// systemd-nspawn refuses a directory without /usr as not an OS tree.
|
|
for _, dir := range []string{"etc", "usr"} {
|
|
if err := os.MkdirAll(filepath.Join(root, dir), 0o755); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if f, err := os.OpenFile(filepath.Join(root, "etc/os-release"), os.O_CREATE, 0o644); err == nil {
|
|
f.Close()
|
|
}
|
|
|
|
envFile, err := l.writeEnvFile(spec)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
args := []string{
|
|
"--unit=" + unitName(spec.Machine),
|
|
"--collect",
|
|
"--quiet",
|
|
"--description=" + spec.Description,
|
|
// Same as systemd-nspawn@.service.
|
|
"--property=Type=notify",
|
|
"--property=KillMode=mixed",
|
|
"--property=Delegate=yes",
|
|
fmt.Sprintf("--property=MemoryMax=%dM", spec.Size.MemoryMB),
|
|
fmt.Sprintf("--property=CPUQuota=%d%%", int(math.Round(spec.Size.CPUs*100))),
|
|
|
|
"systemd-nspawn",
|
|
"--machine=" + spec.Machine,
|
|
"--directory=" + root,
|
|
"--keep-unit",
|
|
"--network-veth", // own network, so every container can use port 1984
|
|
"--bind-ro=/nix/store",
|
|
"--bind-ro=" + envFile + ":" + agentEnvInside,
|
|
"--bind=" + home + ":/root",
|
|
}
|
|
binds := func(list []string, flag string) error {
|
|
for _, b := range list {
|
|
b = strings.ReplaceAll(b, "{machine}", spec.Machine)
|
|
src, _, _ := strings.Cut(b, ":")
|
|
// Missing sources make systemd-nspawn fail; existing ones may be files.
|
|
if _, err := os.Stat(src); errors.Is(err, os.ErrNotExist) {
|
|
if err := os.MkdirAll(src, 0o755); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
args = append(args, flag+b)
|
|
}
|
|
return nil
|
|
}
|
|
if err := binds(l.cfg.ROBinds, "--bind-ro="); err != nil {
|
|
return err
|
|
}
|
|
if err := binds(l.cfg.RWBinds, "--bind="); err != nil {
|
|
return err
|
|
}
|
|
args = append(args, filepath.Join(l.system, "init"))
|
|
|
|
if out, err := exec.CommandContext(ctx, "systemd-run", args...).CombinedOutput(); err != nil {
|
|
return fmt.Errorf("systemd-run: %v: %s", err, strings.TrimSpace(string(out)))
|
|
}
|
|
|
|
// Wait until machined lists it, so the next capacity check counts it.
|
|
for i := 0; i < 50; i++ {
|
|
if l.IsRunning(spec.Machine) {
|
|
return nil
|
|
}
|
|
time.Sleep(200 * time.Millisecond)
|
|
}
|
|
return fmt.Errorf("%s started but never registered with machined", spec.Machine)
|
|
}
|
|
|
|
func (l *Local) writeEnvFile(spec node.StartSpec) (string, error) {
|
|
dir := filepath.Join(l.cfg.StateDir, "env")
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return "", err
|
|
}
|
|
var b strings.Builder
|
|
line := func(k, v string) {
|
|
v = strings.ReplaceAll(v, `\`, `\\`)
|
|
v = strings.ReplaceAll(v, `"`, `\"`)
|
|
fmt.Fprintf(&b, "%s=\"%s\"\n", k, v)
|
|
}
|
|
line("GOOSE_SERVER__SECRET_KEY", spec.Secret)
|
|
for k, v := range spec.Env {
|
|
line(k, v)
|
|
}
|
|
path := filepath.Join(dir, spec.Machine+".env")
|
|
return path, os.WriteFile(path, []byte(b.String()), 0o600)
|
|
}
|
|
|
|
func (l *Local) Poweroff(ctx context.Context, machine string) error {
|
|
return l.machined().CallWithContext(ctx, machinedMgr+".KillMachine", 0, machine, "leader", int32(sigPoweroff)).Err
|
|
}
|
|
|
|
func (l *Local) SetLimits(ctx context.Context, machine string, s model.Size) error {
|
|
type prop struct {
|
|
Name string
|
|
Value dbus.Variant
|
|
}
|
|
props := []prop{
|
|
{"MemoryMax", dbus.MakeVariant(uint64(s.MemoryMB) << 20)},
|
|
{"CPUQuotaPerSecUSec", dbus.MakeVariant(uint64(s.CPUs * 1e6))},
|
|
}
|
|
return l.bus.Object("org.freedesktop.systemd1", "/org/freedesktop/systemd1").
|
|
CallWithContext(ctx, "org.freedesktop.systemd1.Manager.SetUnitProperties", 0, unitName(machine), true, props).Err
|
|
}
|
|
|
|
func (l *Local) SetQuota(ctx context.Context, drive, machine string, mb int64) error {
|
|
if _, err := l.drive(drive); err != nil {
|
|
return err
|
|
}
|
|
if !model.IsOurs(machine) {
|
|
return fmt.Errorf("unexpected machine name %q", machine)
|
|
}
|
|
return storage.SetQuota(filepath.Join(drive, machine), mb)
|
|
}
|
|
|
|
func (l *Local) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) {
|
|
addr, err := l.waitReady(ctx, machine)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return node.DialAgentWS(ctx, addr, secret)
|
|
}
|
|
|
|
func (l *Local) address(machine string) (string, error) {
|
|
var addrs []struct {
|
|
Family int32
|
|
Addr []byte
|
|
}
|
|
if err := l.machined().Call(machinedMgr+".GetMachineAddresses", 0, machine).Store(&addrs); err != nil {
|
|
return "", err
|
|
}
|
|
for _, a := range addrs {
|
|
ip := net.IP(a.Addr)
|
|
if a.Family == syscall.AF_INET && !ip.IsLinkLocalUnicast() {
|
|
return net.JoinHostPort(ip.String(), strconv.Itoa(ServicePort)), nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("%s has no IPv4 address yet", machine)
|
|
}
|
|
|
|
func (l *Local) waitReady(ctx context.Context, machine string) (string, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, l.cfg.BootTimeout)
|
|
defer cancel()
|
|
client := http.Client{Timeout: 3 * time.Second}
|
|
|
|
var lastErr error
|
|
for {
|
|
addr, err := l.address(machine)
|
|
if err == nil {
|
|
var resp *http.Response
|
|
resp, err = client.Get("http://" + addr + "/status")
|
|
if err == nil {
|
|
resp.Body.Close()
|
|
if resp.StatusCode == http.StatusOK {
|
|
return addr, nil
|
|
}
|
|
err = fmt.Errorf("/status returned %s", resp.Status)
|
|
}
|
|
}
|
|
lastErr = err
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return "", fmt.Errorf("goose didn't answer on port %d within %s (last error: %v)", ServicePort, l.cfg.BootTimeout, lastErr)
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
}
|
|
|
|
func (l *Local) Capacity() Capacity { return l.cap }
|