package local import ( "context" "errors" "fmt" "math" "net" "net/http" "os" "os/exec" "path/filepath" "strconv" "strings" "syscall" "time" "github.com/godbus/dbus/v5" "slackerbot/internal/config" "slackerbot/internal/model" "slackerbot/internal/node" "slackerbot/internal/storage" ) const ( ServicePort = 1984 machinedName = "org.freedesktop.machine1" machinedPath = "/org/freedesktop/machine1" machinedMgr = "org.freedesktop.machine1.Manager" systemdName = "org.freedesktop.systemd1" systemdPath = "/org/freedesktop/systemd1" systemdMgr = "org.freedesktop.systemd1.Manager" sigPoweroff = 38 // SIGRTMIN+4, what `machinectl poweroff` sends agentEnvInside = "/var/lib/slackerbot/agent.env" ) type Local struct { cfg *config.Config name string version string cap Capacity bus *dbus.Conn system string exports transfers imports transfers } func New(cfg *config.Config, version string) (*Local, error) { system, err := filepath.EvalSymlinks(cfg.System) if err != nil { return nil, err } if _, err := os.Stat(filepath.Join(system, "init")); err != nil { return nil, fmt.Errorf("%s has no init. Is it a NixOS build with boot.isContainer = true?", system) } for _, d := range cfg.Drives { if err := storage.PrepareDrive(d.Path); err != nil { return nil, err } } capacity, err := DetectCapacity(cfg.HostReserve) if err != nil { return nil, err } bus, err := dbus.ConnectSystemBus() if err != nil { return nil, fmt.Errorf("connecting to the system bus: %w", err) } return &Local{cfg: cfg, name: cfg.Cluster.Name, version: version, cap: capacity, bus: bus, system: system}, nil } func unitName(machine string) string { return "slackerbot-" + machine + ".service" } func (l *Local) Name() string { return l.name } func (l *Local) machined() dbus.BusObject { return l.bus.Object(machinedName, machinedPath) } func (l *Local) Info(ctx context.Context) (node.NodeInfo, error) { info := node.NodeInfo{Name: l.name, Version: l.version, Total: l.cap.Total, Usable: l.cap.Usable} for _, d := range l.cfg.Drives { free, err := storage.FreeMB(d.Path) if err != nil { return info, err } di := node.DriveInfo{Path: d.Path, FreeMB: free, ReserveMB: d.ReserveMB, HomesMB: map[string]int64{}} entries, err := os.ReadDir(d.Path) if err != nil { return info, err } for _, e := range entries { if e.IsDir() && model.IsOurs(e.Name()) { used, _ := storage.UsedMB(filepath.Join(d.Path, e.Name())) di.HomesMB[e.Name()] = used } } info.Drives = append(info.Drives, di) } running, err := l.Running(ctx) info.Running = running return info, err } func (l *Local) Running(ctx context.Context) ([]node.RunningMachine, error) { var list []struct { Name, Class, Service string Path dbus.ObjectPath } if err := l.machined().Call(machinedMgr+".ListMachines", 0).Store(&list); err != nil { return nil, err } var out []node.RunningMachine for _, x := range list { if !model.IsOurs(x.Name) { continue } r := node.RunningMachine{Name: x.Name, Outdated: l.outdated(x.Name)} if v, err := l.bus.Object(machinedName, x.Path).GetProperty("org.freedesktop.machine1.Machine.Timestamp"); err == nil { if us, ok := v.Value().(uint64); ok { r.Since = time.UnixMicro(int64(us)) } } out = append(out, r) } return out, nil } // The unit's command ends with the init of the system the container booted. // When it can't be read, the container counts as up to date. func (l *Local) outdated(machine string) bool { args := l.execStart(unitName(machine)) if len(args) == 0 { return false } init := args[len(args)-1] return filepath.Base(init) == "init" && filepath.Dir(init) != l.system } func (l *Local) execStart(unit string) []string { var path dbus.ObjectPath if err := l.bus.Object(systemdName, systemdPath).Call(systemdMgr+".GetUnit", 0, unit).Store(&path); err != nil { return nil } v, err := l.bus.Object(systemdName, path).GetProperty(systemdName + ".Service.ExecStart") if err != nil { return nil } var cmds []struct { Path string Args []string IgnoreErrors bool StartTime, StartTimeMono, ExitTime, ExitTimeMono uint64 PID uint32 Code, Status int32 } if err := v.Store(&cmds); err != nil || len(cmds) == 0 { return nil } return cmds[0].Args } func (l *Local) IsRunning(machine string) bool { var p dbus.ObjectPath return l.machined().Call(machinedMgr+".GetMachine", 0, machine).Store(&p) == nil } func (l *Local) drive(path string) (config.DriveConfig, error) { for _, d := range l.cfg.Drives { if d.Path == path { return d, nil } } return config.DriveConfig{}, fmt.Errorf("drive %s isn't configured on node %s", path, l.name) } func (l *Local) Start(ctx context.Context, spec node.StartSpec) error { if !model.IsOurs(spec.Machine) { return fmt.Errorf("unexpected machine name %q", spec.Machine) } if _, err := l.drive(spec.Drive); err != nil { return err } home := filepath.Join(spec.Drive, spec.Machine) if _, err := os.Stat(home); errors.Is(err, os.ErrNotExist) { if err := storage.CreateSubvolume(home); err != nil { return err } } else if err != nil { return err } if err := storage.SetQuota(home, spec.StorageMaxMB); err != nil { return err } // Only the home folder persists. NixOS rebuilds the rest on boot. root := filepath.Join(l.cfg.MachinesDir, spec.Machine) // NixOS marks /var/empty immutable, which blocks RemoveAll. _ = exec.Command("chattr", "-i", filepath.Join(root, "var/empty")).Run() if err := os.RemoveAll(root); err != nil { return fmt.Errorf("clearing old container files: %w", err) } // systemd-nspawn refuses a directory without /usr as not an OS tree. for _, dir := range []string{"etc", "usr"} { if err := os.MkdirAll(filepath.Join(root, dir), 0o755); err != nil { return err } } if f, err := os.OpenFile(filepath.Join(root, "etc/os-release"), os.O_CREATE, 0o644); err == nil { f.Close() } envFile, err := l.writeEnvFile(spec) if err != nil { return err } args := []string{ "--unit=" + unitName(spec.Machine), "--collect", "--quiet", "--description=" + spec.Description, // Same as systemd-nspawn@.service. "--property=Type=notify", "--property=KillMode=mixed", "--property=Delegate=yes", fmt.Sprintf("--property=MemoryMax=%dM", spec.Size.MemoryMB), fmt.Sprintf("--property=CPUQuota=%d%%", int(math.Round(spec.Size.CPUs*100))), "systemd-nspawn", "--machine=" + spec.Machine, "--directory=" + root, "--keep-unit", "--network-veth", // own network, so every container can use port 1984 "--bind-ro=/nix/store", "--bind-ro=" + envFile + ":" + agentEnvInside, "--bind=" + home + ":/root", } binds := func(list []string, flag string) error { for _, b := range list { b = strings.ReplaceAll(b, "{machine}", spec.Machine) src, _, _ := strings.Cut(b, ":") // Missing sources make systemd-nspawn fail; existing ones may be files. if _, err := os.Stat(src); errors.Is(err, os.ErrNotExist) { if err := os.MkdirAll(src, 0o755); err != nil { return err } } args = append(args, flag+b) } return nil } if err := binds(l.cfg.ROBinds, "--bind-ro="); err != nil { return err } if err := binds(l.cfg.RWBinds, "--bind="); err != nil { return err } args = append(args, filepath.Join(l.system, "init")) if out, err := exec.CommandContext(ctx, "systemd-run", args...).CombinedOutput(); err != nil { return fmt.Errorf("systemd-run: %v: %s", err, strings.TrimSpace(string(out))) } // Wait until machined lists it, so the next capacity check counts it. for i := 0; i < 50; i++ { if l.IsRunning(spec.Machine) { return nil } time.Sleep(200 * time.Millisecond) } return fmt.Errorf("%s started but never registered with machined", spec.Machine) } func (l *Local) writeEnvFile(spec node.StartSpec) (string, error) { dir := filepath.Join(l.cfg.StateDir, "env") if err := os.MkdirAll(dir, 0o700); err != nil { return "", err } var b strings.Builder line := func(k, v string) { v = strings.ReplaceAll(v, `\`, `\\`) v = strings.ReplaceAll(v, `"`, `\"`) fmt.Fprintf(&b, "%s=\"%s\"\n", k, v) } line("GOOSE_SERVER__SECRET_KEY", spec.Secret) for k, v := range spec.Env { line(k, v) } path := filepath.Join(dir, spec.Machine+".env") return path, os.WriteFile(path, []byte(b.String()), 0o600) } func (l *Local) Poweroff(ctx context.Context, machine string) error { return l.machined().CallWithContext(ctx, machinedMgr+".KillMachine", 0, machine, "leader", int32(sigPoweroff)).Err } func (l *Local) SetLimits(ctx context.Context, machine string, s model.Size) error { type prop struct { Name string Value dbus.Variant } props := []prop{ {"MemoryMax", dbus.MakeVariant(uint64(s.MemoryMB) << 20)}, {"CPUQuotaPerSecUSec", dbus.MakeVariant(uint64(s.CPUs * 1e6))}, } return l.bus.Object("org.freedesktop.systemd1", "/org/freedesktop/systemd1"). CallWithContext(ctx, "org.freedesktop.systemd1.Manager.SetUnitProperties", 0, unitName(machine), true, props).Err } func (l *Local) SetQuota(ctx context.Context, drive, machine string, mb int64) error { if _, err := l.drive(drive); err != nil { return err } if !model.IsOurs(machine) { return fmt.Errorf("unexpected machine name %q", machine) } return storage.SetQuota(filepath.Join(drive, machine), mb) } func (l *Local) DialAgent(ctx context.Context, machine, secret string) (node.MsgConn, error) { addr, err := l.waitReady(ctx, machine) if err != nil { return nil, err } return node.DialAgentWS(ctx, addr, secret) } func (l *Local) address(machine string) (string, error) { var addrs []struct { Family int32 Addr []byte } if err := l.machined().Call(machinedMgr+".GetMachineAddresses", 0, machine).Store(&addrs); err != nil { return "", err } for _, a := range addrs { ip := net.IP(a.Addr) if a.Family == syscall.AF_INET && !ip.IsLinkLocalUnicast() { return net.JoinHostPort(ip.String(), strconv.Itoa(ServicePort)), nil } } return "", fmt.Errorf("%s has no IPv4 address yet", machine) } func (l *Local) waitReady(ctx context.Context, machine string) (string, error) { ctx, cancel := context.WithTimeout(ctx, l.cfg.BootTimeout) defer cancel() client := http.Client{Timeout: 3 * time.Second} var lastErr error for { addr, err := l.address(machine) if err == nil { var resp *http.Response resp, err = client.Get("http://" + addr + "/status") if err == nil { resp.Body.Close() if resp.StatusCode == http.StatusOK { return addr, nil } err = fmt.Errorf("/status returned %s", resp.Status) } } lastErr = err select { case <-ctx.Done(): return "", fmt.Errorf("goose didn't answer on port %d within %s (last error: %v)", ServicePort, l.cfg.BootTimeout, lastErr) case <-time.After(time.Second): } } } func (l *Local) Capacity() Capacity { return l.cap }