0
0
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"maps"
"net"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/app/compose"
"github.com/johnandersen777/socialweb-computer-kcp/internal/app/gateway"
"github.com/johnandersen777/socialweb-computer-kcp/internal/app/marketnetwork"
"github.com/johnandersen777/socialweb-computer-kcp/internal/app/sshgateway"
"github.com/johnandersen777/socialweb-computer-kcp/internal/assets"
"github.com/johnandersen777/socialweb-computer-kcp/internal/bootstrap"
"github.com/johnandersen777/socialweb-computer-kcp/internal/bootstrap/table"
"github.com/johnandersen777/socialweb-computer-kcp/internal/clicfg"
"github.com/johnandersen777/socialweb-computer-kcp/internal/computevm"
"github.com/johnandersen777/socialweb-computer-kcp/internal/devcore"
"github.com/johnandersen777/socialweb-computer-kcp/internal/devinstance"
"github.com/johnandersen777/socialweb-computer-kcp/internal/devsup"
"github.com/johnandersen777/socialweb-computer-kcp/internal/hostos"
"github.com/johnandersen777/socialweb-computer-kcp/internal/hostsvc"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/fsx"
"github.com/johnandersen777/socialweb-computer-kcp/internal/lease/local"
"github.com/johnandersen777/socialweb-computer-kcp/internal/pgunit"
"github.com/johnandersen777/socialweb-computer-kcp/internal/stack/bidder"
"github.com/johnandersen777/socialweb-computer-kcp/internal/stack/gatewaykcp"
"github.com/johnandersen777/socialweb-computer-kcp/internal/versions"
)
const (
devsupReadyTimeout = 15 * time.Second
// devsupStopWait bounds the wait for a stopping devsup to exit (a unit stop takes up to its stop timeout)
devsupStopWait = 120 * time.Second
devsupExitTimeout = 90 * time.Second
networkDownTimeout = 2 * time.Minute
coreStopTimeout = 10 * time.Second
fgDownTimeout = 5 * time.Minute
)
func (a *app) fail(code int, format string, args ...any) int {
fmt.Fprintf(a.stderr, format+"\n", args...)
return code
}
func (a *app) printJSON(v any) {
b, _ := json.MarshalIndent(v, "", " ")
fmt.Fprintln(a.stdout, string(b))
}
func (a *app) layout(vals clicfg.Values) (devinstance.Layout, error) {
return a.layoutOf(vals.String("instance"), vals)
}
func (a *app) layoutOf(name string, vals clicfg.Values) (devinstance.Layout, error) {
cwd, err := os.Getwd()
if err != nil {
return devinstance.Layout{}, err
}
id, err := devinstance.ID(name, cwd, a.env)
if err != nil {
return devinstance.Layout{}, err
}
root, err := devinstance.StateRoot(vals.String("state-dir"), a.env)
if err != nil {
return devinstance.Layout{}, err
}
rt, err := devinstance.RuntimeRoot(root, a.env)
if err != nil {
return devinstance.Layout{}, err
}
return devinstance.NewLayout(id, root, rt)
}
func (a *app) worktree(vals clicfg.Values) string {
if vals.String("instance") != "" {
return ""
}
cwd, err := os.Getwd()
if err != nil {
return ""
}
real, err := filepath.EvalSymlinks(cwd)
if err != nil {
return ""
}
return devinstance.WorktreeRoot(real)
}
func writeEnvFiles(l devinstance.Layout, ex []devinstance.Export) error {
if err := os.WriteFile(filepath.Join(l.StateDir, devinstance.EnvFile), []byte(devinstance.ShellExports(ex)), 0o600); err != nil {
return err
}
return fsx.WriteJSON(filepath.Join(l.StateDir, devinstance.EnvJSONFile), exportMap(ex), 0o600)
}
func exportMap(ex []devinstance.Export) map[string]string {
out := map[string]string{}
for _, e := range ex {
out[e.Name] = e.Value
}
return out
}
func coreSpec(rec *devinstance.Record, exe string, readyTimeout time.Duration) (hostsvc.UnitSpec, error) {
return bootstrap.Config{Layout: rec.Layout, Exe: exe, ReadyTimeout: readyTimeout}.CoreSpec(rec)
}
func (a *app) postgresRoot(vals clicfg.Values, profile string) (devinstance.Executable, error) {
root := vals.String("postgres-root")
if assets.Release() {
if root != "" {
return devinstance.Executable{}, errors.New("postgres-root is a dev-build option; this release build runs its embedded postgres")
}
cache, err := a.assetCache(vals)
if err != nil {
return devinstance.Executable{}, err
}
e, err := assets.Resolve(assets.Options{Cache: cache, Log: a.log}, pgunit.Asset)
if err != nil {
return devinstance.Executable{}, err
}
if root, err = pgunit.RootOf(e.Path); err != nil {
return devinstance.Executable{}, err
}
}
if root == "" {
if bootstrap.NeedsPostgres(profile) {
return devinstance.Executable{}, fmt.Errorf("--profile %s runs kine on postgres: postgres-root must name the absolute root of the extracted postgres asset tree (versions.json postgres; never PATH)", profile)
}
return devinstance.Executable{}, nil
}
in, err := pgunit.FromRoot(root)
if err != nil {
return devinstance.Executable{}, err
}
e, err := hashExe("postgres-root", in.Path("postgres"))
if err != nil {
return devinstance.Executable{}, err
}
return devinstance.Executable{Path: root, SHA256: e.SHA256, Version: in.Version}, nil
}
func selfExe() (string, error) {
p, err := os.Executable()
if err != nil {
return "", err
}
return filepath.EvalSymlinks(p)
}
func checkExe(ctx context.Context, option, path string) (devinstance.Executable, error) {
e, err := hashExe(option, path)
if err != nil {
return e, err
}
vctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
out, _ := exec.CommandContext(vctx, path, "--version").CombinedOutput()
e.Version = strings.TrimSpace(string(out))
return e, nil
}
func hashExe(option, path string) (devinstance.Executable, error) {
if path == "" {
return devinstance.Executable{}, fmt.Errorf("%s is required in this dev build (an absolute path; never PATH)", option)
}
if !filepath.IsAbs(path) {
return devinstance.Executable{}, fmt.Errorf("%s %q must be an absolute path; this build never looks in PATH", option, path)
}
f, err := os.Open(path)
if err != nil {
return devinstance.Executable{}, fmt.Errorf("%s: %w", option, err)
}
defer f.Close()
fi, err := f.Stat()
if err != nil || fi.IsDir() || fi.Mode()&0o111 == 0 {
return devinstance.Executable{}, fmt.Errorf("%s %s is not an executable file", option, path)
}
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return devinstance.Executable{}, err
}
return devinstance.Executable{Path: path, SHA256: hex.EncodeToString(h.Sum(nil))}, nil
}
func lockFile(ctx context.Context, path string, wait time.Duration) (*os.File, error) {
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return nil, err
}
if err := hostos.LockWait(ctx, f, hostos.LockExclusive, wait); err != nil {
f.Close()
return nil, fmt.Errorf("%s is held by another dev up or dev down of this instance: %w", path, err)
}
return f, nil
}
func procConfig(l devinstance.Layout) hostsvc.Config {
return hostsvc.Config{Backend: hostsvc.BackendProc, ProcSocket: l.Socket}
}
func helloDevsup(ctx context.Context, l devinstance.Layout) (hostsvc.ProcClient, hostsvc.ProcHello, error) {
c, err := hostsvc.OpenProcClient(ctx, procConfig(l))
if err != nil {
return nil, hostsvc.ProcHello{}, err
}
h, err := c.Hello(ctx)
if err != nil {
return nil, hostsvc.ProcHello{}, err
}
return c, h, nil
}
var errDevsupDetached = errors.New("the instance is already up with a detached devsup")
func startOrAdoptDevsup(ctx context.Context, a *app, l devinstance.Layout, exe string, dieWith int, fg bool) (hostsvc.ProcClient, devinstance.Devsup, error) {
if c, h, err := helloDevsup(ctx, l); err == nil && h.Stopping {
// the instance's previous devsup is shutting down (its owner stopped the instance, or died mid-stop): its units are
// going away in order, so none is adopted; wait for it to exit and start a new one
c.Close()
a.log.Info("the instance's devsup is stopping; waiting for it to exit before starting", "pid", h.PID, "instance", h.Instance)
wait := time.Now().Add(devsupStopWait)
for {
c2, h2, err2 := helloDevsup(ctx, l)
if err2 != nil || h2.PID != h.PID || !h2.Stopping {
if c2 != nil {
c2.Close()
}
break
}
c2.Close()
if time.Now().After(wait) {
return nil, devinstance.Devsup{}, fmt.Errorf("the instance's devsup (pid %d) is still stopping after %s", h.PID, devsupStopWait)
}
select {
case <-ctx.Done():
return nil, devinstance.Devsup{}, ctx.Err()
case <-time.After(100 * time.Millisecond):
}
}
}
if c, h, err := helloDevsup(ctx, l); err == nil {
if fg {
c.Close()
return nil, devinstance.Devsup{}, fmt.Errorf("%w (devsup pid %d); dev down it first, or run dev up without --fg", errDevsupDetached, h.PID)
}
return c, devinstance.Devsup{Instance: h.Instance, PID: h.PID, StartTicks: h.StartTicks, BootID: h.BootID, Adopted: true}, nil
}
if err := os.MkdirAll(l.SupState, 0o700); err != nil {
return nil, devinstance.Devsup{}, err
}
argv := []string{exe, "internal", "devsup", "--state", l.SupState, "--socket", l.Socket, "--instance", devinstance.ProcessTag(l)}
if dieWith != 0 {
argv = append(argv, "--die-with", strconv.Itoa(dieWith))
}
pid, err := devinstance.SpawnDetached(argv, filepath.Join(l.SupState, "devsup.log"))
if err != nil {
return nil, devinstance.Devsup{}, fmt.Errorf("starting devsup: %w", err)
}
deadline := time.Now().Add(devsupReadyTimeout)
for {
c, h, err := helloDevsup(ctx, l)
if err == nil && h.PID == pid {
a.log.Info("devsup started", "pid", pid, "instance", h.Instance, "socket", l.Socket)
return c, devinstance.Devsup{Instance: h.Instance, PID: h.PID, StartTicks: h.StartTicks, BootID: h.BootID}, nil
}
if time.Now().After(deadline) {
b, _ := os.ReadFile(filepath.Join(l.SupState, "devsup.log"))
return nil, devinstance.Devsup{}, fmt.Errorf("devsup pid %d did not answer on %s: %v\n%s", pid, l.Socket, err, b)
}
time.Sleep(20 * time.Millisecond)
}
}
type upReport struct {
Instance string `json:"instance"`
Profile string `json:"profile"`
StateDir string `json:"stateDir"`
Socket string `json:"socket"`
AdminKubeconfig string `json:"adminKubeconfig"`
KcpURL string `json:"kcpURL"`
KineURL string `json:"kineURL"`
Devsup devinstance.Devsup `json:"devsup"`
Units []devinstance.Unit `json:"units"`
Core devinstance.Core `json:"core"`
StartedAt time.Time `json:"startedAt"`
ReadyAt time.Time `json:"readyAt"`
Seconds float64 `json:"seconds"`
Env map[string]string `json:"env"`
EnvFile string `json:"envFile"`
Marker string `json:"marker,omitempty"`
Foreground bool `json:"foreground,omitempty"`
Executables map[string]devinstance.Executable `json:"executables"`
Network *v1alpha1.MarketNetwork `json:"network,omitempty"`
layout devinstance.Layout
}
func runDevUp(ctx context.Context, a *app, vals clicfg.Values, rest []string) int {
return devUp(ctx, a, vals, false)
}
func runDevHost(ctx context.Context, a *app, vals clicfg.Values, rest []string) int {
return devUp(ctx, a, vals, true)
}
func (a *app) parentOf(ctx context.Context, vals clicfg.Values, child devinstance.Layout) (devinstance.Parent, hostsvc.ProcHello, error) {
pl, err := a.layoutOf(vals.String("parent"), vals)
if err != nil {
return devinstance.Parent{}, hostsvc.ProcHello{}, err
}
if pl.ID == child.ID {
return devinstance.Parent{}, hostsvc.ProcHello{}, fmt.Errorf("instance %s cannot be its own parent", child.ID)
}
var prec devinstance.Record
if err := fsx.ReadJSON(pl.Record, &prec); err != nil {
return devinstance.Parent{}, hostsvc.ProcHello{}, fmt.Errorf("parent %s has no instance record (dev host first): %w", pl.ID, err)
}
if !prec.Host || !prec.Ready || prec.AdminKubeconfig == "" {
return devinstance.Parent{}, hostsvc.ProcHello{}, fmt.Errorf("instance %s is not a ready dev host (dev host --instance %s)", pl.ID, pl.ID)
}
if prec.Parent != nil {
return devinstance.Parent{}, hostsvc.ProcHello{}, fmt.Errorf("parent %s is itself a child; instance trees do not nest", pl.ID)
}
c, h, err := helloDevsup(ctx, pl)
if err != nil {
return devinstance.Parent{}, hostsvc.ProcHello{}, fmt.Errorf("the devsup of parent %s: %w", pl.ID, err)
}
c.Close()
return devinstance.Parent{ID: pl.ID, Socket: pl.Socket, Kubeconfig: prec.AdminKubeconfig, Workspace: devinstance.ChildWorkspace(child.ID)}, h, nil
}
func bringUp(ctx context.Context, a *app, vals clicfg.Values, host bool) (upReport, int) {
started := time.Now().UTC()
verb := "dev up"
if host {
verb = "dev host"
}
if err := devinstance.RefuseRoot(os.Geteuid()); err != nil {
return upReport{}, a.fail(exitRefused, "%s: %v", verb, err)
}
if b := vals.String(backendOpt); b != "" && b != string(hostsvc.BackendProc) {
return upReport{}, a.fail(exitRefused, "%s: host-backend=%s: dev places every unit under devsup (proc); systemd-user is for install", verb, b)
}
profile, standInsFile := vals.String("profile"), vals.String("stand-ins")
parent := vals.String("parent")
if host && profile == "" {
profile = bootstrap.ProfileQuick
}
if parent != "" && standInsFile != "" {
return upReport{}, a.fail(exitRefused, "dev up: --parent places the bootstrap layer through the parent's kcp; stand-ins have no kcp")
}
var standIns []devinstance.StandIn
switch {
case profile != "" && standInsFile != "":
return upReport{}, a.fail(exitRefused, "dev up: --profile %s and --stand-ins name two bootstrap layers; pass one", profile)
case profile == "" && standInsFile == "":
return upReport{}, a.fail(exitRefused, "dev up: name a profile (--profile; built: quick, prodlike) or the units to place (--stand-ins)")
case standInsFile != "":
s, err := bootstrap.ReadStandIns(standInsFile)
if err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
standIns, profile = s, bootstrap.ProfileStandIns
default:
if err := bootstrap.CheckProfile(profile); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
}
composes := profile == bootstrap.ProfileMarketNetwork
if composes && (host || parent != "") {
return upReport{}, a.fail(exitRefused, "%s: --profile %s is an application of its own whose children are workspaces of its kcp; it neither nests under a parent nor is a dev host", verb, profile)
}
var networkOverlay []byte
if f := vals.String("app-values"); f != "" {
if !composes {
return upReport{}, a.fail(exitRefused, "dev up: app-values names values for the children of --profile %s", bootstrap.ProfileMarketNetwork)
}
b, err := os.ReadFile(f)
if err != nil {
return upReport{}, a.fail(exitRefused, "dev up: app-values: %v", err)
}
networkOverlay = b
}
networkTimeout, err := time.ParseDuration(vals.String("network-timeout"))
if err != nil {
return upReport{}, a.fail(exitRefused, "dev up: network-timeout: %v", err)
}
readyTimeout, err := time.ParseDuration(vals.String("ready-timeout"))
if err != nil {
return upReport{}, a.fail(exitRefused, "dev up: ready-timeout: %v", err)
}
if readyTimeout < 0 {
return upReport{}, a.fail(exitRefused, "dev up: ready-timeout: %s is negative", readyTimeout)
}
if composes && vals.Int("bidders") > 0 {
if err := a.vmHostPreflight(ctx, vals); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
}
var kcpExe, kineExe devinstance.Executable
pgExe, err := a.postgresRoot(vals, profile)
if err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
if standIns == nil {
if kcpExe, err = a.devExe(ctx, vals, "kcp", "kcp-path"); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
if want, err := versions.Get("kcp"); err == nil && want.Version != "" && !strings.Contains(kcpExe.Version, want.Version) {
return upReport{}, a.fail(exitRefused, "dev up: kcp-path %s reports %q; versions.json pins %s", kcpExe.Path, kcpExe.Version, want.Version)
}
if kineExe, err = a.devExe(ctx, vals, "kine", "kine-path"); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
}
var baoExe devinstance.Executable
if standIns == nil && (assets.Release() || vals.String("bao-path") != "" || composes) {
if baoExe, err = a.devExe(ctx, vals, "bao", "bao-path"); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
}
var pluginExes map[string]devinstance.Executable
if standIns == nil {
if pluginExes, err = a.openBaoPluginExes(vals); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
}
var dumbpipeExe devinstance.Executable
if standIns == nil && (assets.Release() || vals.String("dumbpipe-path") != "") {
if dumbpipeExe, err = a.devExe(ctx, vals, sshgateway.DumbpipeExecutable, "dumbpipe-path"); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
}
var gatewayExes map[string]devinstance.Executable
if composes && vals.Bool("with-gateway") {
if gatewayExes, err = a.gatewayChildExecutables(vals); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
}
exe, err := selfExe()
if err != nil {
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
name := vals.String("instance")
if host && name == "" {
name = devinstance.HostID
}
l, err := a.layoutOf(name, vals)
if err != nil {
return upReport{}, a.fail(exitRefused, "%s: %v", verb, err)
}
var networkSpec v1alpha1.MarketNetworkSpec
if composes {
imageSource := vals.String("guest-image-source")
if imageSource == "" {
if imageSource, err = publishedGuestImageSource(ctx, vals.String("guest-image-release-base")); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
}
guestNetwork, err := guestNetworkFlags(vals)
if err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
computeKinds, err := computeKindsFlag(vals)
if err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
vm := marketnetwork.BidderVMDefaults{GuestImageSource: imageSource, VmHostBackend: vals.String("vm-host-backend"), GuestNetwork: guestNetwork,
ComputeKinds: computeKinds, CPUOvercommit: int32(vals.Int("cpu-overcommit")), Policy: strings.TrimSpace(vals.String("bidder-policy"))}
if networkSpec, err = marketnetwork.NetworkSpec(l.ID, vals.Int("bidders"), vals.Bool("with-gateway"), networkOverlay, vm); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
if bind := strings.TrimSpace(vals.String("network-bind")); bind != "" {
if net.ParseIP(bind) == nil {
return upReport{}, a.fail(exitRefused, "dev up: network-bind %q is not an IP address", bind)
}
networkSpec.BindAddress = bind
}
networkSpec.Domain = strings.Trim(strings.TrimSpace(vals.String("network-domain")), ".")
networkSpec.AtircURL = strings.TrimRight(strings.TrimSpace(vals.String("network-atirc-url")), "/")
networkSpec.QrOrigin = strings.TrimRight(strings.TrimSpace(vals.String("network-qr-origin")), "/")
knotless, err := knotlessDir(vals.String("with-knotless"), vals.Bool("with-gateway"))
if err != nil {
return upReport{}, a.fail(exitRefused, "dev up: %v", err)
}
for i := range networkSpec.Apps {
if networkSpec.Apps[i].Kind != v1alpha1.ApplicationKindGateway {
continue
}
if networkSpec.Apps[i].ClusterProvider, err = gatewaykcp.ClusterProviderFromEnv(a.env.LookupEnv, networkSpec.Apps[i].ClusterProvider); err != nil {
return upReport{}, a.fail(exitRefused, "dev up: the gateway's clusterprovider settings: %v", err)
}
if knotless != "" {
networkSpec.Apps[i].Options = append(networkSpec.Apps[i].Options, v1alpha1.ApplicationOption{Name: gateway.KnotlessOption, Value: knotless})
}
}
}
var par *devinstance.Parent
var parentHello hostsvc.ProcHello
if parent != "" {
p, h, err := a.parentOf(ctx, vals, l)
if err != nil {
return upReport{}, a.fail(exitRefused, "dev up: --parent %s: %v", parent, err)
}
par, parentHello = &p, h
}
for _, d := range []string{l.StateDir, l.RuntimeDir, l.Logs} {
if err := os.MkdirAll(d, 0o700); err != nil {
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
os.Chmod(d, 0o700)
}
lock, err := lockFile(ctx, l.Lock, 30*time.Second)
if err != nil {
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
fg := vals.Bool("fg")
dieWith := vals.Int("die-with")
if fg {
dieWith = os.Getpid()
}
var m hostsvc.Manager
var sup devinstance.Devsup
if par != nil {
if err := devinstance.EnsureChildWorkspace(ctx, par.Kubeconfig, l.ID, devinstance.ChildWorkspaceReady, a.log); err != nil {
lock.Close()
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
r, err := devinstance.OpenRemote(ctx, *par, a.log)
if err != nil {
lock.Close()
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
m = r
sup = devinstance.Devsup{Instance: parentHello.Instance, PID: parentHello.PID, StartTicks: parentHello.StartTicks,
BootID: parentHello.BootID, Adopted: true}
a.log.Info("the bootstrap layer is placed through the parent's kcp", "parent", par.ID, "workspace", par.Workspace)
} else {
c, s, err := startOrAdoptDevsup(ctx, a, l, exe, dieWith, fg)
if errors.Is(err, errDevsupDetached) {
lock.Close()
return upReport{}, a.fail(exitRefused, "%s: %v", verb, err)
}
if err != nil {
lock.Close()
return upReport{}, a.fail(exitFailed, "%s: %v", verb, err)
}
m, sup = c, s
}
defer m.Close()
var rec devinstance.Record
if err := fsx.ReadJSON(l.Record, &rec); err != nil {
rec = devinstance.Record{Created: started}
}
rec.Layout, rec.Profile, rec.Backend, rec.Devsup, rec.StandIns = l, profile, hostsvc.BackendProc, sup, standIns
rec.Host, rec.Parent = host || composes, par
rec.Worktree = a.worktree(vals)
if host {
rec.Worktree = ""
}
rec.BaseWorkspace = vals.String("kcp-base-workspace")
rec.Role = vals.String(appRoleOpt)
rec.Network = nil
if n := vals.String(coreNetworkOpt); n != "" {
rec.Network = json.RawMessage(n)
}
rec.Executables = map[string]devinstance.Executable{"kcp": kcpExe, "kine": kineExe, "postgres": pgExe, "socialweb-computer": {Path: exe}}
if baoExe.Path != "" {
rec.Executables["bao"] = baoExe
}
if dumbpipeExe.Path != "" {
rec.Executables[sshgateway.DumbpipeExecutable] = dumbpipeExe
}
maps.Copy(rec.Executables, gatewayExes)
maps.Copy(rec.Executables, pluginExes)
rec.OpenBao = nil
if addr := vals.String("openbao-addr"); addr != "" {
tokenFile := vals.String("openbao-token-file")
if !filepath.IsAbs(tokenFile) {
lock.Close()
return upReport{}, a.fail(exitRefused, "dev up: openbao-addr %s needs openbao-token-file as an absolute path, got %q", addr, tokenFile)
}
rec.OpenBao = &devinstance.OpenBao{Address: addr, TokenFile: tokenFile}
}
rec.VM = nil
if !host {
vmExes, err := a.devVMExecutables(vals)
if err != nil {
lock.Close()
return upReport{}, a.fail(exitRefused, "%s: %v", verb, err)
}
maps.Copy(rec.Executables, vmExes)
if rec.VM, err = a.devVM(vals, l); err != nil {
lock.Close()
return upReport{}, a.fail(exitRefused, "%s: %v", verb, err)
}
}
bcfg := bootstrap.Config{Layout: l, KcpPath: kcpExe.Path, KinePath: kineExe.Path, PostgresRoot: pgExe.Path, Exe: exe,
ReadyTimeout: readyTimeout, Log: a.log}
if err := bootstrap.Ensure(ctx, m, bcfg, &rec); err != nil {
fsx.WriteJSON(l.Record, rec, 0o600)
lock.Close()
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
if err := fsx.WriteJSON(l.Record, rec, 0o600); err != nil {
lock.Close()
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
lock.Close()
cspec, err := coreSpec(&rec, exe, readyTimeout)
if err != nil {
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
cst, err := m.Start(ctx, cspec)
if err != nil {
return upReport{}, a.fail(exitFailed, "dev up: placing the core: %v", err)
}
core, err := waitCore(ctx, l, m, cspec.Name, cst.InvocationID, readyTimeout)
if err != nil {
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
fsx.ReadJSON(l.Record, &rec)
if host || composes {
if err := devinstance.WaitHost(ctx, rec.AdminKubeconfig, readyTimeout); err != nil {
return upReport{}, a.fail(exitFailed, "%s: %v", verb, err)
}
}
var network *v1alpha1.MarketNetwork
if composes {
if err := marketnetwork.ApplyNetwork(ctx, rec.AdminKubeconfig, networkSpec, readyTimeout); err != nil {
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
a.log.Info("the MarketNetwork root object is applied; its core composes the children", "bidders", vals.Int("bidders"),
"gateway", vals.Bool("with-gateway"), "apps", len(networkSpec.Apps))
o, err := marketnetwork.WaitNetwork(ctx, rec.AdminKubeconfig, networkTimeout, func(st v1alpha1.MarketNetworkStatus) {
var apps []string
for _, ap := range st.Apps {
apps = append(apps, ap.Name+"="+string(ap.Phase))
}
a.log.Info("market network", "phase", string(st.Phase), "message", st.Message, "apps", strings.Join(apps, " "))
})
network = &o
if err != nil {
a.printJSON(network)
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
}
ex := devinstance.Exports(rec)
if err := writeEnvFiles(l, ex); err != nil {
return upReport{}, a.fail(exitFailed, "dev up: %v", err)
}
marker := ""
if rec.Worktree != "" {
if err := devinstance.LinkMarker(rec.Worktree, l.StateDir); err != nil {
a.log.Warn("worktree marker", "err", err.Error())
} else {
marker = filepath.Join(rec.Worktree, devinstance.Marker)
}
}
ready := time.Now().UTC()
return upReport{Instance: l.ID, Profile: rec.Profile, StateDir: l.StateDir, Socket: l.Socket,
AdminKubeconfig: rec.AdminKubeconfig, KcpURL: rec.KcpURL, KineURL: rec.KineURL, Devsup: sup, Units: rec.Units,
Core: core, StartedAt: started, ReadyAt: ready, Seconds: ready.Sub(started).Seconds(),
Env: exportMap(ex), EnvFile: filepath.Join(l.StateDir, devinstance.EnvFile), Marker: marker, Foreground: fg,
Executables: rec.Executables, Network: network, layout: l}, exitOK
}
func devUp(ctx context.Context, a *app, vals clicfg.Values, host bool) int {
r, code := bringUp(ctx, a, vals, host)
if code != exitOK {
return code
}
a.printJSON(r)
if !r.Foreground {
return exitOK
}
a.log.Info("dev up --fg: the instance lives as long as this process; Ctrl-C is dev down", "instance", r.layout.ID)
<-ctx.Done()
dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), fgDownTimeout)
defer cancel()
d := downInstance(dctx, a, r.layout, vals.Bool("keep-state"), false)
a.printJSON(d)
if len(d.Remaining) > 0 {
return exitFailed
}
return exitOK
}
func waitCore(ctx context.Context, l devinstance.Layout, m hostsvc.Manager, unit, invocation string, timeout time.Duration) (devinstance.Core, error) {
wctx, cancel := context.WithCancel(ctx)
defer cancel()
records, err := local.Watch(wctx, l.CoreRecord)
if err != nil {
return devinstance.Core{}, err
}
units, err := m.Watch(wctx, unit)
if err != nil {
return devinstance.Core{}, err
}
deadline := time.NewTimer(timeout)
defer deadline.Stop()
st, stErr := m.Inspect(ctx, unit)
for {
var c devinstance.Core
if err := fsx.ReadJSON(l.CoreRecord, &c); err == nil && c.InvocationID == invocation && !c.AdoptedAt.IsZero() {
return c, nil
}
if stErr == nil && ((st.InvocationID != "" && st.InvocationID != invocation) ||
st.State == hostsvc.Failed || st.State == hostsvc.Inactive || st.State == hostsvc.NotFound) {
b, _ := os.ReadFile(filepath.Join(l.Logs, "core.log"))
return devinstance.Core{}, fmt.Errorf("the core (invocation %s) ended before it recorded its adoption: unit %s is %s %s (invocation %s, exit status %d)\n%s",
invocation, unit, st.State, st.Result, st.InvocationID, st.ExecMainStatus, lastLines(b, 40))
}
select {
case <-ctx.Done():
return devinstance.Core{}, ctx.Err()
case <-deadline.C:
b, _ := os.ReadFile(filepath.Join(l.Logs, "core.log"))
return devinstance.Core{}, fmt.Errorf("the core (invocation %s) did not record its adoption within %s\n%s", invocation, timeout, b)
case _, ok := <-records:
if !ok {
records = nil
}
case ev, ok := <-units:
if !ok {
units = nil
} else if ev.Name == unit {
st, stErr = ev.Status, nil
}
}
}
}
type statusReport struct {
Instance string `json:"instance"`
Found bool `json:"found"`
Record *devinstance.Record `json:"record,omitempty"`
Core *devinstance.Core `json:"core,omitempty"`
Devsup struct {
Reachable bool `json:"reachable"`
Hello *hostsvc.ProcHello `json:"hello,omitempty"`
Units []hostsvc.UnitStatus `json:"units,omitempty"`
Error string `json:"error,omitempty"`
} `json:"devsup"`
Ports map[string]int `json:"ports,omitempty"`
Network *v1alpha1.MarketNetwork `json:"network,omitempty"`
NetworkError string `json:"networkError,omitempty"`
Apps []appStatusView `json:"apps,omitempty"`
}
func runDevStatus(ctx context.Context, a *app, vals clicfg.Values, rest []string) int {
l, err := a.layout(vals)
if err != nil {
return a.fail(exitRefused, "dev status: %v", err)
}
r := statusReport{Instance: l.ID}
var rec devinstance.Record
if err := fsx.ReadJSON(l.Record, &rec); err == nil {
r.Found, r.Record = true, &rec
r.Ports = map[string]int{}
for _, u := range rec.Units {
r.Ports[u.Role] = u.Port
}
}
var core devinstance.Core
if err := fsx.ReadJSON(l.CoreRecord, &core); err == nil {
r.Core = &core
}
sl := l
if rec.Parent != nil {
sl.Socket = rec.Parent.Socket
}
if c, h, err := helloDevsup(ctx, sl); err == nil {
r.Devsup.Reachable, r.Devsup.Hello = true, &h
for _, prefix := range []string{l.Names().UnitPrefix(), l.Names().CorePrefix()} {
units, err := c.List(ctx, prefix)
if err != nil {
r.Devsup.Error = err.Error()
}
r.Devsup.Units = append(r.Devsup.Units, units...)
}
} else {
r.Devsup.Error = err.Error()
}
if rec.Profile == bootstrap.ProfileMarketNetwork && rec.AdminKubeconfig != "" {
if o, err := marketnetwork.ReadNetwork(ctx, rec.AdminKubeconfig); err == nil {
r.Network = &o
} else {
r.NetworkError = err.Error()
}
r.Apps = appStatuses(ctx, l, rec)
}
a.printJSON(r)
if !r.Found {
return exitFailed
}
return exitOK
}
type downReport struct {
Instance string `json:"instance"`
MarkerRemoved bool `json:"markerRemoved"`
DevsupStopped bool `json:"devsupStopped"`
DevsupPID int `json:"devsupPID,omitempty"`
Swept []devsup.SweepReport `json:"swept,omitempty"`
LeftoverKilled []int `json:"leftoverKilled"`
Remaining []int `json:"remaining"`
VMsStopped []string `json:"vmsStopped,omitempty"`
VMsLeft []string `json:"vmsLeft,omitempty"`
RemovedState bool `json:"removedState"`
RemovedRuntime bool `json:"removedRuntime"`
Seconds float64 `json:"seconds"`
}
func runDevDown(ctx context.Context, a *app, vals clicfg.Values, rest []string) int {
l, err := a.layout(vals)
if err != nil {
return a.fail(exitRefused, "dev down: %v", err)
}
if _, err := os.Stat(l.StateDir); err == nil {
lock, err := lockFile(ctx, l.Lock, 2*time.Minute)
if err != nil {
return a.fail(exitFailed, "dev down: %v", err)
}
defer lock.Close()
}
r := downInstance(ctx, a, l, vals.Bool("keep-state"), false)
a.printJSON(r)
if len(r.Remaining) > 0 {
return exitFailed
}
return exitOK
}
func downInstance(ctx context.Context, a *app, l devinstance.Layout, keepState, retainOnLeftover bool) downReport {
started := time.Now()
r := downReport{Instance: l.ID, LeftoverKilled: []int{}, Remaining: []int{}}
var rec devinstance.Record
fsx.ReadJSON(l.Record, &rec)
vmOwners := []devinstance.Record{rec}
if rec.Profile == bootstrap.ProfileMarketNetwork {
vmOwners = append(vmOwners, networkChildren(l)...)
}
if rec.Profile == bootstrap.ProfileMarketNetwork && rec.AdminKubeconfig != "" {
if err := marketnetwork.DeleteNetwork(ctx, rec.AdminKubeconfig, networkDownTimeout); err != nil {
a.log.Warn("deleting the MarketNetwork before devsup stops; devsup stops what is left", "err", err.Error())
} else {
a.log.Info("the MarketNetwork is deleted: its children, their kcps and its services are gone")
}
}
for _, o := range vmOwners {
stopped, left := a.downVMs(ctx, o)
r.VMsStopped, r.VMsLeft = append(r.VMsStopped, stopped...), append(r.VMsLeft, left...)
}
if rec.Parent != nil {
r.Remaining = downChild(ctx, a, l, *rec.Parent)
r.DevsupPID = rec.Devsup.PID
} else if c, h, err := helloDevsup(ctx, l); err == nil {
r.DevsupPID = h.PID
if err := c.Shutdown(ctx); err != nil {
a.log.Warn("devsup shutdown", "err", err.Error())
}
awaitExit([]int{h.PID}, devsupExitTimeout)
r.DevsupStopped = !devsup.Alive(h.PID)
} else {
a.log.Info("devsup unreachable; sweeping its records", "err", err.Error())
if rep, err := devsup.SweepOffline(l.SupState); err == nil {
r.Swept = rep
}
}
if pids := devsup.KillInstance(devinstance.ProcessTag(l)); len(pids) > 0 {
r.LeftoverKilled = pids
}
if pids := marketnetwork.KillUnder(l.StateDir, 5*time.Second); len(pids) > 0 {
r.LeftoverKilled = append(r.LeftoverKilled, pids...)
}
awaitExit(r.LeftoverKilled, 100*time.Millisecond)
if left := devsup.InstanceProcesses(devinstance.ProcessTag(l)); len(left) > 0 {
r.Remaining = left
}
if len(r.Remaining) > 0 && retainOnLeftover {
r.Seconds = time.Since(started).Seconds()
return r
}
if err := os.RemoveAll(l.RuntimeDir); err == nil {
r.RemovedRuntime = true
os.Remove(filepath.Dir(l.RuntimeDir))
}
if !keepState {
if rec.Worktree != "" {
r.MarkerRemoved = devinstance.UnlinkMarker(rec.Worktree, l.StateDir)
}
if err := assets.RemoveTree(l.StateDir); err == nil {
r.RemovedState = true
}
}
r.Seconds = time.Since(started).Seconds()
return r
}
func networkChildren(l devinstance.Layout) []devinstance.Record {
state, rt := filepath.Join(l.StateDir, compose.AppsDir), filepath.Join(l.RuntimeDir, compose.AppsDir)
entries, err := os.ReadDir(devinstance.DevRoot(state))
if err != nil {
return nil
}
var out []devinstance.Record
for _, e := range entries {
cl, err := devinstance.NewLayout(e.Name(), state, rt)
if err != nil {
continue
}
var rec devinstance.Record
if fsx.ReadJSON(cl.Record, &rec) == nil && rec.VM != nil {
out = append(out, rec)
}
}
return out
}
func downChild(ctx context.Context, a *app, l devinstance.Layout, p devinstance.Parent) []int {
if err := devinstance.DeleteChildWorkspace(ctx, p.Kubeconfig, l.ID); err != nil {
a.log.Warn("deleting the child workspace in the parent", "parent", p.ID, "err", err.Error())
} else {
a.log.Info("deleted the child workspace in the parent; its supervisor stops the units", "parent", p.ID, "workspace", p.Workspace)
}
m, err := hostsvc.Open(ctx, hostsvc.Config{Backend: hostsvc.BackendProc, ProcSocket: p.Socket})
if err != nil {
a.log.Info("the parent devsup is unreachable; nothing of this child runs there", "parent", p.ID, "err", err.Error())
return []int{}
}
defer m.Close()
wctx, cancel := context.WithCancel(ctx)
defer cancel()
events, werr := watchUnits(wctx, m, devinstance.Prefixes(l.ID))
state := map[string]hostsvc.UnitStatus{}
list := func() error {
units, err := devinstance.ChildUnits(ctx, m, l.ID)
if err != nil {
return err
}
clear(state)
for _, u := range units {
state[u.Name] = u
}
return nil
}
err = list()
deadline := time.NewTimer(devsupExitTimeout)
defer deadline.Stop()
for {
var live []int
for _, u := range state {
if u.State == hostsvc.Active || u.State == hostsvc.Activating || u.State == hostsvc.Deactivating {
live = append(live, u.MainPID)
}
}
if err == nil && len(live) == 0 {
return []int{}
}
if werr != nil {
a.log.Warn("the parent devsup offers no unit watch; the child's units are left as listed", "parent", p.ID, "err", werr.Error())
return live
}
select {
case <-deadline.C:
return live
case ev, ok := <-events:
if !ok {
return live
}
if err != nil {
err = list()
} else {
state[ev.Name] = ev.Status
}
}
}
}
func watchUnits(ctx context.Context, m hostsvc.Manager, prefixes []string) (<-chan hostsvc.UnitEvent, error) {
out := make(chan hostsvc.UnitEvent, 64)
var chans []<-chan hostsvc.UnitEvent
for _, prefix := range prefixes {
ch, err := m.Watch(ctx, prefix)
if err != nil {
return nil, err
}
chans = append(chans, ch)
}
var wg sync.WaitGroup
for _, ch := range chans {
wg.Go(func() {
for ev := range ch {
select {
case out <- ev:
case <-ctx.Done():
return
}
}
})
}
go func() {
wg.Wait()
close(out)
}()
return out, nil
}
func (a *app) instanceManager(ctx context.Context, rec devinstance.Record) (hostsvc.Manager, error) {
if rec.Parent != nil {
return devinstance.OpenRemote(ctx, *rec.Parent, a.log)
}
return hostsvc.Open(ctx, procConfig(rec.Layout))
}
func runDevsup(ctx context.Context, a *app, vals clicfg.Values, rest []string) int {
if err := devinstance.RefuseRoot(os.Geteuid()); err != nil {
return a.fail(exitRefused, "internal devsup: %v", err)
}
for _, n := range []string{"state", "socket"} {
if err := a.spec.Require(vals, n); err != nil {
return a.fail(exitRefused, "internal devsup: %v", err)
}
}
err := devsup.Run(ctx, devsup.Options{State: vals.String("state"), Socket: vals.String("socket"),
DieWith: vals.Int("die-with"), Instance: vals.String("instance"), Log: a.log})
if err != nil {
return a.fail(exitFailed, "internal devsup: %v", err)
}
return exitOK
}
func runDevCore(ctx context.Context, a *app, vals clicfg.Values, rest []string) int {
if err := a.spec.Require(vals, "instance-dir"); err != nil {
return a.fail(exitRefused, "internal dev-core: %v", err)
}
var rec devinstance.Record
if err := fsx.ReadJSON(filepath.Join(vals.String("instance-dir"), "instance.json"), &rec); err != nil {
return a.fail(exitFailed, "internal dev-core: %v", err)
}
timeout, err := time.ParseDuration(vals.String("ready-timeout"))
if err != nil {
return a.fail(exitRefused, "internal dev-core: ready-timeout: %v", err)
}
m, err := a.instanceManager(ctx, rec)
if err != nil {
return a.fail(exitFailed, "internal dev-core: %v", err)
}
defer m.Close()
if rec.Gateway != nil {
if err := a.runGatewayChild(ctx, rec, m, timeout); err != nil && !errors.Is(err, context.Canceled) {
return a.fail(exitFailed, "internal dev-core: the gateway child %s: %v", rec.Layout.ID, err)
}
return exitOK
}
if stop, err := a.requesterGatewayOutbound(ctx, rec); err != nil {
return a.fail(exitFailed, "internal dev-core: the requester's gateway outbound: %v", err)
} else {
defer stop()
}
cfg := devcore.Config{Layout: rec.Layout, ReadyTimeout: timeout, Log: a.log, SignInGuard: bidder.CheckSignIn}
if rec.VM == nil && rec.Role == table.RoleBidder && rec.AdminKubeconfig != "" {
if err := computevm.ApplyBidderSets(ctx, computevm.Config{Handle: rec.Layout.ID, Log: a.log}, rec.AdminKubeconfig,
rec.BaseWorkspace); err != nil {
a.log.Warn("applying the bidder's API sets without a VM runtime failed", "err", err.Error())
} else {
a.log.Info("the bidder's API sets are applied: this bidder serves only compute kinds that need no VM runtime")
}
}
if rec.VM != nil {
rt, vm, err := a.instanceVMRuntime(ctx, rec)
if err != nil {
return a.fail(exitFailed, "internal dev-core: the instance is a bidder application instance (handle %s) and its VM runtime does not open: %v", rec.VM.Handle, err)
}
defer vm.Close()
cfg.ComputeVM = &computevm.Config{Handle: rec.VM.Handle, Runtime: rt, Manager: vm, GuestNetwork: rec.VM.GuestNetwork, Log: a.log}
}
if err := devcore.Run(ctx, cfg, m); err != nil {
if errors.Is(err, context.Canceled) {
return exitOK
}
return a.fail(exitFailed, "internal dev-core: %v", err)
}
return exitOK
}
var knotlessBinaries = []string{"knotless-serve", "knotless-run", "knotless", "git-remote-knotless", "dumbpipe"}
func knotlessDir(dir string, withGateway bool) (string, error) {
if dir = strings.TrimSpace(dir); dir == "" {
return "", nil
}
if !withGateway {
return "", errors.New("--with-knotless runs knotless in the gateway: it takes --with-gateway")
}
abs, err := filepath.Abs(dir)
if err != nil {
return "", err
}
for _, b := range knotlessBinaries {
if st, err := os.Stat(filepath.Join(abs, b)); err != nil || st.Mode()&0o111 == 0 {
return "", fmt.Errorf("--with-knotless %s holds no executable %s (the knotless fork's image: docker cp its /usr/local/bin/%s)", abs, b, b)
}
}
return abs, nil
}