johnandersen777.bsky.social/socialweb-computer-kcp

Switch repository

Loading…

package openbaoctl

import (
	"bytes"
	"cmp"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/baosync"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/conds"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/rootvault"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/vault/openbao"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/vaultcert"
)

const (
	RetryAfter       = 2 * time.Second
	TidyRetryAfter   = time.Minute
	PassTimeout      = 30 * time.Second
	DefaultListen    = rootvault.LoopbackHost + ":0"
	DefaultShares    = 1
	DefaultThreshold = 1
	DirName          = "openbaoservers"
	ChildPrefix      = "bao-"
	OwnerLabel       = "socialweb.computer/openbaoserver"

	ConditionReady       = "Ready"
	ConditionInitialized = "Initialized"
	ConditionUnsealed    = "Unsealed"
	ConditionSynced      = "Synced"
	LogCapBytes          = 64 << 20
)

var GVR = v1alpha1.OpenBaoServerGVR

var ErrNotHeld = errors.New(grantNotHeldReason)

type Config struct {
	StateDir string
	Prefix   string
	Env      []v1alpha1.HostProcessEnv
	Resolve  func(v1alpha1.HostProcessAsset) (string, error)
	Log      *slog.Logger
}

var OutboundEnvNames = []string{"HTTPS_PROXY", "HTTP_PROXY", "NO_PROXY", "SSL_CERT_FILE"}

func OutboundEnv(getenv func(string) string) []v1alpha1.HostProcessEnv {
	var out []v1alpha1.HostProcessEnv
	for _, n := range OutboundEnvNames {
		if v := getenv(n); v != "" {
			out = append(out, v1alpha1.HostProcessEnv{Name: n, Value: v})
		}
	}
	return out
}

type Children interface {
	Ensure(ctx context.Context, name string, spec v1alpha1.HostProcessSpec) (v1alpha1.HostProcess, error)
	Delete(ctx context.Context, name string) error
}

type Outcome struct {
	Status v1alpha1.OpenBaoServerStatus

	Grants map[string]v1alpha1.GuestIdentityGrantStatus

	RequeueAfter time.Duration

	Err error
}

type Servers struct {
	c   Config
	log *slog.Logger
}

func New(c Config) (*Servers, error) {
	if !filepath.IsAbs(c.StateDir) {
		return nil, fmt.Errorf("openbaoctl: StateDir %q must be absolute", c.StateDir)
	}
	if c.Prefix == "" {
		return nil, fmt.Errorf("openbaoctl: a unit prefix is required; a controller places only its own instance's servers")
	}
	log := c.Log
	if log == nil {
		log = slog.New(slog.DiscardHandler)
	}
	return &Servers{c: c, log: log.With("component", "openbaoserver")}, nil
}

func ServerDir(stateDir, name string) string { return filepath.Join(stateDir, DirName, name) }

func ChildName(name string) string { return ChildPrefix + name }

func NeedsGrants(o v1alpha1.OpenBaoServer) bool {
	return o.Spec.Declared != nil && o.Spec.Declared.IdentityOIDC != nil
}

func (s *Servers) unit(o v1alpha1.OpenBaoServer) string {
	if o.Spec.Unit != "" {
		return o.Spec.Unit
	}
	return s.c.Prefix + ChildPrefix + o.Name + ".service"
}

func refs(dir string, t *v1alpha1.OpenBaoServerTLS) *v1alpha1.OpenBaoServerRefs {
	ca := rootvault.TLSCAPath(dir)
	if t != nil {
		ca = t.CAFile
	}
	return &v1alpha1.OpenBaoServerRefs{UnsealKeys: "file:" + rootvault.UnsealKeysPath(dir), CA: "file:" + ca,
		ClientCert: "file:" + rootvault.ClientCertPath(dir), ClientKey: "file:" + rootvault.ClientKeyPath(dir),
		CertAuthMount: CertAuthMount, CertAuthRole: CertAuthRole, KVMount: KVMount}
}

var replaceable = map[v1alpha1.HostProcessLostReason]bool{v1alpha1.HostProcessLostUnitGone: true, v1alpha1.HostProcessLostHostRebooted: true,
	v1alpha1.HostProcessLostManagerRestarted: true, v1alpha1.HostProcessLostIdentityMismatch: true}

func restartKey(config []byte, files ...string) string {
	h := sha256.New()
	h.Write(config)
	for _, f := range files {
		if f == "" {
			continue
		}
		b, _ := os.ReadFile(f)
		h.Write([]byte(f))
		h.Write(b)
	}
	return hex.EncodeToString(h.Sum(nil))[:16]
}

const MemoryMax = 512 << 20

func (s *Servers) childSpec(o v1alpha1.OpenBaoServer, dir, key string) v1alpha1.HostProcessSpec {
	rp := o.Spec.RestartPolicy
	rp = cmp.Or(rp, v1alpha1.HostProcessRestartOnFailure)
	spec := v1alpha1.HostProcessSpec{Unit: s.unit(o), Exec: o.Spec.Exec, Argv: []string{"server", "-config", rootvault.ConfigPath(dir)},
		WorkDir: rootvault.Dir(dir), RestartPolicy: rp, Stop: &v1alpha1.HostProcessStop{Signal: "SIGTERM", GraceSeconds: 15},
		Placement: v1alpha1.HostProcessPlacementSupervisor, Slice: o.Spec.Slice, RestartKey: key,
		Limits: &v1alpha1.HostProcessLimits{MemoryMax: strconv.FormatUint(MemoryMax, 10)},
		After:  append([]string(nil), o.Spec.After...), Wants: append([]string(nil), o.Spec.Wants...)}
	if o.Spec.Asset != nil {
		a := *o.Spec.Asset
		spec.Asset = &a
	}
	if o.Spec.Log != "" {
		spec.Output = &v1alpha1.HostProcessOutput{Stdout: o.Spec.Log, Stderr: o.Spec.Log, CapBytes: LogCapBytes}
	}
	spec.Env = append(spec.Env, s.c.Env...)
	return spec
}

func writeIfChanged(path string, data []byte, mode os.FileMode) error {
	if cur, err := os.ReadFile(path); err == nil && bytes.Equal(cur, data) {
		return os.Chmod(path, mode)
	}
	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
		return err
	}
	tmp := path + ".tmp"
	if err := os.WriteFile(tmp, data, mode); err != nil {
		return err
	}
	return os.Rename(tmp, path)
}

func unseal(ctx context.Context, client *openbao.Client, k UnsealKeys) error {
	for i := 0; i < k.Threshold; i++ {
		sealed, err := client.Unseal(ctx, k.Keys[i])
		if err != nil {
			return fmt.Errorf("unsealing with key %d of %d: %w", i+1, k.Threshold, err)
		}
		if !sealed {
			return nil
		}
	}
	return fmt.Errorf("still sealed after %d keys", k.Threshold)
}

type synced struct {
	report  baosync.Report
	pending []string
	how     string
	desired baosync.Desired
	running map[string]string
	tidied  bool
	tidyErr error
	roles   map[string]grantRole
	refused map[string]string
}

func tidyInterval(s v1alpha1.OpenBaoServerSpec) (time.Duration, error) {
	if s.Tidy == nil || s.Tidy.Interval == "" {
		return DefaultTidyInterval, nil
	}
	d, err := time.ParseDuration(s.Tidy.Interval)
	if err != nil || d < time.Minute {
		return 0, fmt.Errorf("spec.tidy.interval %q: a duration of at least 1m", s.Tidy.Interval)
	}
	return d, nil
}

func tidyDue(prev *v1alpha1.OpenBaoServerTidyStatus, interval time.Duration, now time.Time) bool {
	return tidyIn(prev, interval, now) <= 0
}

func tidyIn(prev *v1alpha1.OpenBaoServerTidyStatus, interval time.Duration, now time.Time) time.Duration {
	if prev == nil || prev.LastRunAt == "" {
		return 0
	}
	at, err := time.Parse(time.RFC3339Nano, prev.LastRunAt)
	if err != nil {
		return 0
	}
	return at.Add(interval).Sub(now)
}

func tidyTokensAndLeases(ctx context.Context, admin *openbao.Client) (bool, error) {
	var errs []error
	for _, p := range []string{"auth/token/tidy", "sys/leases/tidy"} {
		if err := admin.Write(ctx, p, map[string]any{}); err != nil {
			errs = append(errs, fmt.Errorf("%s: %w", p, err))
		}
	}
	return len(errs) == 0, errors.Join(errs...)
}

func recordTidy(st *v1alpha1.OpenBaoServerStatus, s synced, interval time.Duration, log *slog.Logger) {
	next := &v1alpha1.OpenBaoServerTidyStatus{Interval: interval.String()}
	if st.Tidy != nil {
		next.LastRunAt, next.Message = st.Tidy.LastRunAt, st.Tidy.Message
	}
	switch {
	case s.tidyErr != nil:
		next.Message = "token and lease tidy failed: " + s.tidyErr.Error()
		log.Warn("token and lease tidy failed", "err", s.tidyErr.Error())
	case s.tidied:
		next.LastRunAt = time.Now().UTC().Format(time.RFC3339Nano)
		next.Message = "token and lease tidy ran"
		log.Info("token and lease tidy ran", "interval", interval.String())
	}
	st.Tidy = next
}

func syncWith(ctx context.Context, admin *openbao.Client, in layoutInput) (synced, error) {
	in.accessors = accessorsOf(ctx, admin)
	d, pending, roles, refused, err := desiredWithIdentity(ctx, in, pkiCA(admin))
	if err != nil {
		return synced{}, err
	}
	rep, err := baosync.Sync(ctx, admin, d)
	generatedRoot := false
	if err == nil && rep.Err == nil {
		generatedRoot, err = ensurePKIRoots(ctx, admin, in.declared)
		if err != nil {
			return synced{}, err
		}
	}
	if err == nil && rep.Err == nil && (generatedRoot || len(pending) > 0 && rep.Changed()) {
		in.accessors = accessorsOf(ctx, admin)
		if d2, pending2, roles2, refused2, derr := desiredWithIdentity(ctx, in, pkiCA(admin)); derr == nil {
			rep2, err2 := baosync.Sync(ctx, admin, d2)
			rep.Applied = append(rep.Applied, rep2.Applied...)
			rep.Conflicts, rep.Err = rep2.Conflicts, errors.Join(rep.Err, rep2.Err)
			d, pending, roles, refused, err = d2, pending2, roles2, refused2, err2
		}
	}
	s := synced{report: rep, pending: pending, desired: d, roles: roles, refused: refused}
	if err == nil && in.tidyDue {
		s.tidied, s.tidyErr = tidyTokensAndLeases(ctx, admin)
	}
	if err == nil && len(in.plugins) > 0 {
		s.running = map[string]string{}
		if auths, aerr := admin.AuthMounts(ctx); aerr == nil {
			for p, m := range auths {
				s.running[p] = m.RunningSHA256
			}
		}
	}
	return s, err
}

const (
	defaultPKIRootCommonName = "socialweb.local"
	defaultPKIRootTTL        = "8760h"
)

func ensurePKIRoots(ctx context.Context, c *openbao.Client, d *v1alpha1.OpenBaoServerDeclared) (bool, error) {
	if d == nil {
		return false, nil
	}
	generated := false
	for _, m := range d.SecretMounts {
		if m.Type != string(baosync.MountPKI) {
			continue
		}
		if _, err := c.CAPEM(ctx, m.Path); err == nil {
			continue
		} else if !pkiRootMissing(err) {
			return false, err
		}
		ttl := strings.TrimSpace(m.MaxLeaseTTL)
		ttl = cmp.Or(ttl, defaultPKIRootTTL)
		if _, err := c.GeneratePKIRoot(ctx, m.Path, defaultPKIRootCommonName, ttl); err != nil {
			return false, fmt.Errorf("generating the pki root under %s: %w", m.Path, err)
		}
		generated = true
	}
	return generated, nil
}

func pkiRootMissing(err error) bool {
	var re *openbao.ResponseError
	return errors.As(err, &re) && (re.Status == 404 || re.Status == 400) || strings.Contains(err.Error(), "holds no certificate")
}

func accessorsOf(ctx context.Context, admin *openbao.Client) map[string]string {
	out := map[string]string{}
	auths, err := admin.AuthMounts(ctx)
	if err != nil {
		return out
	}
	for p, m := range auths {
		if m.Accessor != "" {
			out[p] = m.Accessor
		}
	}
	return out
}

func syncCore(ctx context.Context, client *openbao.Client, keysPath string, in layoutInput) (synced, error) {
	core, err := loginCore(ctx, client)
	if err == nil {
		s, serr := syncWith(ctx, core, in)
		core.RevokeSelf(context.WithoutCancel(ctx))
		if serr == nil || !permissionDenied(serr) {
			s.how = "the core's cert-auth token"
			return s, serr
		}
		err = serr
	} else if !loginRefused(err) {
		return synced{}, err
	}
	k, kerr := ReadUnsealKeys(keysPath)
	if kerr != nil {
		return synced{}, fmt.Errorf("the core cannot reconcile (%v) and %w", err, kerr)
	}
	root, gerr := client.GenerateRoot(ctx, k.Keys, k.Threshold)
	if gerr != nil {
		return synced{}, fmt.Errorf("the core cannot reconcile (%v) and %w", err, gerr)
	}
	defer root.RevokeSelf(context.WithoutCancel(ctx))
	s, serr := syncWith(ctx, root, in)
	s.how = fmt.Sprintf("a root token generated from the unseal keys and revoked after the pass, because the core's own token could not reconcile (%v)", err)
	return s, serr
}

func recordSync(st *v1alpha1.OpenBaoServerStatus, gen int64, s synced, err error, log *slog.Logger) {
	prev := st.Sync
	next := &v1alpha1.OpenBaoServerSync{ObservedGeneration: gen}
	if prev != nil {
		next.SyncedAt, next.LastChange, next.LastChangeAt = prev.SyncedAt, prev.LastChange, prev.LastChangeAt
	}
	now := time.Now().UTC().Format(time.RFC3339Nano)
	if err != nil {
		st.Sync = next
		if prev != nil {
			next.Declared, next.Pending, next.Conflicts = prev.Declared, prev.Pending, prev.Conflicts
		}
		conds.Set(&st.Conditions, gen, ConditionSynced, false, "SyncFailed", err.Error())
		log.Warn("the reconcile of the server against its declaration did not run", "err", err.Error())
		return
	}
	next.Declared, next.Pending, next.Conflicts = s.desired.String(), s.pending, s.report.Conflicts
	if s.report.Changed() {
		next.LastChange, next.LastChangeAt = s.report.Summary(), now
		ops := make([]string, 0, len(s.report.Applied))
		for _, o := range s.report.Applied {
			ops = append(ops, o.String()+" ("+o.Why+")")
		}
		log.Info("the server is brought back to its declaration", "at", now, "ops", ops, "by", s.how)
	}
	if prev == nil || prev.ObservedGeneration != gen || s.report.Changed() {
		next.SyncedAt = now
	}
	st.Sync = next
	switch {
	case s.report.Err != nil:
		conds.Set(&st.Conditions, gen, ConditionSynced, false, "ApplyFailed", s.report.Summary())
	case len(s.report.Conflicts) > 0:
		conds.Set(&st.Conditions, gen, ConditionSynced, false, "Conflict", strings.Join(s.report.Conflicts, "; "))
	default:
		conds.Set(&st.Conditions, gen, ConditionSynced, true, "InSync", "the server holds exactly its declaration: "+s.desired.String())
	}
}

func (s *Servers) Reconcile(ctx context.Context, o v1alpha1.OpenBaoServer, grants []v1alpha1.GuestIdentityGrant, children Children) Outcome {
	log := s.log.With("openbaoserver", o.Name)
	var st v1alpha1.OpenBaoServerStatus
	o.Status.DeepCopyInto(&st)
	gen := o.Generation
	st.ObservedGeneration = gen
	dir := ServerDir(s.c.StateDir, o.Name)
	st.HostProcess = ChildName(o.Name)
	st.Refs = refs(dir, o.Spec.TLS)
	st.Ready = false
	failed := func(format string, args ...any) v1alpha1.OpenBaoServerStatus {
		st.Phase, st.Message = v1alpha1.OpenBaoServerPhaseFailed, fmt.Sprintf(format, args...)
		conds.Set(&st.Conditions, gen, ConditionReady, false, "Failed", st.Message)
		log.Warn("failed", "msg", st.Message)
		return st
	}
	fail := func(format string, args ...any) Outcome {
		return Outcome{Status: failed(format, args...)}
	}
	retry := func(format string, args ...any) Outcome {
		out := Outcome{Status: failed(format, args...)}
		out.Err = errors.New(out.Status.Message)
		return out
	}
	starting := func(after time.Duration, format string, args ...any) Outcome {
		st.Phase, st.Message, st.Address = v1alpha1.OpenBaoServerPhaseStarting, fmt.Sprintf(format, args...), ""
		conds.Set(&st.Conditions, gen, ConditionReady, false, "Starting", st.Message)
		return Outcome{Status: st, RequeueAfter: after}
	}
	shares, threshold := int(o.Spec.Init.Shares), int(o.Spec.Init.Threshold)
	shares = cmp.Or(shares, DefaultShares)
	threshold = cmp.Or(threshold, DefaultThreshold)
	if threshold > shares {
		return fail("init.threshold %d is over init.shares %d", threshold, shares)
	}
	if o.Spec.Asset == nil && o.Spec.Exec == "" {
		return fail("name the bao executable: spec.asset (name bao) or spec.exec (an absolute path)")
	}
	listen := o.Spec.Listen
	listen = cmp.Or(listen, DefaultListen)
	if err := checkClients(o.Spec.Clients, o.Spec.Declared); err != nil {
		return fail("spec.clients: %v", err)
	}
	if o.Spec.Unit != "" && !strings.HasPrefix(o.Spec.Unit, s.c.Prefix) {
		return fail("spec.unit %s is not under this controller's prefix %s", o.Spec.Unit, s.c.Prefix)
	}
	tidyEvery, err := tidyInterval(o.Spec)
	if err != nil {
		return fail("%v", err)
	}
	tlsm, err := vaultcert.EnsureRootVaultTLS(dir, rootvault.LoopbackHost)
	if err != nil {
		return retry("mutual TLS material: %v", err)
	}
	clientsCA, err := vaultcert.EnsureOpenBaoClientsCA(dir, tlsm.CAPEM)
	if err != nil {
		return retry("the clients CA: %v", err)
	}
	st.Clients = nil
	for _, cl := range o.Spec.Clients {
		m, err := clientsCA.EnsureClient(dir, cl.Name)
		if err != nil {
			return retry("client %s: %v", cl.Name, err)
		}
		st.Clients = append(st.Clients, v1alpha1.OpenBaoServerClientRefs{Name: cl.Name, CertAuthMount: CertAuthMount,
			CertAuthRole: cl.Name, Policy: strings.Join(clientTokenPolicies(cl), ","), KVMount: cl.KVMount, ClientCert: "file:" + m.CertFile,
			ClientKey: "file:" + m.KeyFile})
	}
	plugins, err := s.placePlugins(o, dir)
	if err != nil {
		st.Plugins = nil
		return retry("spec.plugins: %v", err)
	}
	if !NeedsGrants(o) {
		grants = nil
	}
	layout := layoutInput{caPEM: tlsm.CAPEM, clientsCAPEM: clientsCA.CAPEM, clients: o.Spec.Clients, declared: o.Spec.Declared, plugins: plugins, grants: grants,
		tidyDue: tidyDue(st.Tidy, tidyEvery, time.Now())}
	opts, err := serverOptions(o.Spec, clientsCA.BundlePath)
	if err != nil {
		return fail("%v", err)
	}
	if len(plugins) > 0 {
		opts.PluginDirectory = PluginDir(dir)
	}
	body, err := rootvault.ServerConfigWith(dir, listen, opts)
	if err != nil {
		return fail("%v", err)
	}
	if err := os.MkdirAll(rootvault.StorageDir(dir), 0o700); err != nil {
		return retry("storage dir: %v", err)
	}
	if err := writeIfChanged(rootvault.ConfigPath(dir), []byte(body), 0o600); err != nil {
		return retry("server config: %v", err)
	}
	certFile := opts.CertFile
	if certFile == "" {
		certFile = rootvault.ServerCertPath(dir)
	}
	key := restartKey([]byte(body), certFile, caFile(o.Spec, dir), clientsCA.BundlePath)
	child, err := children.Ensure(ctx, ChildName(o.Name), s.childSpec(o, dir, key))
	if err != nil {
		return retry("HostProcess %s: %v", ChildName(o.Name), err)
	}
	if rt := child.Status.Runtime; rt != nil {
		st.Runtime = &v1alpha1.OpenBaoServerRuntime{Backend: rt.Backend, Ref: rt.Unit, Instance: rt.InvocationID, Pid: rt.Pid,
			StartTime: rt.ActiveSince, HostBootID: rt.HostBootID}
	}
	if child.Status.Phase == v1alpha1.HostProcessPhaseLost && replaceable[child.Status.LostReason] && child.DeletionTimestamp == nil {
		if err := children.Delete(ctx, child.Name); err != nil {
			return retry("HostProcess %s is Lost (%s) and deleting it failed: %v", child.Name, child.Status.LostReason, err)
		}
		log.Info("the server's HostProcess was Lost; it is placed again on the same storage", "hostprocess", child.Name,
			"reason", child.Status.LostReason)
		return starting(0, "HostProcess %s was Lost (%s); placing it again from its storage", child.Name, child.Status.LostReason)
	}
	switch child.Status.Phase {
	case v1alpha1.HostProcessPhaseFailed, v1alpha1.HostProcessPhaseLost:
		return fail("HostProcess %s is %s %s: %s", child.Name, child.Status.Phase, child.Status.LostReason, child.Status.Message)
	case v1alpha1.HostProcessPhaseRunning:
	default:
		return starting(0, "HostProcess %s is %q: %s", child.Name, child.Status.Phase, child.Status.Message)
	}
	if len(child.Status.Listeners) == 0 {
		return starting(0, "HostProcess %s runs and reports no bound address yet", child.Name)
	}
	if child.Status.RestartKey != key {
		return starting(0, "HostProcess %s restarts in place for its new config (restartKey %s)", child.Name, key)
	}
	addr := "https://" + child.Status.Listeners[0].Address
	tlsConf, err := tlsm.ClientTLSConfig()
	if err != nil {
		return retry("%v", err)
	}
	pool := tlsConf.RootCAs
	if o.Spec.TLS != nil {
		raw, err := os.ReadFile(o.Spec.TLS.CAFile)
		if err != nil {
			return retry("spec.tls.caFile: %v", err)
		}
		if pool, err = openbao.CertPoolFromPEM(raw); err != nil {
			return retry("spec.tls.caFile %s: %v", o.Spec.TLS.CAFile, err)
		}
	}
	client := openbao.New(addr, "", openbao.WithCACertPool(pool), openbao.WithClientCertificate(tlsConf.Certificates[0]))
	h, err := client.Health(ctx)
	if err != nil {
		return starting(RetryAfter, "the server at %s does not answer yet: %v", addr, err)
	}
	st.Address = addr
	keysPath := rootvault.UnsealKeysPath(dir)
	instance := ""
	if st.Runtime != nil {
		instance = st.Runtime.Instance
	}
	if !h.Initialized {
		if _, err := os.Lstat(keysPath); err == nil {
			return fail("the storage at %s is not initialized and unseal keys already exist at %s: refusing to initialize over them",
				rootvault.StorageDir(dir), keysPath)
		}
		res, root, err := client.Init(ctx, shares, threshold)
		if err != nil {
			return retry("init: %v", err)
		}
		k := UnsealKeys{Threshold: threshold, Shares: shares, Keys: res.KeysBase64}
		if err := WriteUnsealKeys(keysPath, k); err != nil {
			return fail("initialized, and writing the unseal keys to %s failed (%v): the storage at %s cannot be unsealed again",
				keysPath, err, rootvault.StorageDir(dir))
		}
		log.Info("initialized; unseal keys written to the state dir (0600), never to kcp", "shares", shares, "threshold", threshold,
			"keys", st.Refs.UnsealKeys)
		if err := unseal(ctx, client, k); err != nil {
			return retry("%v", err)
		}
		st.UnsealedInstance, st.UnsealedAt = instance, time.Now().UTC().Format(time.RFC3339Nano)
		sy, serr := syncWith(ctx, root, layout)
		sy.how = "the initial root token"
		if err := root.RevokeSelf(ctx); err != nil {
			log.Warn("revoking the initial root token", "err", err.Error())
		} else {
			log.Info("the initial root token is revoked; the core logs in with its client certificate")
		}
		recordSync(&st, gen, sy, serr, log)
		recordTidy(&st, sy, tidyEvery, log)
		layout.tidyDue = tidyDue(st.Tidy, tidyEvery, time.Now())
	} else if h.Sealed {
		k, err := ReadUnsealKeys(keysPath)
		if err != nil {
			return retry("sealed: %v", err)
		}
		if err := unseal(ctx, client, k); err != nil {
			return retry("%v", err)
		}
		st.UnsealedInstance, st.UnsealedAt = instance, time.Now().UTC().Format(time.RFC3339Nano)
		log.Info("unsealed", "instance", instance, "address", addr)
	}
	sy, serr := syncCore(ctx, client, keysPath, layout)
	recordSync(&st, gen, sy, serr, log)
	recordTidy(&st, sy, tidyEvery, log)
	var held error
	switch {
	case serr != nil:
		held = serr
	case sy.report.Err != nil:
		held = sy.report.Err
	case len(sy.pending) > 0:
		held = ErrNotHeld
	}
	out := Outcome{}
	if NeedsGrants(o) {
		reason := ""
		switch {
		case held != nil:
			reason = held.Error()
		case len(sy.report.Conflicts) > 0:
			reason = grantNotHeldReason
		}
		out.Grants = grantOutcomes(sy.roles, sy.refused, grants, reason == "", reason)
	}
	if len(plugins) > 0 {
		perr := serr
		if perr == nil {
			perr = sy.report.Err
		}
		st.Plugins = pluginStatus(plugins, sy.running, perr)
		if perr == nil {
			if removed := prunePlugins(dir, plugins); len(removed) > 0 {
				log.Info("removed plugin binaries no catalog entry runs", "files", removed)
			}
		}
	} else {
		st.Plugins = nil
	}
	h, err = client.Health(ctx)
	if err != nil {
		out.Status = starting(RetryAfter, "the server at %s stopped answering: %v", addr, err).Status
		out.RequeueAfter = RetryAfter
		return out
	}
	st.Initialized, st.Sealed, st.Version = h.Initialized, h.Sealed, h.Version
	if ca, err := os.ReadFile(FileRef(st.Refs.CA)); err == nil {
		st.CABundle = string(ca)
	}
	conds.Set(&st.Conditions, gen, ConditionInitialized, h.Initialized, "Health", "")
	conds.Set(&st.Conditions, gen, ConditionUnsealed, !h.Sealed, "Health", "")
	if !h.Initialized || h.Sealed {
		st.Phase, st.Message = v1alpha1.OpenBaoServerPhaseSealed, "sealed"
		conds.Set(&st.Conditions, gen, ConditionReady, false, "Sealed", st.Message)
		out.Status, out.RequeueAfter = st, RetryAfter
		return out
	}
	st.Phase, st.Message, st.Ready = v1alpha1.OpenBaoServerPhaseReady, "initialized and unsealed", true
	conds.Set(&st.Conditions, gen, ConditionReady, true, "Ready", st.Message)
	out.Status, out.Err = st, held
	out.RequeueAfter = tidyIn(st.Tidy, tidyEvery, time.Now())
	if out.RequeueAfter <= 0 {
		out.RequeueAfter = TidyRetryAfter
	}
	return out
}