~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
898 lines · 25.8 KB Raw History
package requester

import (
	"bytes"
	"cmp"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net"
	"net/http"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/bluesky-social/indigo/atproto/identity"

	commonv1 "github.com/johnandersen777/socialweb-computer-kcp/api/common/v1alpha1"
	"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/atproto/cloudinit"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/atproto/market"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/tunnel"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/netretry"
)

const (
	SecretsNSID = "com.publicdomainrelay.temp.compute.secrets"

	SecretsRoute = cloudinit.GetSecretsRoute

	SecretsAPIAudience = "ATProto"

	FedproxyRBACNSID = "com.fedproxy.rbac"
)

const (
	SecretsChannelIroh = "iroh"

	SecretsChannelWebRTC = "webrtc"

	SecretsChannelHTTPS = "https"
)

type secretsChannelKey struct{}

func WithSecretsChannel(ctx context.Context, channel string) context.Context {
	return context.WithValue(ctx, secretsChannelKey{}, channel)
}

func secretsChannelOf(ctx context.Context) string {
	if channel, _ := ctx.Value(secretsChannelKey{}).(string); channel != "" {
		return channel
	}
	return SecretsChannelHTTPS
}

type SecretEntry struct {
	Path string `json:"path"`

	Value string `json:"value"`

	Mode string `json:"mode,omitempty"`
}

func (e *SecretEntry) UnmarshalJSON(raw []byte) error {
	var in struct {
		Path string `json:"path"`

		Value json.RawMessage `json:"value"`

		Mode string `json:"mode,omitempty"`
	}
	if err := json.Unmarshal(raw, &in); err != nil {
		return err
	}
	value, err := secretValueText(in.Value)
	if err != nil {
		return err
	}
	*e = SecretEntry{Path: in.Path, Value: value, Mode: in.Mode}
	return nil
}

func secretValueText(raw json.RawMessage) (string, error) {
	if len(raw) == 0 {
		return "", nil
	}
	var text string
	if err := json.Unmarshal(raw, &text); err == nil {
		return text, nil
	}
	var out bytes.Buffer
	if err := json.Compact(&out, raw); err != nil {
		return "", err
	}
	return out.String(), nil
}

var secretModeRE = regexp.MustCompile(`^0?[0-7]{1,3}$`)

type secretsState struct {
	mu sync.Mutex

	name string

	log *slog.Logger

	entries []SecretEntry

	issuer string

	audience string

	subject string

	verifier TokenVerifier

	fetchedAt string

	fetchedVia string

	fetched chan struct{}

	once sync.Once

	unminted map[string]bool
}

func (s *secretsState) withholdUnminted(paths ...string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	for _, path := range paths {
		if s.unminted == nil {
			s.unminted = map[string]bool{}
		}
		s.unminted[path] = true
	}
}

func (s *secretsState) minted(path string) bool {
	if s == nil {
		return false
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	if !s.unminted[path] {
		return false
	}
	delete(s.unminted, path)
	return true
}

func (s *secretsState) servable(entries []SecretEntry) []SecretEntry {
	if s == nil {
		return entries
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.servableLocked(entries)
}

func (s *secretsState) servableLocked(entries []SecretEntry) []SecretEntry {
	if len(s.unminted) == 0 {
		return entries
	}
	out := make([]SecretEntry, 0, len(entries))
	for _, entry := range entries {
		if s.unminted[entry.Path] {
			continue
		}
		out = append(out, entry)
	}
	return out
}

func newSecretsState(name string, log *slog.Logger) *secretsState {
	if log == nil {
		log = slog.New(slog.DiscardHandler)
	}
	return &secretsState{name: name, log: log, fetched: make(chan struct{})}
}

func (s *secretsState) install(issuer, audience, subject string, verifier TokenVerifier) {
	s.mu.Lock()
	s.issuer, s.audience, s.subject, s.verifier = issuer, audience, subject, verifier
	s.mu.Unlock()
}

func (s *secretsState) addEntries(entries ...SecretEntry) {
	s.mu.Lock()
	for _, entry := range entries {
		replaced := false
		for i := range s.entries {
			if s.entries[i].Path == entry.Path {
				s.entries[i] = entry
				replaced = true
				break
			}
		}
		if !replaced {
			s.entries = append(s.entries, entry)
		}
	}
	s.mu.Unlock()
}

func (s *secretsState) fetchedAtValue() string {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.fetchedAt
}

func (s *secretsState) fetchedViaValue() string {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.fetchedVia
}

func (s *secretsState) markFetched(now, via string) bool {
	s.mu.Lock()
	if s.fetchedAt != "" {
		s.mu.Unlock()
		return false
	}
	s.fetchedAt, s.fetchedVia = now, via
	s.mu.Unlock()
	s.once.Do(func() { close(s.fetched) })
	return true
}

func (s *secretsState) fetch(ctx context.Context, authorization string) ([]SecretEntry, error) {
	s.mu.Lock()
	issuer, audience, subject, verifier := s.issuer, s.audience, s.subject, s.verifier
	already := s.fetchedAt
	s.mu.Unlock()
	if verifier == nil || issuer == "" || audience == "" || subject == "" {
		s.log.Warn("the secrets server is not ready", "contract", s.name)
		return nil, &RequestError{Status: http.StatusServiceUnavailable, Code: "NotReady"}
	}
	token, ok := strings.CutPrefix(strings.TrimSpace(authorization), "Bearer ")
	if !ok || strings.TrimSpace(token) == "" {
		s.log.Warn("the guest's secrets request carries no bearer token", "contract", s.name, "err", "requester: no bearer token")
		return nil, &RequestError{Status: http.StatusUnauthorized, Code: "AuthenticationRequired"}
	}
	claims, err := verifier.VerifyAudience(ctx, strings.TrimSpace(token), audience)
	if err != nil {
		s.log.Warn("the guest's secrets token is not accepted", "contract", s.name, "issuer", issuer, "err", err.Error())
		return nil, &RequestError{Status: http.StatusUnauthorized, Code: "InvalidToken"}
	}
	if got, _ := claims[cloudinit.NodeJWTUserClaim].(string); got != subject {
		s.log.Warn("the guest's secrets token names another subject", "contract", s.name, "subject", subject)
		return nil, &RequestError{Status: http.StatusUnauthorized, Code: "InvalidToken"}
	}
	s.mu.Lock()
	entries := s.servableLocked(append([]SecretEntry(nil), s.entries...))
	s.mu.Unlock()
	channel := secretsChannelOf(ctx)
	if already == "" {
		s.markFetched(time.Now().UTC().Format(time.RFC3339Nano), channel)
	}
	s.log.Info("requester_guest_secrets_fetched", "contract", s.name, "entries", len(entries), "channel", channel)
	return entries, nil
}

func (q *Requester) Secrets(ctx context.Context, contract, authorization string) ([]SecretEntry, error) {
	q.secretsMu.Lock()
	state := q.secrets[contract]
	q.secretsMu.Unlock()
	if state == nil {
		return nil, &RequestError{Status: http.StatusNotFound, Code: "NotFound"}
	}
	return state.fetch(ctx, authorization)
}

func (q *Requester) registerSecrets(name string, state *secretsState) {
	q.secretsMu.Lock()
	q.secrets[name] = state
	q.secretsMu.Unlock()
}

func (q *Requester) unregisterSecrets(name string) {
	q.secretsMu.Lock()
	delete(q.secrets, name)
	q.secretsMu.Unlock()
}

func (q *Requester) readSecretsBundle(ctx context.Context, ref commonv1.SecretRef) ([]SecretEntry, error) {
	if q.deps.Vault == nil {
		return nil, errors.New("requester: the contract names a secrets bundle and the component names no OpenBaoServer")
	}
	kv, err := q.deps.Vault(ctx, ref)
	if err != nil {
		return nil, retryable(err)
	}
	data, err := kv.Read(ctx, ref.Mount, ref.Path, 0)
	if err != nil {
		return nil, retryable(fmt.Errorf("requester: reading the secrets bundle %s/%s: %w", ref.Mount, ref.Path, err))
	}
	text, _ := data["bundle"].(string)
	if text == "" {
		return nil, fmt.Errorf("requester: the secrets bundle %s/%s carries no bundle", ref.Mount, ref.Path)
	}
	entries, err := ParseSecretsBundle(text)
	if err != nil {
		return nil, fmt.Errorf("requester: the secrets bundle %s/%s: %w", ref.Mount, ref.Path, err)
	}
	return entries, nil
}

func ParseSecretsBundle(text string) ([]SecretEntry, error) {
	entries := []SecretEntry{}
	if err := json.Unmarshal([]byte(text), &entries); err != nil {
		return nil, errors.New("the bundle is not a JSON array of {path, value, mode}")
	}
	for i := range entries {
		if err := CheckSecretEntry(entries[i].Path, entries[i].Mode); err != nil {
			return nil, fmt.Errorf("entry %d: %w", i, err)
		}
	}
	return entries, nil
}

func CheckSecretEntry(path, mode string) error {
	if path == "" || path == "/" || !strings.HasPrefix(path, "/") || strings.ContainsFunc(path, secretPathControl) || filepath.Clean(path) != path {
		return errors.New("the path is not a clean absolute path")
	}
	if mode != "" && !secretModeRE.MatchString(mode) {
		return errors.New("the mode is not an octal string of at most 0777")
	}
	return nil
}

func secretPathControl(r rune) bool {
	return r < 0x20 || r == 0x7f
}

func (q *Requester) ingressExposureURL(ctx context.Context) (string, error) {
	name := "svc-" + strings.ToLower(KindRequester) + "-" + strings.TrimSpace(q.cfg.Name) + "-ingress"
	if raw := q.deps.Inputs.AwaitAddress(ctx, name, DefaultExposureWait, nil); raw != "" {
		return raw, nil
	}
	if err := ctx.Err(); err != nil {
		return "", err
	}
	return "", retryable(fmt.Errorf("requester: the %s exposure reports no url", name))
}

func (q *Requester) guestIrohURL(ctx context.Context) string {
	ref := q.cfg.Spec.GuestIrohExposureRef
	if ref == nil || strings.TrimSpace(ref.Name) == "" {
		return ""
	}
	raw := q.deps.Inputs.AwaitAddress(ctx, ref.Name, DefaultExposureWait, nil)
	if raw == "" {
		q.log.Warn("requester_guest_iroh_unavailable", "exposure", ref.Name)
	}
	return raw
}

func (q *Requester) issuerVerifier(issuer string) TokenVerifier {
	issuer = strings.TrimRight(issuer, "/")
	client := q.deps.HTTP
	return q.deps.Guests.Verifier(issuer, func(ctx context.Context) ([]byte, error) {
		req, err := http.NewRequestWithContext(ctx, http.MethodGet, issuer+"/.well-known/keys", nil)
		if err != nil {
			return nil, err
		}
		res, err := client.Do(req)
		if err != nil {
			return nil, err
		}
		defer res.Body.Close()
		if res.StatusCode != http.StatusOK {
			io.Copy(io.Discard, io.LimitReader(res.Body, 4096))
			return nil, fmt.Errorf("requester: the bidder OIDC keys answered HTTP %d", res.StatusCode)
		}
		return io.ReadAll(io.LimitReader(res.Body, 1<<20))
	})
}

var errSecretsNeedIngress = errors.New("secrets need the requester ingress (getSecrets) or secretsFallback: Sealed")

var errSecretsDirectOnly = errors.New("the requester has an inbound endpoint, so secrets go over getSecrets only and are never sealed into a record")

func secretsAudience(requesterDID string) string {
	return "api://" + SecretsAPIAudience + "?actx=" + requesterDID
}

func secretsRouteOf(contract string) string {
	return SecretsRoute + "/" + contract
}

func secretsRBACKey(contract string) string {
	safe := strings.Map(func(r rune) rune {
		switch {
		case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
			return r
		case r == '.', r == '_', r == '~', r == ':', r == '-':
			return r
		default:
			return '-'
		}
	}, "node."+contract)
	if len(safe) > 512 {
		return safe[:512]
	}
	return safe
}

type bidConfig struct {
	IssuerURI string

	Actx string
}

func resolveBidConfig(ctx context.Context, reader market.RecordResolver, bidderDID string, record map[string]any) (bidConfig, error) {
	raw, _ := record["config"].(map[string]any)
	uri, _ := raw["uri"].(string)
	cid, _ := raw["cid"].(string)
	if uri == "" {
		return bidConfig{}, errors.New("the winning bid carries no bid config reference")
	}
	if market.ATURIAuthority(uri) != bidderDID {
		return bidConfig{}, fmt.Errorf("the winning bid's config record %s is not in the bidder %s's repo", uri, bidderDID)
	}
	resolved, err := reader.Resolve(ctx, market.RecordRef{URI: uri, CID: cid})
	if err != nil {
		return bidConfig{}, fmt.Errorf("reading the winning bid's config record: %w", err)
	}
	issuer, _ := resolved.Value["issuer_uri"].(string)
	actx, _ := resolved.Value["actx"].(string)
	if issuer == "" {
		return bidConfig{}, errors.New("the winning bid's config record carries no issuer_uri")
	}
	return bidConfig{IssuerURI: issuer, Actx: actx}, nil
}

func (q *Requester) nodeGrant(bidderDID, requesterDID, role, cluster string, cfg bidConfig) (*v1alpha1.NodeClaimGrant, error) {
	subject, err := q.deps.Guests.Subject(bidderDID, requesterDID, role)
	if err != nil {
		return nil, err
	}
	return &v1alpha1.NodeClaimGrant{
		Issuer:   cfg.IssuerURI,
		Subject:  subject,
		Audience: cloudinit.NodeAudience(cluster, role),
		Role:     role,
	}, nil
}

func nodeClaimDocument(cluster, node, issuer string) map[string]any {
	return map[string]any{
		"node":     node,
		"cluster":  cluster,
		"jwtMount": cloudinit.NodeJWTMount(issuer),
		"role":     node,
		"audience": cloudinit.NodeAudience(cluster, node),
		"kvMount":  cloudinit.NodeKVMount,
		"kvPath":   node,
	}
}

func nodeRBACRecord(role, issuer, subject, secretsAud, nodeAud string, nodeJoin bool) map[string]any {
	roles := map[string]any{
		role: map[string]any{
			"role_name": role,
			"definition": map[string]any{
				"iss":      issuer,
				"aud":      secretsAud,
				"sub":      subject,
				"policies": []string{},
			},
		},
	}
	if nodeJoin {
		nodeRole := role + "-node"
		roles[nodeRole] = map[string]any{
			"role_name": nodeRole,
			"definition": map[string]any{
				"iss":      issuer,
				"aud":      nodeAud,
				"sub":      subject,
				"policies": []string{},
			},
		}
	}
	return map[string]any{
		"$type":     FedproxyRBACNSID,
		"roles":     roles,
		"policies":  map[string]any{},
		"createdAt": time.Now().UTC().Format(time.RFC3339),
	}
}

func meshdPinOf(meshd *v1alpha1.ComputeContractMeshd) *cloudinit.MeshdPin {
	if meshd == nil || meshd.DownloadBase == "" {
		return nil
	}
	out := &cloudinit.MeshdPin{DownloadBase: meshd.DownloadBase, Sha256: map[string]string{}}
	for _, pin := range meshd.Pins {
		out.Sha256[pin.Arch] = pin.Sha256
	}
	return out
}

func nodeAgentPinFromJoin(join *v1alpha1.ComputeContractNodeJoin) *cloudinit.NodeAgentPin {
	if join == nil || join.NodeAgent == nil || join.NodeAgent.DownloadBase == "" {
		return nil
	}
	out := &cloudinit.NodeAgentPin{DownloadBase: join.NodeAgent.DownloadBase, Sha256: map[string]string{}}
	for _, pin := range join.NodeAgent.Pins {
		out.Sha256[pin.Arch] = pin.Sha256
	}
	return out
}

func contractAcceptPath(contract v1alpha1.ComputeContract) string {
	if contract.Spec.NodeJoin != nil {
		return cloudinit.NodeSecretsDir + "/accept.json"
	}
	return "/root/secrets/publicdomainrelay.com/market/accept.json"
}

func (r *contractRunner) ensureSecrets(ctx context.Context, contract v1alpha1.ComputeContract) error {
	if contract.Spec.SecretsRef == nil || r.secrets != nil {
		return nil
	}
	inbound, sealed := r.q.cfg.InboundXRPC(), r.q.cfg.SealedSecrets()
	if !inbound && !sealed {
		return errSecretsNeedIngress
	}
	entries, err := r.q.readSecretsBundle(ctx, *contract.Spec.SecretsRef)
	if err != nil {
		return err
	}
	state := newSecretsState(r.name, r.q.log)
	state.addEntries(entries...)
	if envFile, err := agentEnvFile(contract); err == nil && envFile != "" && !agentPeerReady(entries, envFile) {
		state.withholdUnminted(envFile, cloudinit.AgentIrohKeyFile)
	}
	if contract.Spec.NodeJoin == nil {
		if err := r.installResumedVerifier(ctx, contract, state); err != nil {
			return err
		}
	}
	url, iroh := "", ""
	if inbound {
		if url, err = r.q.ingressExposureURL(ctx); err != nil {
			return err
		}
		iroh = r.q.guestIrohURL(ctx)
		r.q.registerSecrets(r.name, state)
	}
	r.mu.Lock()
	r.secrets = state
	r.secretsURL = url
	r.secretsIrohURL = iroh
	r.mu.Unlock()
	if grant := contract.Status.Grant; grant != nil && contract.Spec.NodeJoin != nil {
		audience := secretsAudience(r.q.callbackDID())
		state.install(grant.Issuer, audience, grant.Subject, r.q.issuerVerifier(grant.Issuer))
		doc, err := json.Marshal(nodeClaimDocument(contract.Spec.NodeJoin.ClusterRef.Name, grant.Role, grant.Issuer))
		if err != nil {
			return err
		}
		state.addEntries(SecretEntry{Path: cloudinit.NodeClaimDocPath, Value: string(doc) + "\n"})
		record := nodeRBACRecord(grant.Role, grant.Issuer, grant.Subject, audience, grant.Audience, true)
		key := r.rbacKeyOf(contract.Status.Contract)
		if key == "" {
			return nil
		}
		if err := r.putRBAC(ctx, key, record); err != nil {
			return fmt.Errorf("requester: rewriting the contract's RBAC record: %w", err)
		}
	}
	return nil
}

func transientOnly(err error) error {
	if netretry.Transient(err) || isRetryable(err) || errors.Is(err, identity.ErrDIDResolutionFailed) {
		return retryable(err)
	}
	return err
}

func (r *contractRunner) installResumedVerifier(ctx context.Context, contract v1alpha1.ComputeContract, state *secretsState) error {
	st := contract.Status.Contract
	if st == nil || st.Accept == nil || st.Bid == nil || st.Bid.URI == "" || winnerDID(st) == "" {
		return nil
	}
	reader := market.NewHTTPReader(r.q.deps.Directory)
	reader.Client = r.q.deps.HTTP
	bid, err := reader.Resolve(ctx, market.RecordRef{URI: st.Bid.URI, CID: st.Bid.CID})
	if err != nil {
		return transientOnly(fmt.Errorf("requester: the guest token verifier of a resumed contract: reading the winning bid: %w", err))
	}
	cfg, err := resolveBidConfig(ctx, reader, winnerDID(st), bid.Value)
	if err != nil {
		return transientOnly(fmt.Errorf("requester: the guest token verifier of a resumed contract: %w", err))
	}
	subject, err := r.q.deps.Guests.Subject(winnerDID(st), r.q.callbackDID(), cmp.Or(st.VMName, contract.Spec.VMName))
	if err != nil {
		return fmt.Errorf("requester: the guest token verifier of a resumed contract: %w", err)
	}
	state.install(cfg.IssuerURI, secretsAudience(r.q.callbackDID()), subject, r.q.issuerVerifier(cfg.IssuerURI))
	r.log().Info("requester_guest_secrets_resumed", "contract", r.name)
	return nil
}

func (r *contractRunner) rbacKeyOf(state *v1alpha1.NodeClaimContract) string {
	switch {
	case state == nil:
		return ""
	case state.Accept != nil && state.Accept.URI != "":
		return market.RBACRKeyOfAccept(state.Accept.URI)
	case state.RFP != nil && state.RFP.URI != "" && state.Bid != nil && state.Bid.URI != "":
		return market.RBACRKey(acceptRKeyFor(r.q.repoDID, r.name, state.RFP.URI, state.RFP.CID, state.Bid.URI, state.Bid.CID))
	}
	return ""
}

func acceptRKeyFor(repoDID, contract, rfpURI, rfpCID, bidURI, bidCID string) string {
	return sessionRKey("accept", repoDID, contract, rfpURI, rfpCID, bidURI, bidCID)
}

func (r *contractRunner) putRBAC(ctx context.Context, key string, record map[string]any) error {
	_, err := r.putSetupRecordAt(ctx, "putRBAC", FedproxyRBACNSID, key, record)
	if settled, ok := errors.AsType[retryableError](err); ok {
		return settled.error
	}
	if err != nil {
		return err
	}
	r.mu.Lock()
	r.rbacRKey = key
	r.mu.Unlock()
	r.deleteLegacyRBAC(ctx)
	return nil
}

func (r *contractRunner) deleteLegacyRBAC(ctx context.Context) {
	key := secretsRBACKey(r.name)
	if _, err := r.q.repo.GetRecord(ctx, r.q.repoDID, FedproxyRBACNSID, key); err != nil {
		if !rbacGone(err) {
			r.log().Warn("requester_rbac_legacy_unread", "rkey", key, "err", err)
		}
		return
	}
	if err := r.q.repo.DeleteRecord(ctx, r.q.repoDID, FedproxyRBACNSID, key); err != nil && !rbacGone(err) {
		r.log().Warn("requester_rbac_delete_failed", "rkey", key, "err", err)
	}
}

func (r *contractRunner) contractModules(userData string, contract v1alpha1.ComputeContract) (string, error) {
	modules := []any{}
	values := cloudinit.Context{}
	if contract.Spec.NodeJoin != nil {
		opts := cloudinit.K3sJoinOptions{Meshd: meshdPinOf(contract.Spec.NodeJoin.Meshd),
			NodeAgent: nodeAgentPinFromJoin(contract.Spec.NodeJoin)}
		modules = append(modules, "k3s", cloudinit.K3sJoinModule(opts))
	}
	if len(contract.Spec.Tunnels) > 0 {
		opts, err := tunnelsOptions(contract)
		if err != nil {
			return "", fmt.Errorf("the contract's tunnels: %w", err)
		}
		modules = append(modules, cloudinit.TunnelsModule(opts))
	}
	envFile, err := agentEnvFile(contract)
	if err != nil {
		return "", fmt.Errorf("the contract's agent: %w", err)
	}
	if envFile != "" {
		agentPLC, _ := guestNetworkOf(r.q.cfg.Spec.Network)
		modules = append(modules, cloudinit.AgentModule(cloudinit.AgentOptions{EnvFile: envFile, PLC: agentPLC}))
	}
	if a := contract.Spec.Association; a != nil && a.FromGuest == (strings.TrimSpace(a.Subject) != "") {
		return "", errors.New("spec.association takes exactly one of a subject or fromGuest")
	}
	if fromGuest(contract) {
		if contract.Spec.SecretsRef == nil {
			return "", errors.New("spec.association.fromGuest needs spec.secretsRef: the association ref reaches the guest in the secrets bundle")
		}
		refFile, err := associationRefFile(contract)
		if err != nil {
			return "", err
		}
		guestPLC, guestIngress := guestNetworkOf(r.q.cfg.Spec.Network)
		modules = append(modules, cloudinit.AssociationModule(refFile, guestPLC, guestIngress))
	}
	if len(contract.Spec.GuestTools) > 0 {
		modules = append(modules, cloudinit.GuestToolsModule(contract.Spec.GuestTools))
	}
	if r.secrets != nil {
		inbound := r.q.cfg.InboundXRPC()
		if inbound {
			modules = append(modules, "secrets")
			values.SecretsURL = r.secretsURLValue()
			values.SecretsRoute = secretsRouteOf(r.name)
			values.SecretsAud = secretsAudience(r.q.callbackDID())
			values.SecretsAcceptPath = contractAcceptPath(contract)
			peer := cloudinit.GuestRequester{Iroh: r.secretsIrohURLValue()}
			if r.q.guestWebRTC() {
				peer.WebRTC = tunnel.ID(r.q.repoDID)
			}
			if peer.Iroh != "" || peer.WebRTC != "" {
				modules = append(modules, cloudinit.GuestRequesterModule(peer))
			}
		}
		if r.q.cfg.SealedSecrets() {
			modules = append(modules, cloudinit.GuestSecretsModule())
		}
	}
	if len(modules) == 0 {
		return userData, nil
	}
	return cloudinit.BuildUserData(cloudinit.BuildOptions{Base: userData, Modules: modules, Ctx: values})
}

func (r *contractRunner) prepareWorkloadGrant(ctx context.Context, state *v1alpha1.NodeClaimContract, winner *BidRecord, acceptRKey string) error {
	if winner == nil || (r.nodeJoin == nil && r.secrets == nil) {
		return nil
	}
	reader := market.NewHTTPReader(r.q.deps.Directory)
	reader.Client = r.q.deps.HTTP
	var cfg bidConfig
	err := r.untilSettled(ctx, "readBidConfig", func() error {
		var readErr error
		cfg, readErr = resolveBidConfig(ctx, reader, winner.DID, winner.Record)
		return transientRetryable(readErr)
	})
	if err != nil {
		return err
	}
	vmName := state.VMName
	if vmName == "" {
		vmName = r.snapshot().Spec.VMName
	}
	subject := ""
	if r.nodeJoin != nil {
		grant, err := r.q.nodeGrant(winner.DID, r.q.callbackDID(), vmName, r.nodeJoin.ClusterRef.Name, cfg)
		if err != nil {
			return err
		}
		r.mu.Lock()
		r.grant = grant
		r.mu.Unlock()
		subject = grant.Subject
	} else {
		subject, err = r.q.deps.Guests.Subject(winner.DID, r.q.callbackDID(), vmName)
		if err != nil {
			return err
		}
	}
	if r.secrets == nil {
		return nil
	}
	audience := secretsAudience(r.q.callbackDID())
	r.secrets.install(cfg.IssuerURI, audience, subject, r.q.issuerVerifier(cfg.IssuerURI))
	if r.nodeJoin != nil {
		doc, err := json.Marshal(nodeClaimDocument(r.nodeJoin.ClusterRef.Name, vmName, cfg.IssuerURI))
		if err != nil {
			return err
		}
		r.secrets.addEntries(SecretEntry{Path: cloudinit.NodeClaimDocPath, Value: string(doc) + "\n"})
	}
	nodeAud := ""
	if r.nodeJoin != nil {
		nodeAud = cloudinit.NodeAudience(r.nodeJoin.ClusterRef.Name, vmName)
	}
	record := nodeRBACRecord(vmName, cfg.IssuerURI, subject, audience, nodeAud, r.nodeJoin != nil)
	if err := r.putRBAC(ctx, market.RBACRKey(acceptRKey), record); err != nil {
		return fmt.Errorf("requester: writing the contract's RBAC record: %w", err)
	}
	return nil
}

func (r *contractRunner) cleanup() {
	r.mu.Lock()
	secrets := r.secrets
	r.mu.Unlock()
	if secrets != nil {
		r.q.unregisterSecrets(r.name)
	}
	r.q.forgetGuestWebRTC(r.name)
}

func (r *contractRunner) deleteRBAC(ctx context.Context) {
	r.mu.Lock()
	held := r.rbacRKey
	r.mu.Unlock()
	if r.q.repo == nil || (held == "" && r.snapshot().Spec.SecretsRef == nil) {
		return
	}
	deleteCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()
	keys := []string{held}
	if key := r.rbacKeyOf(r.stateCopy()); key != held {
		keys = append(keys, key)
	}
	for _, key := range keys {
		if key == "" {
			continue
		}
		if err := r.q.repo.DeleteRecord(deleteCtx, r.q.repoDID, FedproxyRBACNSID, key); err != nil && !rbacGone(err) {
			r.log().Warn("requester_rbac_delete_failed", "rkey", key, "err", err)
		}
	}
	r.deleteLegacyRBAC(deleteCtx)
	r.mu.Lock()
	r.rbacRKey = ""
	r.mu.Unlock()
}

func rbacGone(err error) bool {
	return errors.Is(err, market.ErrNotFound) || strings.Contains(err.Error(), "not found")
}

func (r *contractRunner) secretsFetched() string {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.secretsFetchedAt
}

func (r *contractRunner) setSecretsFetched(at, via string) {
	if at == "" {
		return
	}
	r.mu.Lock()
	r.secretsFetchedAt, r.secretsFetchedVia = at, via
	r.mu.Unlock()
}

func (r *contractRunner) secretsFetchedViaValue() string {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.secretsFetchedVia
}

func secretsFetchedReason(via string) string {
	switch via {
	case SecretsChannelIroh:
		return "GetSecretsIroh"
	case SecretsChannelWebRTC:
		return "GetSecretsWebRTC"
	case SecretsChannelHTTPS:
		return "GetSecretsHTTPS"
	}
	return "GetSecrets"
}

func (r *contractRunner) secretsIrohURLValue() string {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.secretsIrohURL
}

func (r *contractRunner) secretsURLValue() string {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.secretsURL
}

func (r *contractRunner) grantValue() *v1alpha1.NodeClaimGrant {
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.grant == nil {
		return nil
	}
	grant := *r.grant
	return &grant
}

// guestNetworkOf: the PLC and the ingress a guest of this requester uses, the requester's own (a network guest's
// requester serves its guests the network it is on, plan 0050); a *.localhost network names a separate guest PLC (its
// direct address is loopback); outside a network, none (plc.directory and the public ingress)
func guestNetworkOf(n *v1alpha1.MarketNetworkValue) (plc, ingress string) {
	if n == nil {
		return "", ""
	}
	plc = strings.TrimRight(strings.TrimSpace(n.PLCDirectoryURL), "/")
	if plc == "" || plc == v1alpha1.PublicPLCDirectoryURL {
		return "", ""
	}
	if g := strings.TrimSpace(n.GuestPLCDirectoryURL); g != "" {
		plc = g
	}
	if host, _, err := net.SplitHostPort(n.IngressProxyHost); err == nil {
		ingress = host
	} else {
		ingress = strings.TrimSpace(n.IngressProxyHost)
	}
	return plc, ingress
}