0
0
package infra
import (
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/bluesky-social/indigo/atproto/syntax"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
commonv1 "github.com/johnandersen777/socialweb-computer-kcp/api/common/v1alpha1"
eventsv1alpha1 "github.com/johnandersen777/socialweb-computer-kcp/api/events/v1alpha1"
exposurev1alpha1 "github.com/johnandersen777/socialweb-computer-kcp/api/exposure/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/bus"
"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/atprotorepo"
"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/events"
ctlruntime "github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/kube"
"github.com/johnandersen777/socialweb-computer-kcp/internal/eventstream"
"github.com/johnandersen777/socialweb-computer-kcp/internal/infra"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/binding"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/conds"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/netretry"
)
const RepoKind = "AtprotoRepo"
var ErrRepoNoDID = errors.New("reports no did yet")
var ErrRecordUnresolved = errors.New("infra: the record's repo cannot be resolved")
type unresolved struct {
reason string
err error
}
func (e unresolved) Error() string { return e.err.Error() }
func (e unresolved) Unwrap() []error { return []error{ErrRecordUnresolved, e.err} }
func Unresolved(err error) (string, bool) {
var u unresolved
if errors.As(err, &u) {
return u.reason, true
}
return "", false
}
type readRetrying struct {
err error
}
func (e readRetrying) Error() string { return e.err.Error() }
func (e readRetrying) Unwrap() error { return e.err }
func ReadFailed(err error) error {
if netretry.Transient(err) {
return readRetrying{err: err}
}
return err
}
func ReadRetrying(err error) bool {
var r readRetrying
return errors.As(err, &r)
}
type recordSync struct {
action infra.Action
uri string
cid string
rkey string
repo commonv1.ObjectRef
declaration infra.Declaration
render infra.Declaration
publish bool
createdAt time.Time
status *v1alpha1.InfraRecordStatus
conditions []metav1.Condition
write bool
swapCid string
recorded infra.Decoded
}
func recordURI(did, nsid, rkey string) string {
return "at://" + did + "/" + nsid + "/" + rkey
}
func RepoRef(workspace string, ref commonv1.ObjectRef) (commonv1.ObjectRef, error) {
kind := strings.TrimSpace(ref.Kind)
if kind != "" && !strings.EqualFold(kind, RepoKind) {
return commonv1.ObjectRef{}, unresolved{reason: "RepoRefInvalid", err: fmt.Errorf("infra: the repoRef must name an %s, not %s", RepoKind, kind)}
}
if strings.TrimSpace(ref.Name) == "" {
return commonv1.ObjectRef{}, unresolved{reason: "RepoRefInvalid", err: errors.New("infra: the repoRef names no AtprotoRepo")}
}
return commonv1.ObjectRef{Group: v1alpha1.GroupName, Kind: RepoKind, Name: strings.TrimSpace(ref.Name), Workspace: ref.InWorkspace(workspace)}, nil
}
func Repo(ctx context.Context, b *bus.Bus, dynamic kube.Factory, self, ref commonv1.ObjectRef, nsids ...string) (string, atprotorepo.RecordReader, error) {
dyn, err := dynamic(ref.Workspace)
if err != nil {
return "", nil, err
}
repo, err := kube.Get[v1alpha1.AtprotoRepo](ctx, dyn.Resource(v1alpha1.AtprotoRepoGVR), ref.Name)
if apierrors.IsNotFound(err) {
return "", nil, unresolved{reason: "RepoNotFound", err: fmt.Errorf("infra: the AtprotoRepo %s does not exist", ref.String())}
}
if err != nil {
return "", nil, fmt.Errorf("infra: reading the AtprotoRepo %s: %w", ref.String(), err)
}
did := strings.TrimSpace(repo.Status.DID)
if did == "" {
if backendGone(ctx, dynamic, ref.Workspace, repo.Spec.BackendRef) {
return "", nil, unresolved{reason: "RepoBackendGone", err: fmt.Errorf("infra: the AtprotoRepo %s has no DID and its backend %s %s does not exist",
ref.String(), repo.Spec.BackendRef.Kind, repo.Spec.BackendRef.Name)}
}
return "", nil, fmt.Errorf("infra: the AtprotoRepo %s %w", ref.String(), ErrRepoNoDID)
}
reader, err := bus.Resolve[atprotorepo.RecordReader](ctx, b, self, ref, bus.Capability{Name: atprotorepo.CapabilityRead, Scope: nsids})
if err != nil {
return "", nil, unresolved{reason: "RepoUnreadable", err: fmt.Errorf("infra: the repo.read handle of %s: %w", ref.String(), err)}
}
return did, reader, nil
}
func backendGone(ctx context.Context, dynamic kube.Factory, workspace string, ref commonv1.ObjectRef) bool {
var gvr schema.GroupVersionResource
switch strings.TrimSpace(ref.Kind) {
case "LocalRepo":
gvr = v1alpha1.LocalRepoGVR
case "RemoteRepo":
gvr = v1alpha1.RemoteRepoGVR
default:
return false
}
dyn, err := dynamic(ref.InWorkspace(workspace))
if err != nil {
return false
}
_, err = kube.Get[metav1.PartialObjectMetadata](ctx, dyn.Resource(gvr), strings.TrimSpace(ref.Name))
return apierrors.IsNotFound(err)
}
func (r *reconciler[T]) syncRecord(ctx context.Context, req ctlruntime.Request, v view, now time.Time) (*recordSync, error) {
b := v.spec.Record
nsid := v.kind.NSID()
repo, err := RepoRef(req.Workspace, b.RepoRef)
if err != nil {
return nil, err
}
r.c.bindRepo(string(v.kind), req, repo)
did, reader, err := Repo(ctx, r.c.bus, r.c.dynamic, r.self(req), repo, nsid)
if err != nil {
return nil, err
}
name := v.object.GetName()
uid := string(v.object.GetUID())
rkey := name
stored := v.status.Record
fresh := stored == nil || strings.TrimSpace(stored.URI) == "" || stored.Direction == v1alpha1.InfraSyncRetained
follow := strings.TrimSpace(b.URI) != ""
if follow {
uri, err := syntax.ParseATURI(strings.TrimSpace(b.URI))
if err != nil {
return nil, fmt.Errorf("infra: spec.record.uri: %w", err)
}
if uri.Authority().String() != did || uri.Collection().String() != nsid {
return nil, fmt.Errorf("infra: spec.record.uri %s is not a %s record of the repo %s", b.URI, nsid, did)
}
rkey = uri.RecordKey().String()
} else if !fresh {
if uri, err := syntax.ParseATURI(strings.TrimSpace(stored.URI)); err == nil && uri.Authority().String() == did &&
uri.Collection().String() == nsid && infra.InSuccession(name, uid, uri.RecordKey().String()) {
rkey = uri.RecordKey().String()
}
}
publish := Publishing(b)
render := v.declaration
if publish {
render = render.Published()
}
out := &recordSync{rkey: rkey, repo: repo, uri: recordURI(did, nsid, rkey), createdAt: v.object.GetCreationTimestamp().Time,
render: render, publish: publish}
gen := v.object.GetGeneration()
if v.kind == infra.KindInfra && rkey == infra.ReservedRkey {
out.action = infra.ActionReserved
out.status = &v1alpha1.InfraRecordStatus{URI: out.uri}
out.conditions = []metav1.Condition{
conds.New(v1alpha1.InfraConditionRecordReserved, true, "Reserved", "rkey self holds the account's index of live deployments; nothing is written", gen),
conds.New(v1alpha1.InfraConditionSynced, false, "Reserved", "rkey self is reserved", gen)}
return out, nil
}
var got atprotorepo.Record
var present bool
var decoded infra.Decoded
var derr error
for n := 0; ; n++ {
got, err = reader.GetRecord(ctx, did, nsid, rkey)
present = err == nil
if err != nil && !errors.Is(err, atprotorepo.ErrNotFound) {
return nil, ReadFailed(fmt.Errorf("infra: reading %s: %w", out.uri, err))
}
decoded, derr = infra.Decoded{}, nil
if present {
decoded, derr = infra.Decode(v.kind, got.Value)
}
if follow || !fresh || !present || !(decoded.Over(v.kind, now) || errors.Is(derr, infra.ErrReserved)) {
break
}
if n+1 > infra.MaxSuccessors {
return nil, fmt.Errorf("infra: %d successor rkeys of %s all hold ended records", infra.MaxSuccessors, name)
}
next := infra.Successor(name, uid, n+1)
r.c.log.Info("infra_record_successor", "kind", v.kind, "workspace", req.Workspace, "name", req.Name, "ended", out.uri,
"endedAt", decoded.EndedAt, "successor", next)
rkey = next
out.rkey, out.uri = rkey, recordURI(did, nsid, rkey)
}
if err := r.ensureWatch(ctx, req, v, repo, rkey, b.EventSourceRefs); err != nil {
return nil, err
}
observed := infra.Record{Present: present, CID: got.CID}
if present {
observed.Hash, observed.Err = decoded.Declaration.Hash(), derr
observed.Ended, observed.Reserved = decoded.Ended(), errors.Is(derr, infra.ErrReserved)
out.declaration, out.recorded = decoded.Declaration, decoded
if !decoded.CreatedAt.IsZero() {
out.createdAt = decoded.CreatedAt
}
}
out.cid = got.CID
r.c.index(string(v.kind), req, out.uri, got.CID)
specHash := render.Hash()
out.action = infra.Decide(infra.SyncInput{Follow: follow, Publish: publish, Stored: stored, URI: out.uri, SpecHash: specHash, Record: observed})
endHeld := stored != nil && stored.URI == out.uri && stored.Direction != v1alpha1.InfraSyncRetained &&
meta.IsStatusConditionTrue(v.status.Conditions, v1alpha1.InfraConditionRecordEnded)
endLost := false
switch out.action {
case infra.ActionEnded, infra.ActionDelete, infra.ActionReserved:
default:
if endHeld {
r.c.log.Warn("infra_record_end_lost", "kind", v.kind, "workspace", req.Workspace, "name", req.Name, "uri", out.uri, "cid", got.CID,
"action", out.action)
out.action, endLost = infra.ActionEnded, true
}
}
keep := v1alpha1.InfraRecordStatus{URI: out.uri}
if stored != nil && stored.URI == out.uri {
keep = *stored
}
synced := func(direction v1alpha1.InfraSyncDirection) *v1alpha1.InfraRecordStatus {
return &v1alpha1.InfraRecordStatus{URI: out.uri, CID: got.CID, DeclarationHash: observed.Hash, SyncedGeneration: gen,
SyncedAt: now.Format(time.RFC3339), Direction: direction}
}
valid := conds.New(v1alpha1.InfraConditionRecordValid, true, "Decoded", "the record decodes into a declaration", gen)
switch out.action {
case infra.ActionInSync:
out.status = &keep
out.conditions = []metav1.Condition{valid, conds.New(v1alpha1.InfraConditionSynced, true, "InSync", "the record holds the declaration", gen)}
note, pulled := r.c.takePull(string(v.kind), req, observed.Hash)
rebound := keep.Direction == v1alpha1.InfraSyncRetained
if keep.CID != got.CID || keep.DeclarationHash != observed.Hash || rebound {
direction := keep.Direction
if pulled {
direction = v1alpha1.InfraSyncPulled
}
out.status = synced(direction)
if out.status.Direction == "" || rebound {
out.status.Direction = v1alpha1.InfraSyncInSync
}
}
out.conditions = append(out.conditions, note...)
out.write, out.swapCid = true, got.CID
case infra.ActionCreate:
if publish && keep.CID != "" {
r.c.log.Warn("infra_record_rewritten", "kind", v.kind, "workspace", req.Workspace, "name", req.Name, "uri", out.uri, "goneCid", keep.CID)
}
out.status = &v1alpha1.InfraRecordStatus{URI: out.uri, Direction: v1alpha1.InfraSyncCreated}
out.write = true
out.conditions = []metav1.Condition{conds.New(v1alpha1.InfraConditionSynced, false, "Creating", "writing the record", gen)}
case infra.ActionPush:
keep.Direction = v1alpha1.InfraSyncPushed
out.status = &keep
out.write, out.swapCid = true, got.CID
out.conditions = []metav1.Condition{valid, conds.New(v1alpha1.InfraConditionSynced, false, "Pushing", "writing the kcp declaration to the record", gen)}
if publish && (observed.Err != nil || observed.Hash != keep.DeclarationHash) {
if observed.Err != nil {
out.conditions[0] = conds.New(v1alpha1.InfraConditionRecordValid, false, "Invalid", observed.Err.Error(), gen)
}
out.conditions[1] = conds.New(v1alpha1.InfraConditionSynced, false, "Overwriting",
"the record is published from kcp only; a change made elsewhere is written over", gen)
r.c.log.Warn("infra_record_overwritten", "kind", v.kind, "workspace", req.Workspace, "name", req.Name, "uri", out.uri, "cid", got.CID,
"invalid", observed.Err != nil)
}
case infra.ActionPull:
out.status = synced(v1alpha1.InfraSyncPulled)
out.conditions = []metav1.Condition{valid, conds.New(v1alpha1.InfraConditionSynced, true, "Pulled", "the record's declaration is applied to the spec", gen)}
case infra.ActionConflict:
out.status = synced(v1alpha1.InfraSyncPulled)
message := fmt.Sprintf("generation %d of the spec and record cid %s both changed since the last sync; the record won", gen, got.CID)
out.conditions = []metav1.Condition{valid,
conds.New(v1alpha1.InfraConditionSynced, true, "Pulled", "the record's declaration is applied to the spec", gen),
conds.New(v1alpha1.InfraConditionSyncConflict, true, "RecordWon", message, gen)}
r.c.log.Warn("infra_record_conflict", "kind", v.kind, "workspace", req.Workspace, "name", req.Name, "uri", out.uri, "cid", got.CID,
"generation", gen)
case infra.ActionEnded:
direction := keep.Direction
if direction == "" || direction == v1alpha1.InfraSyncRetained {
direction = v1alpha1.InfraSyncInSync
}
if endLost {
out.status = &keep
out.conditions = []metav1.Condition{conds.New(v1alpha1.InfraConditionSynced, false, "EndLost",
"the record no longer shows the end this object recorded; the object stays ended and writes nothing", gen)}
break
}
out.status = synced(direction)
message := fmt.Sprintf("the record ended at %s (%s); it is kept and never written again", decoded.EndedAt, strings.ToLower(string(decoded.EndReason)))
out.conditions = []metav1.Condition{
conds.New(v1alpha1.InfraConditionSynced, true, "Ended", message, gen),
conds.New(v1alpha1.InfraConditionRecordEnded, true, string(decoded.EndReason), message, gen)}
case infra.ActionReserved:
out.status = &keep
out.conditions = []metav1.Condition{
conds.New(v1alpha1.InfraConditionRecordReserved, true, "Reserved", "the record holds the account's index of live deployments; nothing is written", gen),
conds.New(v1alpha1.InfraConditionSynced, false, "Reserved", "the record is the index, not a deployment", gen)}
case infra.ActionInvalid:
out.status = &keep
out.conditions = []metav1.Condition{conds.New(v1alpha1.InfraConditionRecordValid, false, "Invalid", observed.Err.Error(), gen),
conds.New(v1alpha1.InfraConditionSynced, false, "RecordInvalid", "the record is neither pulled nor overwritten", gen)}
}
return out, nil
}
func (r *reconciler[T]) settled(ctx context.Context, req ctlruntime.Request, v view, finished bool) bool {
stored := v.status.Record
if !finished || stored == nil || strings.TrimSpace(stored.URI) == "" || stored.CID == "" {
return false
}
synced := meta.FindStatusCondition(v.status.Conditions, v1alpha1.InfraConditionSynced)
if synced == nil || synced.Status != metav1.ConditionTrue || synced.ObservedGeneration != v.object.GetGeneration() {
return false
}
k := keyOf(req, v)
r.c.mu.Lock()
cid, indexed := r.c.bound[stored.URI][k]
_, moved := r.c.moved[k]
unwritten := r.c.unwritten[k]
r.c.mu.Unlock()
switch {
case moved || unwritten:
return false
case indexed:
return cid == stored.CID
}
repo, err := RepoRef(req.Workspace, v.spec.Record.RepoRef)
if err != nil {
return false
}
r.c.bindRepo(string(v.kind), req, repo)
uri, err := syntax.ParseATURI(strings.TrimSpace(stored.URI))
if err != nil {
return false
}
if err := r.ensureWatch(ctx, req, v, repo, uri.RecordKey().String(), v.spec.Record.EventSourceRefs); err != nil {
return false
}
if changed, err := r.watchedChange(ctx, req, v, stored); err != nil || changed {
return false
}
r.c.index(string(v.kind), req, stored.URI, stored.CID)
r.c.log.Info("infra_record_settled", "kind", v.kind, "workspace", req.Workspace, "name", req.Name, "uri", stored.URI, "cid", stored.CID)
return true
}
func (r *reconciler[T]) watchedChange(ctx context.Context, req ctlruntime.Request, v view, stored *v1alpha1.InfraRecordStatus) (bool, error) {
name := infra.ChildName(v.kind, v.object.GetName())
dyn, err := r.c.dynamic(req.Workspace)
if err != nil {
return false, err
}
watch, err := kube.Get[v1alpha1.AtprotoRecordWatch](ctx, dyn.Resource(v1alpha1.AtprotoRecordWatchGVR), name)
if err != nil {
return false, err
}
i := slices.IndexFunc(watch.Status.Records, func(rec v1alpha1.WatchedRecord) bool { return rec.URI == stored.URI })
if i >= 0 {
return watch.Status.Records[i].Cid != stored.CID, nil
}
return slices.Contains(watch.Status.Deleted, stored.URI), nil
}
func (r *reconciler[T]) pull(ctx context.Context, req ctlruntime.Request, v view, d infra.Declaration) error {
contracts := d.Contracts
if contracts == nil {
contracts = []v1alpha1.InfraContract{}
}
spec := map[string]any{"description": d.Description, "contracts": contracts}
gvr := v1alpha1.InfraGVR
if v.kind == infra.KindEphemeralInfra {
gvr = v1alpha1.EphemeralInfraGVR
spec["ttlSeconds"], spec["endWhen"] = d.TTLSeconds, d.EndWhen
}
dyn, err := r.c.dynamic(req.Workspace)
if err != nil {
return err
}
doc := kube.Doc{"apiVersion": v1alpha1.GroupVersion.String(), "kind": string(v.kind), "metadata": map[string]any{"name": req.Name}, "spec": spec}
if _, err := kube.Apply[kube.Doc](ctx, dyn.Resource(gvr), doc, kube.ApplyOptions{FieldManager: r.pullManager, Force: true}); err != nil {
return fmt.Errorf("infra: applying the record's declaration to %s %s: %w", v.kind, req.Name, err)
}
return nil
}
func Publishing(b *v1alpha1.InfraRecordBinding) bool {
return b != nil && b.Sync == v1alpha1.InfraRecordSyncPublish
}
func (r *reconciler[T]) writeRecord(ctx context.Context, req ctlruntime.Request, v view, rec *recordSync, observed infra.Observed) error {
if rec.publish {
observed = observed.Published()
}
observed.EndedAt, observed.EndReason = "", ""
value, err := infra.Render(v.kind, rec.render, observed, rec.createdAt)
if err != nil {
return err
}
return r.applyRecord(ctx, req, v, rec.repo, rec.rkey, value, rec.swapCid)
}
func (r *reconciler[T]) applyRecord(ctx context.Context, req ctlruntime.Request, v view, repo commonv1.ObjectRef, rkey string, value json.RawMessage, swapCid string) error {
if repo.Workspace == req.Workspace {
repo.Workspace = ""
}
ar := &v1alpha1.AtprotoRecord{TypeMeta: metav1.TypeMeta{APIVersion: v1alpha1.GroupVersion.String(), Kind: "AtprotoRecord"},
ObjectMeta: metav1.ObjectMeta{Name: infra.ChildName(v.kind, v.object.GetName()), Labels: childLabels(v),
OwnerReferences: []metav1.OwnerReference{ownerRef(v)}},
Spec: v1alpha1.AtprotoRecordSpec{RepoRef: &repo, Collection: v.kind.NSID(), Rkey: rkey, Record: value, SwapCid: swapCid,
DeletionPolicy: v1alpha1.RecordDeletionRetain}}
dyn, err := r.c.dynamic(req.Workspace)
if err != nil {
return err
}
if _, err := kube.Apply[v1alpha1.AtprotoRecord](ctx, dyn.Resource(v1alpha1.AtprotoRecordGVR), ar, kube.ApplyOptions{FieldManager: r.manager, Force: true}); err != nil {
return fmt.Errorf("infra: applying the AtprotoRecord %s: %w", ar.Name, err)
}
return nil
}
func RecordOrphaned(err error) bool {
reason, ok := Unresolved(err)
return ok && (reason == "RepoNotFound" || reason == "RepoBackendGone")
}
func (r *reconciler[T]) finalObserved(v view) infra.Observed {
at := v.object.GetDeletionTimestamp()
out := infra.Observed{Phase: v1alpha1.InfraPhaseEnded, EndReason: v1alpha1.EphemeralInfraEndDeleted}
for _, c := range v.status.Contracts {
switch c.Phase {
case v1alpha1.InfraContractPhaseStaged, v1alpha1.InfraContractPhaseReleased, v1alpha1.InfraContractPhaseFailed:
default:
if v.object.GetDeletionTimestamp() != nil {
c.Phase = v1alpha1.InfraContractPhaseReleased
}
}
out.Contracts = append(out.Contracts, c)
}
if at != nil {
out.EndedAt = at.UTC().Format(time.RFC3339)
}
if v.ephemeral != nil {
out.ExpiresAt = v.ephemeral.ExpiresAt
if v.ephemeral.EndedAt != "" {
out.EndedAt, out.EndReason = v.ephemeral.EndedAt, cmp.Or(v.ephemeral.EndReason, v1alpha1.EphemeralInfraEndDeleted)
}
}
if out.EndedAt == "" {
out.EndedAt = time.Now().UTC().Format(time.RFC3339)
}
if v.spec.Record == nil || Publishing(v.spec.Record) {
out = out.Published()
}
return out
}
func (r *reconciler[T]) owns(v view, rkey string) bool {
if v.spec.Record != nil && strings.TrimSpace(v.spec.Record.URI) != "" {
return false
}
for _, o := range v.object.GetOwnerReferences() {
if o.Controller != nil && *o.Controller && (o.Kind == CollectionKind || o.Kind == EphemeralCollectionKind) {
return false
}
}
return infra.InSuccession(v.object.GetName(), string(v.object.GetUID()), rkey)
}
func arInSync(ar *v1alpha1.AtprotoRecord) bool {
c := meta.FindStatusCondition(ar.Status.Conditions, v1alpha1.ConditionRecordInSync)
return c != nil && c.Status == metav1.ConditionTrue && c.ObservedGeneration == ar.Generation && ar.Status.ObservedGeneration == ar.Generation
}
func pendingEnd(ar *v1alpha1.AtprotoRecord) string {
var value struct {
Status struct {
EndedAt string `json:"endedAt"`
} `json:"status"`
}
if json.Unmarshal(ar.Spec.Record, &value) != nil {
return ""
}
return strings.TrimSpace(value.Status.EndedAt)
}
func (r *reconciler[T]) finalWrite(ctx context.Context, req ctlruntime.Request, v view) (bool, error) {
stored := v.status.Record
if stored == nil || stored.Direction == v1alpha1.InfraSyncRetained || strings.TrimSpace(stored.URI) == "" {
return true, nil
}
uri, err := syntax.ParseATURI(strings.TrimSpace(stored.URI))
if err != nil || uri.Collection().String() != v.kind.NSID() {
return true, nil
}
rkey := uri.RecordKey().String()
if v.kind == infra.KindInfra && rkey == infra.ReservedRkey {
return true, nil
}
attrs := []any{"kind", v.kind, "workspace", req.Workspace, "name", req.Name, "uri", stored.URI}
if !r.owns(v, rkey) {
r.c.log.Info("infra_record_not_owned", attrs...)
return true, nil
}
dyn, err := r.c.dynamic(req.Workspace)
if err != nil {
return false, err
}
ar, err := kube.Get[v1alpha1.AtprotoRecord](ctx, dyn.Resource(v1alpha1.AtprotoRecordGVR), infra.ChildName(v.kind, v.object.GetName()))
switch {
case apierrors.IsNotFound(err):
ar = nil
case err != nil:
return false, err
}
var ref commonv1.ObjectRef
switch {
case v.spec.Record != nil:
ref = v.spec.Record.RepoRef
case ar != nil && ar.Spec.RepoRef != nil:
ref = *ar.Spec.RepoRef
default:
r.c.log.Info("infra_record_end_skipped", append(attrs, "reason", "no repoRef")...)
return true, nil
}
repo, err := RepoRef(req.Workspace, ref)
if err != nil {
return false, err
}
r.c.bindRepo(string(v.kind), req, repo)
_, reader, err := Repo(ctx, r.c.bus, r.c.dynamic, r.self(req), repo, v.kind.NSID())
if err != nil {
return false, err
}
got, err := reader.GetRecord(ctx, uri.Authority().String(), uri.Collection().String(), rkey)
switch {
case errors.Is(err, atprotorepo.ErrNotFound):
r.c.log.Info("infra_record_absent_at_end", attrs...)
return true, nil
case err != nil:
return false, ReadFailed(fmt.Errorf("infra: reading %s for its final status: %w", stored.URI, err))
}
attrs = append(attrs, "cid", got.CID)
decoded, derr := infra.Decode(v.kind, got.Value)
switch {
case errors.Is(derr, infra.ErrReserved):
return true, nil
case decoded.Ended() && !mirror(v):
r.c.log.Info("infra_record_ended", append(attrs, "endedAt", decoded.EndedAt, "endReason", decoded.EndReason)...)
return true, nil
}
if mirror(v) {
return r.dropMirror(ctx, req, v, ar, repo, rkey, got, attrs)
}
own := []string{strings.TrimSpace(stored.CID)}
if ar != nil && arInSync(ar) {
own = append(own, strings.TrimSpace(ar.Status.Cid))
}
if !slices.Contains(own, got.CID) {
r.c.log.Info("infra_record_replaced", append(attrs, "storedCid", stored.CID)...)
return true, nil
}
observed := r.finalObserved(v)
if ar != nil {
if at := pendingEnd(ar); at != "" {
observed.EndedAt = at
if ar.Status.Phase == v1alpha1.AtprotoRecordPhaseFailed && ar.Status.ObservedGeneration == ar.Generation {
return false, fmt.Errorf("infra: the final status write of %s failed: %s", stored.URI, ar.Status.Message)
}
}
}
value, err := infra.Final(v.kind, got.Value, observed)
if err != nil {
return false, err
}
if err := r.applyRecord(ctx, req, v, repo, rkey, value, got.CID); err != nil {
return false, err
}
r.c.log.Info("infra_record_ending", attrs...)
return false, nil
}
func (r *reconciler[T]) unbindRecord(ctx context.Context, req ctlruntime.Request, v view) (bool, error) {
dyn, err := r.c.dynamic(req.Workspace)
if err != nil {
return false, err
}
name := infra.ChildName(v.kind, v.object.GetName())
records := dyn.Resource(v1alpha1.AtprotoRecordGVR)
ar, err := kube.Get[v1alpha1.AtprotoRecord](ctx, records, name)
switch {
case apierrors.IsNotFound(err):
return true, r.dropWatch(ctx, dyn, name)
case err != nil:
return false, err
case !ar.DeletionTimestamp.IsZero():
return false, nil
}
if err := r.retain(ctx, records, ar); err != nil {
return false, err
}
if err := records.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
return false, err
}
r.c.log.Info("infra_record_unbound", "kind", v.kind, "workspace", req.Workspace, "name", req.Name, "atprotoRecord", name, "uri", ar.Status.URI)
return false, nil
}
func (r *reconciler[T]) retain(ctx context.Context, records kube.Resource, ar *v1alpha1.AtprotoRecord) error {
if ar.Spec.DeletionPolicy == v1alpha1.RecordDeletionRetain {
return nil
}
retained := &v1alpha1.AtprotoRecord{TypeMeta: metav1.TypeMeta{APIVersion: v1alpha1.GroupVersion.String(), Kind: "AtprotoRecord"},
ObjectMeta: metav1.ObjectMeta{Name: ar.Name, Labels: ar.Labels, OwnerReferences: ar.OwnerReferences}, Spec: ar.Spec}
retained.Spec.DeletionPolicy = v1alpha1.RecordDeletionRetain
if _, err := kube.Apply[v1alpha1.AtprotoRecord](ctx, records, retained, kube.ApplyOptions{FieldManager: r.manager, Force: true}); err != nil {
return fmt.Errorf("infra: retaining the record of the AtprotoRecord %s: %w", ar.Name, err)
}
return nil
}
func (r *reconciler[T]) retainChild(ctx context.Context, req ctlruntime.Request, v view) error {
dyn, err := r.c.dynamic(req.Workspace)
if err != nil {
return err
}
records := dyn.Resource(v1alpha1.AtprotoRecordGVR)
ar, err := kube.Get[v1alpha1.AtprotoRecord](ctx, records, infra.ChildName(v.kind, v.object.GetName()))
switch {
case apierrors.IsNotFound(err):
return nil
case err != nil:
return err
case !ar.DeletionTimestamp.IsZero():
return nil
}
return r.retain(ctx, records, ar)
}
func (r *reconciler[T]) retainOnce(ctx context.Context, req ctlruntime.Request, v view) error {
if v.spec.Record == nil && (v.status.Record == nil || v.status.Record.Direction == v1alpha1.InfraSyncRetained) {
return nil
}
k := keyOf(req, v)
r.c.mu.Lock()
done := r.c.retained[k]
r.c.mu.Unlock()
if done {
return nil
}
if err := r.retainChild(ctx, req, v); err != nil {
return err
}
r.c.mu.Lock()
r.c.retained[k] = true
r.c.mu.Unlock()
return nil
}
func (r *reconciler[T]) dropWatch(ctx context.Context, dyn *kube.Client, name string) error {
for _, child := range []struct {
gvr schema.GroupVersionResource
name string
}{{v1alpha1.AtprotoRecordWatchGVR, name}, {exposurev1alpha1.ComponentLinkGVR, name + "-deliver"}} {
if err := dyn.Resource(child.gvr).Delete(ctx, child.name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
return err
}
}
return nil
}
func (r *reconciler[T]) ensureWatch(ctx context.Context, req ctlruntime.Request, v view, repo commonv1.ObjectRef, rkey string, sources []commonv1.ObjectRef) error {
return EnsureWatch(ctx, r.c.dynamic, r.manager, req.Workspace, infra.ChildName(v.kind, v.object.GetName()), childLabels(v), ownerRef(v),
repo, v.kind.NSID(), []string{rkey}, sources)
}
func EnsureWatch(ctx context.Context, dynamic kube.Factory, manager, workspace, name string, labels map[string]string, owner metav1.OwnerReference,
repo commonv1.ObjectRef, collection string, rkeys []string, sources []commonv1.ObjectRef) error {
dyn, err := dynamic(workspace)
if err != nil {
return err
}
if repo.Workspace == workspace {
repo.Workspace = ""
}
watch := &v1alpha1.AtprotoRecordWatch{TypeMeta: metav1.TypeMeta{APIVersion: v1alpha1.GroupVersion.String(), Kind: events.RecordWatchKind},
ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels, OwnerReferences: []metav1.OwnerReference{owner}},
Spec: v1alpha1.AtprotoRecordWatchSpec{RepoRef: &repo, Collection: collection, Rkeys: rkeys,
Source: v1alpha1.AtprotoRecordWatchSource{EventSourceRefs: sources}}}
if _, err := kube.Apply[v1alpha1.AtprotoRecordWatch](ctx, dyn.Resource(v1alpha1.AtprotoRecordWatchGVR), watch,
kube.ApplyOptions{FieldManager: manager, Force: true}); err != nil {
return fmt.Errorf("infra: applying the AtprotoRecordWatch %s: %w", name, err)
}
link := &exposurev1alpha1.ComponentLink{TypeMeta: metav1.TypeMeta{APIVersion: exposurev1alpha1.GroupVersion.String(), Kind: "ComponentLink"},
ObjectMeta: metav1.ObjectMeta{Name: name + "-deliver", Labels: labels, OwnerReferences: []metav1.OwnerReference{owner}},
Spec: exposurev1alpha1.ComponentLinkSpec{
From: exposurev1alpha1.ComponentLinkFrom{Selector: &exposurev1alpha1.ComponentLinkSelector{Group: eventsv1alpha1.GroupName,
Kind: "EventStream", Workspace: workspace, MatchLabels: map[string]string{eventstream.RecordWatchLabel: name}}},
To: commonv1.ObjectRef{Group: v1alpha1.GroupName, Kind: events.RecordWatchKind, Name: name, Workspace: workspace},
Capabilities: []exposurev1alpha1.LinkCapability{{Name: events.CapabilityDeliver, Scope: []string{name}}}}}
if _, err := kube.Apply[exposurev1alpha1.ComponentLink](ctx, dyn.Resource(exposurev1alpha1.ComponentLinkGVR), link,
kube.ApplyOptions{FieldManager: manager, Force: true}); err != nil {
return fmt.Errorf("infra: applying the ComponentLink %s: %w", link.Name, err)
}
return nil
}
type pullNote struct {
hash string
conditions []metav1.Condition
}
func (c *Controller) notePull(kind string, req ctlruntime.Request, hash string, conditions []metav1.Condition) {
var keep []metav1.Condition
for _, cond := range conditions {
if cond.Type == v1alpha1.InfraConditionSyncConflict {
keep = append(keep, cond)
}
}
c.mu.Lock()
defer c.mu.Unlock()
c.pulls[indexKey{kind: kind, cluster: req.Cluster, name: req.Name}] = pullNote{hash: hash, conditions: keep}
}
func (c *Controller) takePull(kind string, req ctlruntime.Request, hash string) ([]metav1.Condition, bool) {
k := indexKey{kind: kind, cluster: req.Cluster, name: req.Name}
c.mu.Lock()
defer c.mu.Unlock()
note, ok := c.pulls[k]
if !ok || note.hash != hash {
return nil, false
}
delete(c.pulls, k)
return note.conditions, true
}
type indexKey struct {
kind string
cluster string
name string
}
func (c *Controller) index(kind string, req ctlruntime.Request, uri, cid string) {
k := indexKey{kind: kind, cluster: req.Cluster, name: req.Name}
c.mu.Lock()
defer c.mu.Unlock()
c.dropLocked(k)
if moved, ok := c.moved[k]; ok && moved == cid {
delete(c.moved, k)
}
users := c.bound[uri]
if users == nil {
users = map[indexKey]string{}
c.bound[uri] = users
}
users[k] = cid
}
func (c *Controller) forget(kind string, req ctlruntime.Request) {
c.repos[kind].Set(binding.Target{Cluster: req.Cluster, Workspace: req.Workspace, Name: req.Name}, nil)
c.mu.Lock()
defer c.mu.Unlock()
k := indexKey{kind: kind, cluster: req.Cluster, name: req.Name}
c.dropLocked(k)
delete(c.pulls, k)
delete(c.moved, k)
delete(c.unwritten, k)
delete(c.retained, k)
delete(c.wants, k)
delete(c.indexFailed, k)
c.retries.Reset(k)
c.indexRetries.Reset(k)
}
func (c *Controller) bindRepo(kind string, req ctlruntime.Request, repo commonv1.ObjectRef) {
c.repos[kind].Set(binding.Target{Cluster: req.Cluster, Workspace: req.Workspace, Name: req.Name}, &struct{}{},
binding.Key(repo.Workspace, RepoKind, repo.Name))
}
func (c *Controller) repoUsers(kind, workspace, name string) []ctlruntime.Request {
var out []ctlruntime.Request
for _, t := range c.repos[kind].Users(binding.Key(workspace, RepoKind, name)) {
out = append(out, ctlruntime.Request{Cluster: t.Cluster, Workspace: t.Workspace, Name: t.Name})
}
return out
}
func (c *Controller) dropLocked(k indexKey) {
for u, users := range c.bound {
delete(users, k)
if len(users) == 0 {
delete(c.bound, u)
}
}
}
func (c *Controller) watched(cluster, kind string, watch *v1alpha1.AtprotoRecordWatch) []ctlruntime.Request {
c.mu.Lock()
defer c.mu.Unlock()
var out []ctlruntime.Request
present := map[string]bool{}
add := func(uri, now string) {
for key, cid := range c.bound[uri] {
if key.kind != kind || key.cluster != cluster {
continue
}
if cid == now {
delete(c.moved, key)
continue
}
c.moved[key] = now
out = append(out, ctlruntime.Request{Cluster: key.cluster, Name: key.name})
}
}
for _, rec := range watch.Status.Records {
present[rec.URI] = true
add(rec.URI, rec.Cid)
}
for _, uri := range watch.Status.Deleted {
if !present[uri] {
add(uri, "")
}
}
return out
}
func (c *Controller) setUnwritten(k indexKey, pending bool) {
c.mu.Lock()
defer c.mu.Unlock()
if pending {
c.unwritten[k] = true
return
}
delete(c.unwritten, k)
}
func mirror(v view) bool {
return v.kind == infra.KindInfra && v.spec.ClusterRef != nil
}
func (r *reconciler[T]) dropMirror(ctx context.Context, req ctlruntime.Request, v view, ar *v1alpha1.AtprotoRecord, repo commonv1.ObjectRef,
rkey string, got atprotorepo.Record, attrs []any) (bool, error) {
cid := got.CID
if ar != nil {
if !ar.DeletionTimestamp.IsZero() {
return false, nil
}
dyn, err := r.c.dynamic(req.Workspace)
if err != nil {
return false, err
}
records := dyn.Resource(v1alpha1.AtprotoRecordGVR)
held := &v1alpha1.AtprotoRecord{TypeMeta: metav1.TypeMeta{APIVersion: v1alpha1.GroupVersion.String(), Kind: "AtprotoRecord"},
ObjectMeta: metav1.ObjectMeta{Name: ar.Name, Labels: ar.Labels, OwnerReferences: ar.OwnerReferences}, Spec: ar.Spec}
held.Spec.DeletionPolicy, held.Spec.SwapCid, held.Spec.Record = v1alpha1.RecordDeletionRetain, cid, got.Value
if _, err := kube.Apply[v1alpha1.AtprotoRecord](ctx, records, held, kube.ApplyOptions{FieldManager: r.manager, Force: true}); err != nil {
return false, fmt.Errorf("infra: holding the AtprotoRecord %s at cid %s: %w", ar.Name, cid, err)
}
if err := records.Delete(ctx, ar.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
return false, err
}
r.c.log.Info("infra_record_mirror_unbinding", append(attrs, "atprotoRecord", ar.Name)...)
return false, nil
}
writer, err := bus.Resolve[atprotorepo.RecordWriter](ctx, r.c.bus, r.self(req), repo,
bus.Capability{Name: atprotorepo.CapabilityWrite, Scope: []string{v.kind.NSID()}})
if err != nil {
return false, err
}
if d, ok := writer.(atprotorepo.SwapDeleter); ok {
err = d.DeleteRecordSwap(ctx, v.kind.NSID(), rkey, cid)
} else {
err = writer.DeleteRecord(ctx, v.kind.NSID(), rkey)
}
if err != nil && !errors.Is(err, atprotorepo.ErrNotFound) {
return false, fmt.Errorf("infra: deleting the cluster mirror record %s: %w", v.status.Record.URI, err)
}
r.c.log.Info("infra_record_deleted", append(attrs, "reason", "the record mirrors the nodes of a cluster")...)
return true, nil
}