0
0
package infracollection
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log/slog"
"slices"
"strings"
"sync"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/utils/ptr"
commonv1 "github.com/johnandersen777/socialweb-computer-kcp/api/common/v1alpha1"
exposurev1alpha1 "github.com/johnandersen777/socialweb-computer-kcp/api/exposure/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/bus"
"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/atprotorepo"
infractl "github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/infra"
ctlruntime "github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/kube"
"github.com/johnandersen777/socialweb-computer-kcp/internal/infra"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/binding"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/conds"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/netretry"
)
const (
Name = "infracollection"
EphemeralName = "ephemeralinfracollection"
)
type Options struct {
Bus *bus.Bus
Dynamic kube.Factory
Scope func(workspace string) bool
Log *slog.Logger
}
type Controller struct {
bus *bus.Bus
dynamic kube.Factory
log *slog.Logger
repos map[string]*binding.Table[struct{}]
retries *netretry.Tracker[retryKey]
mu sync.Mutex
over map[retryKey]map[string]bool
absent map[retryKey]absentAt
}
type absentAt struct {
index string
uris map[string]bool
}
type retryKey struct {
kind string
cluster string
name string
}
func Register(m *ctlruntime.Manager, opts Options) (*Controller, error) {
if opts.Bus == nil {
return nil, errors.New("infracollection: the bus is required")
}
if opts.Dynamic == nil {
return nil, errors.New("infracollection: the dynamic client factory is required")
}
if err := m.AddToScheme(v1alpha1.AddToScheme, exposurev1alpha1.AddToScheme); err != nil {
return nil, err
}
log := opts.Log
if log == nil {
log = slog.New(slog.DiscardHandler)
}
c := &Controller{bus: opts.Bus, dynamic: opts.Dynamic, log: log.With("controller", Name), retries: netretry.NewTracker[retryKey](), over: map[retryKey]map[string]bool{}, absent: map[retryKey]absentAt{}, repos: map[string]*binding.Table[struct{}]{
infractl.CollectionKind: binding.New[struct{}](), infractl.EphemeralCollectionKind: binding.New[struct{}]()}}
if err := register[*v1alpha1.InfraCollection, *v1alpha1.Infra](m, c, Name, infraOps, opts.Scope); err != nil {
return nil, err
}
if err := register[*v1alpha1.EphemeralInfraCollection, *v1alpha1.EphemeralInfra](m, c, EphemeralName, ephemeralOps, opts.Scope); err != nil {
return nil, err
}
return c, nil
}
func register[T, C ctlruntime.Object](m *ctlruntime.Manager, c *Controller, name string, ops collectionOps[T], scope func(string) bool) error {
r := &reconciler[T]{c: c, ops: ops, manager: ctlruntime.FieldManagerPrefix + name}
options := []ctlruntime.Option{ctlruntime.Export(infractl.Export), ctlruntime.Owns[C](),
ctlruntime.WatchesIn(infractl.RecordExport, func(_ context.Context, _ ctlruntime.Request, obj *v1alpha1.AtprotoRecordWatch) []ctlruntime.Request {
labels := obj.GetLabels()
if labels[v1alpha1.InfraKindLabel] != ops.kind || labels[v1alpha1.InfraNameLabel] == "" {
return nil
}
return []ctlruntime.Request{{Name: labels[v1alpha1.InfraNameLabel]}}
}),
ctlruntime.WatchesFiltered(infractl.RepoExport, ctlruntime.WatchFilter[*v1alpha1.AtprotoRepo]{Changed: func(old, new *v1alpha1.AtprotoRepo) bool {
return old.Status.DID != new.Status.DID
}}, func(_ context.Context, src ctlruntime.Request, obj *v1alpha1.AtprotoRepo) []ctlruntime.Request {
return c.repoUsers(ops.kind, src.Workspace, obj.Name)
}),
infractl.RepoLinkWatch(func(workspace, name string) []ctlruntime.Request { return c.repoUsers(ops.kind, workspace, name) }),
ctlruntime.Events(infractl.RepoBindEvents(c.bus, func(workspace, name string) []ctlruntime.Request { return c.repoUsers(ops.kind, workspace, name) })),
ctlruntime.Gone(func(_ context.Context, req ctlruntime.Request) {
c.repos[ops.kind].Set(binding.Target{Cluster: req.Cluster, Workspace: req.Workspace, Name: req.Name}, nil)
c.retries.Reset(retryKey{kind: ops.kind, cluster: req.Cluster, name: req.Name})
c.forget(retryKey{kind: ops.kind, cluster: req.Cluster, name: req.Name})
})}
if scope != nil {
options = append(options, ctlruntime.InWorkspaces(scope))
}
if err := ctlruntime.For[T](m, name, r, options...); err != nil {
return fmt.Errorf("infracollection: registering the %s controller: %w", ops.kind, err)
}
return nil
}
func (c *Controller) bindRepo(kind string, req ctlruntime.Request, repo commonv1.ObjectRef) {
c.repos[kind].Set(binding.Target{Cluster: req.Cluster, Workspace: req.Workspace, Name: req.Name}, &struct{}{},
binding.Key(repo.Workspace, infractl.RepoKind, repo.Name))
}
func (c *Controller) repoUsers(kind, workspace, name string) []ctlruntime.Request {
var out []ctlruntime.Request
for _, t := range c.repos[kind].Users(binding.Key(workspace, infractl.RepoKind, name)) {
out = append(out, ctlruntime.Request{Cluster: t.Cluster, Workspace: t.Workspace, Name: t.Name})
}
return out
}
type child struct {
name string
owners []metav1.OwnerReference
record *v1alpha1.InfraRecordBinding
uri string
}
type collectionOps[T ctlruntime.Object] struct {
kind string
child infra.Kind
view func(T) (v1alpha1.InfraCollectionSpec, v1alpha1.InfraCollectionStatus)
statusObject func(name string, status v1alpha1.InfraCollectionStatus) T
children func(ctx context.Context, dyn *kube.Client) ([]child, error)
newChild func(meta metav1.ObjectMeta, binding v1alpha1.InfraRecordBinding) ctlruntime.Object
}
func childOf(meta metav1.ObjectMeta, spec v1alpha1.InfraSpec, status v1alpha1.InfraStatus) child {
out := child{name: meta.Name, owners: meta.OwnerReferences, record: spec.Record}
if status.Record != nil {
out.uri = status.Record.URI
}
return out
}
var infraOps = collectionOps[*v1alpha1.InfraCollection]{
kind: infractl.CollectionKind,
child: infra.KindInfra,
view: func(o *v1alpha1.InfraCollection) (v1alpha1.InfraCollectionSpec, v1alpha1.InfraCollectionStatus) {
return o.Spec, o.Status
},
statusObject: func(name string, status v1alpha1.InfraCollectionStatus) *v1alpha1.InfraCollection {
return &v1alpha1.InfraCollection{ObjectMeta: metav1.ObjectMeta{Name: name}, Status: status}
},
children: func(ctx context.Context, dyn *kube.Client) ([]child, error) {
items, err := kube.List[v1alpha1.Infra](ctx, dyn.Resource(v1alpha1.InfraGVR), metav1.ListOptions{})
var out []child
for _, o := range items {
out = append(out, childOf(o.ObjectMeta, o.Spec, o.Status))
}
return out, err
},
newChild: func(meta metav1.ObjectMeta, binding v1alpha1.InfraRecordBinding) ctlruntime.Object {
return &v1alpha1.Infra{ObjectMeta: meta, Spec: v1alpha1.InfraSpec{Record: &binding}}
},
}
var ephemeralOps = collectionOps[*v1alpha1.EphemeralInfraCollection]{
kind: infractl.EphemeralCollectionKind,
child: infra.KindEphemeralInfra,
view: func(o *v1alpha1.EphemeralInfraCollection) (v1alpha1.InfraCollectionSpec, v1alpha1.InfraCollectionStatus) {
return o.Spec, o.Status
},
statusObject: func(name string, status v1alpha1.InfraCollectionStatus) *v1alpha1.EphemeralInfraCollection {
return &v1alpha1.EphemeralInfraCollection{ObjectMeta: metav1.ObjectMeta{Name: name}, Status: status}
},
children: func(ctx context.Context, dyn *kube.Client) ([]child, error) {
items, err := kube.List[v1alpha1.EphemeralInfra](ctx, dyn.Resource(v1alpha1.EphemeralInfraGVR), metav1.ListOptions{})
var out []child
for _, o := range items {
out = append(out, childOf(o.ObjectMeta, o.Spec.InfraSpec, o.Status.InfraStatus))
}
return out, err
},
newChild: func(meta metav1.ObjectMeta, binding v1alpha1.InfraRecordBinding) ctlruntime.Object {
return &v1alpha1.EphemeralInfra{ObjectMeta: meta, Spec: v1alpha1.EphemeralInfraSpec{InfraSpec: v1alpha1.InfraSpec{Record: &binding}}}
},
}
type reconciler[T ctlruntime.Object] struct {
c *Controller
ops collectionOps[T]
manager string
}
func ChildName(rkey string) string {
if len(validation.IsDNS1123Subdomain(rkey)) == 0 {
return rkey
}
sum := sha256.Sum256([]byte(rkey))
return "r-" + hex.EncodeToString(sum[:6])
}
func (r *reconciler[T]) Reconcile(ctx context.Context, req ctlruntime.Request, obj T) (ctlruntime.Result, error) {
if !obj.GetDeletionTimestamp().IsZero() {
return ctlruntime.Result{}, nil
}
spec, previous := r.ops.view(obj)
nsid := r.ops.child.NSID()
gen := obj.GetGeneration()
status := v1alpha1.InfraCollectionStatus{ObservedGeneration: gen, Collection: nsid, Did: previous.Did, Records: previous.Records,
Children: previous.Children, Conditions: slices.Clone(previous.Conditions)}
meta.RemoveStatusCondition(&status.Conditions, v1alpha1.InfraCollectionConditionListed)
attrs := []any{"kind", r.ops.kind, "workspace", req.Workspace, "name", req.Name}
key := retryKey{kind: r.ops.kind, cluster: req.Cluster, name: req.Name}
fail := func(err error) (ctlruntime.Result, error) {
status.Message = err.Error()
reason, retry := "IndexReadFailed", err
unresolvedReason, unresolved := infractl.Unresolved(err)
delay := infractl.ReadRetry(r.c.log, r.c.retries, key, err, attrs...)
switch {
case errors.Is(err, infractl.ErrRepoNoDID):
reason, retry = "RepoPending", nil
case unresolved:
reason, retry = unresolvedReason, nil
infractl.NoteUnresolved(r.c.log, previous.Conditions, &status.Conditions, gen, unresolvedReason, err, attrs...)
case delay > 0:
reason, retry = v1alpha1.InfraReasonRecordReadRetrying, nil
status.Message = v1alpha1.InfraMessageRecordReadRetrying
}
conds.Set(&status.Conditions, gen, v1alpha1.InfraCollectionConditionIndexed, false, reason, status.Message)
return ctlruntime.Result{Backoff: unresolved, RequeueAfter: delay}, errors.Join(retry, req.ApplyStatus(ctx, r.ops.statusObject(req.Name, status)))
}
repo, err := infractl.RepoRef(req.Workspace, spec.RepoRef)
if err != nil {
return fail(err)
}
r.c.bindRepo(r.ops.kind, req, repo)
self := commonv1.ObjectRef{Group: v1alpha1.GroupName, Kind: r.ops.kind, Name: req.Name, Workspace: req.Workspace}
did, reader, err := infractl.Repo(ctx, r.c.bus, r.c.dynamic, self, repo, infra.KindInfra.NSID(), nsid)
if err != nil {
return fail(err)
}
infractl.NoteUnresolved(r.c.log, previous.Conditions, &status.Conditions, gen, "", nil, attrs...)
owner := metav1.OwnerReference{APIVersion: v1alpha1.GroupVersion.String(), Kind: r.ops.kind, Name: req.Name, UID: obj.GetUID(), Controller: ptr.To(true)}
labels := map[string]string{v1alpha1.InfraKindLabel: r.ops.kind, v1alpha1.InfraNameLabel: req.Name}
watchName := infra.ChildName(r.ops.child, "coll", req.Name)
if err := infractl.EnsureWatch(ctx, r.c.dynamic, r.manager, req.Workspace, watchName, labels, owner,
repo, infra.KindInfra.NSID(), []string{infra.ReservedRkey}, spec.EventSourceRefs); err != nil {
return fail(err)
}
dyn, err := r.c.dynamic(req.Workspace)
if err != nil {
return fail(err)
}
raw, indexCID, present, err := readIndex(ctx, dyn, reader, watchName, did)
if err != nil {
return fail(infractl.ReadFailed(err))
}
now := time.Now().UTC()
var live []infra.IndexEntry
invalid := ""
if present {
entries, err := infra.DecodeIndex(raw)
if err != nil {
invalid = err.Error()
}
for _, e := range entries {
if e.Collection == nsid && !e.Expired(now) {
live = append(live, e)
}
}
}
r.c.retries.Reset(key)
kids, err := r.ops.children(ctx, dyn)
if err != nil {
return fail(err)
}
byURI := map[string]child{}
byName := map[string]child{}
for _, k := range kids {
byName[k.name] = k
if uri := boundURI(k, req.Workspace, repo, did, nsid); uri != "" {
byURI[uri] = k
}
}
var children []v1alpha1.InfraCollectionChild
created := 0
for _, e := range live {
uri := "at://" + did + "/" + nsid + "/" + e.Rkey
entry := v1alpha1.InfraCollectionChild{Rkey: e.Rkey, URI: uri}
if k, ok := byURI[uri]; ok {
if ownedBy(k, obj.GetUID()) {
entry.Name = k.name
} else {
entry.BoundBy = k.name
}
children = append(children, entry)
continue
}
if r.c.ended(key, uri) {
entry.Message = "Ended: the record ended; it is kept and not adopted"
children = append(children, entry)
continue
}
name := ChildName(e.Rkey)
if k, ok := byName[name]; ok {
entry.Message = fmt.Sprintf("NameConflict: %s %s exists and binds no record of this collection", r.ops.child, k.name)
children = append(children, entry)
continue
}
if r.c.isAbsent(key, indexCID, uri) {
entry.Message = "NotFound: the index names a record the repo does not hold"
children = append(children, entry)
continue
}
got, err := reader.GetRecord(ctx, did, nsid, e.Rkey)
if errors.Is(err, atprotorepo.ErrNotFound) {
r.c.noteAbsent(key, indexCID, uri)
entry.Message = "NotFound: the index names a record the repo does not hold"
children = append(children, entry)
continue
}
if err != nil {
return fail(infractl.ReadFailed(fmt.Errorf("infracollection: reading %s: %w", uri, err)))
}
entry.CID = got.CID
decoded, derr := infra.Decode(r.ops.child, got.Value)
if errors.Is(derr, infra.ErrReserved) || decoded.Over(r.ops.child, now) {
r.c.noteEnded(key, uri)
entry.Message = "Ended: the record ended; it is kept and not adopted"
children = append(children, entry)
continue
}
meta := metav1.ObjectMeta{Name: name, Labels: map[string]string{v1alpha1.InfraCollectionLabel: req.Name},
OwnerReferences: []metav1.OwnerReference{owner}}
binding := v1alpha1.InfraRecordBinding{RepoRef: spec.RepoRef, URI: uri, EventSourceRefs: spec.EventSourceRefs}
if err := req.Apply(ctx, r.ops.newChild(meta, binding)); err != nil {
return fail(fmt.Errorf("infracollection: creating %s %s for %s: %w", r.ops.child, name, uri, err))
}
r.c.log.Info("infra_collection_child_created", "kind", r.ops.kind, "workspace", req.Workspace, "name", req.Name, "child", name,
"uri", uri)
entry.Name = name
created++
children = append(children, entry)
}
status.Did, status.Records, status.Children = did, len(live), children
reason := "Indexed"
status.Message = fmt.Sprintf("%d live %s entr(ies) in %s, %d child object(s) created this pass", len(live), nsid, infra.IndexURI(did), created)
switch {
case invalid != "":
reason, status.Message = "IndexInvalid", "the index record does not decode: "+invalid
case !present:
reason, status.Message = "IndexEmpty", "the repo holds no index at "+infra.IndexURI(did)
case len(live) == 0:
reason = "IndexEmpty"
}
conds.Set(&status.Conditions, gen, v1alpha1.InfraCollectionConditionIndexed, invalid == "", reason, status.Message)
if err := req.ApplyStatus(ctx, r.ops.statusObject(req.Name, status)); err != nil {
return ctlruntime.Result{}, err
}
return ctlruntime.Result{}, nil
}
func readIndex(ctx context.Context, dyn *kube.Client, reader atprotorepo.RecordReader, watchName, did string) (json.RawMessage, string, bool, error) {
uri := infra.IndexURI(did)
watch, err := kube.Get[v1alpha1.AtprotoRecordWatch](ctx, dyn.Resource(v1alpha1.AtprotoRecordWatchGVR), watchName)
if err != nil && !apierrors.IsNotFound(err) {
return nil, "", false, err
}
if err == nil {
if i := slices.IndexFunc(watch.Status.Records, func(rec v1alpha1.WatchedRecord) bool { return rec.URI == uri }); i >= 0 {
return watch.Status.Records[i].Value, watch.Status.Records[i].Cid, true, nil
}
if slices.Contains(watch.Status.Deleted, uri) {
return nil, "", false, nil
}
}
got, err := reader.GetRecord(ctx, did, infra.KindInfra.NSID(), infra.ReservedRkey)
if errors.Is(err, atprotorepo.ErrNotFound) {
return nil, "", false, nil
}
if err != nil {
return nil, "", false, fmt.Errorf("infracollection: reading %s: %w", uri, err)
}
return got.Value, got.CID, true, nil
}
func (c *Controller) isAbsent(key retryKey, index, uri string) bool {
c.mu.Lock()
defer c.mu.Unlock()
memo, ok := c.absent[key]
return ok && index != "" && memo.index == index && memo.uris[uri]
}
func (c *Controller) noteAbsent(key retryKey, index, uri string) {
if index == "" {
return
}
c.mu.Lock()
defer c.mu.Unlock()
memo, ok := c.absent[key]
if !ok || memo.index != index {
memo = absentAt{index: index, uris: map[string]bool{}}
}
memo.uris[uri] = true
c.absent[key] = memo
}
func (c *Controller) ended(key retryKey, uri string) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.over[key][uri]
}
func (c *Controller) noteEnded(key retryKey, uri string) {
c.mu.Lock()
defer c.mu.Unlock()
if c.over[key] == nil {
c.over[key] = map[string]bool{}
}
c.over[key][uri] = true
}
func (c *Controller) forget(key retryKey) {
c.mu.Lock()
defer c.mu.Unlock()
delete(c.over, key)
delete(c.absent, key)
}
func ownedBy(k child, uid types.UID) bool {
for _, o := range k.owners {
if o.UID == uid {
return true
}
}
return false
}
func boundURI(k child, workspace string, repo commonv1.ObjectRef, did, nsid string) string {
if k.uri != "" {
return k.uri
}
if k.record == nil {
return ""
}
if uri := strings.TrimSpace(k.record.URI); uri != "" {
return uri
}
ref, err := infractl.RepoRef(workspace, k.record.RepoRef)
if err != nil || ref.Name != repo.Name || ref.Workspace != repo.Workspace {
return ""
}
return "at://" + did + "/" + nsid + "/" + k.name
}