0
0
package atprotorecordwatch
import (
"cmp"
"context"
"errors"
"fmt"
"log/slog"
"reflect"
"slices"
"strings"
"sync"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
commonv1 "github.com/johnandersen777/socialweb-computer-kcp/api/common/v1alpha1"
eventsv1 "github.com/johnandersen777/socialweb-computer-kcp/api/events/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/bus"
"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/events"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/atprotorecord"
ctlruntime "github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime"
"github.com/johnandersen777/socialweb-computer-kcp/internal/events/delivery"
"github.com/johnandersen777/socialweb-computer-kcp/internal/eventstream"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/binding"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/conds"
"k8s.io/apimachinery/pkg/runtime/schema"
)
const (
Name = "atprotorecordwatch"
Export = "marketidentities"
EventsExport = "events"
RepoExport = "applications"
FieldManager = ctlruntime.FieldManagerPrefix + Name
Kind = events.RecordWatchKind
RepoKind = eventstream.AtprotoRepoKind
LocalRepoKind = "LocalRepo"
RemoteRepoKind = "RemoteRepo"
VerifyInterval = 5 * time.Minute
VerifyRetry = 30 * time.Second
requestQueue = 256
)
var ErrNotReconciled = fmt.Errorf("%w", bus.ErrNoProvider)
type Children interface {
ApplyEventStream(ctx context.Context, workspace string, obj *eventsv1.EventStream) error
ApplyEventSource(ctx context.Context, workspace string, obj *eventsv1.EventSource) error
ApplyPdsEventSource(ctx context.Context, workspace string, obj *eventsv1.PdsEventSource) error
EventStreams(ctx context.Context, workspace, watch string) ([]eventsv1.EventStream, error)
DeleteEventStream(ctx context.Context, workspace, name string) error
}
type Repos interface {
AtprotoRepo(ctx context.Context, workspace, name string) (*v1alpha1.AtprotoRepo, error)
LocalRepo(ctx context.Context, workspace, name string) (*v1alpha1.LocalRepo, error)
RemoteRepo(ctx context.Context, workspace, name string) (*v1alpha1.RemoteRepo, error)
}
type ValueReader func(ctx context.Context, workspace string, from v1alpha1.AtprotoValueFrom) (map[string]any, error)
type Options struct {
Bus *bus.Bus
Backend *eventstream.Watches
Children Children
Repos Repos
Values ValueReader
Scope func(workspace string) bool
Log *slog.Logger
}
type Controller struct {
bus *bus.Bus
backend *eventstream.Watches
children Children
repos Repos
values ValueReader
log *slog.Logger
refs *binding.Table[struct{}]
requests chan ctlruntime.Request
mu sync.Mutex
watches map[string]*watch
}
type watch struct {
uid types.UID
cluster string
name string
matches []eventstream.Match
matched bool
status v1alpha1.AtprotoRecordWatchStatus
verifiedGeneration int64
verifyAt time.Time
sink *eventstream.Sink
persisted *v1alpha1.AtprotoRecordWatchStatus
cursorsAt time.Time
}
func New(opts Options) (*Controller, error) {
if opts.Backend == nil {
return nil, errors.New("atprotorecordwatch: a record watch backend is required")
}
if opts.Children == nil {
return nil, errors.New("atprotorecordwatch: an events children writer is required")
}
log := opts.Log
if log == nil {
log = slog.New(slog.DiscardHandler)
}
return &Controller{bus: opts.Bus, backend: opts.Backend, children: opts.Children, repos: opts.Repos, values: opts.Values,
log: log.With("component", "atprotorecordwatch-controller"), refs: binding.New[struct{}](),
requests: make(chan ctlruntime.Request, requestQueue), watches: map[string]*watch{}}, nil
}
func Register(m *ctlruntime.Manager, opts Options) (*Controller, error) {
if err := m.AddToScheme(v1alpha1.AddToScheme, eventsv1.AddToScheme); err != nil {
return nil, err
}
ctl, err := New(opts)
if err != nil {
return nil, err
}
ctl.Provide()
options := []ctlruntime.Option{ctlruntime.Export(Export),
ctlruntime.WatchesIn(EventsExport, func(_ context.Context, _ ctlruntime.Request, obj *eventsv1.EventStream) []ctlruntime.Request {
return owner(obj)
}),
ctlruntime.WatchesIn(EventsExport, func(_ context.Context, _ ctlruntime.Request, obj *eventsv1.EventSource) []ctlruntime.Request {
return owner(obj)
}),
ctlruntime.WatchesIn(RepoExport, func(_ context.Context, src ctlruntime.Request, obj *v1alpha1.AtprotoRepo) []ctlruntime.Request {
return ctl.users(src.Workspace, RepoKind, obj.Name)
}),
ctlruntime.WatchesIn(RepoExport, func(_ context.Context, src ctlruntime.Request, obj *v1alpha1.LocalRepo) []ctlruntime.Request {
return ctl.users(src.Workspace, LocalRepoKind, obj.Name)
}),
ctlruntime.WatchesIn(RepoExport, func(_ context.Context, src ctlruntime.Request, obj *v1alpha1.RemoteRepo) []ctlruntime.Request {
return ctl.users(src.Workspace, RemoteRepoKind, obj.Name)
}),
ctlruntime.Watches(func(_ context.Context, src ctlruntime.Request, obj *v1alpha1.MarketIdentity) []ctlruntime.Request {
return ctl.users(src.Workspace, atprotorecord.MarketIdentityKind, obj.Name)
}),
ctlruntime.Watches(func(_ context.Context, src ctlruntime.Request, obj *v1alpha1.AtprotoRecord) []ctlruntime.Request {
return ctl.users(src.Workspace, atprotorecord.Kind, obj.Name)
}),
ctlruntime.Events(ctl.requests),
ctlruntime.Gone(ctl.gone)}
if opts.Scope != nil {
options = append(options, ctlruntime.InWorkspaces(opts.Scope))
}
if err := ctlruntime.For[*v1alpha1.AtprotoRecordWatch](m, Name, ctl, options...); err != nil {
return nil, fmt.Errorf("atprotorecordwatch: registering the AtprotoRecordWatch controller: %w", err)
}
return ctl, nil
}
func (c *Controller) Provide() {
bus.Provide[events.Sink](c.bus, schema.GroupKind{Group: v1alpha1.GroupName, Kind: Kind}, events.CapabilityDeliver, c.open)
}
func (c *Controller) open(_ context.Context, to commonv1.ObjectRef, _ []string) (events.Sink, error) {
c.mu.Lock()
defer c.mu.Unlock()
w := c.watches[binding.Key(to.Workspace, to.Name)]
if w == nil {
return nil, fmt.Errorf("%w: the AtprotoRecordWatch %s is not reconciled", ErrNotReconciled, to.String())
}
return w.sink, nil
}
func owner(obj metav1.Object) []ctlruntime.Request {
if name := strings.TrimSpace(obj.GetLabels()[eventstream.RecordWatchLabel]); name != "" {
return []ctlruntime.Request{{Name: name}}
}
for _, ref := range obj.GetOwnerReferences() {
if ref.Kind == Kind && strings.HasPrefix(ref.APIVersion, v1alpha1.GroupName+"/") {
return []ctlruntime.Request{{Name: ref.Name}}
}
}
return nil
}
func (c *Controller) users(workspace, kind, name string) []ctlruntime.Request {
var out []ctlruntime.Request
for _, t := range c.refs.Users(binding.Key(workspace, kind, name)) {
out = append(out, ctlruntime.Request{Cluster: t.Cluster, Workspace: t.Workspace, Name: t.Name})
}
return out
}
func (c *Controller) bind(req ctlruntime.Request, obj *v1alpha1.AtprotoRecordWatch) *watch {
key := binding.Key(req.Workspace, req.Name)
c.mu.Lock()
w := c.watches[key]
created := w == nil
if created {
w = &watch{name: req.Name, cluster: req.Cluster}
w.sink = c.backend.NewSink(req.Name, func() ([]eventstream.Match, bool) {
c.mu.Lock()
defer c.mu.Unlock()
return w.matches, w.matched
}, func() {
c.mu.Lock()
target := ctlruntime.Request{Cluster: w.cluster, Name: w.name}
c.mu.Unlock()
c.requests <- target
})
c.watches[key] = w
}
w.cluster = req.Cluster
if w.uid != obj.UID {
w.uid = obj.UID
w.status = v1alpha1.AtprotoRecordWatchStatus{Records: slices.Clone(obj.Status.Records), Deleted: slices.Clone(obj.Status.Deleted),
Cursors: slices.Clone(obj.Status.Cursors)}
w.matches, w.matched = nil, false
w.verifiedGeneration, w.verifyAt = 0, time.Time{}
w.persisted, w.cursorsAt = nil, time.Time{}
w.sink.Take()
}
c.mu.Unlock()
if created {
c.bus.NotifyBound(bus.Key{Workspace: req.Workspace, Kind: Kind, Name: req.Name})
}
return w
}
func (c *Controller) setMatches(w *watch, matches []eventstream.Match, ok bool) {
c.mu.Lock()
defer c.mu.Unlock()
w.matches, w.matched = matches, ok
}
func (c *Controller) gone(ctx context.Context, req ctlruntime.Request) {
c.release(req)
streams, err := c.children.EventStreams(ctx, req.Workspace, req.Name)
if err != nil {
c.log.Warn("listing the EventStreams of a gone AtprotoRecordWatch", "workspace", req.Workspace, "name", req.Name, "err", err.Error())
return
}
for _, stream := range streams {
if err := c.children.DeleteEventStream(ctx, req.Workspace, stream.Name); err != nil && !apierrors.IsNotFound(err) {
c.log.Warn("collecting an EventStream", "workspace", req.Workspace, "name", stream.Name, "err", err.Error())
}
}
}
func (c *Controller) release(req ctlruntime.Request) {
c.refs.Set(binding.Target{Cluster: req.Cluster, Workspace: req.Workspace, Name: req.Name}, nil)
key := binding.Key(req.Workspace, req.Name)
c.mu.Lock()
w := c.watches[key]
delete(c.watches, key)
c.mu.Unlock()
if w != nil {
w.sink.Close()
}
}
func (c *Controller) Reconcile(ctx context.Context, req ctlruntime.Request, obj *v1alpha1.AtprotoRecordWatch) (ctlruntime.Result, error) {
if !obj.DeletionTimestamp.IsZero() {
c.gone(ctx, req)
return ctlruntime.Result{}, nil
}
w := c.bind(req, obj)
w.sink.SetRkeys(obj.Spec.Rkeys)
w.status = eventstream.KeepRkeys(eventstream.MergeStatus(w.status, w.sink.Take()), obj.Spec.Rkeys)
generation := obj.Generation
repo, repoKeys, err := c.repo(ctx, req.Workspace, obj.Spec)
c.refs.Set(binding.Target{Cluster: req.Cluster, Workspace: req.Workspace, Name: req.Name}, &struct{}{},
append(repoKeys, matchKeys(req.Workspace, obj.Spec)...)...)
if err == nil {
var plans []eventstream.SourcePlan
if plans, err = eventstream.PlanSources(req.Workspace, obj.Name, obj.Spec, repo); err == nil {
return c.sync(ctx, req, obj, w, plans)
}
}
var transient transientError
if errors.As(err, &transient) {
return ctlruntime.Result{}, errors.Join(err, c.apply(ctx, req, obj, w, failedConditions("ReconcileFailed", err, generation)))
}
return ctlruntime.Result{}, c.apply(ctx, req, obj, w, failedConditions("InvalidSpec", err, generation))
}
func (c *Controller) sync(ctx context.Context, req ctlruntime.Request, obj *v1alpha1.AtprotoRecordWatch, w *watch,
plans []eventstream.SourcePlan) (ctlruntime.Result, error) {
generation := obj.Generation
unwatched := false
matches, err := eventstream.ResolveMatches(ctx, obj.Spec.Match, func(ctx context.Context, from v1alpha1.AtprotoValueFrom) (map[string]any, error) {
if object, ok, err := atprotorecord.ReadOwn(ctx, req, from); ok {
return object, err
}
unwatched = true
if c.values == nil {
return nil, fmt.Errorf("no reader for %s %s", from.Kind, from.Name)
}
return c.values(ctx, cmp.Or(strings.TrimSpace(from.Workspace), req.Workspace), from)
})
if err != nil {
c.setMatches(w, nil, false)
pending := []metav1.Condition{
conds.New(eventstream.RecordWatchConditionReady, false, "MatchPending", err.Error(), generation),
conds.New(eventstream.RecordWatchConditionProgressing, true, "MatchPending", err.Error(), generation),
conds.New(eventstream.RecordWatchConditionFailed, false, "NoError", err.Error(), generation),
conds.New(eventstream.RecordWatchConditionVerified, false, "MatchPending", err.Error(), generation),
}
applyErr := c.apply(ctx, req, obj, w, pending)
if unwatched {
return ctlruntime.Result{}, errors.Join(err, applyErr)
}
return ctlruntime.Result{}, applyErr
}
c.setMatches(w, matches, true)
ownerRef := metav1.OwnerReference{APIVersion: v1alpha1.GroupVersion.String(), Kind: Kind, Name: obj.Name, UID: obj.UID}
wanted := map[string]bool{}
for _, plan := range plans {
wanted[plan.Name] = true
if err := c.applyChildren(ctx, req.Workspace, ownerRef, plan); err != nil {
return ctlruntime.Result{}, errors.Join(err, c.apply(ctx, req, obj, w, failedConditions("ReconcileFailed", err, generation)))
}
}
listed, err := c.children.EventStreams(ctx, req.Workspace, obj.Name)
if err != nil {
return ctlruntime.Result{}, err
}
streams := map[string]eventsv1.EventStream{}
for _, stream := range listed {
if wanted[stream.Name] {
streams[stream.Name] = stream
continue
}
if err := c.children.DeleteEventStream(ctx, req.Workspace, stream.Name); err != nil && !apierrors.IsNotFound(err) {
c.log.Warn("collecting a stale EventStream", "workspace", req.Workspace, "name", stream.Name, "err", err.Error())
}
}
conditions := eventstream.StreamConditions(plans, streams, generation)
verified, reason, message := c.verify(ctx, obj, w, matches)
if ready := meta.FindStatusCondition(conditions, eventstream.RecordWatchConditionReady); ready == nil || ready.Status != metav1.ConditionTrue {
verified, reason, message = false, "StreamNotReady", "the EventStreams are starting"
}
conditions = append(conditions, conds.New(eventstream.RecordWatchConditionVerified, verified, reason, message, generation))
if err := c.apply(ctx, req, obj, w, conditions); err != nil {
return ctlruntime.Result{}, err
}
return ctlruntime.Result{RequeueAfter: max(time.Until(w.verifyAt), time.Second)}, nil
}
func (c *Controller) verify(ctx context.Context, obj *v1alpha1.AtprotoRecordWatch, w *watch, matches []eventstream.Match) (bool, string, string) {
if w.verifiedGeneration == obj.Generation && time.Now().Before(w.verifyAt) {
if existing := meta.FindStatusCondition(obj.Status.Conditions, eventstream.RecordWatchConditionVerified); existing != nil &&
existing.Reason != "StreamNotReady" && existing.Reason != "MatchPending" {
return existing.Status == metav1.ConditionTrue, existing.Reason, existing.Message
}
return true, "Verified", eventstream.Verification{Records: w.status.Records}.Message()
}
result := c.backend.Verify(ctx, w.status.Records, matches)
w.status.Records = result.Records
w.status.Deleted = eventstream.NoteDeleted(w.status.Deleted, result.Gone)
w.verifiedGeneration = obj.Generation
if !result.Verified() {
w.verifyAt = time.Now().Add(VerifyRetry)
return false, "VerifyFailed", result.Message()
}
w.verifyAt = time.Now().Add(VerifyInterval)
return true, "Verified", result.Message()
}
func (c *Controller) applyChildren(ctx context.Context, workspace string, ownerRef metav1.OwnerReference, plan eventstream.SourcePlan) error {
if backend, abstract, ok := eventstream.PdsSourcesFor(ownerRef, workspace, plan); ok {
if err := c.children.ApplyPdsEventSource(ctx, workspace, backend); err != nil {
return fmt.Errorf("applying PdsEventSource %s: %w", backend.Name, err)
}
if err := c.children.ApplyEventSource(ctx, workspace, abstract); err != nil {
return fmt.Errorf("applying EventSource %s: %w", abstract.Name, err)
}
}
stream := eventstream.EventStreamFor(ownerRef, workspace, plan)
if err := c.children.ApplyEventStream(ctx, workspace, stream); err != nil {
return fmt.Errorf("applying EventStream %s: %w", stream.Name, err)
}
return nil
}
func (c *Controller) apply(ctx context.Context, req ctlruntime.Request, obj *v1alpha1.AtprotoRecordWatch, w *watch, conditions []metav1.Condition) error {
next := obj.DeepCopy()
next.Status = v1alpha1.AtprotoRecordWatchStatus{ObservedGeneration: obj.Generation, Records: w.status.Records, Deleted: w.status.Deleted,
Cursors: w.status.Cursors, Conditions: slices.Clone(obj.Status.Conditions)}
for _, condition := range conditions {
conds.Set(&next.Status.Conditions, condition.ObservedGeneration, condition.Type, condition.Status == metav1.ConditionTrue,
condition.Reason, condition.Message)
}
now := time.Now()
cursorsMoved := w.persisted == nil || !slices.Equal(next.Status.Cursors, w.persisted.Cursors)
if w.persisted != nil {
held := next.Status
held.Cursors = w.persisted.Cursors
if sameStatus(held, *w.persisted) {
if !cursorsMoved {
return nil
}
if now.Sub(w.cursorsAt) < delivery.ProgressDelay {
w.sink.MarkProgress()
return nil
}
}
}
if err := req.ApplyStatus(ctx, next); err != nil {
return err
}
persisted := *next.Status.DeepCopy()
w.persisted = &persisted
if cursorsMoved {
w.cursorsAt = now
}
return nil
}
func sameStatus(a, b v1alpha1.AtprotoRecordWatchStatus) bool {
strip := func(in []metav1.Condition) []metav1.Condition {
out := make([]metav1.Condition, len(in))
for i, c := range in {
c.LastTransitionTime = metav1.Time{}
out[i] = c
}
return out
}
a.Conditions, b.Conditions = strip(a.Conditions), strip(b.Conditions)
return reflect.DeepEqual(a, b)
}
func failedConditions(reason string, err error, generation int64) []metav1.Condition {
return []metav1.Condition{
conds.New(eventstream.RecordWatchConditionReady, false, reason, err.Error(), generation),
conds.New(eventstream.RecordWatchConditionProgressing, false, reason, err.Error(), generation),
conds.New(eventstream.RecordWatchConditionFailed, true, reason, err.Error(), generation),
conds.New(eventstream.RecordWatchConditionVerified, false, reason, err.Error(), generation),
}
}