johnandersen777.bsky.social/socialweb-computer-kcp

Switch repository

Loading…

package eventstream

import (
	"cmp"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"log/slog"
	"reflect"
	"slices"
	"strings"
	"sync"
	"time"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	commonv1 "github.com/johnandersen777/socialweb-computer-kcp/api/common/v1alpha1"
	eventsv1 "github.com/johnandersen777/socialweb-computer-kcp/api/events/v1alpha1"
	v1alpha1 "github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/bus"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/events"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/atprotorecordwatch"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/eventsource"
	ctlruntime "github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/events/delivery"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/conds"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/debounce"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/supervise"
)

const (
	Name = "eventstream"

	Export = "events"

	Finalizer = "events.socialweb.computer/eventstream"

	ReasonInvalidSpec = "InvalidSpec"

	ReasonSourceMissing = "SourceMissing"

	ReasonSourceUnbound = "SourceUnbound"

	ReasonResolveFailed = "ResolveFailed"

	ReasonSourceNotReady = "SourceNotReady"

	ReasonSinkNotReady = "SinkNotReady"

	ReasonSuspended = "Suspended"

	statusTimeout = 10 * time.Second

	boundQueue = 64
)

type Options struct {
	Bus *bus.Bus

	Cursors events.CursorStore

	FlushDelay time.Duration

	Log *slog.Logger
}

type Controller struct {
	bus *bus.Bus

	cursors events.CursorStore

	flushDelay time.Duration

	log *slog.Logger

	root context.Context

	cancelRoot context.CancelFunc

	mu sync.Mutex

	streams map[string]*stream

	sources refIndex

	sinks refIndex

	bound chan ctlruntime.Request
}

type refIndex struct {
	by map[string]map[string]ctlruntime.Request

	of map[string]string
}

func newRefIndex() refIndex {
	return refIndex{by: map[string]map[string]ctlruntime.Request{}, of: map[string]string{}}
}

func (x refIndex) users(target string) []ctlruntime.Request {
	out := make([]ctlruntime.Request, 0, len(x.by[target]))
	for _, r := range x.by[target] {
		out = append(out, r)
	}
	return out
}

func (x refIndex) set(streamKey, target string, req ctlruntime.Request) {
	if prior, ok := x.of[streamKey]; ok {
		delete(x.by[prior], streamKey)
		if len(x.by[prior]) == 0 {
			delete(x.by, prior)
		}
		delete(x.of, streamKey)
	}
	if target == "" {
		return
	}
	if x.by[target] == nil {
		x.by[target] = map[string]ctlruntime.Request{}
	}
	x.by[target][streamKey] = ctlruntime.Request{Cluster: req.Cluster, Namespace: req.Namespace, Name: req.Name}
	x.of[streamKey] = target
}

func Register(ctx context.Context, m *ctlruntime.Manager, opts Options) (*Controller, error) {
	log := opts.Log
	if log == nil {
		log = slog.New(slog.DiscardHandler)
	}
	cursors := opts.Cursors
	if cursors == nil {
		cursors = delivery.CursorStore{}
	}
	root, cancel := context.WithCancel(ctx)
	c := &Controller{bus: opts.Bus, cursors: cursors, flushDelay: opts.FlushDelay, log: log.With("component", "eventstream-controller"),
		root: root, cancelRoot: cancel, streams: map[string]*stream{}, sources: newRefIndex(), sinks: newRefIndex(),
		bound: make(chan ctlruntime.Request, boundQueue)}
	c.wakeOnBind()
	err := ctlruntime.For[*eventsv1.EventStream](m, Name, c, ctlruntime.Export(Export), ctlruntime.Finalizer(Finalizer), ctlruntime.Events(c.bound),
		ctlruntime.Watches[*eventsv1.EventSource](func(_ context.Context, src ctlruntime.Request, _ *eventsv1.EventSource) []ctlruntime.Request {
			return c.streamsOf(key(src.Workspace, src.Name))
		}))
	if err != nil {
		cancel()
		return nil, fmt.Errorf("eventstream: registering the EventStream controller: %w", err)
	}
	return c, nil
}

func (c *Controller) wakeOnBind() {
	c.bus.Subscribe(func(k bus.Key) {
		var reqs []ctlruntime.Request
		switch k.Kind {
		case eventsv1.EventSourceKind:
			reqs = c.streamsOf(key(k.Workspace, k.Name))
		case events.RecordWatchKind:
			reqs = c.streamsTo(key(k.Workspace, k.Name))
		}
		for _, req := range reqs {
			select {
			case c.bound <- req:
			default:
			}
		}
	})
}

func (c *Controller) Run(ctx context.Context) error {
	<-ctx.Done()
	c.cancelRoot()
	c.mu.Lock()
	streams := make([]*stream, 0, len(c.streams))
	for k, s := range c.streams {
		streams = append(streams, s)
		delete(c.streams, k)
	}
	c.mu.Unlock()
	for _, s := range streams {
		s.stop(true)
	}
	return nil
}

func key(workspace, name string) string {
	return strings.TrimSpace(workspace) + "/" + strings.TrimSpace(name)
}

func (c *Controller) streamsOf(source string) []ctlruntime.Request {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.sources.users(source)
}

func (c *Controller) streamsTo(sink string) []ctlruntime.Request {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.sinks.users(sink)
}

func (c *Controller) indexSource(streamKey, source string, req ctlruntime.Request) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.sources.set(streamKey, source, req)
}

func (c *Controller) indexSink(streamKey, sink string, req ctlruntime.Request) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.sinks.set(streamKey, sink, req)
}

func SourceRef(workspace string, ref commonv1.ObjectRef) commonv1.ObjectRef {
	return commonv1.ObjectRef{Group: eventsv1.GroupName, Kind: eventsv1.EventSourceKind, Name: strings.TrimSpace(ref.Name), Workspace: ref.InWorkspace(workspace)}
}

func SinkRef(workspace string, sink eventsv1.EventStreamSink) (commonv1.ObjectRef, error) {
	name := strings.TrimSpace(sink.Name)
	if name == "" {
		return commonv1.ObjectRef{}, errors.New("spec.sink needs a name")
	}
	switch sink.Kind {
	case eventsv1.EventStreamSinkRecordWatch:
		return commonv1.ObjectRef{Group: v1alpha1.GroupName, Kind: events.RecordWatchKind, Name: name,
			Workspace: cmp.Or(strings.TrimSpace(sink.Namespace), workspace)}, nil
	case eventsv1.EventStreamSinkHandler:
		return commonv1.ObjectRef{Group: eventsv1.GroupName, Kind: events.HandlerKind, Name: name, Workspace: workspace}, nil
	default:
		return commonv1.ObjectRef{}, fmt.Errorf("unknown sink kind %q", sink.Kind)
	}
}

func (c *Controller) Reconcile(ctx context.Context, req ctlruntime.Request, es *eventsv1.EventStream) (ctlruntime.Result, error) {
	k := key(req.Workspace, es.Name)
	if !es.DeletionTimestamp.IsZero() {
		c.indexSource(k, "", req)
		c.indexSink(k, "", req)
		c.stop(k, false)
		return ctlruntime.Result{}, nil
	}
	if strings.TrimSpace(es.Spec.SourceRef.Name) == "" {
		c.stop(k, true)
		return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhaseFailed, false, true, ReasonInvalidSpec, "spec.sourceRef needs a name")
	}
	src := SourceRef(req.Workspace, es.Spec.SourceRef)
	c.indexSource(k, key(src.Workspace, src.Name), req)
	if es.Spec.Suspend {
		c.stop(k, true)
		return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhaseSuspended, false, false, ReasonSuspended, "suspended")
	}
	sinkRef, err := SinkRef(req.Workspace, es.Spec.Sink)
	if err != nil {
		c.stop(k, true)
		return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhaseFailed, false, true, ReasonInvalidSpec, err.Error())
	}
	sourceState := ""
	var rewind int64
	if src.Workspace == req.Workspace {
		var live eventsv1.EventSource
		err := req.Get(ctx, "", src.Name, &live)
		switch {
		case apierrors.IsNotFound(err):
			c.stop(k, true)
			return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhasePending, false, false, ReasonSourceMissing,
				"the EventSource "+src.Name+" does not exist")
		case err != nil:
			return ctlruntime.Result{}, err
		}
		bound := meta.FindStatusCondition(live.Status.Conditions, eventsv1.EventSourceConditionBackendBound)
		if bound == nil || bound.Status != metav1.ConditionTrue || bound.ObservedGeneration != live.Generation {
			c.stop(k, true)
			return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhasePending, false, false, ReasonSourceUnbound,
				"the EventSource "+src.Name+" is not bound to its backend")
		}
		sourceState = fmt.Sprintf("%s/%d/%s", live.UID, live.Generation, live.Spec.BackendRef.String())
		rewind = rewindFor(live.Spec.BackendRef.Kind)
	}
	fingerprint, err := fingerprintOf(es.Spec, src, sinkRef, sourceState)
	if err != nil {
		return ctlruntime.Result{}, err
	}
	c.mu.Lock()
	current := c.streams[k]
	c.mu.Unlock()
	if current != nil && current.uid == string(es.UID) && current.fingerprint == fingerprint {
		current.update(req, es)
		return ctlruntime.Result{}, nil
	}
	c.stop(k, true)
	self := commonv1.ObjectRef{Group: eventsv1.GroupName, Kind: "EventStream", Name: es.Name, Workspace: req.Workspace}
	filter := events.Filter{Collections: es.Spec.Filter.Collections, DIDs: es.Spec.Filter.DIDs}
	source, err := bus.Resolve[events.Source](ctx, c.bus, self, src, bus.Capability{Name: events.CapabilitySubscribe, Scope: filter.Scope()})
	if errors.Is(err, eventsource.ErrNotReconciled) {
		c.log.Info("an EventStream waits for its EventSource", "workspace", req.Workspace, "name", es.Name, "err", err.Error())
		return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhasePending, false, false, ReasonSourceNotReady,
			"the EventSource "+src.String()+" is not reconciled yet")
	}
	if err != nil {
		return ctlruntime.Result{}, errors.Join(err, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhaseFailed, false, true, ReasonResolveFailed,
			"resolving the EventSource "+src.String()+": "+err.Error()))
	}
	c.indexSink(k, key(sinkRef.Workspace, sinkRef.Name), req)
	sink, err := bus.Resolve[events.Sink](ctx, c.bus, self, sinkRef, bus.Capability{Name: events.CapabilityDeliver, Scope: []string{sinkRef.Name}})
	if errors.Is(err, atprotorecordwatch.ErrNotReconciled) {
		c.log.Info("an EventStream waits for its sink", "workspace", req.Workspace, "name", es.Name, "err", err.Error())
		return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhasePending, false, false, ReasonSinkNotReady,
			"the sink "+sinkRef.String()+" is not reconciled yet")
	}
	if err != nil {
		return ctlruntime.Result{}, errors.Join(err, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhaseFailed, false, true, ReasonResolveFailed,
			"resolving the sink "+sinkRef.String()+": "+err.Error()))
	}
	s := c.start(ctx, k, req, es, fingerprint, filter, rewind, source, sink)
	s.flush.Mark()
	return ctlruntime.Result{}, s.publish(ctx, false)
}

func rewindFor(kind string) int64 {
	switch kind {
	case eventsv1.RelayEventSourceKind:
		return delivery.RelayRewindEvents
	case eventsv1.JetstreamEventSourceKind:
		return delivery.JetstreamRewindMicros
	}
	return 0
}

func fingerprintOf(spec eventsv1.EventStreamSpec, src, sink commonv1.ObjectRef, sourceState string) (string, error) {
	raw, err := json.Marshal(struct {
		Spec   eventsv1.EventStreamSpec
		Source commonv1.ObjectRef
		Sink   commonv1.ObjectRef
		State  string
	}{spec, src, sink, sourceState})
	if err != nil {
		return "", fmt.Errorf("eventstream: fingerprinting the spec: %w", err)
	}
	sum := sha256.Sum256(raw)
	return hex.EncodeToString(sum[:]), nil
}

func (c *Controller) stop(k string, flush bool) {
	c.mu.Lock()
	s := c.streams[k]
	delete(c.streams, k)
	c.mu.Unlock()
	if s != nil {
		s.stop(flush)
	}
}

func (c *Controller) applyStatus(ctx context.Context, req ctlruntime.Request, es *eventsv1.EventStream, phase eventsv1.EventStreamPhase,
	ready, failed bool, reason, message string) error {
	status := eventsv1.EventStreamStatus{ObservedGeneration: es.Generation, Phase: phase, Cursor: es.Status.Cursor, Lag: es.Status.Lag,
		LastEventAt: es.Status.LastEventAt, ServerFilter: es.Status.ServerFilter, Conditions: append([]metav1.Condition(nil), es.Status.Conditions...)}
	setConditions(&status, es.Generation, ready, false, failed, reason, message)
	next := es.DeepCopy()
	next.Status = status
	return req.ApplyStatus(ctx, next)
}

func setConditions(status *eventsv1.EventStreamStatus, generation int64, ready, lagging, failed bool, reason, message string) {
	readyReason := reason
	if ready {
		readyReason = delivery.ReasonStreamOpen
	}
	conds.Set(&status.Conditions, generation, eventsv1.EventStreamConditionReady, ready, readyReason, message)
	lagReason, lagMessage := "UpToDate", "the stream is current"
	if lagging {
		lagReason, lagMessage = "CursorGap", fmt.Sprintf("%d events behind", status.Lag)
	}
	conds.Set(&status.Conditions, generation, eventsv1.EventStreamConditionLagging, lagging, lagReason, lagMessage)
	failedReason, failedMessage := "NoError", "the stream is healthy"
	if failed {
		failedReason, failedMessage = reason, message
	}
	conds.Set(&status.Conditions, generation, eventsv1.EventStreamConditionFailed, failed, failedReason, failedMessage)
}

type stream struct {
	c *Controller

	key string

	uid string

	fingerprint string

	loop *delivery.Loop

	group *supervise.Group

	flush *debounce.Debounce

	progress *debounce.Debounce

	rewind int64

	mu sync.Mutex

	req ctlruntime.Request

	es *eventsv1.EventStream

	publishMu sync.Mutex

	last *eventsv1.EventStreamStatus
}

func (c *Controller) start(ctx context.Context, k string, req ctlruntime.Request, es *eventsv1.EventStream, fingerprint string,
	filter events.Filter, rewind int64, source events.Source, sink events.Sink) *stream {
	s := &stream{c: c, key: k, uid: string(es.UID), fingerprint: fingerprint, rewind: rewind, req: req, es: es.DeepCopy()}
	s.flush = debounce.New(c.flushDelay, s.flushNow)
	s.progress = debounce.New(delivery.ProgressDelay, s.flushProgress)
	s.loop = delivery.New(delivery.Config{Filter: filter, Rewind: rewind, Live: es.Spec.Resume == eventsv1.EventStreamResumeLive},
		delivery.Deps{Source: source, Sink: sink, Resume: s.resume, Changed: s.flush.Mark, Progress: s.progress.Mark,
			Log: c.log.With("stream", k)})
	streamCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
	stopRoot := context.AfterFunc(c.root, cancel)
	s.group = supervise.New(streamCtx)
	s.group.Go("eventstream "+k, func(ctx context.Context) error {
		defer stopRoot()
		defer cancel()
		s.loop.Run(ctx)
		return nil
	})
	c.mu.Lock()
	c.streams[k] = s
	c.mu.Unlock()
	return s
}

func (s *stream) update(req ctlruntime.Request, es *eventsv1.EventStream) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.req, s.es = req, es.DeepCopy()
}

func (s *stream) stop(flush bool) {
	s.flush.Stop()
	s.progress.Stop()
	_ = s.group.Stop()
	if flush {
		s.flushAt(true)
	}
}

func (s *stream) resume() *int64 {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.es.Spec.Resume == eventsv1.EventStreamResumeLive {
		return nil
	}
	recent := s.es.Spec.Resume == eventsv1.EventStreamResumeRecent
	if saved, ok := s.c.cursors.Load(s.req.Workspace, s.es.Name, s.uid); ok {
		if recent && !delivery.Recent(saved.SavedAt, time.Now()) {
			return nil
		}
		return delivery.Rewound(saved.Cursor, s.rewind)
	}
	if s.es.Status.Cursor > 0 {
		if recent && (s.es.Status.LastEventAt == nil || !delivery.Recent(s.es.Status.LastEventAt.Time, time.Now())) {
			return nil
		}
		return delivery.Rewound(s.es.Status.Cursor, s.rewind)
	}
	if s.es.Spec.Cursor != nil {
		v := *s.es.Spec.Cursor
		return &v
	}
	return nil
}

func (s *stream) flushNow() {
	s.flushAt(false)
}

func (s *stream) flushProgress() {
	s.flushAt(true)
}

func (s *stream) flushAt(progress bool) {
	ctx, cancel := context.WithTimeout(context.WithoutCancel(s.c.root), statusTimeout)
	defer cancel()
	if err := s.publish(ctx, progress); err != nil {
		s.c.log.Warn("writing the EventStream status", "stream", s.key, "err", err.Error())
	}
	state := s.loop.State()
	s.mu.Lock()
	workspace, name, live := s.req.Workspace, s.es.Name, s.es.Spec.Resume == eventsv1.EventStreamResumeLive
	s.mu.Unlock()
	if state.Started && !live {
		if err := s.c.cursors.Save(workspace, name, s.uid, state.Cursor); err != nil {
			s.c.log.Warn("saving the EventStream cursor", "stream", s.key, "err", err.Error())
		}
	}
}

func (s *stream) publish(ctx context.Context, progress bool) error {
	s.publishMu.Lock()
	defer s.publishMu.Unlock()
	state := s.loop.State()
	s.mu.Lock()
	req, es := s.req, s.es
	s.mu.Unlock()
	status := eventsv1.EventStreamStatus{ObservedGeneration: es.Generation, Phase: eventsv1.EventStreamPhase(state.Phase), Cursor: es.Status.Cursor,
		Lag: state.Lag, LastEventAt: es.Status.LastEventAt, ServerFilter: es.Status.ServerFilter, Conditions: append([]metav1.Condition(nil), es.Status.Conditions...)}
	if s.last != nil {
		status.Conditions = append([]metav1.Condition(nil), s.last.Conditions...)
		status.Cursor, status.LastEventAt, status.ServerFilter = s.last.Cursor, s.last.LastEventAt, s.last.ServerFilter
	}
	if f := state.ServerFilter; f != nil {
		status.ServerFilter = &eventsv1.EventStreamServerFilter{Collections: slices.Clone(f.Collections), DIDs: slices.Clone(f.DIDs),
			Supported: eventsv1.EventStreamServerFilterSupport(f.Supported)}
	}
	if state.Started {
		status.Cursor = state.Cursor
		at := metav1.NewTime(state.LastEventAt.UTC().Truncate(time.Second))
		status.LastEventAt = &at
	}
	setConditions(&status, es.Generation, state.Ready, state.Lag > 0, state.Failed, state.Reason, state.Message)
	if s.last != nil && statusEqual(*s.last, status) {
		return nil
	}
	if !progress && s.last != nil && statusEqual(withoutProgress(*s.last), withoutProgress(status)) {
		s.progress.Mark()
		return nil
	}
	next := es.DeepCopy()
	next.Status = status
	if err := req.ApplyStatus(ctx, next); err != nil {
		if apierrors.IsNotFound(err) {
			return nil
		}
		return err
	}
	s.last = &status
	return nil
}

func withoutProgress(status eventsv1.EventStreamStatus) eventsv1.EventStreamStatus {
	status.Cursor, status.Lag, status.LastEventAt = 0, 0, nil
	status.Conditions = slices.DeleteFunc(slices.Clone(status.Conditions), func(c metav1.Condition) bool {
		return c.Type == eventsv1.EventStreamConditionLagging
	})
	return status
}

func statusEqual(a, b eventsv1.EventStreamStatus) bool {
	if a.ObservedGeneration != b.ObservedGeneration || a.Phase != b.Phase || a.Cursor != b.Cursor || a.Lag != b.Lag {
		return false
	}
	if (a.LastEventAt == nil) != (b.LastEventAt == nil) || (a.LastEventAt != nil && !a.LastEventAt.Equal(b.LastEventAt)) {
		return false
	}
	if !reflect.DeepEqual(a.ServerFilter, b.ServerFilter) {
		return false
	}
	if len(a.Conditions) != len(b.Conditions) {
		return false
	}
	for i := range a.Conditions {
		x, y := a.Conditions[i], b.Conditions[i]
		if x.Type != y.Type || x.Status != y.Status || x.Reason != y.Reason || x.Message != y.Message || x.ObservedGeneration != y.ObservedGeneration {
			return false
		}
	}
	return true
}