0
0
package eventstream
import (
"cmp"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log/slog"
"reflect"
"slices"
"strings"
"sync"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
commonv1 "github.com/johnandersen777/socialweb-computer-kcp/api/common/v1alpha1"
eventsv1 "github.com/johnandersen777/socialweb-computer-kcp/api/events/v1alpha1"
v1alpha1 "github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/bus"
"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/events"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/atprotorecordwatch"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/eventsource"
ctlruntime "github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime"
"github.com/johnandersen777/socialweb-computer-kcp/internal/events/delivery"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/conds"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/debounce"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/supervise"
)
const (
Name = "eventstream"
Export = "events"
Finalizer = "events.socialweb.computer/eventstream"
ReasonInvalidSpec = "InvalidSpec"
ReasonSourceMissing = "SourceMissing"
ReasonSourceUnbound = "SourceUnbound"
ReasonResolveFailed = "ResolveFailed"
ReasonSourceNotReady = "SourceNotReady"
ReasonSinkNotReady = "SinkNotReady"
ReasonSuspended = "Suspended"
statusTimeout = 10 * time.Second
boundQueue = 64
)
type Options struct {
Bus *bus.Bus
Cursors events.CursorStore
FlushDelay time.Duration
Log *slog.Logger
}
type Controller struct {
bus *bus.Bus
cursors events.CursorStore
flushDelay time.Duration
log *slog.Logger
root context.Context
cancelRoot context.CancelFunc
mu sync.Mutex
streams map[string]*stream
sources refIndex
sinks refIndex
bound chan ctlruntime.Request
}
type refIndex struct {
by map[string]map[string]ctlruntime.Request
of map[string]string
}
func newRefIndex() refIndex {
return refIndex{by: map[string]map[string]ctlruntime.Request{}, of: map[string]string{}}
}
func (x refIndex) users(target string) []ctlruntime.Request {
out := make([]ctlruntime.Request, 0, len(x.by[target]))
for _, r := range x.by[target] {
out = append(out, r)
}
return out
}
func (x refIndex) set(streamKey, target string, req ctlruntime.Request) {
if prior, ok := x.of[streamKey]; ok {
delete(x.by[prior], streamKey)
if len(x.by[prior]) == 0 {
delete(x.by, prior)
}
delete(x.of, streamKey)
}
if target == "" {
return
}
if x.by[target] == nil {
x.by[target] = map[string]ctlruntime.Request{}
}
x.by[target][streamKey] = ctlruntime.Request{Cluster: req.Cluster, Namespace: req.Namespace, Name: req.Name}
x.of[streamKey] = target
}
func Register(ctx context.Context, m *ctlruntime.Manager, opts Options) (*Controller, error) {
log := opts.Log
if log == nil {
log = slog.New(slog.DiscardHandler)
}
cursors := opts.Cursors
if cursors == nil {
cursors = delivery.CursorStore{}
}
root, cancel := context.WithCancel(ctx)
c := &Controller{bus: opts.Bus, cursors: cursors, flushDelay: opts.FlushDelay, log: log.With("component", "eventstream-controller"),
root: root, cancelRoot: cancel, streams: map[string]*stream{}, sources: newRefIndex(), sinks: newRefIndex(),
bound: make(chan ctlruntime.Request, boundQueue)}
c.wakeOnBind()
err := ctlruntime.For[*eventsv1.EventStream](m, Name, c, ctlruntime.Export(Export), ctlruntime.Finalizer(Finalizer), ctlruntime.Events(c.bound),
ctlruntime.Watches[*eventsv1.EventSource](func(_ context.Context, src ctlruntime.Request, _ *eventsv1.EventSource) []ctlruntime.Request {
return c.streamsOf(key(src.Workspace, src.Name))
}))
if err != nil {
cancel()
return nil, fmt.Errorf("eventstream: registering the EventStream controller: %w", err)
}
return c, nil
}
func (c *Controller) wakeOnBind() {
c.bus.Subscribe(func(k bus.Key) {
var reqs []ctlruntime.Request
switch k.Kind {
case eventsv1.EventSourceKind:
reqs = c.streamsOf(key(k.Workspace, k.Name))
case events.RecordWatchKind:
reqs = c.streamsTo(key(k.Workspace, k.Name))
}
for _, req := range reqs {
select {
case c.bound <- req:
default:
}
}
})
}
func (c *Controller) Run(ctx context.Context) error {
<-ctx.Done()
c.cancelRoot()
c.mu.Lock()
streams := make([]*stream, 0, len(c.streams))
for k, s := range c.streams {
streams = append(streams, s)
delete(c.streams, k)
}
c.mu.Unlock()
for _, s := range streams {
s.stop(true)
}
return nil
}
func key(workspace, name string) string {
return strings.TrimSpace(workspace) + "/" + strings.TrimSpace(name)
}
func (c *Controller) streamsOf(source string) []ctlruntime.Request {
c.mu.Lock()
defer c.mu.Unlock()
return c.sources.users(source)
}
func (c *Controller) streamsTo(sink string) []ctlruntime.Request {
c.mu.Lock()
defer c.mu.Unlock()
return c.sinks.users(sink)
}
func (c *Controller) indexSource(streamKey, source string, req ctlruntime.Request) {
c.mu.Lock()
defer c.mu.Unlock()
c.sources.set(streamKey, source, req)
}
func (c *Controller) indexSink(streamKey, sink string, req ctlruntime.Request) {
c.mu.Lock()
defer c.mu.Unlock()
c.sinks.set(streamKey, sink, req)
}
func SourceRef(workspace string, ref commonv1.ObjectRef) commonv1.ObjectRef {
return commonv1.ObjectRef{Group: eventsv1.GroupName, Kind: eventsv1.EventSourceKind, Name: strings.TrimSpace(ref.Name), Workspace: ref.InWorkspace(workspace)}
}
func SinkRef(workspace string, sink eventsv1.EventStreamSink) (commonv1.ObjectRef, error) {
name := strings.TrimSpace(sink.Name)
if name == "" {
return commonv1.ObjectRef{}, errors.New("spec.sink needs a name")
}
switch sink.Kind {
case eventsv1.EventStreamSinkRecordWatch:
return commonv1.ObjectRef{Group: v1alpha1.GroupName, Kind: events.RecordWatchKind, Name: name,
Workspace: cmp.Or(strings.TrimSpace(sink.Namespace), workspace)}, nil
case eventsv1.EventStreamSinkHandler:
return commonv1.ObjectRef{Group: eventsv1.GroupName, Kind: events.HandlerKind, Name: name, Workspace: workspace}, nil
default:
return commonv1.ObjectRef{}, fmt.Errorf("unknown sink kind %q", sink.Kind)
}
}
func (c *Controller) Reconcile(ctx context.Context, req ctlruntime.Request, es *eventsv1.EventStream) (ctlruntime.Result, error) {
k := key(req.Workspace, es.Name)
if !es.DeletionTimestamp.IsZero() {
c.indexSource(k, "", req)
c.indexSink(k, "", req)
c.stop(k, false)
return ctlruntime.Result{}, nil
}
if strings.TrimSpace(es.Spec.SourceRef.Name) == "" {
c.stop(k, true)
return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhaseFailed, false, true, ReasonInvalidSpec, "spec.sourceRef needs a name")
}
src := SourceRef(req.Workspace, es.Spec.SourceRef)
c.indexSource(k, key(src.Workspace, src.Name), req)
if es.Spec.Suspend {
c.stop(k, true)
return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhaseSuspended, false, false, ReasonSuspended, "suspended")
}
sinkRef, err := SinkRef(req.Workspace, es.Spec.Sink)
if err != nil {
c.stop(k, true)
return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhaseFailed, false, true, ReasonInvalidSpec, err.Error())
}
sourceState := ""
var rewind int64
if src.Workspace == req.Workspace {
var live eventsv1.EventSource
err := req.Get(ctx, "", src.Name, &live)
switch {
case apierrors.IsNotFound(err):
c.stop(k, true)
return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhasePending, false, false, ReasonSourceMissing,
"the EventSource "+src.Name+" does not exist")
case err != nil:
return ctlruntime.Result{}, err
}
bound := meta.FindStatusCondition(live.Status.Conditions, eventsv1.EventSourceConditionBackendBound)
if bound == nil || bound.Status != metav1.ConditionTrue || bound.ObservedGeneration != live.Generation {
c.stop(k, true)
return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhasePending, false, false, ReasonSourceUnbound,
"the EventSource "+src.Name+" is not bound to its backend")
}
sourceState = fmt.Sprintf("%s/%d/%s", live.UID, live.Generation, live.Spec.BackendRef.String())
rewind = rewindFor(live.Spec.BackendRef.Kind)
}
fingerprint, err := fingerprintOf(es.Spec, src, sinkRef, sourceState)
if err != nil {
return ctlruntime.Result{}, err
}
c.mu.Lock()
current := c.streams[k]
c.mu.Unlock()
if current != nil && current.uid == string(es.UID) && current.fingerprint == fingerprint {
current.update(req, es)
return ctlruntime.Result{}, nil
}
c.stop(k, true)
self := commonv1.ObjectRef{Group: eventsv1.GroupName, Kind: "EventStream", Name: es.Name, Workspace: req.Workspace}
filter := events.Filter{Collections: es.Spec.Filter.Collections, DIDs: es.Spec.Filter.DIDs}
source, err := bus.Resolve[events.Source](ctx, c.bus, self, src, bus.Capability{Name: events.CapabilitySubscribe, Scope: filter.Scope()})
if errors.Is(err, eventsource.ErrNotReconciled) {
c.log.Info("an EventStream waits for its EventSource", "workspace", req.Workspace, "name", es.Name, "err", err.Error())
return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhasePending, false, false, ReasonSourceNotReady,
"the EventSource "+src.String()+" is not reconciled yet")
}
if err != nil {
return ctlruntime.Result{}, errors.Join(err, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhaseFailed, false, true, ReasonResolveFailed,
"resolving the EventSource "+src.String()+": "+err.Error()))
}
c.indexSink(k, key(sinkRef.Workspace, sinkRef.Name), req)
sink, err := bus.Resolve[events.Sink](ctx, c.bus, self, sinkRef, bus.Capability{Name: events.CapabilityDeliver, Scope: []string{sinkRef.Name}})
if errors.Is(err, atprotorecordwatch.ErrNotReconciled) {
c.log.Info("an EventStream waits for its sink", "workspace", req.Workspace, "name", es.Name, "err", err.Error())
return ctlruntime.Result{}, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhasePending, false, false, ReasonSinkNotReady,
"the sink "+sinkRef.String()+" is not reconciled yet")
}
if err != nil {
return ctlruntime.Result{}, errors.Join(err, c.applyStatus(ctx, req, es, eventsv1.EventStreamPhaseFailed, false, true, ReasonResolveFailed,
"resolving the sink "+sinkRef.String()+": "+err.Error()))
}
s := c.start(ctx, k, req, es, fingerprint, filter, rewind, source, sink)
s.flush.Mark()
return ctlruntime.Result{}, s.publish(ctx, false)
}
func rewindFor(kind string) int64 {
switch kind {
case eventsv1.RelayEventSourceKind:
return delivery.RelayRewindEvents
case eventsv1.JetstreamEventSourceKind:
return delivery.JetstreamRewindMicros
}
return 0
}
func fingerprintOf(spec eventsv1.EventStreamSpec, src, sink commonv1.ObjectRef, sourceState string) (string, error) {
raw, err := json.Marshal(struct {
Spec eventsv1.EventStreamSpec
Source commonv1.ObjectRef
Sink commonv1.ObjectRef
State string
}{spec, src, sink, sourceState})
if err != nil {
return "", fmt.Errorf("eventstream: fingerprinting the spec: %w", err)
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}
func (c *Controller) stop(k string, flush bool) {
c.mu.Lock()
s := c.streams[k]
delete(c.streams, k)
c.mu.Unlock()
if s != nil {
s.stop(flush)
}
}
func (c *Controller) applyStatus(ctx context.Context, req ctlruntime.Request, es *eventsv1.EventStream, phase eventsv1.EventStreamPhase,
ready, failed bool, reason, message string) error {
status := eventsv1.EventStreamStatus{ObservedGeneration: es.Generation, Phase: phase, Cursor: es.Status.Cursor, Lag: es.Status.Lag,
LastEventAt: es.Status.LastEventAt, ServerFilter: es.Status.ServerFilter, Conditions: append([]metav1.Condition(nil), es.Status.Conditions...)}
setConditions(&status, es.Generation, ready, false, failed, reason, message)
next := es.DeepCopy()
next.Status = status
return req.ApplyStatus(ctx, next)
}
func setConditions(status *eventsv1.EventStreamStatus, generation int64, ready, lagging, failed bool, reason, message string) {
readyReason := reason
if ready {
readyReason = delivery.ReasonStreamOpen
}
conds.Set(&status.Conditions, generation, eventsv1.EventStreamConditionReady, ready, readyReason, message)
lagReason, lagMessage := "UpToDate", "the stream is current"
if lagging {
lagReason, lagMessage = "CursorGap", fmt.Sprintf("%d events behind", status.Lag)
}
conds.Set(&status.Conditions, generation, eventsv1.EventStreamConditionLagging, lagging, lagReason, lagMessage)
failedReason, failedMessage := "NoError", "the stream is healthy"
if failed {
failedReason, failedMessage = reason, message
}
conds.Set(&status.Conditions, generation, eventsv1.EventStreamConditionFailed, failed, failedReason, failedMessage)
}
type stream struct {
c *Controller
key string
uid string
fingerprint string
loop *delivery.Loop
group *supervise.Group
flush *debounce.Debounce
progress *debounce.Debounce
rewind int64
mu sync.Mutex
req ctlruntime.Request
es *eventsv1.EventStream
publishMu sync.Mutex
last *eventsv1.EventStreamStatus
}
func (c *Controller) start(ctx context.Context, k string, req ctlruntime.Request, es *eventsv1.EventStream, fingerprint string,
filter events.Filter, rewind int64, source events.Source, sink events.Sink) *stream {
s := &stream{c: c, key: k, uid: string(es.UID), fingerprint: fingerprint, rewind: rewind, req: req, es: es.DeepCopy()}
s.flush = debounce.New(c.flushDelay, s.flushNow)
s.progress = debounce.New(delivery.ProgressDelay, s.flushProgress)
s.loop = delivery.New(delivery.Config{Filter: filter, Rewind: rewind, Live: es.Spec.Resume == eventsv1.EventStreamResumeLive},
delivery.Deps{Source: source, Sink: sink, Resume: s.resume, Changed: s.flush.Mark, Progress: s.progress.Mark,
Log: c.log.With("stream", k)})
streamCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
stopRoot := context.AfterFunc(c.root, cancel)
s.group = supervise.New(streamCtx)
s.group.Go("eventstream "+k, func(ctx context.Context) error {
defer stopRoot()
defer cancel()
s.loop.Run(ctx)
return nil
})
c.mu.Lock()
c.streams[k] = s
c.mu.Unlock()
return s
}
func (s *stream) update(req ctlruntime.Request, es *eventsv1.EventStream) {
s.mu.Lock()
defer s.mu.Unlock()
s.req, s.es = req, es.DeepCopy()
}
func (s *stream) stop(flush bool) {
s.flush.Stop()
s.progress.Stop()
_ = s.group.Stop()
if flush {
s.flushAt(true)
}
}
func (s *stream) resume() *int64 {
s.mu.Lock()
defer s.mu.Unlock()
if s.es.Spec.Resume == eventsv1.EventStreamResumeLive {
return nil
}
recent := s.es.Spec.Resume == eventsv1.EventStreamResumeRecent
if saved, ok := s.c.cursors.Load(s.req.Workspace, s.es.Name, s.uid); ok {
if recent && !delivery.Recent(saved.SavedAt, time.Now()) {
return nil
}
return delivery.Rewound(saved.Cursor, s.rewind)
}
if s.es.Status.Cursor > 0 {
if recent && (s.es.Status.LastEventAt == nil || !delivery.Recent(s.es.Status.LastEventAt.Time, time.Now())) {
return nil
}
return delivery.Rewound(s.es.Status.Cursor, s.rewind)
}
if s.es.Spec.Cursor != nil {
v := *s.es.Spec.Cursor
return &v
}
return nil
}
func (s *stream) flushNow() {
s.flushAt(false)
}
func (s *stream) flushProgress() {
s.flushAt(true)
}
func (s *stream) flushAt(progress bool) {
ctx, cancel := context.WithTimeout(context.WithoutCancel(s.c.root), statusTimeout)
defer cancel()
if err := s.publish(ctx, progress); err != nil {
s.c.log.Warn("writing the EventStream status", "stream", s.key, "err", err.Error())
}
state := s.loop.State()
s.mu.Lock()
workspace, name, live := s.req.Workspace, s.es.Name, s.es.Spec.Resume == eventsv1.EventStreamResumeLive
s.mu.Unlock()
if state.Started && !live {
if err := s.c.cursors.Save(workspace, name, s.uid, state.Cursor); err != nil {
s.c.log.Warn("saving the EventStream cursor", "stream", s.key, "err", err.Error())
}
}
}
func (s *stream) publish(ctx context.Context, progress bool) error {
s.publishMu.Lock()
defer s.publishMu.Unlock()
state := s.loop.State()
s.mu.Lock()
req, es := s.req, s.es
s.mu.Unlock()
status := eventsv1.EventStreamStatus{ObservedGeneration: es.Generation, Phase: eventsv1.EventStreamPhase(state.Phase), Cursor: es.Status.Cursor,
Lag: state.Lag, LastEventAt: es.Status.LastEventAt, ServerFilter: es.Status.ServerFilter, Conditions: append([]metav1.Condition(nil), es.Status.Conditions...)}
if s.last != nil {
status.Conditions = append([]metav1.Condition(nil), s.last.Conditions...)
status.Cursor, status.LastEventAt, status.ServerFilter = s.last.Cursor, s.last.LastEventAt, s.last.ServerFilter
}
if f := state.ServerFilter; f != nil {
status.ServerFilter = &eventsv1.EventStreamServerFilter{Collections: slices.Clone(f.Collections), DIDs: slices.Clone(f.DIDs),
Supported: eventsv1.EventStreamServerFilterSupport(f.Supported)}
}
if state.Started {
status.Cursor = state.Cursor
at := metav1.NewTime(state.LastEventAt.UTC().Truncate(time.Second))
status.LastEventAt = &at
}
setConditions(&status, es.Generation, state.Ready, state.Lag > 0, state.Failed, state.Reason, state.Message)
if s.last != nil && statusEqual(*s.last, status) {
return nil
}
if !progress && s.last != nil && statusEqual(withoutProgress(*s.last), withoutProgress(status)) {
s.progress.Mark()
return nil
}
next := es.DeepCopy()
next.Status = status
if err := req.ApplyStatus(ctx, next); err != nil {
if apierrors.IsNotFound(err) {
return nil
}
return err
}
s.last = &status
return nil
}
func withoutProgress(status eventsv1.EventStreamStatus) eventsv1.EventStreamStatus {
status.Cursor, status.Lag, status.LastEventAt = 0, 0, nil
status.Conditions = slices.DeleteFunc(slices.Clone(status.Conditions), func(c metav1.Condition) bool {
return c.Type == eventsv1.EventStreamConditionLagging
})
return status
}
func statusEqual(a, b eventsv1.EventStreamStatus) bool {
if a.ObservedGeneration != b.ObservedGeneration || a.Phase != b.Phase || a.Cursor != b.Cursor || a.Lag != b.Lag {
return false
}
if (a.LastEventAt == nil) != (b.LastEventAt == nil) || (a.LastEventAt != nil && !a.LastEventAt.Equal(b.LastEventAt)) {
return false
}
if !reflect.DeepEqual(a.ServerFilter, b.ServerFilter) {
return false
}
if len(a.Conditions) != len(b.Conditions) {
return false
}
for i := range a.Conditions {
x, y := a.Conditions[i], b.Conditions[i]
if x.Type != y.Type || x.Status != y.Status || x.Reason != y.Reason || x.Message != y.Message || x.ObservedGeneration != y.ObservedGeneration {
return false
}
}
return true
}