package runtime import ( "context" "fmt" "log/slog" "reflect" "sync" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" ) const ( engagedSourceStart = "Starting EventSource" reflectorFailed = "Failed to watch" ) type runtimeLogHandler struct { next slog.Handler unbound *unservedWatches } type unservedWatches struct { mu sync.Mutex since map[string]time.Time last map[string]time.Time } func newRuntimeLogHandler(next slog.Handler) slog.Handler { return runtimeLogHandler{next: next, unbound: &unservedWatches{since: map[string]time.Time{}, last: map[string]time.Time{}}} } func (w *unservedWatches) retrying(kind string, at time.Time) (time.Duration, bool) { w.mu.Lock() defer w.mu.Unlock() if last, seen := w.last[kind]; !seen || at.Sub(last) >= CacheSyncTimeout { w.since[kind] = at } w.last[kind] = at failing := at.Sub(w.since[kind]) return failing, failing < CacheSyncTimeout } func unservedKind(r slog.Record) (string, bool) { var kind string var unserved bool r.Attrs(func(a slog.Attr) bool { switch a.Key { case "type": kind = a.Value.String() case "err": if err, ok := a.Value.Any().(error); ok { unserved = apierrors.IsNotFound(err) } } return true }) return kind, unserved && kind != "" } func (h runtimeLogHandler) Enabled(ctx context.Context, level slog.Level) bool { return h.next.Enabled(ctx, level) } func (h runtimeLogHandler) Handle(ctx context.Context, r slog.Record) error { level := r.Level if r.Message == engagedSourceStart && level == slog.LevelInfo { level = slog.LevelDebug if !h.next.Enabled(ctx, level) { return nil } } var failing time.Duration escalation := false if r.Message == reflectorFailed && level == slog.LevelError { if kind, ok := unservedKind(r); ok { var retrying bool failing, retrying = h.unbound.retrying(kind, r.Time) escalation = true if retrying { level = slog.LevelWarn } } } out := slog.NewRecord(r.Time, level, r.Message, r.PC) r.Attrs(func(a slog.Attr) bool { out.AddAttrs(encodableAttr(a)) return true }) if escalation { out.AddAttrs(slog.String("failingFor", failing.Round(time.Second).String()), slog.String("errorAfter", CacheSyncTimeout.String())) } return h.next.Handle(ctx, out) } func (h runtimeLogHandler) WithAttrs(attrs []slog.Attr) slog.Handler { return runtimeLogHandler{next: h.next.WithAttrs(encodableAttrs(attrs)), unbound: h.unbound} } func (h runtimeLogHandler) WithGroup(name string) slog.Handler { return runtimeLogHandler{next: h.next.WithGroup(name), unbound: h.unbound} } func encodableAttrs(attrs []slog.Attr) []slog.Attr { out := make([]slog.Attr, len(attrs)) for i, a := range attrs { out[i] = encodableAttr(a) } return out } func encodableAttr(a slog.Attr) slog.Attr { switch a.Value.Kind() { case slog.KindGroup: return slog.Attr{Key: a.Key, Value: slog.GroupValue(encodableAttrs(a.Value.Group())...)} case slog.KindAny: v := a.Value.Any() switch reflect.ValueOf(v).Kind() { case reflect.Func, reflect.Chan, reflect.UnsafePointer: return slog.String(a.Key, fmt.Sprintf("%T", v)) } } return a }