0
0
package kcpclient
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"slices"
"strings"
"sync"
"time"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
corev1alpha1 "github.com/kcp-dev/sdk/apis/core/v1alpha1"
kcpv1alpha1 "github.com/kcp-dev/sdk/apis/tenancy/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/clusterhome"
"github.com/johnandersen777/socialweb-computer-kcp/internal/gateway"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/clock"
"github.com/johnandersen777/socialweb-computer-kcp/internal/revocation"
)
var _ gateway.Feed = (*Client)(nil)
const (
memberClusterLabel = "socialweb.computer/cluster"
memberNameLabel = "socialweb.computer/member"
watchTimeoutSeconds = "300"
shortWatch = time.Second
maxShortWatches = 3
)
var errWatchGone = errors.New("kcpclient: the watch's resource version is gone")
type listOf[T any] struct {
Metadata struct {
ResourceVersion string `json:"resourceVersion"`
} `json:"metadata"`
Items []T `json:"items"`
}
type watchFrame struct {
Type string `json:"type"`
Object json.RawMessage `json:"object"`
}
type source[T any] struct {
r rest.Interface
resource string
label string
field string
where string
}
func (s source[T]) request() *rest.Request {
req := s.r.Get().Resource(s.resource)
if s.label != "" {
req = req.Param("labelSelector", s.label)
}
if s.field != "" {
req = req.Param("fieldSelector", s.field)
}
return req
}
func (s source[T]) list(ctx context.Context) ([]T, string, error) {
raw, err := s.request().DoRaw(ctx)
if err != nil {
return nil, "", fmt.Errorf("kcpclient: list %s in %s: %w", s.resource, s.where, err)
}
var out listOf[T]
if err := json.Unmarshal(raw, &out); err != nil {
return nil, "", fmt.Errorf("kcpclient: list %s in %s: %w", s.resource, s.where, err)
}
return out.Items, out.Metadata.ResourceVersion, nil
}
func (s source[T]) watch(ctx context.Context, rv string, on func(deleted bool, obj T) error) (string, error) {
stream, err := s.request().
Param("watch", "true").
Param("resourceVersion", rv).
Param("allowWatchBookmarks", "true").
Param("timeoutSeconds", watchTimeoutSeconds).
Stream(ctx)
if err != nil {
return rv, fmt.Errorf("kcpclient: watch %s in %s: %w", s.resource, s.where, err)
}
defer stream.Close()
dec := json.NewDecoder(stream)
for {
var f watchFrame
if err := dec.Decode(&f); err != nil {
if errors.Is(err, io.EOF) || ctx.Err() != nil {
return rv, nil
}
return rv, fmt.Errorf("kcpclient: watch %s in %s: %w", s.resource, s.where, err)
}
if f.Type == "ERROR" {
var st metav1.Status
_ = json.Unmarshal(f.Object, &st)
if st.Code == http.StatusGone {
return rv, errWatchGone
}
return rv, fmt.Errorf("kcpclient: watch %s in %s: %s", s.resource, s.where, st.Message)
}
var head struct {
Metadata struct {
ResourceVersion string `json:"resourceVersion"`
} `json:"metadata"`
}
if err := json.Unmarshal(f.Object, &head); err != nil {
return rv, fmt.Errorf("kcpclient: watch %s in %s: %w", s.resource, s.where, err)
}
if head.Metadata.ResourceVersion != "" {
rv = head.Metadata.ResourceVersion
}
if f.Type == "BOOKMARK" {
continue
}
var obj T
if err := json.Unmarshal(f.Object, &obj); err != nil {
return rv, fmt.Errorf("kcpclient: watch %s in %s: %w", s.resource, s.where, err)
}
if err := on(f.Type == "DELETED", obj); err != nil {
return rv, err
}
}
}
func follow[T any](ctx context.Context, s source[T], rv string, on func(deleted bool, obj T) error, resync func([]T) error) error {
short := 0
for {
began := time.Now()
next, err := s.watch(ctx, rv, on)
if ctx.Err() != nil {
return ctx.Err()
}
switch {
case err == nil:
if time.Since(began) < shortWatch {
short++
if short >= maxShortWatches {
return fmt.Errorf("kcpclient: the watch of %s in %s closed %d times in a row within %s",
s.resource, s.where, short, shortWatch)
}
} else {
short = 0
}
rv = next
case errors.Is(err, errWatchGone):
items, lrv, lerr := s.list(ctx)
if lerr != nil {
return lerr
}
if err := resync(items); err != nil {
return err
}
rv = lrv
default:
return err
}
}
}
type feed struct {
c *Client
did string
name string
emit gateway.Emit
mu sync.Mutex
sent map[string]string
homes map[string]context.CancelFunc
errc chan error
wg sync.WaitGroup
}
func (f *feed) fail(err error) {
select {
case f.errc <- err:
default:
}
}
func (f *feed) send(key, typ string, fingerprint string, payload any) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.sent[key] == fingerprint {
return nil
}
f.sent[key] = fingerprint
return f.emit(gateway.Event{Type: typ, Payload: payload})
}
func (f *feed) gone(key, typ string, payload any) error {
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.sent[key]; !ok {
return nil
}
delete(f.sent, key)
return f.emit(gateway.Event{Type: typ, Payload: payload})
}
func (f *feed) keysUnder(prefix string) []string {
f.mu.Lock()
defer f.mu.Unlock()
var out []string
for k := range f.sent {
if strings.HasPrefix(k, prefix) {
out = append(out, k)
}
}
return out
}
var errAccountReady = errors.New("kcpclient: the account workspace is ready")
func (c *Client) waitForAccount(ctx context.Context, did string) error {
rRoot, err := c.resource(clusterhome.RootWorkspace, tenancyResource)
if err != nil {
return err
}
spaces := source[kcpv1alpha1.Workspace]{r: rRoot, resource: "workspaces",
field: "metadata.name=" + clusterhome.AccountHash(did), where: clusterhome.RootWorkspace}
ready := func(ws kcpv1alpha1.Workspace) bool {
return ws.DeletionTimestamp == nil && ws.Status.Phase == corev1alpha1.LogicalClusterPhaseReady
}
anyReady := func(items []kcpv1alpha1.Workspace) error {
if slices.ContainsFunc(items, ready) {
return errAccountReady
}
return nil
}
items, rv, err := spaces.list(ctx)
if err != nil {
return err
}
if anyReady(items) != nil {
return nil
}
err = follow(ctx, spaces, rv, func(deleted bool, ws kcpv1alpha1.Workspace) error {
if !deleted && ready(ws) {
return errAccountReady
}
return nil
}, anyReady)
if errors.Is(err, errAccountReady) {
return nil
}
return err
}
func (c *Client) Follow(ctx context.Context, caller gateway.Caller, name string, emit gateway.Emit) error {
did := string(caller)
if did == "" {
return gateway.ErrUnauthorized
}
state, err := c.Lookup(ctx, did)
if err != nil {
return err
}
snapshotSent := false
if state == revocation.DenyFirstLogin {
if name != "" {
return &gateway.XRPCError{Name: "ClusterNotFound", Status: 404,
Message: "this account has no clusters yet: call createCluster, then subscribe"}
}
if err := emit(gateway.Event{Type: gateway.EventInfo, Payload: gateway.InfoEvent{
Name: gateway.InfoSnapshotComplete,
Message: "this account has no clusters yet; events start when createCluster makes the first one",
}}); err != nil {
return err
}
snapshotSent = true
if err := c.waitForAccount(ctx, did); err != nil {
return err
}
if state, err = c.Lookup(ctx, did); err != nil {
return err
}
}
switch state {
case revocation.DenyGranted, revocation.DenyFirstLogin:
default:
return gateway.ErrIdentityRevoked
}
ctx, cancel := context.WithCancel(ctx)
f := &feed{c: c, did: did, name: name, emit: emit,
sent: map[string]string{}, homes: map[string]context.CancelFunc{}, errc: make(chan error, 1)}
defer func() {
cancel()
f.wg.Wait()
}()
account := clusterhome.AccountPath(did)
rTenancy, err := c.resource(account, tenancyResource)
if err != nil {
return err
}
rAccount, err := c.resource(account, socialwebResource)
if err != nil {
return err
}
identities := source[v1alpha1.AtprotoIdentity]{r: rAccount, resource: "atprotoidentities",
field: "metadata.name=" + IdentityRecordName(did), where: account}
workspaces := source[kcpv1alpha1.Workspace]{r: rTenancy, resource: "workspaces", where: account}
_, idRV, err := identities.list(ctx)
if err != nil {
return err
}
spaces, wsRV, err := workspaces.list(ctx)
if err != nil {
return err
}
found := name == ""
var starts []func()
for _, ws := range spaces {
if !f.wanted(ws) {
continue
}
found = true
if ws.Status.Phase != corev1alpha1.LogicalClusterPhaseReady || ws.DeletionTimestamp != nil {
continue
}
start, err := f.open(ctx, ws.Name, !snapshotSent)
if err != nil {
return err
}
starts = append(starts, start)
}
if !found {
return gateway.ErrClusterNotFound
}
if !snapshotSent {
if err := emit(gateway.Event{Type: gateway.EventInfo, Payload: gateway.InfoEvent{
Name: gateway.InfoSnapshotComplete,
Message: "the snapshot is complete; what follows are live changes",
}}); err != nil {
return err
}
}
for _, start := range starts {
start()
}
f.wg.Go(func() {
f.fail(follow(ctx, identities, idRV, f.onIdentity, f.resyncIdentity))
})
f.wg.Go(func() {
f.fail(follow(ctx, workspaces, wsRV, func(deleted bool, ws kcpv1alpha1.Workspace) error {
return f.onWorkspace(ctx, deleted, ws)
}, func(items []kcpv1alpha1.Workspace) error {
return f.resyncWorkspaces(ctx, items)
}))
})
select {
case err := <-f.errc:
return err
case <-ctx.Done():
return ctx.Err()
}
}
func (f *feed) wanted(ws kcpv1alpha1.Workspace) bool {
return ws.Spec.Mount == nil && (f.name == "" || ws.Name == f.name)
}
func (f *feed) onIdentity(deleted bool, id v1alpha1.AtprotoIdentity) error {
if deleted || id.Spec.Revoked || id.Spec.DID != f.did {
return gateway.ErrIdentityRevoked
}
return nil
}
func (f *feed) resyncIdentity(items []v1alpha1.AtprotoIdentity) error {
if len(items) == 0 {
return gateway.ErrIdentityRevoked
}
return f.onIdentity(false, items[0])
}
func (f *feed) onWorkspace(ctx context.Context, deleted bool, ws kcpv1alpha1.Workspace) error {
if !f.wanted(ws) {
return nil
}
if deleted {
return f.closeHome(ws.Name)
}
if ws.Status.Phase != corev1alpha1.LogicalClusterPhaseReady || ws.DeletionTimestamp != nil {
return nil
}
f.mu.Lock()
_, open := f.homes[ws.Name]
f.mu.Unlock()
if open {
return nil
}
start, err := f.open(ctx, ws.Name, false)
if err != nil {
return err
}
start()
return nil
}
func (f *feed) resyncWorkspaces(ctx context.Context, items []kcpv1alpha1.Workspace) error {
present := map[string]bool{}
for _, ws := range items {
if f.wanted(ws) {
present[ws.Name] = true
}
if err := f.onWorkspace(ctx, false, ws); err != nil {
return err
}
}
f.mu.Lock()
var missing []string
for name := range f.homes {
if !present[name] {
missing = append(missing, name)
}
}
f.mu.Unlock()
for _, name := range missing {
if err := f.closeHome(name); err != nil {
return err
}
}
return nil
}
func (f *feed) closeHome(cluster string) error {
f.mu.Lock()
cancel, ok := f.homes[cluster]
delete(f.homes, cluster)
f.mu.Unlock()
if ok {
cancel()
}
for _, k := range f.keysUnder("member/" + cluster + "/") {
f.mu.Lock()
delete(f.sent, k)
f.mu.Unlock()
}
for _, k := range f.keysUnder("claim/" + cluster + "/") {
f.mu.Lock()
delete(f.sent, k)
f.mu.Unlock()
}
return f.clusterGone(cluster)
}
func (f *feed) clusterGone(cluster string) error {
if err := f.gone("cluster/"+cluster, gateway.EventCluster, gateway.ClusterEvent{
Name: cluster, Phase: string(v1alpha1.ClusterUnavailable), Deleted: true, Time: clock.NowISO(),
}); err != nil {
return err
}
if f.name != "" {
return &gateway.XRPCError{Name: "ClusterNotFound", Status: 404, Message: "the cluster was deleted"}
}
return nil
}
type memberView struct {
member, phase, reason, message string
}
type stream[T any] struct {
src source[T]
prefix string
key func(T) string
on func(bool, T) error
}
func (s stream[T]) prime(ctx context.Context, f *feed) (func(context.Context) error, error) {
items, rv, err := s.src.list(ctx)
if err != nil {
return nil, err
}
if err := resyncSet(f, s.prefix, items, s.key, s.on); err != nil {
return nil, err
}
return func(hctx context.Context) error {
return follow(hctx, s.src, rv, s.on, func(items []T) error {
return resyncSet(f, s.prefix, items, s.key, s.on)
})
}, nil
}
func memberStream[T any](f *feed, cluster string, src source[T], name func(T) string, view func(T) memberView, snapshot *bool) stream[T] {
prefix := "member/" + cluster + "/" + src.resource + "/"
key := func(o T) string { return prefix + name(o) }
return stream[T]{src: src, prefix: prefix, key: key, on: func(deleted bool, o T) error {
return f.sendMember(key(o), cluster, view(o), deleted, *snapshot)
}}
}
func labelOr(labels map[string]string, fallback string) string {
if m := labels[memberNameLabel]; m != "" {
return m
}
return fallback
}
func (f *feed) open(ctx context.Context, cluster string, snapshot bool) (func(), error) {
home := clusterhome.HomePath(f.did, cluster)
key, err := clusterhome.Key(home)
if err != nil {
return nil, err
}
rHome, err := f.c.resource(home, socialwebResource)
if err != nil {
return nil, err
}
rHosts, err := f.c.resource(f.c.hostProcessWorkspace, socialwebResource)
if err != nil {
return nil, err
}
ws := f.c.hostProcessWorkspace
selector := memberClusterLabel + "=" + key
clusters := source[v1alpha1.Cluster]{r: rHome, resource: "clusters", field: "metadata.name=" + cluster, where: home}
onCluster := func(deleted bool, o v1alpha1.Cluster) error {
if deleted {
return f.clusterGone(cluster)
}
return f.sendCluster(o, snapshot)
}
cl, clRV, err := clusters.list(ctx)
if err != nil {
return nil, err
}
if err := f.resyncClusters(cluster, cl, onCluster); err != nil {
return nil, err
}
var runs []func(context.Context) error
for _, prime := range []func() (func(context.Context) error, error){
func() (func(context.Context) error, error) {
return memberStream(f, cluster, source[v1alpha1.Postgres]{r: rHosts, resource: "postgreses", label: selector, where: ws},
func(o v1alpha1.Postgres) string { return o.Name },
func(o v1alpha1.Postgres) memberView {
return memberView{labelOr(o.Labels, "postgres"), string(o.Status.Phase), "", o.Status.Message}
}, &snapshot).prime(ctx, f)
},
func() (func(context.Context) error, error) {
return memberStream(f, cluster, source[v1alpha1.OpenBaoServer]{r: rHosts, resource: "openbaoservers", label: selector, where: ws},
func(o v1alpha1.OpenBaoServer) string { return o.Name },
func(o v1alpha1.OpenBaoServer) memberView {
return memberView{labelOr(o.Labels, "openbao"), string(o.Status.Phase), "", o.Status.Message}
}, &snapshot).prime(ctx, f)
},
func() (func(context.Context) error, error) {
return memberStream(f, cluster, source[v1alpha1.HostProcess]{r: rHosts, resource: "hostprocesses", label: selector, where: ws},
func(o v1alpha1.HostProcess) string { return o.Name },
func(o v1alpha1.HostProcess) memberView {
return memberView{labelOr(o.Labels, o.Name), string(o.Status.Phase), string(o.Status.LostReason), o.Status.Message}
}, &snapshot).prime(ctx, f)
},
func() (func(context.Context) error, error) {
return stream[v1alpha1.NodeClaim]{
src: source[v1alpha1.NodeClaim]{r: rHome, resource: "nodeclaims", where: home},
prefix: "claim/" + cluster + "/",
key: func(o v1alpha1.NodeClaim) string { return "claim/" + cluster + "/" + o.Name },
on: func(deleted bool, o v1alpha1.NodeClaim) error {
return f.sendClaim(cluster, o, deleted, snapshot)
},
}.prime(ctx, f)
},
} {
run, err := prime()
if err != nil {
return nil, err
}
runs = append(runs, run)
}
snapshot = false
hctx, cancel := context.WithCancel(ctx)
f.mu.Lock()
f.homes[cluster] = cancel
f.mu.Unlock()
runs = append(runs, func(hctx context.Context) error {
return follow(hctx, clusters, clRV, onCluster, func(items []v1alpha1.Cluster) error {
return f.resyncClusters(cluster, items, onCluster)
})
})
return func() {
for _, run := range runs {
f.wg.Go(func() {
if err := run(hctx); err != nil && hctx.Err() == nil {
f.fail(err)
}
})
}
}, nil
}
func (f *feed) resyncClusters(cluster string, items []v1alpha1.Cluster, on func(bool, v1alpha1.Cluster) error) error {
for _, o := range items {
if o.Name == cluster {
return on(false, o)
}
}
f.mu.Lock()
_, had := f.sent["cluster/"+cluster]
f.mu.Unlock()
if had {
return f.clusterGone(cluster)
}
return nil
}
func resyncSet[T any](f *feed, prefix string, items []T, key func(T) string, on func(bool, T) error) error {
present := map[string]T{}
for _, o := range items {
present[key(o)] = o
if err := on(false, o); err != nil {
return err
}
}
for _, k := range f.keysUnder(prefix) {
if _, ok := present[k]; ok {
continue
}
if err := f.forget(k); err != nil {
return err
}
}
return nil
}
func (f *feed) forget(key string) error {
f.mu.Lock()
fp, ok := f.sent[key]
delete(f.sent, key)
f.mu.Unlock()
if !ok {
return nil
}
var stale struct {
Type string
Payload json.RawMessage
}
if err := json.Unmarshal([]byte(fp), &stale); err != nil {
return nil
}
switch stale.Type {
case gateway.EventMember:
var ev gateway.MemberEvent
if json.Unmarshal(stale.Payload, &ev) == nil {
ev.Deleted, ev.Snapshot, ev.Time = true, false, clock.NowISO()
return f.emitLocked(gateway.Event{Type: gateway.EventMember, Payload: ev})
}
case gateway.EventNodeClaim:
var ev gateway.NodeClaimEvent
if json.Unmarshal(stale.Payload, &ev) == nil {
ev.Deleted, ev.Snapshot, ev.Time = true, false, clock.NowISO()
return f.emitLocked(gateway.Event{Type: gateway.EventNodeClaim, Payload: ev})
}
}
return nil
}
func (f *feed) emitLocked(ev gateway.Event) error {
f.mu.Lock()
defer f.mu.Unlock()
return f.emit(ev)
}
func fingerprint(typ string, payload any) string {
b, _ := json.Marshal(struct {
Type string
Payload any
}{typ, payload})
return string(b)
}
func (f *feed) sendCluster(o v1alpha1.Cluster, snapshot bool) error {
reason, message := explain(o.Status.Conditions)
phase := string(o.Status.Phase)
if phase == "" {
phase = string(v1alpha1.ClusterUnavailable)
}
ev := gateway.ClusterEvent{
Name: o.Name,
Phase: phase,
Running: o.Spec.Running,
Instance: o.Status.Instance,
Server: o.Status.URL,
Reason: reason,
Message: message,
TeardownBlocked: conditionTrue(o.Status.Conditions, v1alpha1.ConditionTeardownBlocked),
}
fp := fingerprint(gateway.EventCluster, ev)
ev.Snapshot, ev.Time = snapshot, clock.NowISO()
return f.send("cluster/"+o.Name, gateway.EventCluster, fp, ev)
}
func explain(conds []metav1.Condition) (string, string) {
for _, t := range []string{v1alpha1.ConditionIdentityVerificationFailed, v1alpha1.ConditionTeardownBlocked} {
if conditionTrue(conds, t) {
c := meta.FindStatusCondition(conds, t)
return c.Reason, c.Message
}
}
if c := meta.FindStatusCondition(conds, "Ready"); c != nil {
return c.Reason, c.Message
}
return "", ""
}
func (f *feed) sendMember(key, cluster string, v memberView, deleted, snapshot bool) error {
phase := v.phase
if phase == "" {
phase = "Pending"
}
ev := gateway.MemberEvent{Cluster: cluster, Member: v.member, Phase: phase, Reason: v.reason, Message: v.message}
if deleted {
ev.Deleted, ev.Time = true, clock.NowISO()
return f.gone(key, gateway.EventMember, ev)
}
fp := fingerprint(gateway.EventMember, ev)
ev.Snapshot, ev.Time = snapshot, clock.NowISO()
return f.send(key, gateway.EventMember, fp, ev)
}
func (f *feed) sendClaim(cluster string, o v1alpha1.NodeClaim, deleted, snapshot bool) error {
phase := string(o.Status.Phase)
if phase == "" {
phase = string(v1alpha1.NodeClaimPending)
}
ev := gateway.NodeClaimEvent{
Cluster: cluster,
Name: o.Name,
NodeName: o.Spec.NodeName,
Phase: phase,
Message: o.Status.Message,
ContractPhase: string(o.Status.Contract.Phase),
Outcome: string(o.Status.Contract.Outcome),
FQDN: o.Status.Fqdn,
}
if w := o.Status.Contract.Winner; w != nil {
ev.Bidder = w.DID
}
if ev.Message == "" {
ev.Message = o.Status.Contract.Error
}
key := "claim/" + cluster + "/" + o.Name
if deleted {
ev.Deleted, ev.Time = true, clock.NowISO()
return f.gone(key, gateway.EventNodeClaim, ev)
}
fp := fingerprint(gateway.EventNodeClaim, ev)
ev.Snapshot, ev.Time = snapshot, clock.NowISO()
return f.send(key, gateway.EventNodeClaim, fp, ev)
}