0
0
package nodeclaim
import (
"context"
"errors"
"fmt"
"log/slog"
"path"
"reflect"
"strings"
"sync"
"time"
kcpv1alpha1 "github.com/kcp-dev/sdk/apis/tenancy/v1alpha1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
commonv1 "github.com/johnandersen777/socialweb-computer-kcp/api/common/v1alpha1"
exposurev1alpha1 "github.com/johnandersen777/socialweb-computer-kcp/api/exposure/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/api/kcpgvr"
"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/bus"
contract "github.com/johnandersen777/socialweb-computer-kcp/internal/contract/nodeclaim"
"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/vault"
ctlruntime "github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kcpclient"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kcplibs/common/ref"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kcplibs/impl/kcpstore"
)
const (
Name = "nodeclaim"
Export = "clusters"
ContractExport = "computecontracts"
ExposureExport = "exposures"
homeIndexRefresh = 2 * time.Second
lookupTimeout = 30 * time.Second
resyncTimeout = 2 * time.Minute
)
var _ contract.Binder = (*Controller)(nil)
type Options struct {
Bus *bus.Bus
Log *slog.Logger
}
type known struct {
workspace string
links []string
cluster string
infra string
entry string
contract string
}
type Controller struct {
bus *bus.Bus
log *slog.Logger
events chan ctlruntime.Request
resyncs chan struct{}
mu sync.Mutex
binding *contract.Binding
store *kcpstore.Store
clusters map[string]string
claims map[ctlruntime.Request]known
byContract map[string]ctlruntime.Request
infraMu sync.Mutex
waiting map[ctlruntime.Request]bool
homes map[string]bool
homesAt time.Time
}
func New(opts Options) *Controller {
log := opts.Log
if log == nil {
log = slog.New(slog.DiscardHandler)
}
return &Controller{bus: opts.Bus, log: log.With("controller", Name), events: make(chan ctlruntime.Request, 64), resyncs: make(chan struct{}, 1),
clusters: map[string]string{}, claims: map[ctlruntime.Request]known{}, byContract: map[string]ctlruntime.Request{},
waiting: map[ctlruntime.Request]bool{}}
}
func Register(m *ctlruntime.Manager, c *Controller, cfg *rest.Config) error {
if c == nil {
return errors.New("nodeclaim: a controller is required")
}
if c.bus == nil {
return errors.New("nodeclaim: the bus is required")
}
if cfg == nil || cfg.Host == "" {
return errors.New("nodeclaim: a kcp rest config is required")
}
store, err := kcpstore.New(kcpstore.Options{Host: cfg.Host, RestConfig: cfg})
if err != nil {
return fmt.Errorf("nodeclaim: the kcp store: %w", err)
}
if err := m.AddToScheme(v1alpha1.AddToScheme, exposurev1alpha1.AddToScheme); err != nil {
return err
}
c.mu.Lock()
c.store = store
c.mu.Unlock()
if err := ctlruntime.For[*v1alpha1.NodeClaim](m, Name, c,
ctlruntime.Export(Export),
ctlruntime.FieldManager(contract.FieldManager),
ctlruntime.SharedStatus(),
ctlruntime.Finalizer(v1alpha1.FinalizerNodeClaim),
ctlruntime.InWorkspacesContext(c.inScope),
ctlruntime.Gone(c.gone),
ctlruntime.Events(c.events),
ctlruntime.WatchesIn(ContractExport, c.contractClaims),
ctlruntime.WatchesIn(ContractExport, c.infraClaims),
ctlruntime.Watches(c.clusterClaims),
ctlruntime.Watches(c.linkClaims),
ctlruntime.Watches(joinedClaims),
ctlruntime.WatchesIn(ExposureExport, c.exposureClaims),
); err != nil {
return fmt.Errorf("nodeclaim: registering the NodeClaim controller: %w", err)
}
if err := m.Go(Name+": resync", c.resyncLoop); err != nil {
return fmt.Errorf("nodeclaim: the resync loop: %w", err)
}
c.wake()
return nil
}
func (c *Controller) wake() {
select {
case c.resyncs <- struct{}{}:
default:
}
}
func (c *Controller) resyncLoop(ctx context.Context) error {
for {
select {
case <-ctx.Done():
return nil
case <-c.resyncs:
}
if b, _ := c.current(); b != nil {
c.resync(ctx, *b)
}
}
}
func (c *Controller) Bind(b contract.Binding) {
if b.Mapper == nil {
return
}
b.Homes = append([]string(nil), b.Homes...)
c.mu.Lock()
c.binding = &b
c.homes, c.homesAt = nil, time.Time{}
c.mu.Unlock()
c.log.Info("node_claim_bound", "owner", b.Owner.Workspace+"/"+b.Owner.Name, "homes", b.Homes,
"indexWorkspace", b.IndexWorkspace, "indexNamespace", b.IndexNamespace, "contractWorkspace", b.ContractWorkspace)
c.wake()
}
func (c *Controller) Unbind(owner commonv1.ObjectRef) {
c.mu.Lock()
defer c.mu.Unlock()
if c.binding != nil && c.binding.Owner == owner {
c.binding = nil
c.homes, c.homesAt = nil, time.Time{}
c.log.Info("node_claim_unbound", "owner", owner.Workspace+"/"+owner.Name)
}
}
func (c *Controller) current() (*contract.Binding, *kcpstore.Store) {
c.mu.Lock()
defer c.mu.Unlock()
return c.binding, c.store
}
func (c *Controller) Reconcile(ctx context.Context, req ctlruntime.Request, obj *v1alpha1.NodeClaim) (ctlruntime.Result, error) {
b, store := c.current()
if b == nil || store == nil {
return ctlruntime.Result{Waiting: true}, nil
}
claim := contract.Claim{Workspace: req.Workspace, Object: *obj}
at := entryOf(b, claim)
openBao, apiserver := b.Mapper.LinkNames(claim)
cluster, owned := clusterRef(claim)
c.learn(req, known{workspace: req.Workspace, links: []string{openBao, apiserver}, cluster: cluster.Name,
infra: at.Name, entry: at.Contract, contract: at.ComputeContract})
cc, err := readContract(ctx, store, at)
if err != nil {
return ctlruntime.Result{}, err
}
alive := true
var cl *v1alpha1.Cluster
if owned {
if cl, err = readCluster(ctx, store, cluster); err != nil {
return ctlruntime.Result{}, err
}
alive = cl != nil && cl.DeletionTimestamp.IsZero()
}
if !alive {
if err := c.deleteInfra(ctx, store, at.Workspace, at.Name, "the cluster is deleting"); err != nil {
return ctlruntime.Result{}, err
}
}
if !obj.DeletionTimestamp.IsZero() {
return c.settle(ctx, req, store, b, &claim, at, cc, cl, alive)
}
if cc == nil {
if !alive {
c.wait(req, false)
return ctlruntime.Result{}, nil
}
return c.ensure(ctx, req, store, b, &claim, at, cl)
}
c.wait(req, false)
if alive {
if err := c.declare(ctx, store, claim, at, nil, cl); err != nil && !errors.Is(err, errInfraDeleting) {
return ctlruntime.Result{}, err
}
}
next := b.Mapper.Status(claim, *cc)
if next == nil {
if claim.Object.Status.Infra != nil {
return ctlruntime.Result{}, nil
}
same := statusCopy(claim.Object.Status)
next = &same
}
if err := c.status(ctx, req, &claim.Object, at, *next); err != nil {
return ctlruntime.Result{}, err
}
c.log.Info("node_claim_mapped", "claim", claim.Key(), "infra", at.Name, "entry", at.Contract, "contract", at.ComputeContract, "phase", next.Phase)
return ctlruntime.Result{}, nil
}
func (c *Controller) ensure(ctx context.Context, req ctlruntime.Request, store *kcpstore.Store, b *contract.Binding, claim *contract.Claim,
at v1alpha1.NodeClaimInfra, cl *v1alpha1.Cluster) (ctlruntime.Result, error) {
obj := &claim.Object
ensure, adopt := b.Mapper.Wants(*claim)
if !ensure {
c.wait(req, false)
return ctlruntime.Result{}, c.waitable(c.declare(ctx, store, *claim, at, nil, cl))
}
bases, ok := b.Mapper.Bases(ctx, adopt)
if !ok {
c.wait(req, true)
staged := c.entry(b, *claim, at, adopt, contract.Bases{})
return ctlruntime.Result{}, c.waitable(c.declare(ctx, store, *claim, at, &staged, cl))
}
c.wait(req, false)
key := claim.Key()
if !adopt {
data, err := b.Mapper.Bundle(*claim, c.links(ctx, req, b, *claim))
if err == nil {
secrets := b.Mapper.BundleRef(at.ComputeContract)
var kv vault.KV
if kv, err = c.kv(ctx, b, claim.Ref(), secrets); err != nil {
return ctlruntime.Result{}, err
}
err = kv.Write(ctx, secrets.Mount, secrets.Path, data)
}
if err != nil {
c.log.Error("node_claim_bundle_failed", "claim", key, "err", err)
return ctlruntime.Result{}, c.status(ctx, req, obj, at, b.Mapper.Failed(obj.Status, err))
}
}
if err := c.status(ctx, req, obj, at, b.Mapper.Requested(obj.Status, adopt)); err != nil {
return ctlruntime.Result{}, err
}
requested := c.entry(b, *claim, at, adopt, bases)
if err := c.declare(ctx, store, *claim, at, &requested, cl); err != nil {
if errors.Is(err, errInfraDeleting) {
c.log.Info("node_claim_infra_deleting", "claim", key, "infra", at.Name)
return ctlruntime.Result{}, nil
}
c.log.Error("node_claim_entry_failed", "claim", key, "infra", at.Name, "entry", at.Contract, "err", err)
return ctlruntime.Result{}, err
}
c.log.Info("node_claim_entry_requested", "claim", key, "infra", at.Name, "entry", at.Contract, "contract", at.ComputeContract,
"adopt", adopt, "workspace", at.Workspace)
return ctlruntime.Result{}, nil
}
func (c *Controller) waitable(err error) error {
if errors.Is(err, errInfraDeleting) {
return nil
}
return err
}
func (c *Controller) settle(ctx context.Context, req ctlruntime.Request, store *kcpstore.Store, b *contract.Binding, claim *contract.Claim,
at v1alpha1.NodeClaimInfra, cc *v1alpha1.ComputeContract, cl *v1alpha1.Cluster, alive bool) (ctlruntime.Result, error) {
obj := &claim.Object
c.wait(req, false)
if !controllerutil.ContainsFinalizer(obj, v1alpha1.FinalizerNodeClaim) {
return ctlruntime.Result{}, nil
}
key := claim.Key()
if alive {
if err := c.declare(ctx, store, *claim, at, nil, cl); c.waitable(err) != nil {
return ctlruntime.Result{Waiting: true}, err
}
}
s := b.Mapper.Settle(*claim, cc)
if s.Status != nil {
if err := c.status(ctx, req, obj, at, *s.Status); err != nil {
c.log.Warn("node_claim_status_write_failed", "claim", key, "err", err)
}
}
if !s.Done {
return ctlruntime.Result{Waiting: true}, nil
}
if s.RemoveBundle {
if err := c.removeBundle(ctx, b, claim.Ref(), at.ComputeContract); err != nil {
c.log.Warn("node_claim_bundle_remove_failed", "claim", key, "err", err)
}
}
c.log.Info("node_claim_released", "claim", key, "infra", at.Name, "entry", at.Contract, "contract", at.ComputeContract, "phase", phaseOf(s.Status))
return ctlruntime.Result{}, nil
}
func (c *Controller) gone(ctx context.Context, req ctlruntime.Request) {
k, ok := c.forget(req)
if !ok || k.infra == "" {
return
}
b, store := c.current()
if b == nil || store == nil {
return
}
if err := c.dropEntry(ctx, store, b.ContractWorkspace, k.infra, k.entry); err != nil {
c.log.Warn("node_claim_entry_drop_failed", "claim", req.Workspace+"/"+req.Name, "infra", k.infra, "entry", k.entry, "err", err)
}
}
func (c *Controller) status(ctx context.Context, req ctlruntime.Request, obj *v1alpha1.NodeClaim, at v1alpha1.NodeClaimInfra,
next v1alpha1.NodeClaimStatus) error {
next.Infra = &at
if reflect.DeepEqual(obj.Status, next) {
return nil
}
own := next
own.JoinedAt = ""
if err := req.ApplyStatus(ctx, &v1alpha1.NodeClaim{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}, Status: own}); err != nil {
return err
}
obj.Status = next
return nil
}
func (c *Controller) links(ctx context.Context, req ctlruntime.Request, b *contract.Binding, claim contract.Claim) contract.Links {
openBao, apiserver := b.Mapper.LinkNames(claim)
return contract.Links{OpenBao: readLink(ctx, req, openBao), Apiserver: readLink(ctx, req, apiserver)}
}
func readLink(ctx context.Context, req ctlruntime.Request, name string) contract.Link {
var link v1alpha1.ClusterLink
err := req.Get(ctx, "", name, &link)
switch {
case apierrors.IsNotFound(err):
return contract.Link{Name: name}
case err != nil:
return contract.Link{Name: name, Err: err}
}
return contract.Link{Name: name, Status: &link.Status}
}
func (c *Controller) kv(ctx context.Context, b *contract.Binding, from commonv1.ObjectRef, secrets commonv1.SecretRef) (vault.KV, error) {
scope := strings.Trim(secrets.Mount, "/") + "/" + path.Dir(strings.Trim(secrets.Path, "/"))
kv, err := bus.Resolve[vault.KV](ctx, c.bus, from, b.Vault, bus.Capability{Name: vault.CapabilityWrite, Scope: []string{scope}})
if err != nil {
return nil, fmt.Errorf("nodeclaim: the vault handle for %s: %w", refKey(&from), err)
}
return kv, nil
}
func (c *Controller) removeBundle(ctx context.Context, b *contract.Binding, from commonv1.ObjectRef, name string) error {
secrets := b.Mapper.BundleRef(name)
kv, err := c.kv(ctx, b, from, secrets)
if err != nil {
return err
}
return kv.Delete(ctx, secrets.Mount, secrets.Path)
}
func (c *Controller) inScope(ctx context.Context, workspace string) bool {
c.mu.Lock()
b, store := c.binding, c.store
if b == nil || store == nil {
c.mu.Unlock()
return false
}
if c.homes[workspace] || time.Since(c.homesAt) < homeIndexRefresh {
in := c.homes[workspace]
c.mu.Unlock()
return in
}
c.mu.Unlock()
ctx, cancel := context.WithTimeout(ctx, lookupTimeout)
defer cancel()
homes := c.listHomes(ctx, store, *b)
set := make(map[string]bool, len(homes))
for _, home := range homes {
set[home] = true
}
c.mu.Lock()
defer c.mu.Unlock()
if c.binding == b {
c.homes, c.homesAt = set, time.Now()
}
return set[workspace]
}
func (c *Controller) listHomes(ctx context.Context, store *kcpstore.Store, b contract.Binding) []string {
items, err := kcpstore.Of[corev1.ConfigMap](store, kcpgvr.ConfigMaps).ListIn(ctx, b.IndexWorkspace, b.IndexNamespace)
if err != nil {
c.log.Warn("node_claim_home_index_unreadable", "workspace", b.IndexWorkspace, "namespace", b.IndexNamespace, "err", err)
return append([]string(nil), b.Homes...)
}
seen := map[string]bool{}
homes := []string{}
for i := range items {
cm := &items[i]
if cm.Labels[kcpclient.NodeClaimHomeLabel] != "true" {
continue
}
home := cm.Data[kcpclient.NodeClaimHomeKey]
if home == "" || seen[home] {
continue
}
seen[home] = true
homes = append(homes, home)
}
return homes
}
func (c *Controller) learn(req ctlruntime.Request, k known) {
key := ctlruntime.Request{Cluster: req.Cluster, Name: req.Name}
c.mu.Lock()
defer c.mu.Unlock()
if req.Workspace != "" && req.Cluster != "" {
c.clusters[req.Workspace] = req.Cluster
}
if old, ok := c.claims[key]; ok && old.contract != k.contract {
delete(c.byContract, old.contract)
}
c.claims[key] = k
if k.contract != "" {
c.byContract[k.contract] = key
}
}
func (c *Controller) forget(req ctlruntime.Request) (known, bool) {
key := ctlruntime.Request{Cluster: req.Cluster, Name: req.Name}
c.mu.Lock()
defer c.mu.Unlock()
k, ok := c.claims[key]
delete(c.claims, key)
delete(c.waiting, key)
if ok && c.byContract[k.contract] == key {
delete(c.byContract, k.contract)
}
return k, ok
}
func (c *Controller) wait(req ctlruntime.Request, waiting bool) {
key := ctlruntime.Request{Cluster: req.Cluster, Name: req.Name}
c.mu.Lock()
defer c.mu.Unlock()
if waiting {
c.waiting[key] = true
return
}
delete(c.waiting, key)
}
func (c *Controller) contractClaims(_ context.Context, src ctlruntime.Request, obj *v1alpha1.ComputeContract) []ctlruntime.Request {
b, _ := c.current()
if b == nil || src.Workspace != b.ContractWorkspace {
return nil
}
c.mu.Lock()
defer c.mu.Unlock()
if key, ok := c.byContract[obj.Name]; ok {
return []ctlruntime.Request{key}
}
return nil
}
func (c *Controller) infraClaims(_ context.Context, src ctlruntime.Request, obj *v1alpha1.Infra) []ctlruntime.Request {
b, _ := c.current()
if b == nil || src.Workspace != b.ContractWorkspace {
return nil
}
return c.claimsWhere(func(_ ctlruntime.Request, k known) bool { return k.infra == obj.Name })
}
func (c *Controller) clusterClaims(_ context.Context, src ctlruntime.Request, obj *v1alpha1.Cluster) []ctlruntime.Request {
return c.claimsWhere(func(key ctlruntime.Request, k known) bool { return key.Cluster == src.Cluster && k.cluster == obj.Name })
}
func (c *Controller) claimsWhere(match func(ctlruntime.Request, known) bool) []ctlruntime.Request {
c.mu.Lock()
defer c.mu.Unlock()
var out []ctlruntime.Request
for key, k := range c.claims {
if match(key, k) {
out = append(out, key)
}
}
return out
}
func (c *Controller) linkClaims(_ context.Context, src ctlruntime.Request, obj *v1alpha1.ClusterLink) []ctlruntime.Request {
c.mu.Lock()
defer c.mu.Unlock()
var out []ctlruntime.Request
for key, k := range c.claims {
if key.Cluster != src.Cluster {
continue
}
for _, link := range k.links {
if link == obj.Name {
out = append(out, key)
break
}
}
}
return out
}
func joinedClaims(_ context.Context, src ctlruntime.Request, obj *v1alpha1.NodeClaim) []ctlruntime.Request {
if obj.Status.Phase != v1alpha1.NodeClaimJoining || obj.Status.JoinedAt == "" {
return nil
}
return []ctlruntime.Request{{Namespace: src.Namespace, Name: src.Name}}
}
func (c *Controller) exposureClaims(_ context.Context, src ctlruntime.Request, _ *exposurev1alpha1.Exposure) []ctlruntime.Request {
b, _ := c.current()
if b == nil || src.Workspace != b.Workspace {
return nil
}
c.mu.Lock()
defer c.mu.Unlock()
out := make([]ctlruntime.Request, 0, len(c.waiting))
for key := range c.waiting {
out = append(out, key)
}
return out
}
func (c *Controller) clusterOf(ctx context.Context, workspace string) (string, error) {
c.mu.Lock()
cluster, ok := c.clusters[workspace]
store := c.store
c.mu.Unlock()
if ok {
return cluster, nil
}
if store == nil {
return "", errors.New("nodeclaim: no kcp store yet")
}
i := strings.LastIndex(workspace, ":")
if i <= 0 || i == len(workspace)-1 {
return "", fmt.Errorf("nodeclaim: the workspace %q has no parent", workspace)
}
ctx, cancel := context.WithTimeout(ctx, lookupTimeout)
defer cancel()
ws, err := kcpstore.OfClusterScoped[kcpv1alpha1.Workspace](store, kcpgvr.Workspaces).Get(ctx, ref.New(workspace[:i], "", workspace[i+1:]))
if err != nil {
return "", err
}
if ws.Spec.Cluster == "" {
return "", fmt.Errorf("nodeclaim: the workspace %s has no logical cluster yet", workspace)
}
c.mu.Lock()
c.clusters[workspace] = ws.Spec.Cluster
c.mu.Unlock()
return ws.Spec.Cluster, nil
}
func (c *Controller) resync(ctx context.Context, b contract.Binding) {
ctx, cancel := context.WithTimeout(ctx, resyncTimeout)
defer cancel()
_, store := c.current()
if store == nil {
return
}
var out []ctlruntime.Request
claims := kcpstore.OfClusterScoped[v1alpha1.NodeClaim](store, v1alpha1.NodeClaimGVR)
for _, home := range c.listHomes(ctx, store, b) {
cluster, err := c.clusterOf(ctx, home)
if err != nil {
c.log.Warn("node_claim_workspace_unresolved", "workspace", home, "err", err)
continue
}
items, err := claims.List(ctx, home)
if err != nil {
if !kcpstore.IsNotFound(err) {
c.log.Warn("node_claim_home_unlisted", "workspace", home, "err", err)
}
continue
}
for i := range items {
out = append(out, ctlruntime.Request{Cluster: cluster, Name: items[i].Name})
}
}
c.sweep(ctx, store, b)
for _, r := range out {
select {
case c.events <- r:
case <-ctx.Done():
return
}
}
}
func refKey(r *commonv1.ObjectRef) string {
if r == nil {
return ""
}
return r.Workspace + "/" + r.Name
}
func phaseOf(status *v1alpha1.NodeClaimStatus) v1alpha1.NodeClaimPhase {
if status == nil {
return ""
}
return status.Phase
}