0
0
package kcpclient
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"net/http"
"sort"
"strings"
"sync"
"time"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
corev1alpha1 "github.com/kcp-dev/sdk/apis/core/v1alpha1"
kcpv1alpha1 "github.com/kcp-dev/sdk/apis/tenancy/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/clusterhome"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/kcpapply"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/manifests"
"github.com/johnandersen777/socialweb-computer-kcp/internal/gateway"
"github.com/johnandersen777/socialweb-computer-kcp/internal/revocation"
commonv1 "github.com/johnandersen777/socialweb-computer-kcp/api/common/v1alpha1"
)
var (
_ gateway.Store = (*Client)(nil)
_ revocation.Deny = (*Client)(nil)
_ gateway.Revoker = (*Client)(nil)
)
var (
socialwebResource = v1alpha1.GroupVersion
tenancyResource = kcpv1alpha1.SchemeGroupVersion
)
type GrantInput struct {
Caller gateway.Caller
Cluster string
Server string
TTL time.Duration
NoExpiry bool
}
type KubeconfigGranter interface {
Grant(ctx context.Context, in GrantInput) (string, error)
}
type TenantConfig func(ctx context.Context, did, cluster string) (*rest.Config, error)
type Options struct {
Host string
PublicHost string
Kubeconfigs KubeconfigGranter
Tenant TenantConfig
NodeClaimAgent *ServiceAccountRef
HostProcessWorkspace string
NodeClaimQuota int64
RestConfig *rest.Config
Transport http.RoundTripper
}
type Client struct {
cfg *rest.Config
http *http.Client
codecs runtime.NegotiatedSerializer
host string
publicHost string
kubeconfigs KubeconfigGranter
tenant TenantConfig
agent *agent
hostProcessWorkspace string
nodeClaimQuota int64
certMu sync.Mutex
certWatches map[string]*certWatch
applier *kcpapply.Applier
}
var ErrNoKubeconfigGranter = errors.New(
"kcpclient: no kubeconfig granter: getKubeconfig has no issuer for the client certificate it" +
" hands back, so it hands out nothing")
const ShardAdminContext = "system:admin"
func ConfigFromContext(kubeconfigPath, contextName string) (*rest.Config, error) {
if kubeconfigPath == "" || contextName == "" {
return nil, errors.New(
"kcpclient: ConfigFromContext needs both a kubeconfig path and a context name: the " +
"identity a write is made as comes from exactly those two, and a missing one of " +
"them would be filled in by whatever the file's current-context happens to be")
}
cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
&clientcmd.ConfigOverrides{CurrentContext: contextName},
).ClientConfig()
if err != nil {
return nil, fmt.Errorf("kcpclient: the context %q of %s: %w", contextName, kubeconfigPath, err)
}
return cfg, nil
}
func New(opts Options) (*Client, error) {
if opts.Host == "" {
return nil, errors.New("kcpclient: Host is required")
}
cfg := &rest.Config{}
if opts.RestConfig != nil {
cfg = rest.CopyConfig(opts.RestConfig)
}
cfg.Host = strings.TrimSuffix(opts.Host, "/")
cfg.ContentType = "application/json"
cfg.AcceptContentTypes = "application/json"
if opts.Transport != nil {
cfg.Transport = opts.Transport
}
httpClient, err := rest.HTTPClientFor(cfg)
if err != nil {
return nil, fmt.Errorf("kcpclient: %w", err)
}
scheme, err := newScheme()
if err != nil {
return nil, err
}
codecs := serializer.NewCodecFactory(scheme).WithoutConversion()
c := &Client{
cfg: cfg, http: httpClient, codecs: codecs, host: cfg.Host,
publicHost: strings.TrimSuffix(opts.PublicHost, "/"),
kubeconfigs: opts.Kubeconfigs,
tenant: opts.Tenant,
hostProcessWorkspace: opts.HostProcessWorkspace,
nodeClaimQuota: opts.NodeClaimQuota,
}
applier, err := kcpapply.New(cfg, kcpapply.Options{})
if err != nil {
return nil, fmt.Errorf("kcpclient: %w", err)
}
c.applier = applier
if c.hostProcessWorkspace == "" {
c.hostProcessWorkspace = clusterhome.RootWorkspace
}
if opts.NodeClaimAgent != nil {
c.agent = &agent{ref: opts.NodeClaimAgent}
}
return c, nil
}
func newScheme() (*runtime.Scheme, error) {
s := runtime.NewScheme()
if err := v1alpha1.AddToScheme(s); err != nil {
return nil, fmt.Errorf("kcpclient: %w", err)
}
if err := kcpv1alpha1.AddToScheme(s); err != nil {
return nil, fmt.Errorf("kcpclient: %w", err)
}
metav1.AddToGroupVersion(s, schema.GroupVersion{Version: "v1"})
return s, nil
}
func (c *Client) resource(logicalCluster string, gv schema.GroupVersion) (rest.Interface, error) {
return c.bind(logicalCluster, gv, "/apis")
}
func (c *Client) coreResource(logicalCluster string) (rest.Interface, error) {
return c.bind(logicalCluster, corev1.SchemeGroupVersion, "/api")
}
var ErrNoTenantConfig = errors.New(
"kcpclient: no tenant credential source: a tenant apiserver is reached directly at its own " +
"front and never through the kcp mount, so without one no tenant call can be made")
func (c *Client) tenantCore(ctx context.Context, did, cluster string) (rest.Interface, error) {
if c.tenant == nil {
return nil, ErrNoTenantConfig
}
cfg, err := c.tenant(ctx, did, cluster)
if err != nil {
return nil, err
}
cfg = rest.CopyConfig(cfg)
cfg.GroupVersion = &corev1.SchemeGroupVersion
cfg.APIPath = "/api"
cfg.ContentType = "application/json"
cfg.AcceptContentTypes = "application/json"
cfg.NegotiatedSerializer = c.codecs
return rest.RESTClientFor(cfg)
}
func (c *Client) bind(logicalCluster string, gv schema.GroupVersion, endpoint string) (rest.Interface, error) {
cfg := rest.CopyConfig(c.cfg)
cfg.GroupVersion = &gv
cfg.APIPath = "/clusters/" + logicalCluster + endpoint
cfg.NegotiatedSerializer = c.codecs
r, err := rest.RESTClientForConfigAndClient(cfg, c.http)
if err != nil {
return nil, fmt.Errorf("kcpclient: %s: %w", logicalCluster, err)
}
return r, nil
}
var (
workspaceReadyTimeout = 5 * time.Minute
workspaceReadyRetry = 250 * time.Millisecond
)
var (
errWorkspaceReady = errors.New("kcpclient: the workspace is ready")
errWorkspaceDeleting = errors.New("is being deleted; ask again once it is gone")
errNodeClaimGone = errors.New("kcpclient: the nodeclaim is gone")
)
func (c *Client) homePath(caller gateway.Caller, name string) (string, error) {
if caller == "" {
return "", errors.New("kcpclient: no caller: the identity comes from the verified token and never from a request field")
}
if name == "" {
return "", errors.New("kcpclient: no cluster name")
}
return clusterhome.HomePath(string(caller), name), nil
}
func (c *Client) Ensure(ctx context.Context, caller gateway.Caller, name string) error {
permit, err := revocation.Admit(ctx, c, string(caller))
if err != nil {
return err
}
return c.ensure(ctx, permit, name)
}
func (c *Client) ensure(ctx context.Context, permit revocation.Permit, name string) error {
if name == "" {
return errors.New("kcpclient: no cluster name")
}
did := permit.Admitted()
home := clusterhome.HomePath(did, name)
if err := c.ensureWorkspace(ctx, clusterhome.RootWorkspace, clusterhome.AccountHash(did), kcpv1alpha1.WorkspaceSpec{
Type: &kcpv1alpha1.WorkspaceTypeReference{Name: AccountWorkspaceType, Path: clusterhome.RootWorkspace},
}); err != nil {
return fmt.Errorf("kcpclient: account workspace %s: %w", clusterhome.AccountHash(did), err)
}
if err := c.createGrant(ctx, clusterhome.AccountPath(did), IdentityRecordName(did), did); err != nil {
return fmt.Errorf("kcpclient: account identity record %s: %w", IdentityRecordName(did), err)
}
if err := c.ensureWorkspace(ctx, clusterhome.AccountPath(did), name, kcpv1alpha1.WorkspaceSpec{
Type: &kcpv1alpha1.WorkspaceTypeReference{Name: HomeWorkspaceType, Path: clusterhome.RootWorkspace},
}); err != nil {
return fmt.Errorf("kcpclient: home workspace %s: %w", home, err)
}
created, err := c.createCluster(ctx, home, name, did)
if err != nil {
return fmt.Errorf("kcpclient: cluster %s: %w", home, err)
}
if !created {
if cl, err := c.readCluster(ctx, home, name); err == nil && cl.DeletionTimestamp != nil {
return fmt.Errorf("kcpclient: cluster %s is being deleted; ask again once it is gone", home)
}
}
if err := c.ensureNodeClaimQuota(ctx, home); err != nil {
return err
}
return c.grantAgentHome(ctx, home)
}
const (
DefaultNodeClaimQuota = 64
NodeClaimQuotaName = "socialweb-bounded"
nodeClaimQuotaKey = "count/nodeclaims.socialweb.computer"
)
func (c *Client) ensureNodeClaimQuota(ctx context.Context, home string) error {
limit := c.nodeClaimQuota
if limit < 0 {
return nil
}
if limit == 0 {
limit = DefaultNodeClaimQuota
}
r, err := c.coreResource(home)
if err != nil {
return err
}
body, err := json.Marshal(map[string]any{"apiVersion": "v1", "kind": "ResourceQuota",
"metadata": map[string]any{"name": NodeClaimQuotaName, "namespace": "default"},
"spec": map[string]any{"hard": map[string]string{nodeClaimQuotaKey: fmt.Sprint(limit)}}})
if err != nil {
return err
}
err = r.Post().Namespace("default").Resource("resourcequotas").Body(body).Do(ctx).Error()
if err != nil && !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("kcpclient: the NodeClaim quota of %s: %w", home, err)
}
return nil
}
var ErrNoMountURL = errors.New(
"kcpclient: the mount child addresses the cluster's apiserver and carries the URL in " +
"spec.URL, which kcp copies out of the mounted object's status ONCE. A child written " +
"without a URL keeps an empty one for ever, and the mount then serves kcp itself: an " +
"admin token is accepted, the placeholder Node the provider wrote is listed, and every " +
"object written to the tenant apiserver is invisible")
type MountChild struct{ c *Client }
func NewMountChild(opts Options) (*MountChild, error) {
c, err := New(opts)
if err != nil {
return nil, err
}
return &MountChild{c: c}, nil
}
func (m *MountChild) EnsureMountChild(ctx context.Context, home, name, url string) error {
if home == "" || name == "" {
return errors.New("kcpclient: EnsureMountChild needs the cluster's home workspace and its name")
}
if url == "" {
return ErrNoMountURL
}
child, err := m.c.readWorkspace(ctx, home, clusterhome.ChildLabel)
switch {
case apierrors.IsNotFound(err):
return m.c.createWorkspace(ctx, home, clusterhome.ChildLabel, mountChildSpec(name, url))
case err != nil:
return err
case child.Spec.URL == "":
return m.c.patchWorkspaceURL(ctx, home, clusterhome.ChildLabel, url)
}
return nil
}
func mountChildSpec(name, url string) kcpv1alpha1.WorkspaceSpec {
return kcpv1alpha1.WorkspaceSpec{
URL: url,
Mount: &kcpv1alpha1.Mount{Reference: kcpv1alpha1.ObjectReference{
APIVersion: socialwebResource.String(),
Kind: "Cluster",
Name: name,
}},
}
}
func (c *Client) patchWorkspaceURL(ctx context.Context, parent, name, url string) error {
r, err := c.resource(parent, tenancyResource)
if err != nil {
return err
}
doc := map[string]any{"apiVersion": tenancyResource.String(), "kind": "Workspace", "metadata": map[string]any{"name": name},
"spec": map[string]any{"URL": url}}
if err := applyDoc(ctx, r.Patch(types.ApplyPatchType).Resource("workspaces").Name(name), fieldManagerClusterProvider, true, doc, false); err != nil {
return fmt.Errorf("kcpclient: apply workspace %s spec.URL: %w", name, err)
}
return nil
}
func (c *Client) ensureWorkspace(ctx context.Context, parent, name string, spec kcpv1alpha1.WorkspaceSpec) error {
if err := c.createWorkspace(ctx, parent, name, spec); err != nil {
return err
}
return c.waitForWorkspaceReady(ctx, parent, name)
}
func (c *Client) waitForWorkspaceReady(ctx context.Context, parent, name string) error {
path := parent + ":" + name
r, err := c.resource(parent, tenancyResource)
if err != nil {
return err
}
wctx, cancel := context.WithTimeout(ctx, workspaceReadyTimeout)
defer cancel()
src := source[kcpv1alpha1.Workspace]{r: r, resource: "workspaces", field: "metadata.name=" + name, where: parent}
last := "never read"
judge := func(deleted bool, ws kcpv1alpha1.Workspace) error {
switch {
case deleted:
last = "deleted"
case ws.DeletionTimestamp != nil:
return fmt.Errorf("kcpclient: workspace %s %w", path, errWorkspaceDeleting)
case ws.Status.Phase == corev1alpha1.LogicalClusterPhaseReady:
return errWorkspaceReady
case ws.Status.Phase == corev1alpha1.LogicalClusterPhaseInitializing && len(ws.Status.Initializers) > 0:
last = fmt.Sprintf("phase %s with initializers %v", ws.Status.Phase, ws.Status.Initializers)
default:
last = "phase " + string(ws.Status.Phase)
}
return nil
}
resync := func(items []kcpv1alpha1.Workspace) error {
if len(items) == 0 {
last = "not found"
}
for _, ws := range items {
if err := judge(false, ws); err != nil {
return err
}
}
return nil
}
for {
items, rv, err := src.list(wctx)
if err == nil {
if err = resync(items); err == nil {
err = follow(wctx, src, rv, judge, resync)
}
}
switch {
case errors.Is(err, errWorkspaceReady):
return nil
case ctx.Err() != nil:
return ctx.Err()
case wctx.Err() != nil:
return fmt.Errorf("kcpclient: workspace %s did not become %s within %s "+
"(last read: %s). Nothing may be written into a workspace kcp cannot address, "+
"and the create would be refused with `workspaces.tenancy.kcp.io %q not found`",
path, corev1alpha1.LogicalClusterPhaseReady, workspaceReadyTimeout, last, path)
case errors.Is(err, errWorkspaceDeleting):
return err
case err != nil:
last = err.Error()
}
select {
case <-wctx.Done():
case <-time.After(workspaceReadyRetry):
}
}
}
func (c *Client) createGrant(ctx context.Context, logicalCluster, name, did string) error {
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return err
}
rec := v1alpha1.AtprotoIdentity{
TypeMeta: metav1.TypeMeta{
APIVersion: v1alpha1.GroupVersion.String(),
Kind: "AtprotoIdentity",
},
ObjectMeta: metav1.ObjectMeta{Name: name},
Spec: v1alpha1.AtprotoIdentitySpec{
DID: did,
Scope: v1alpha1.IdentityScopeAccount,
Revoked: false,
},
}
body, err := json.Marshal(rec)
if err != nil {
return fmt.Errorf("kcpclient: grant: marshal: %w", err)
}
err = r.Post().Resource("atprotoidentities").Body(body).Do(ctx).Error()
if apierrors.IsAlreadyExists(err) {
return nil
}
return err
}
func (c *Client) createWorkspace(ctx context.Context, parent, name string, spec kcpv1alpha1.WorkspaceSpec) error {
body, err := json.Marshal(map[string]any{"apiVersion": tenancyResource.String(), "kind": "Workspace",
"metadata": map[string]any{"name": name}, "spec": spec})
if err != nil {
return fmt.Errorf("kcpclient: workspace %s: marshal: %w", name, err)
}
objs, err := manifests.Decode("kcpclient workspace "+parent+":"+name, body)
if err != nil {
return fmt.Errorf("kcpclient: workspace %s: %w", name, err)
}
if _, err := c.applier.ApplyObjects(ctx, parent, objs); err != nil {
return fmt.Errorf("kcpclient: workspace %s in %s: %w", name, parent, err)
}
return nil
}
func (c *Client) createCluster(ctx context.Context, home, name, did string) (bool, error) {
r, err := c.resource(home, socialwebResource)
if err != nil {
return false, err
}
obj := &v1alpha1.Cluster{
TypeMeta: metav1.TypeMeta{APIVersion: socialwebResource.String(), Kind: "Cluster"},
ObjectMeta: metav1.ObjectMeta{Name: name, Finalizers: []string{v1alpha1.FinalizerCluster}},
Spec: v1alpha1.ClusterSpec{Account: did, Running: false},
}
var out v1alpha1.Cluster
err = r.Post().Resource("clusters").Body(obj).Do(ctx).Into(&out)
if apierrors.IsAlreadyExists(err) {
return false, nil
}
if err != nil {
return false, err
}
return true, nil
}
func (c *Client) Get(ctx context.Context, caller gateway.Caller, name string) (gateway.ClusterView, bool, error) {
home, err := c.homePath(caller, name)
if err != nil {
return gateway.ClusterView{}, false, err
}
cl, err := c.readCluster(ctx, home, name)
if apierrors.IsNotFound(err) {
return gateway.ClusterView{}, false, nil
}
if err != nil {
return gateway.ClusterView{}, false, err
}
n, err := c.liveNodes(ctx, home, name)
if err != nil {
return gateway.ClusterView{}, false, err
}
return viewOf(cl, n), true, nil
}
func (c *Client) List(ctx context.Context, caller gateway.Caller) ([]gateway.ClusterView, error) {
if caller == "" {
return nil, errors.New("kcpclient: no caller: the identity comes from the verified token and never from a request field")
}
names, err := c.clusterLabels(ctx, string(caller))
if err != nil {
return nil, err
}
views := make([]gateway.ClusterView, 0, len(names))
for _, name := range names {
view, ok, err := c.Get(ctx, caller, name)
if err != nil {
return nil, err
}
if ok {
views = append(views, view)
}
}
return views, nil
}
func (c *Client) clusterLabels(ctx context.Context, did string) ([]string, error) {
r, err := c.resource(clusterhome.AccountPath(did), tenancyResource)
if err != nil {
return nil, err
}
var list kcpv1alpha1.WorkspaceList
if err := r.Get().Resource("workspaces").Do(ctx).Into(&list); err != nil {
if _, rerr := c.readWorkspace(ctx, clusterhome.RootWorkspace, clusterhome.AccountHash(did)); apierrors.IsNotFound(rerr) {
return []string{}, nil
}
return nil, fmt.Errorf("kcpclient: list clusters for %s: %w", clusterhome.AccountPath(did), err)
}
names := make([]string, 0, len(list.Items))
for _, ws := range list.Items {
if ws.Spec.Mount == nil {
names = append(names, ws.Name)
}
}
sort.Strings(names)
return names, nil
}
func (c *Client) SetRunning(ctx context.Context, caller gateway.Caller, name string, running bool) (gateway.ClusterView, error) {
return c.applySpec(ctx, caller, name, "set running", map[string]any{"running": running})
}
func (c *Client) applySpec(ctx context.Context, caller gateway.Caller, name, verb string, spec map[string]any) (gateway.ClusterView, error) {
home, err := c.homePath(caller, name)
if err != nil {
return gateway.ClusterView{}, err
}
r, err := c.resource(home, socialwebResource)
if err != nil {
return gateway.ClusterView{}, err
}
have, err := c.readCluster(ctx, home, name)
if err != nil {
return gateway.ClusterView{}, fmt.Errorf("kcpclient: %s %s: %w", verb, name, err)
}
owned := map[string]any{"running": have.Spec.Running}
maps.Copy(owned, spec)
doc := map[string]any{"apiVersion": socialwebResource.String(), "kind": "Cluster", "metadata": map[string]any{"name": name},
"spec": owned}
if err := applyDoc(ctx, r.Patch(types.ApplyPatchType).Resource("clusters").Name(name), fieldManagerGateway, true, doc, false); err != nil {
return gateway.ClusterView{}, fmt.Errorf("kcpclient: %s %s: %w", verb, name, err)
}
out, err := c.readCluster(ctx, home, name)
if err != nil {
return gateway.ClusterView{}, fmt.Errorf("kcpclient: %s %s: %w", verb, name, err)
}
n, err := c.liveNodes(ctx, home, name)
if err != nil {
return gateway.ClusterView{}, err
}
return viewOf(out, n), nil
}
func (c *Client) Delete(ctx context.Context, caller gateway.Caller, name string) (gateway.ClusterView, error) {
home, err := c.homePath(caller, name)
if err != nil {
return gateway.ClusterView{}, err
}
cl, err := c.readCluster(ctx, home, name)
if err != nil {
return gateway.ClusterView{}, err
}
n, err := c.liveNodes(ctx, home, name)
if err != nil {
return gateway.ClusterView{}, err
}
r, err := c.resource(home, socialwebResource)
if err != nil {
return gateway.ClusterView{}, err
}
if err := r.Delete().Resource("clusters").Name(name).Do(ctx).Error(); err != nil && !apierrors.IsNotFound(err) {
return gateway.ClusterView{}, fmt.Errorf("kcpclient: delete %s: %w", name, err)
}
return viewOf(cl, n), nil
}
func (c *Client) Kubeconfig(ctx context.Context, caller gateway.Caller, name string, ttl time.Duration, noExpiry bool) (string, error) {
if _, err := c.homePath(caller, name); err != nil {
return "", err
}
if c.kubeconfigs == nil {
return "", ErrNoKubeconfigGranter
}
host := c.host
if c.publicHost != "" {
host = c.publicHost
}
return c.kubeconfigs.Grant(ctx, GrantInput{
Caller: caller,
Cluster: name,
Server: host + "/clusters/" + clusterhome.MountPath(string(caller), name),
TTL: ttl,
NoExpiry: noExpiry,
})
}
func (c *Client) Lookup(ctx context.Context, did string) (revocation.DenyState, error) {
if did == "" {
return "", revocation.ErrNoDID
}
ws, err := c.readWorkspace(ctx, clusterhome.RootWorkspace, clusterhome.AccountHash(did))
switch {
case apierrors.IsNotFound(err):
return revocation.DenyFirstLogin, nil
case err != nil:
return "", fmt.Errorf("kcpclient: deny: account workspace %s: %w", clusterhome.AccountPath(did), err)
case ws.DeletionTimestamp != nil:
return "", fmt.Errorf("kcpclient: deny: account workspace %s is terminating", clusterhome.AccountPath(did))
}
rec, err := c.readIdentity(ctx, clusterhome.AccountPath(did), IdentityRecordName(did))
switch {
case apierrors.IsNotFound(err):
return revocation.DenyRecordMissing, nil
case err != nil:
return "", fmt.Errorf("kcpclient: deny: %s: %w", IdentityRecordName(did), err)
}
if rec.Spec.DID != did || rec.Spec.Scope != v1alpha1.IdentityScopeAccount {
return revocation.DenyRecordMissing, nil
}
if rec.Spec.Revoked {
return revocation.DenyRevoked, nil
}
return revocation.DenyGranted, nil
}
func (c *Client) WriteDeny(ctx context.Context, did string) error {
if did == "" {
return revocation.ErrNoDID
}
if _, err := c.readWorkspace(ctx, clusterhome.RootWorkspace, clusterhome.AccountHash(did)); err != nil {
return fmt.Errorf("kcpclient: deny: account workspace %s: %w", clusterhome.AccountPath(did), err)
}
r, err := c.resource(clusterhome.AccountPath(did), socialwebResource)
if err != nil {
return err
}
name := IdentityRecordName(did)
scope := v1alpha1.IdentityScopeAccount
rec, err := c.readIdentity(ctx, clusterhome.AccountPath(did), name)
switch {
case apierrors.IsNotFound(err):
case err != nil:
return fmt.Errorf("kcpclient: deny: read %s: %w", name, err)
case rec.Spec.Revoked:
return nil
case rec.Spec.Scope != "":
scope = rec.Spec.Scope
}
doc := map[string]any{"apiVersion": v1alpha1.GroupVersion.String(), "kind": "AtprotoIdentity", "metadata": map[string]any{"name": name},
"spec": map[string]any{"did": did, "scope": string(scope), "revoked": true}}
if err := applyDoc(ctx, r.Patch(types.ApplyPatchType).Resource("atprotoidentities").Name(name), fieldManagerRevocation, true, doc, false); err != nil {
return fmt.Errorf("kcpclient: deny: apply %s: %w", name, err)
}
return nil
}
func (c *Client) FanOut(ctx context.Context, did string) ([]revocation.ClusterView, error) {
if did == "" {
return nil, revocation.ErrNoDID
}
names, err := c.clusterLabels(ctx, did)
if err != nil {
return nil, err
}
views := make([]revocation.ClusterView, 0, len(names))
for _, name := range names {
view, err := c.observeServiceAccount(ctx, did, name)
if err != nil {
return nil, err
}
views = append(views, view)
}
return views, nil
}
func (c *Client) observeServiceAccount(ctx context.Context, did, cluster string) (revocation.ClusterView, error) {
view := revocation.ClusterView{Name: cluster}
r, err := c.tenantCore(ctx, did, cluster)
if err != nil {
if errors.Is(err, ErrNoTenantConfig) {
return view, err
}
view.Cold = true
return view, nil
}
err = r.Get().Namespace(TenantNamespace).Resource("serviceaccounts").
Name(ServiceAccountName(did)).Do(ctx).Error()
switch {
case err == nil:
view.ServiceAccountPresent = true
case apierrors.IsNotFound(err):
default:
view.Cold = true
}
return view, nil
}
func (c *Client) DeleteServiceAccount(ctx context.Context, did, cluster string) error {
if did == "" {
return revocation.ErrNoDID
}
r, err := c.tenantCore(ctx, did, cluster)
if err != nil {
return fmt.Errorf("kcpclient: the tenant apiserver of %s: %w", cluster, err)
}
name := ServiceAccountName(did)
err = r.Delete().Namespace(TenantNamespace).Resource("serviceaccounts").
Name(name).Do(ctx).Error()
switch {
case err == nil:
return nil
case apierrors.IsNotFound(err):
if found, ok := err.(apierrors.APIStatus); ok {
d := found.Status().Details
if d != nil && d.Kind == "serviceaccounts" && d.Name == name {
return nil
}
}
return fmt.Errorf("kcpclient: %s in %s answered NotFound without naming the "+
"ServiceAccount, so the tenant apiserver did NOT answer for it and this is not a "+
"completed delete: %w", name, cluster, err)
default:
return fmt.Errorf("kcpclient: delete the ServiceAccount for %s in %s: %w", did, cluster, err)
}
}
func (c *Client) readWorkspace(ctx context.Context, parent, name string) (*kcpv1alpha1.Workspace, error) {
r, err := c.resource(parent, tenancyResource)
if err != nil {
return nil, err
}
var out kcpv1alpha1.Workspace
if err := r.Get().Resource("workspaces").Name(name).Do(ctx).Into(&out); err != nil {
return nil, err
}
return &out, nil
}
func (c *Client) readIdentity(ctx context.Context, logicalCluster, name string) (*v1alpha1.AtprotoIdentity, error) {
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return nil, err
}
var out v1alpha1.AtprotoIdentity
if err := r.Get().Resource("atprotoidentities").Name(name).Do(ctx).Into(&out); err != nil {
return nil, err
}
return &out, nil
}
type ClusterGate struct {
Account string
Serving bool
Deleted bool
}
func (c *Client) ClusterGate(ctx context.Context, home, name string) (ClusterGate, error) {
cl, err := c.readCluster(ctx, home, name)
switch {
case apierrors.IsNotFound(err):
return ClusterGate{}, nil
case err != nil:
return ClusterGate{}, fmt.Errorf("kcpclient: the Cluster %s in %s: %w", name, home, err)
}
return gateOf(cl), nil
}
func (c *Client) readCluster(ctx context.Context, logicalCluster, name string) (*v1alpha1.Cluster, error) {
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return nil, err
}
var out v1alpha1.Cluster
if err := r.Get().Resource("clusters").Name(name).Do(ctx).Into(&out); err != nil {
return nil, err
}
return &out, nil
}
func (c *Client) liveNodes(ctx context.Context, logicalCluster, cluster string) (int, error) {
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return 0, err
}
var list v1alpha1.NodeList
err = r.Get().Resource("nodes").Do(ctx).Into(&list)
if apierrors.IsNotFound(err) {
return 0, nil
}
if err != nil {
return 0, fmt.Errorf("kcpclient: list nodes in %s: %w", logicalCluster, err)
}
n := 0
for _, node := range list.Items {
if node.Spec.ClusterRef.Name == cluster && node.Status.Phase != v1alpha1.NodeDraining {
n++
}
}
return n, nil
}
func viewOf(cl *v1alpha1.Cluster, nodes int) gateway.ClusterView {
return gateway.ClusterView{
Name: cl.Name,
Phase: string(cl.Status.Phase),
URL: cl.Status.URL,
Instance: cl.Status.Instance,
Running: cl.Spec.Running,
NodeCount: nodes,
TeardownBlocked: conditionTrue(cl.Status.Conditions, v1alpha1.ConditionTeardownBlocked),
}
}
func conditionTrue(conds []metav1.Condition, t string) bool {
for _, c := range conds {
if c.Type == t {
return c.Status == metav1.ConditionTrue
}
}
return false
}
func (c *Client) WriteNode(ctx context.Context, logicalCluster, cluster, nodeName string, status v1alpha1.NodeStatus) error {
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return err
}
return WriteNodeTo(ctx, r, cluster, nodeName, status)
}
func WriteNodeTo(ctx context.Context, r rest.Interface, cluster, nodeName string, status v1alpha1.NodeStatus) error {
if nodeName == "" {
return errors.New("kcpclient: WriteNode needs a node name")
}
if cluster == "" {
return errors.New("kcpclient: WriteNode needs the cluster it belongs to")
}
var existing v1alpha1.Node
err := r.Get().Resource("nodes").Name(nodeName).Do(ctx).Into(&existing)
switch {
case apierrors.IsNotFound(err):
return createNode(ctx, r, cluster, nodeName, status)
case err != nil:
return fmt.Errorf("kcpclient: read node %s: %w", nodeName, err)
}
if sameNodeStatus(existing.Status, status) {
return nil
}
if err := applyNodeStatus(ctx, r, nodeName, status); err != nil {
return fmt.Errorf("kcpclient: apply node %s to phase %q: %w", nodeName, status.Phase, err)
}
return nil
}
func sameNodeStatus(have, want v1alpha1.NodeStatus) bool {
if have.Phase != want.Phase {
return false
}
switch {
case have.NodeExpiry == nil && want.NodeExpiry == nil:
return true
case have.NodeExpiry == nil || want.NodeExpiry == nil:
return false
}
return *have.NodeExpiry == *want.NodeExpiry
}
var nodeType = metav1.TypeMeta{APIVersion: v1alpha1.GroupVersion.String(), Kind: "Node"}
func applyNodeStatus(ctx context.Context, r rest.Interface, nodeName string, status v1alpha1.NodeStatus) error {
obj := v1alpha1.Node{TypeMeta: nodeType, ObjectMeta: metav1.ObjectMeta{Name: nodeName}, Status: status}
return applyDoc(ctx, r.Patch(types.ApplyPatchType).SubResource("status").Resource("nodes").Name(nodeName),
fieldManagerClusterProvider, true, &obj, true)
}
func createNode(ctx context.Context, r rest.Interface, cluster, nodeName string, status v1alpha1.NodeStatus) error {
node := v1alpha1.Node{TypeMeta: nodeType, ObjectMeta: metav1.ObjectMeta{Name: nodeName},
Spec: v1alpha1.NodeSpec{ClusterRef: commonv1.LocalObjectRef{Name: cluster}}}
if err := applyDoc(ctx, r.Patch(types.ApplyPatchType).Resource("nodes").Name(nodeName), fieldManagerClusterProvider, true, &node, false); err != nil {
return fmt.Errorf("kcpclient: create node %s: %w", nodeName, err)
}
if status.Phase == "" && status.NodeExpiry == nil {
return nil
}
if err := applyNodeStatus(ctx, r, nodeName, status); err != nil {
return fmt.Errorf("kcpclient: set the status of the node %s it just created: %w -- the "+
"schema declares a status subresource, so a status posted inline is DISCARDED and the "+
"node exists with neither a phase nor an expiry at all", nodeName, err)
}
return nil
}
func (c *Client) WriteNodeClaim(ctx context.Context, logicalCluster string, claim v1alpha1.NodeClaim) error {
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return err
}
return WriteNodeClaimTo(ctx, r, claim)
}
func WriteNodeClaimTo(ctx context.Context, r rest.Interface, claim v1alpha1.NodeClaim) error {
if claim.Name == "" {
return errors.New("kcpclient: WriteNodeClaim needs a claim name")
}
if claim.Spec.ClusterRef.Name == "" {
return errors.New("kcpclient: WriteNodeClaim needs the cluster the claimed node belongs to")
}
if claim.Spec.NodeName == "" {
return errors.New("kcpclient: WriteNodeClaim needs the node name the claimed node joins as")
}
var existing v1alpha1.NodeClaim
err := r.Get().Resource("nodeclaims").Name(claim.Name).Do(ctx).Into(&existing)
switch {
case apierrors.IsNotFound(err):
return createNodeClaim(ctx, r, claim)
case err != nil:
return fmt.Errorf("kcpclient: read nodeclaim %s: %w", claim.Name, err)
}
if sameNodeClaimSpec(existing.Spec, claim.Spec) && (len(existing.OwnerReferences) > 0 || len(claim.OwnerReferences) == 0) {
return nil
}
if err := applyNodeClaim(ctx, r, claim); err != nil {
return fmt.Errorf("kcpclient: apply nodeclaim %s: %w -- the claim is the only handle on a "+
"machine the market is being asked to rent, so a spec that did not land is a node "+
"nobody can account for", claim.Name, err)
}
return nil
}
func (c *Client) ReadNodeClaim(ctx context.Context, logicalCluster, name string) (*v1alpha1.NodeClaim, error) {
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return nil, err
}
if name == "" {
return nil, errors.New("kcpclient: ReadNodeClaim needs a claim name")
}
var claim v1alpha1.NodeClaim
err = r.Get().Resource("nodeclaims").Name(name).Do(ctx).Into(&claim)
if apierrors.IsNotFound(err) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("kcpclient: read nodeclaim %s: %w", name, err)
}
return &claim, nil
}
func (c *Client) ReportNodeClaimJoined(ctx context.Context, logicalCluster, name string) error {
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return err
}
if name == "" {
return errors.New("kcpclient: ReportNodeClaimJoined needs a claim name")
}
body, err := json.Marshal(map[string]any{
"apiVersion": socialwebResource.String(),
"kind": "NodeClaim",
"metadata": map[string]any{"name": name},
"status": map[string]any{"joinedAt": time.Now().UTC().Format(time.RFC3339)},
})
if err != nil {
return err
}
err = r.Patch(types.ApplyPatchType).SubResource("status").Resource("nodeclaims").Name(name).
Param("fieldManager", v1alpha1.FieldManagerNodeMarket).Param("force", "true").Body(body).Do(ctx).Error()
if apierrors.IsNotFound(err) {
return nil
}
if err != nil {
return fmt.Errorf("kcpclient: report nodeclaim %s joined: %w", name, err)
}
return nil
}
func (c *Client) AwaitNodeClaimGone(ctx context.Context, logicalCluster, name string) error {
if name == "" {
return errors.New("kcpclient: AwaitNodeClaimGone needs a claim name")
}
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return err
}
src := source[v1alpha1.NodeClaim]{r: r, resource: "nodeclaims", field: "metadata.name=" + name, where: logicalCluster}
gone := func(deleted bool, _ v1alpha1.NodeClaim) error {
if deleted {
return errNodeClaimGone
}
return nil
}
resync := func(items []v1alpha1.NodeClaim) error {
if len(items) == 0 {
return errNodeClaimGone
}
return nil
}
items, rv, err := src.list(ctx)
if err == nil {
if err = resync(items); err == nil {
err = follow(ctx, src, rv, gone, resync)
}
}
if errors.Is(err, errNodeClaimGone) {
return nil
}
return err
}
func (c *Client) DeleteNodeClaim(ctx context.Context, logicalCluster, name string) error {
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return err
}
if name == "" {
return errors.New("kcpclient: DeleteNodeClaim needs a claim name")
}
err = r.Delete().Resource("nodeclaims").Name(name).Do(ctx).Error()
if apierrors.IsNotFound(err) {
return nil
}
if err != nil {
return fmt.Errorf("kcpclient: delete nodeclaim %s: %w", name, err)
}
return nil
}
func (c *Client) ListNodeClaims(ctx context.Context, logicalCluster string) ([]v1alpha1.NodeClaim, error) {
r, err := c.resource(logicalCluster, socialwebResource)
if err != nil {
return nil, err
}
var list v1alpha1.NodeClaimList
if err := r.Get().Resource("nodeclaims").Do(ctx).Into(&list); err != nil {
return nil, fmt.Errorf("kcpclient: list nodeclaims in %s: %w", logicalCluster, err)
}
return list.Items, nil
}
func sameNodeClaimSpec(have, want v1alpha1.NodeClaimSpec) bool {
if have.ClusterRef.Name != want.ClusterRef.Name || have.NodeName != want.NodeName ||
have.OwnerDid != want.OwnerDid || have.ApiserverPort != want.ApiserverPort ||
have.AdvertiseAddress != want.AdvertiseAddress || have.TokenFile != want.TokenFile ||
have.NodePasswordFile != want.NodePasswordFile || have.ServerCAFile != want.ServerCAFile {
return false
}
if want.AdoptJoinedNode && !have.AdoptJoinedNode {
return false
}
if len(have.DenyBidderDids) != len(want.DenyBidderDids) {
return false
}
for i := range have.DenyBidderDids {
if have.DenyBidderDids[i] != want.DenyBidderDids[i] {
return false
}
}
return true
}
func applyNodeClaim(ctx context.Context, r rest.Interface, claim v1alpha1.NodeClaim) error {
obj := v1alpha1.NodeClaim{
TypeMeta: metav1.TypeMeta{APIVersion: v1alpha1.GroupVersion.String(), Kind: "NodeClaim"},
ObjectMeta: metav1.ObjectMeta{Name: claim.Name, Annotations: claim.Annotations, OwnerReferences: claim.OwnerReferences},
Spec: claim.Spec,
}
return applyDoc(ctx, r.Patch(types.ApplyPatchType).Resource("nodeclaims").Name(claim.Name), v1alpha1.FieldManagerNodeMarket, true, &obj, false)
}
func createNodeClaim(ctx context.Context, r rest.Interface, claim v1alpha1.NodeClaim) error {
if err := applyNodeClaim(ctx, r, claim); err != nil {
if apierrors.IsForbidden(err) && strings.Contains(err.Error(), "exceeded quota") {
return fmt.Errorf("kcpclient: AtCapacity: the home workspace's NodeClaim quota is used up, so nodeclaim %s is not created and the next pass asks again: %w", claim.Name, err)
}
return fmt.Errorf("kcpclient: create nodeclaim %s: %w", claim.Name, err)
}
return nil
}