/* Copyright 2022 The kcp Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package server import ( "context" "crypto/tls" "crypto/x509" "fmt" "net/http" "net/http/httputil" "net/url" "os" "sync" "time" apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authentication/authenticator" authenticatorunion "k8s.io/apiserver/pkg/authentication/request/union" "k8s.io/apiserver/pkg/endpoints/filters" "k8s.io/apiserver/pkg/informerfactoryhack" "k8s.io/apiserver/pkg/quota/v1/generic" genericapiserver "k8s.io/apiserver/pkg/server" serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/util/compatibility" "k8s.io/apiserver/pkg/util/webhook" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controlplane" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" "k8s.io/kubernetes/pkg/controlplane/apiserver/miniaggregator" generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" quotainstall "k8s.io/kubernetes/pkg/quota/v1/install" kcpapiextensionsclientset "github.com/kcp-dev/client-go/apiextensions/client" kcpapiextensionsinformers "github.com/kcp-dev/client-go/apiextensions/informers" kcpdynamic "github.com/kcp-dev/client-go/dynamic" kcpkubernetesinformers "github.com/kcp-dev/client-go/informers" kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" embeddedetcd "github.com/kcp-dev/kcp/pkg/server/embeddedshim" "github.com/kcp-dev/logicalcluster/v3" apisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2" corev1alpha1 "github.com/kcp-dev/sdk/apis/core/v1alpha1" tenancyv1alpha1 "github.com/kcp-dev/sdk/apis/tenancy/v1alpha1" kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" kcpinformers "github.com/kcp-dev/sdk/client/informers/externalversions" kcpadmissioninitializers "github.com/kcp-dev/kcp/pkg/admission/initializers" "github.com/kcp-dev/kcp/pkg/authentication" "github.com/kcp-dev/kcp/pkg/authorization" bootstrappolicy "github.com/kcp-dev/kcp/pkg/authorization/bootstrap" "github.com/kcp-dev/kcp/pkg/contextmanager" kcpfeatures "github.com/kcp-dev/kcp/pkg/features" "github.com/kcp-dev/kcp/pkg/indexers" "github.com/kcp-dev/kcp/pkg/informer" "github.com/kcp-dev/kcp/pkg/network" "github.com/kcp-dev/kcp/pkg/objectcount" "github.com/kcp-dev/kcp/pkg/reconciler/dynamicrestmapper" "github.com/kcp-dev/kcp/pkg/reconciler/migration/logicalclustermigration" "github.com/kcp-dev/kcp/pkg/server/aggregatingcrdversiondiscovery" "github.com/kcp-dev/kcp/pkg/server/bootstrap" kcpfilters "github.com/kcp-dev/kcp/pkg/server/filters" "github.com/kcp-dev/kcp/pkg/server/migrationdump" "github.com/kcp-dev/kcp/pkg/server/openapiv3" kcpserveroptions "github.com/kcp-dev/kcp/pkg/server/options" "github.com/kcp-dev/kcp/pkg/server/options/batteries" "github.com/kcp-dev/kcp/pkg/server/virtualresources" "github.com/kcp-dev/kcp/pkg/shardlookup" _ "net/http/pprof" ) type Config struct { Options kcpserveroptions.CompletedOptions EmbeddedEtcd *embeddedetcd.Config GenericConfig *genericapiserver.Config // the config embedded into MiniAggregator, the head of the delegation chain MiniAggregator *miniaggregator.MiniAggregatorConfig Apis *controlplaneapiserver.Config ApiExtensions *apiextensionsapiserver.Config VirtualResources *virtualresources.Config AggregatingCRDVersionDiscovery *aggregatingcrdversiondiscovery.Config OptionalVirtual *VirtualConfig ExtraConfig } type ExtraConfig struct { // resolveIdentities is to be called on server start until it succeeds. It injects the kcp // resource identities into the rest.Config used by the client. Only after it succeeds, // the clients can wildcard-list/watch most kcp resources. resolveIdentities func(ctx context.Context) error IdentityConfig *rest.Config // authentication kcpAdminToken, shardAdminToken, userToken string shardAdminTokenHash []byte // clients DynamicClusterClient kcpdynamic.ClusterInterface KubeClusterClient kcpkubernetesclientset.ClusterInterface DeepSARClient kcpkubernetesclientset.ClusterInterface ApiExtensionsClusterClient kcpapiextensionsclientset.ClusterInterface CacheApiExtensionsClusterClient kcpapiextensionsclientset.ClusterInterface KcpClusterClient kcpclientset.ClusterInterface KcpCacheClusterClient kcpclientset.ClusterInterface RootShardKcpClusterClient kcpclientset.ClusterInterface BootstrapDynamicClusterClient kcpdynamic.ClusterInterface BootstrapApiExtensionsClusterClient kcpapiextensionsclientset.ClusterInterface CacheDynamicClient kcpdynamic.ClusterInterface LogicalClusterAdminConfig *rest.Config // client config connecting directly to shards, skipping the front proxy ExternalLogicalClusterAdminConfig *rest.Config // client config connecting to the front proxy // misc preHandlerChainMux *handlerChainMuxes quotaAdmissionStopCh chan struct{} ClusterContextManager *contextmanager.Manager[logicalcluster.Path] MigratingLogicalClusters *logicalclustermigration.MigratingLogicalClusters MigrationDumpHandler *migrationdump.Handler openAPIv3Controller *openapiv3.Controller openAPIv3ServiceCache *openapiv3.ServiceCache // ObjectCountRegistry tracks the total object count per logical cluster on // this shard, fed by a periodic etcd scan and admission-time deltas. It is // shared between the objectcountlimit admission plugin and the scanner. ObjectCountRegistry *objectcount.Registry // URL getters depending on genericspiserver.ExternalAddress which is initialized on server run ShardBaseURL func() string ShardExternalURL func() string ShardVirtualWorkspaceURL func() string // informers KcpSharedInformerFactory kcpinformers.SharedInformerFactory KubeSharedInformerFactory kcpkubernetesinformers.SharedInformerFactory ApiExtensionsSharedInformerFactory kcpapiextensionsinformers.SharedInformerFactory PartialMetadataDDSIF *informer.DiscoveringDynamicSharedInformerFactory CachePartialMetadataDDSIF *informer.DiscoveringDynamicSharedInformerFactory CacheApiExtensionsSharedInformerFactory kcpapiextensionsinformers.SharedInformerFactory CacheKcpSharedInformerFactory kcpinformers.SharedInformerFactory CacheKubeSharedInformerFactory kcpkubernetesinformers.SharedInformerFactory // DynRESTMapper is a workspace-aware REST mapper, backed by a reconciler, // which dynamically loads all bound resources through every type associated // with an APIBinding in the workspace into the mapper. Another controller can // use this to resolve the Kind/Resource of the objects. DynamicRESTMapper *dynamicrestmapper.DynamicRESTMapper } type completedConfig struct { Options kcpserveroptions.CompletedOptions GenericConfig genericapiserver.CompletedConfig EmbeddedEtcd embeddedetcd.CompletedConfig MiniAggregator miniaggregator.CompletedMiniAggregatorConfig Apis controlplaneapiserver.CompletedConfig ApiExtensions apiextensionsapiserver.CompletedConfig VirtualResources virtualresources.CompletedConfig AggregatingCRDVersionDiscovery aggregatingcrdversiondiscovery.CompletedConfig OptionalVirtual CompletedVirtualConfig ExtraConfig } type CompletedConfig struct { // Embed a private pointer that cannot be instantiated outside of this package. *completedConfig } // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. func (c *Config) Complete() (CompletedConfig, error) { miniAggregator := c.MiniAggregator.Complete() return CompletedConfig{&completedConfig{ Options: c.Options, GenericConfig: c.GenericConfig.Complete(informerfactoryhack.Wrap(c.KubeSharedInformerFactory)), EmbeddedEtcd: c.EmbeddedEtcd.Complete(), MiniAggregator: miniAggregator, Apis: c.Apis.Complete(), ApiExtensions: c.ApiExtensions.Complete(), AggregatingCRDVersionDiscovery: c.AggregatingCRDVersionDiscovery.Complete(), VirtualResources: c.VirtualResources.Complete(), OptionalVirtual: c.OptionalVirtual.Complete( miniAggregator.GenericConfig.Authentication, miniAggregator.GenericConfig.AuditPolicyRuleEvaluator, miniAggregator.GenericConfig.AuditBackend, c.GenericConfig.ExternalAddress, ), ExtraConfig: c.ExtraConfig, }}, nil } const KcpBootstrapperUserName = "system:kcp:bootstrapper" func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Config, error) { c := &Config{ Options: opts, } if opts.Extra.ProfilerAddress != "" { //nolint:errcheck,gosec go http.ListenAndServe(opts.Extra.ProfilerAddress, nil) } if opts.EmbeddedEtcd.Enabled { var err error c.EmbeddedEtcd, err = embeddedetcd.NewConfig(opts.EmbeddedEtcd, opts.GenericControlPlane.Etcd.EnableWatchCache) if err != nil { return nil, err } c.EmbeddedEtcd.LogLevel = "error" } var err error var storageFactory *serverstorage.DefaultStorageFactory c.GenericConfig, c.KubeSharedInformerFactory, storageFactory, err = controlplaneapiserver.BuildGenericConfig( opts.GenericControlPlane, []*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme}, controlplane.DefaultAPIResourceConfigSource(), generatedopenapi.GetOpenAPIDefinitions, ) if err != nil { return nil, err } // break connections on the tcp layer. Setting the client timeout would // also apply to watches, which we don't want. // To prevent data races when wrapping the default transport in // multiple goroutines the wrapping is done in a once. var wrapDefaultTransportWrapper = sync.Once{} c.GenericConfig.LoopbackClientConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper { wrapDefaultTransportWrapper.Do(func() { rt = network.DefaultTransportWrapper(rt) }) return rt }) // Re-install the in-process dialer after the network wrapper swapped // DialContext. This is a no-op for native builds that never set one. c.GenericConfig.LoopbackClientConfig.Wrap(wrapLoopbackDial) // Set effective version to the default kube version of the vendored libs. c.GenericConfig.EffectiveVersion = compatibility.DefaultBuildEffectiveVersion() c.KubeClusterClient, err = kcpkubernetesclientset.NewForConfig(rest.CopyConfig(c.GenericConfig.LoopbackClientConfig)) if err != nil { return nil, err } cacheClientConfig, err := c.Options.Cache.Client.RestConfig(rest.CopyConfig(c.GenericConfig.LoopbackClientConfig)) if err != nil { return nil, err } cacheKcpClusterClient, err := kcpclientset.NewForConfig(cacheClientConfig) if err != nil { return nil, err } c.KcpCacheClusterClient = cacheKcpClusterClient cacheKubeClusterClient, err := kcpkubernetesclientset.NewForConfig(cacheClientConfig) if err != nil { return nil, err } c.CacheKcpSharedInformerFactory = kcpinformers.NewSharedInformerFactoryWithOptions( cacheKcpClusterClient, resyncPeriod, ) c.CacheKubeSharedInformerFactory = kcpkubernetesinformers.NewSharedInformerFactoryWithOptions( cacheKubeClusterClient, resyncPeriod, ) c.CacheDynamicClient, err = kcpdynamic.NewForConfig(cacheClientConfig) if err != nil { return nil, err } // Setup kcp * informers, but those will need the identities for the APIExports used to make the APIs available. // The identities are not known before we can get them from the APIExports via the loopback client or from the root shard in case this is a non-root shard, // hence we postpone this to getOrCreateKcpIdentities() in the kcp-start-informers post-start hook. // The informers here are not used before the informers are actually started (i.e. no race). if len(c.Options.Extra.RootShardKubeconfigFile) > 0 { // TODO(p0lyn0mial): use kcp-admin instead of system:admin nonIdentityRootKcpShardBaseConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: c.Options.Extra.RootShardKubeconfigFile}, &clientcmd.ConfigOverrides{CurrentContext: "shard-base"}).ClientConfig() if err != nil { return nil, fmt.Errorf("failed to load the kubeconfig from: %s, for the root shard, err: %w", c.Options.Extra.RootShardKubeconfigFile, err) } var kcpShardIdentityRoundTripper func(rt http.RoundTripper) http.RoundTripper kcpShardIdentityRoundTripper, c.resolveIdentities = bootstrap.NewWildcardIdentitiesWrappingRoundTripper(bootstrap.KcpRootGroupExportNames, bootstrap.KcpRootGroupResourceExportNames, nonIdentityRootKcpShardBaseConfig, c.KubeClusterClient) rootKcpShardIdentityConfig := rest.CopyConfig(nonIdentityRootKcpShardBaseConfig) rootKcpShardIdentityConfig.Wrap(kcpShardIdentityRoundTripper) c.RootShardKcpClusterClient, err = kcpclientset.NewForConfig(rootKcpShardIdentityConfig) if err != nil { return nil, err } c.IdentityConfig = rest.CopyConfig(c.GenericConfig.LoopbackClientConfig) c.IdentityConfig.Wrap(kcpShardIdentityRoundTripper) c.KcpClusterClient, err = kcpclientset.NewForConfig(c.IdentityConfig) // this is now generic to be used for all kcp API groups if err != nil { return nil, err } } else { // The informers here are not used before the informers are actually started (i.e. no race). c.IdentityConfig, c.resolveIdentities = bootstrap.NewConfigWithWildcardIdentities(c.GenericConfig.LoopbackClientConfig, bootstrap.KcpRootGroupExportNames, bootstrap.KcpRootGroupResourceExportNames, nil) c.KcpClusterClient, err = kcpclientset.NewForConfig(c.IdentityConfig) // this is now generic to be used for all kcp API groups if err != nil { return nil, err } c.RootShardKcpClusterClient = c.KcpClusterClient } informerConfig := rest.CopyConfig(c.IdentityConfig) informerConfig.UserAgent = "kcp-informers" informerKcpClient, err := kcpclientset.NewForConfig(informerConfig) if err != nil { return nil, err } c.KcpSharedInformerFactory = kcpinformers.NewSharedInformerFactoryWithOptions( informerKcpClient, resyncPeriod, ) c.DeepSARClient, err = kcpkubernetesclientset.NewForConfig(authorization.WithDeepSARConfig(rest.CopyConfig(c.GenericConfig.LoopbackClientConfig))) if err != nil { return nil, err } c.LogicalClusterAdminConfig = rest.CopyConfig(c.GenericConfig.LoopbackClientConfig) if len(c.Options.Extra.LogicalClusterAdminKubeconfig) > 0 { c.LogicalClusterAdminConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: c.Options.Extra.LogicalClusterAdminKubeconfig}, nil).ClientConfig() if err != nil { return nil, fmt.Errorf("failed to load the logical cluster admin kubeconfig from %q: %w", c.Options.Extra.LogicalClusterAdminKubeconfig, err) } } c.ExternalLogicalClusterAdminConfig = rest.CopyConfig(c.GenericConfig.LoopbackClientConfig) if len(c.Options.Extra.ExternalLogicalClusterAdminKubeconfig) > 0 { c.ExternalLogicalClusterAdminConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: c.Options.Extra.ExternalLogicalClusterAdminKubeconfig}, nil).ClientConfig() if err != nil { return nil, fmt.Errorf("failed to load the external logical cluster admin kubeconfig from %q: %w", c.Options.Extra.ExternalLogicalClusterAdminKubeconfig, err) } } externalKubeClient, err := kcpkubernetesclientset.NewForConfig(c.ExternalLogicalClusterAdminConfig) if err != nil { return nil, err } opts.Extra.ServiceAccountCache.SetKubeShardClient(externalKubeClient) opts.Extra.ServiceAccountCache.SetInformers(c.KcpSharedInformerFactory) // Setup apiextensions * informers c.ApiExtensionsClusterClient, err = kcpapiextensionsclientset.NewForConfig(c.GenericConfig.LoopbackClientConfig) if err != nil { return nil, err } c.ApiExtensionsSharedInformerFactory = kcpapiextensionsinformers.NewSharedInformerFactoryWithOptions( c.ApiExtensionsClusterClient, resyncPeriod, ) c.CacheApiExtensionsClusterClient, err = kcpapiextensionsclientset.NewForConfig(cacheClientConfig) if err != nil { return nil, err } c.CacheApiExtensionsSharedInformerFactory = kcpapiextensionsinformers.NewSharedInformerFactoryWithOptions( c.CacheApiExtensionsClusterClient, resyncPeriod, ) // Setup dynamic client c.DynamicClusterClient, err = kcpdynamic.NewForConfig(c.GenericConfig.LoopbackClientConfig) if err != nil { return nil, err } if err := opts.Authorization.ApplyTo(ctx, c.GenericConfig, c.KubeSharedInformerFactory, c.CacheKubeSharedInformerFactory, c.KcpSharedInformerFactory, c.CacheKcpSharedInformerFactory); err != nil { return nil, err } var userToken string if sets.New[string](opts.Extra.BatteriesIncluded...).Has(batteries.Admin) { c.kcpAdminToken, c.shardAdminToken, userToken, c.shardAdminTokenHash, err = opts.AdminAuthentication.ApplyTo(c.GenericConfig) if err != nil { return nil, err } if sets.New[string](opts.Extra.BatteriesIncluded...).Has(batteries.User) { c.userToken = userToken } } bootstrapConfig := rest.CopyConfig(c.GenericConfig.LoopbackClientConfig) bootstrapConfig.Impersonate.UserName = KcpBootstrapperUserName bootstrapConfig.Impersonate.Groups = []string{bootstrappolicy.SystemKcpWorkspaceBootstrapper} bootstrapConfig = rest.AddUserAgent(bootstrapConfig, "kcp-bootstrapper") c.BootstrapApiExtensionsClusterClient, err = kcpapiextensionsclientset.NewForConfig(bootstrapConfig) if err != nil { return nil, err } c.BootstrapDynamicClusterClient, err = kcpdynamic.NewForConfig(bootstrapConfig) if err != nil { return nil, err } if err := opts.GenericControlPlane.Audit.ApplyTo(c.GenericConfig); err != nil { return nil, err } var shardVirtualWorkspaceURL *url.URL if !opts.Virtual.Enabled && opts.Extra.ShardVirtualWorkspaceURL != "" { shardVirtualWorkspaceURL, err = url.Parse(opts.Extra.ShardVirtualWorkspaceURL) if err != nil { return nil, err } } var virtualWorkspaceServerProxyTransport http.RoundTripper var virtualWorkspaceTranposportTLSClientConfig *tls.Config if opts.Extra.ShardClientCertFile != "" && opts.Extra.ShardClientKeyFile != "" && opts.Extra.ShardVirtualWorkspaceCAFile != "" { caCert, err := os.ReadFile(opts.Extra.ShardVirtualWorkspaceCAFile) if err != nil { return nil, fmt.Errorf("failed to read CA file %q: %w", opts.Extra.ShardVirtualWorkspaceCAFile, err) } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) cert, err := tls.LoadX509KeyPair(opts.Extra.ShardClientCertFile, opts.Extra.ShardClientKeyFile) if err != nil { return nil, fmt.Errorf("failed to load client certificate %q or key %q: %w", opts.Extra.ShardClientCertFile, opts.Extra.ShardClientKeyFile, err) } transport := http.DefaultTransport.(*http.Transport).Clone() virtualWorkspaceTranposportTLSClientConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, RootCAs: caCertPool, } transport.TLSClientConfig = virtualWorkspaceTranposportTLSClientConfig virtualWorkspaceServerProxyTransport = transport } // Transport used by the local proxy when it forwards a request for a mounted // workspace back to the front-proxy. It presents a requestheader-CA-signed client // certificate so the front-proxy authenticates the forwarded identity headers // (via its requestheader authenticator) instead of clearing them. var mountProxyTransport http.RoundTripper if opts.Extra.MountProxyClientCertFile != "" && opts.Extra.MountProxyClientKeyFile != "" { cert, err := tls.LoadX509KeyPair(opts.Extra.MountProxyClientCertFile, opts.Extra.MountProxyClientKeyFile) if err != nil { return nil, fmt.Errorf("failed to load mount proxy client certificate %q or key %q: %w", opts.Extra.MountProxyClientCertFile, opts.Extra.MountProxyClientKeyFile, err) } transport := http.DefaultTransport.(*http.Transport).Clone() transport.TLSClientConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, // TODO: verify the front-proxy serving cert here instead of skipping, once a CA is wired through in dev mode. InsecureSkipVerify: true, } mountProxyTransport = transport } // Prepare a local cluster index that can be used both by the LocalProxy, as well as the authenticator // (the authenticator cannot always use the data the localproxy puts into the context because the // authentication rest storage provider calls the authenticator outside of the handler chain). clusterIndex := newLocalClusterIndex( opts.Extra.ShardName, opts.Extra.ShardBaseURL, c.KcpSharedInformerFactory.Tenancy().V1alpha1().Workspaces(), c.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters(), ) // Prepare an authentication index to be used later by a middleware. We start it early // because it can potentially fail and the BuildHandlerChainFunc() has no way to return // an error. var authIndex authentication.AuthenticatorIndex if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.WorkspaceAuthentication) { localWSTIndexer := c.KcpSharedInformerFactory.Tenancy().V1alpha1().WorkspaceTypes().Informer().GetIndexer() _ = localWSTIndexer.AddIndexers(cache.Indexers{ indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, }) cacheWSTIndexer := c.CacheKcpSharedInformerFactory.Tenancy().V1alpha1().WorkspaceTypes().Informer().GetIndexer() _ = cacheWSTIndexer.AddIndexers(cache.Indexers{ indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, }) logicalClusterLister := c.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters().Lister() wacLister := c.KcpSharedInformerFactory.Tenancy().V1alpha1().WorkspaceAuthenticationConfigurations().Lister() externalKcpClient, err := kcpclientset.NewForConfig(c.ExternalLogicalClusterAdminConfig) if err != nil { return nil, fmt.Errorf("failed to create external kcp client for workspace authentication: %w", err) } wacCache := shardlookup.NewTTLCache[*tenancyv1alpha1.WorkspaceAuthenticationConfiguration]() wacCache.StartWithContext(ctx) wacLookup := shardlookup.NewLookup( wacCache, func(clusterName logicalcluster.Name, _, _ string) bool { _, err := logicalClusterLister.Cluster(clusterName).Get(corev1alpha1.LogicalClusterName) return err == nil }, func(clusterName logicalcluster.Name, _, name string) (*tenancyv1alpha1.WorkspaceAuthenticationConfiguration, error) { return wacLister.Cluster(clusterName).Get(name) }, func(clusterName logicalcluster.Name, _, name string) (*tenancyv1alpha1.WorkspaceAuthenticationConfiguration, error) { // Enforce a timeout in case the front-proxy or targeted shard do not answer. ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) defer cancel() return externalKcpClient.TenancyV1alpha1().WorkspaceAuthenticationConfigurations().Cluster(clusterName.Path()).Get(ctx, name, metav1.GetOptions{}) }, ) authIndex = authentication.NewLazyIndex( ctx, c.GenericConfig.Authentication.APIAudiences, localWSTIndexer, cacheWSTIndexer, func(clusterName logicalcluster.Name, name string) (*tenancyv1alpha1.WorkspaceAuthenticationConfiguration, error) { return wacLookup.Get(clusterName, "", name) }, ) } c.ExtraConfig.ClusterContextManager = contextmanager.New[logicalcluster.Path](ctx) // preHandlerChainMux is called before the actual handler chain. Note that BuildHandlerChainFunc below // is called multiple times, but only one of the handler chain will actually be used. Hence, we wrap it // to give handlers below one mux.Handle func to call. // // NOTE: kcp fully replaces BuildHandlerChainFunc here, which means any handler chain customization // done earlier in the setup pipeline (e.g. genericConfig.BuildHandlerChainFunc = // genericapiserver.BuildHandlerChainWithStorageVersionPrecondition in CreateAggregatorConfig when // StorageVersionAPI + APIServerIdentity feature gates are enabled) is overwritten and never applied. // WithStorageVersionPrecondition is therefore applied explicitly below, gated behind the // StorageVersionAPI feature gate (alpha, off by default) to match upstream k8s behaviour. // See: https://github.com/kcp-dev/kubernetes/pull/185 c.preHandlerChainMux = &handlerChainMuxes{} c.GenericConfig.BuildHandlerChainFunc = func(apiHandler http.Handler, genericConfig *genericapiserver.Config) (secure http.Handler) { apiHandler = openapiv3.WithOpenAPIv3(apiHandler, c.openAPIv3ServiceCache) // will be initialized further down after apiextensions-apiserver apiHandler = kcpfilters.WithWildcardListWatchGuard(apiHandler) apiHandler = kcpfilters.WithResourceIdentity(apiHandler) apiHandler = authorization.WithSubjectAccessReviewAuditAnnotations(apiHandler) apiHandler = authorization.WithDeepSubjectAccessReview(apiHandler) // WithStorageVersionPrecondition blocks write requests to resources whose storage versions // have not yet converged across all kcp shards during a rolling upgrade. It must wrap the // API handler before the authz chain runs, and relies on WithRequestInfo // (in DefaultBuildHandlerChainFromStartToBeforeImpersonation below) being present in the // outer chain at request time. // Gated behind StorageVersionAPI (alpha, off by default) to match upstream k8s behaviour. if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.StorageVersionAPI) { apiHandler = filters.WithStorageVersionPrecondition(apiHandler, genericConfig.StorageVersionManager, genericConfig.Serializer) } // The following ensures that only the default main api handler chain executes authorizers which log audit messages. // All other invocations of the same authorizer chain still work but do not produce audit log entries. // This compromises audit log size and information overflow vs. having audit reasons for the main api handler only. // First, remember authorizer chain with audit logging disabled. authorizerWithoutAudit := genericConfig.Authorization.Authorizer // configure audit logging enabled authorizer chain and build the apiHandler using this configuration. genericConfig.Authorization.Authorizer = authorization.WithAuditLogging("request.auth.kcp.io", genericConfig.Authorization.Authorizer) apiHandler = genericapiserver.DefaultBuildHandlerChainFromAuthzToCompletion(apiHandler, genericConfig) // reset authorizer chain with audit logging disabled. genericConfig.Authorization.Authorizer = authorizerWithoutAudit if opts.HomeWorkspaces.Enabled { apiHandler, err = WithHomeWorkspaces( apiHandler, genericConfig.Authorization.Authorizer, c.KubeClusterClient, c.KcpClusterClient, c.KubeSharedInformerFactory, c.KcpSharedInformerFactory, c.GenericConfig.ExternalAddress, ) if err != nil { panic(err) // shouldn't happen due to flag validation } } // Only include this when the virtual workspace server is external if shardVirtualWorkspaceURL != nil && virtualWorkspaceServerProxyTransport != nil { proxy := &httputil.ReverseProxy{ Director: func(r *http.Request) { r.URL.Scheme = shardVirtualWorkspaceURL.Scheme r.URL.Host = shardVirtualWorkspaceURL.Host delete(r.Header, "X-Forwarded-For") }, } // Note: this has to come after DefaultBuildHandlerChainBeforeAuthz because it needs the user info, which // is only available after DefaultBuildHandlerChainBeforeAuthz. apiHandler = WithVirtualWorkspacesProxy(apiHandler, shardVirtualWorkspaceURL, virtualWorkspaceServerProxyTransport, proxy) } // Wrap authenticator with a per-workspace authenticator if desired. This authenticator // requires the WorkspaceAuth middleware to have looked up and injected the relevant // authenticator into the request context already. if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.WorkspaceAuthentication) { genericConfig.Authentication.Authenticator = authenticatorunion.New( genericConfig.Authentication.Authenticator, authentication.NewStandaloneWorkspaceAuthenticator(clusterIndex, authIndex), ) } // There is ordering here in play: // 1. Default handlers up to impersonation gatekeeper preventing impersonation of the privileged user. // 2. Rest of the handlers up to Authz // 3. Scoping handlers to ensure that the request is scoped to the user's clusters before authz is done. // 4. Rest of the handlers. if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.LogicalClusterMigration) { apiHandler = kcpfilters.WithMigrationDumpHandler(apiHandler, c.MigrationDumpHandler) apiHandler = kcpfilters.WithBlockMigratingLogicalClusters(apiHandler, c.MigratingLogicalClusters.IsMigrating) } apiHandler = kcpfilters.WithImpersonationScoping(apiHandler) apiHandler = genericapiserver.DefaultBuildHandlerChainFromImpersonationToAuthz(apiHandler, genericConfig) apiHandler = kcpfilters.WithImpersonationGatekeeper(apiHandler) apiHandler = genericapiserver.DefaultBuildHandlerChainFromStartToBeforeImpersonation(apiHandler, genericConfig) // When workspace auth is enabled, it depends on the target cluster whether // a custom authenticator exists or not. This needs to be determined before // the authentication middleware can run, as it needs to know about the // workspace authenticator. if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.WorkspaceAuthentication) { apiHandler = authentication.WithWorkspaceAuthResolver(apiHandler, authIndex) } // this will be replaced in DefaultBuildHandlerChain. So at worst we get twice as many warning. // But this is not harmful as the kcp warnings are not many. apiHandler = filters.WithWarningRecorder(apiHandler) apiHandler = kcpfilters.WithAuditEventClusterAnnotation(apiHandler, c.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters()) apiHandler = kcpfilters.WithPerClusterContext(apiHandler, c.ClusterContextManager) apiHandler = kcpfilters.WithBlockInactiveLogicalClusters(apiHandler, c.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters()) // Add a mux before the chain, for other handlers with their own handler chain to hook in. For example, when // the virtual workspace server is running as part of kcp, it registers /services with the mux so it can handle // that path itself, instead of the rest of the handler chain above handling it. mux := http.NewServeMux() mux.Handle("/", apiHandler) *c.preHandlerChainMux = []*http.ServeMux{mux} apiHandler = mux apiHandler = filters.WithAuditInit(apiHandler) // Must run before any audit annotation is made // Defense-in-depth: after WithLocalProxy has resolved the workspace path to // a logical cluster name, reject any request whose cluster.Name is still // path-shaped. Prevents path strings from leaking into etcd keys if the // localproxy logic ever regresses. apiHandler = kcpfilters.WithClusterNameShapeInvariant(apiHandler) // After WithLocalProxy has set (or left empty) the cluster context, decide // whether shard-wide URLs like /metrics may be served. Workspace-scoped // requests are 501'd; top-level requests are evaluated against root RBAC. apiHandler = kcpfilters.WithShardLevelPaths(apiHandler) // The local proxy forwards requests for mounted workspaces before the // authentication filter runs, so it authenticates those itself and only // forwards the resulting identity, never the inbound identity headers. mountAuthenticator := authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) { if genericConfig.Authentication.Authenticator == nil { return nil, false, nil } if auds := genericConfig.Authentication.APIAudiences; len(auds) > 0 { req = req.WithContext(authenticator.WithAudiences(req.Context(), auds)) } return genericConfig.Authentication.Authenticator.AuthenticateRequest(req) }) apiHandler, err = WithLocalProxy(apiHandler, opts.Extra.ShardName, opts.Extra.AdditionalMappingsFile, clusterIndex, mountProxyTransport, mountAuthenticator) if err != nil { panic(err) // shouldn't happen due to flag validation } apiHandler = kcpfilters.WithInClusterServiceAccountRequestRewrite(apiHandler) apiHandler = kcpfilters.WithAcceptHeader(apiHandler) apiHandler = kcpfilters.WithUserAgent(apiHandler) return apiHandler } // TODO(ncdc): find a way to support the default configuration. For now, don't use it, because it is difficult // to get support for the special evaluators for pods/services/pvcs. // quotaConfiguration := quotainstall.NewQuotaConfigurationForAdmission() quotaConfiguration := generic.NewConfiguration(nil, quotainstall.DefaultIgnoredResources()) c.ExtraConfig.quotaAdmissionStopCh = make(chan struct{}) c.ExtraConfig.MigratingLogicalClusters = logicalclustermigration.NewMigratingLogicalClusters() c.ExtraConfig.ObjectCountRegistry = objectcount.NewRegistry(opts.Extra.LogicalClusterTotalObjectLimit) // DynamicRESTMapper is initialized here, but it starts to be populated only once its controller starts. c.DynamicRESTMapper = dynamicrestmapper.NewDynamicRESTMapper() admissionPluginInitializers := []admission.PluginInitializer{ //nolint:prealloc kcpadmissioninitializers.NewKcpInformersInitializer(c.KcpSharedInformerFactory, c.CacheKcpSharedInformerFactory), kcpadmissioninitializers.NewObjectCountRegistryInitializer(c.ObjectCountRegistry), kcpadmissioninitializers.NewKubeInformersInitializer(c.KubeSharedInformerFactory, c.CacheKubeSharedInformerFactory), kcpadmissioninitializers.NewKubeClusterClientInitializer(c.KubeClusterClient), kcpadmissioninitializers.NewKcpClusterClientInitializer(c.KcpClusterClient), kcpadmissioninitializers.NewDeepSARClientInitializer(c.DeepSARClient), // The external address is provided as a function, as its value may be updated // with the default secure port, when the config is later completed. kcpadmissioninitializers.NewKubeQuotaConfigurationInitializer(quotaConfiguration), kcpadmissioninitializers.NewServerShutdownInitializer(c.quotaAdmissionStopCh), kcpadmissioninitializers.NewDynamicClusterClientInitializer(c.DynamicClusterClient), kcpadmissioninitializers.NewDynamicRESTMapperInitializer(c.DynamicRESTMapper), kcpadmissioninitializers.NewClusterContextManagerInitializer(c.ClusterContextManager), } c.ShardBaseURL = func() string { if opts.Extra.ShardBaseURL != "" { return opts.Extra.ShardBaseURL } return "https://" + c.GenericConfig.ExternalAddress } c.ShardExternalURL = func() string { if opts.Extra.ShardExternalURL != "" { return opts.Extra.ShardExternalURL } return c.ShardBaseURL() } // thisShardLabels reports the labels of this shard's own Shard object, which // is what an endpoint's shard selector is matched against. Every shard // labels itself with its name when it registers, so a selector can name a // shard without the installation having to label anything. thisShardLabels := func() labels.Set { shard, err := indexers.ShardByName( c.CacheKcpSharedInformerFactory.Core().V1alpha1().Shards().Lister(), opts.Extra.ShardName, ) if err != nil { // Before the Shard object exists or syncs there is nothing to match // on. Selective endpoints will not match, which fails loudly, and // endpoints without a selector are unaffected. return nil } return labels.Set(shard.Labels) } c.ShardVirtualWorkspaceURL = func() string { if opts.Extra.ShardVirtualWorkspaceURL != "" { return opts.Extra.ShardVirtualWorkspaceURL } return c.ShardBaseURL() } serviceResolver := webhook.NewDefaultServiceResolver() kubeControlPlane, kubePluginInitializer, err := controlplaneapiserver.CreateConfig( opts.GenericControlPlane, c.GenericConfig, c.KubeSharedInformerFactory, storageFactory, serviceResolver, admissionPluginInitializers, ) if err != nil { return nil, err } c.Apis = kubeControlPlane admissionPluginInitializers = append(admissionPluginInitializers, kubePluginInitializer...) authInfoResolver := webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeControlPlane.ProxyTransport, kubeControlPlane.Generic.EgressSelector, kubeControlPlane.Generic.LoopbackClientConfig, kubeControlPlane.Generic.TracerProvider) conversionFactory, err := NewCRConverterFactory(serviceResolver, authInfoResolver) if err != nil { return nil, err } apiextGenericConfig := *c.GenericConfig apiextGenericConfig.SkipOpenAPIInstallation = true // we run our own OpenAPI service c.ApiExtensions, err = controlplaneapiserver.CreateAPIExtensionsConfig( apiextGenericConfig, informerfactoryhack.Wrap(c.KubeSharedInformerFactory), admissionPluginInitializers, opts.GenericControlPlane, opts.Extra.APIExtensionsMasterCount(), conversionFactory) if err != nil { return nil, fmt.Errorf("error configuring api extensions: %w", err) } // make sure the informer gets started, otherwise conversions will not work! _ = c.KcpSharedInformerFactory.Apis().V1alpha1().APIConversions().Informer() _ = c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions().Informer().GetIndexer().AddIndexers(cache.Indexers{byGroupResourceName: indexCRDByGroupResourceName}) _ = c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Informer().GetIndexer().AddIndexers(cache.Indexers{ indexers.APIBindingByIdentityAndGroupResource: indexers.IndexAPIBindingByIdentityGroupResource, indexers.APIBindingByBoundResources: indexers.IndexAPIBindingByBoundResources, }) _ = c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer().AddIndexers(cache.Indexers{ indexers.APIExportByVirtualResourceFingerprint: indexers.IndexAPIExportByVirtualResourceFingerprint, }) _ = c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer().AddIndexers(cache.Indexers{ indexers.APIExportByVirtualResourceFingerprint: indexers.IndexAPIExportByVirtualResourceFingerprint, }) apiBindingAwareCRDClusterLister := &apiBindingAwareCRDClusterLister{ kcpClusterClient: c.KcpClusterClient, crdLister: c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions().Lister(), crdIndexer: c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions().Informer().GetIndexer(), workspaceLister: c.KcpSharedInformerFactory.Tenancy().V1alpha1().Workspaces().Lister(), apiBindingLister: c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Lister(), apiBindingIndexer: c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Informer().GetIndexer(), getAPIExportByPath: func(clusterPath logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { return indexers.ByPathAndNameWithFallback[*apisv1alpha2.APIExport]( apisv1alpha2.Resource("apiexports"), c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer(), c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer(), clusterPath, name, ) }, } c.ApiExtensions.ExtraConfig.ClusterAwareCRDLister = apiBindingAwareCRDClusterLister c.ApiExtensions.ExtraConfig.Client = c.ApiExtensionsClusterClient c.ApiExtensions.ExtraConfig.Informers = c.ApiExtensionsSharedInformerFactory c.ApiExtensions.ExtraConfig.TableConverterProvider = NewTableConverterProvider() if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.CacheAPIs) { // vwClientConfig is used by the aggregating CRD-version discovery server and the virtual // resources server to forward requests to the virtual workspace endpoint URLs advertised // in ClusterCachedResourceEndpointSlices. vwClientConfig := rest.CopyConfig(c.GenericConfig.LoopbackClientConfig) if opts.Extra.ShardClientCertFile != "" && opts.Extra.ShardClientKeyFile != "" { // On a real, multi-component deployment the advertised endpoint URL is the serving // shard's own external virtual-workspace URL (Shard.Spec.VirtualWorkspaceURL, e.g. // https://shard.example:8443/services/replication/...), which is reached through the // cluster ingress that routes by TLS SNI directly to that shard — not the in-cluster // Service address and not loopback, even when the virtual workspace runs embedded in // this shard. // // The loopback config therefore cannot be used: its bearer token is only valid on the // loopback listener (the forwarded request would authenticate as anonymous and be // denied) and its ServerName is the loopback name (so the ingress cannot SNI-route the // connection and resets it). Authenticate with the shard client certificate against a // CA that trusts the shard serving certificate, drop the loopback bearer token, and // leave ServerName empty so the SNI is derived from the target host. virtualWorkspaceCAFile := opts.Extra.ShardVirtualWorkspaceCAFile if virtualWorkspaceCAFile == "" { // With embedded virtual workspaces --shard-virtual-workspace-ca-file is typically // not set; fall back to the root CA, which signs the shard serving certificate. virtualWorkspaceCAFile = opts.Controllers.SAController.RootCAFile } vwClientConfig.BearerToken = "" vwClientConfig.BearerTokenFile = "" vwClientConfig.TLSClientConfig = rest.TLSClientConfig{ CAFile: virtualWorkspaceCAFile, CertFile: opts.Extra.ShardClientCertFile, KeyFile: opts.Extra.ShardClientKeyFile, } } // Otherwise (no shard client certificate, e.g. an in-process test server) the endpoint URL // points back at this same server, so the loopback config is correct and is kept as-is. // We need an aggregating version discovery for CRDs that is RESTstorage-aware. // The apiextensions apiserver sources its data from the apiBindingAwareCRDClusterLister. // With the onset of virtual resources, not all bound CRDs use CRD storage, and as a consequence, // the verbs such a resource supports may also be different -- compared to what apiextensions // thinks a CRD should support. This special version discovery server is aware of the storage // defined in the APIExport of the bound CRD, and will advertise correct set of verbs in // APIResourceList for CRD- or virtual-backed resources. aggregatingVersionDiscoveryConfig := *c.GenericConfig c.AggregatingCRDVersionDiscovery, err = aggregatingcrdversiondiscovery.NewConfig( &aggregatingVersionDiscoveryConfig, c.DynamicRESTMapper, c.CacheDynamicClient, vwClientConfig, c.Options.Extra.ShardName, c.ShardVirtualWorkspaceURL, thisShardLabels, c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions(), apiBindingAwareCRDClusterLister, c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings(), c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports(), c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports(), c.CacheKcpSharedInformerFactory.Core().V1alpha1().Shards(), ) if err != nil { return nil, fmt.Errorf("failed to create config for aggregating version discovery server: %v", err) } // The virtual resources apiserver serves resources from a virtual workspace. virtualResourcesConfig := *c.GenericConfig virtualResourcesConfig.SkipOpenAPIInstallation = true c.VirtualResources, err = virtualresources.NewConfig( &virtualResourcesConfig, vwClientConfig, c.CacheDynamicClient, c.Options.Extra.ShardName, c.ShardVirtualWorkspaceURL, thisShardLabels, c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions(), c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings(), c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports(), c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports(), ) if err != nil { return nil, fmt.Errorf("failed to create config for virtual resources server: %v", err) } } if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.LogicalClusterMigration) && !isMemoryStorageConfig(c.Options.GenericControlPlane.Etcd.StorageConfig.Transport) { etcdClient, err := newEtcdClient(c.Options.GenericControlPlane.Etcd.StorageConfig.Transport) if err != nil { return nil, fmt.Errorf("failed to create etcd client for migration dump handler: %w", err) } c.ExtraConfig.MigrationDumpHandler = migrationdump.NewHandler( etcdClient, c.Options.GenericControlPlane.Etcd.StorageConfig.Prefix, c.ExtraConfig.MigratingLogicalClusters, ) } c.openAPIv3Controller = openapiv3.NewController(c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions()) c.openAPIv3ServiceCache = openapiv3.NewServiceCache(c.GenericConfig.OpenAPIV3Config, c.ApiExtensions.ExtraConfig.ClusterAwareCRDLister, c.openAPIv3Controller, openapiv3.DefaultServiceCacheSize) c.MiniAggregator = &miniaggregator.MiniAggregatorConfig{ GenericConfig: *c.GenericConfig, } // same as in kube-aggregator, remove hooks and OpenAPI c.MiniAggregator.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} c.MiniAggregator.GenericConfig.SkipOpenAPIInstallation = true if opts.Virtual.Enabled { virtualWorkspacesConfig := rest.CopyConfig(c.GenericConfig.LoopbackClientConfig) virtualWorkspacesConfig = rest.AddUserAgent(virtualWorkspacesConfig, "virtual-workspaces") // externalLogicalClusterAdminConfig (front-proxy) is used by VWs that // need to reach clusters hosted on other shards. Defaults to the // loopback when no dedicated kubeconfig was supplied; the SAR will // fall back to local-only resolution which works in single-shard // deployments. externalLogicalClusterAdminConfig := rest.CopyConfig(c.ExternalLogicalClusterAdminConfig) externalLogicalClusterAdminConfig = rest.AddUserAgent(externalLogicalClusterAdminConfig, "virtual-workspaces-external") c.OptionalVirtual, err = newVirtualConfig( opts, virtualWorkspacesConfig, cacheClientConfig, externalLogicalClusterAdminConfig, c.KubeSharedInformerFactory, c.KcpSharedInformerFactory, c.CacheKcpSharedInformerFactory, ) if err != nil { return nil, err } } return c, nil }