~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/socialweb-computer-console

console.socialweb.computer: kcp, kubectl and ssh compiled to wasm in a browser tab with a terminal; a kcp fork

0
0
main
967 lines · 46.0 KB Raw History
/*
Copyright 2022 The kcp Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"sync"
	"time"

	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/admission"
	"k8s.io/apiserver/pkg/authentication/authenticator"
	authenticatorunion "k8s.io/apiserver/pkg/authentication/request/union"
	"k8s.io/apiserver/pkg/endpoints/filters"
	"k8s.io/apiserver/pkg/informerfactoryhack"
	"k8s.io/apiserver/pkg/quota/v1/generic"
	genericapiserver "k8s.io/apiserver/pkg/server"
	serverstorage "k8s.io/apiserver/pkg/server/storage"
	"k8s.io/apiserver/pkg/util/compatibility"
	"k8s.io/apiserver/pkg/util/webhook"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
	"k8s.io/kubernetes/pkg/controlplane"
	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
	"k8s.io/kubernetes/pkg/controlplane/apiserver/miniaggregator"
	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
	quotainstall "k8s.io/kubernetes/pkg/quota/v1/install"

	kcpapiextensionsclientset "github.com/kcp-dev/client-go/apiextensions/client"
	kcpapiextensionsinformers "github.com/kcp-dev/client-go/apiextensions/informers"
	kcpdynamic "github.com/kcp-dev/client-go/dynamic"
	kcpkubernetesinformers "github.com/kcp-dev/client-go/informers"
	kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes"
	embeddedetcd "github.com/kcp-dev/kcp/pkg/server/embeddedshim"
	"github.com/kcp-dev/logicalcluster/v3"
	apisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2"
	corev1alpha1 "github.com/kcp-dev/sdk/apis/core/v1alpha1"
	tenancyv1alpha1 "github.com/kcp-dev/sdk/apis/tenancy/v1alpha1"
	kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster"
	kcpinformers "github.com/kcp-dev/sdk/client/informers/externalversions"

	kcpadmissioninitializers "github.com/kcp-dev/kcp/pkg/admission/initializers"
	"github.com/kcp-dev/kcp/pkg/authentication"
	"github.com/kcp-dev/kcp/pkg/authorization"
	bootstrappolicy "github.com/kcp-dev/kcp/pkg/authorization/bootstrap"
	"github.com/kcp-dev/kcp/pkg/contextmanager"
	kcpfeatures "github.com/kcp-dev/kcp/pkg/features"
	"github.com/kcp-dev/kcp/pkg/indexers"
	"github.com/kcp-dev/kcp/pkg/informer"
	"github.com/kcp-dev/kcp/pkg/network"
	"github.com/kcp-dev/kcp/pkg/objectcount"
	"github.com/kcp-dev/kcp/pkg/reconciler/dynamicrestmapper"
	"github.com/kcp-dev/kcp/pkg/reconciler/migration/logicalclustermigration"
	"github.com/kcp-dev/kcp/pkg/server/aggregatingcrdversiondiscovery"
	"github.com/kcp-dev/kcp/pkg/server/bootstrap"
	kcpfilters "github.com/kcp-dev/kcp/pkg/server/filters"
	"github.com/kcp-dev/kcp/pkg/server/migrationdump"
	"github.com/kcp-dev/kcp/pkg/server/openapiv3"
	kcpserveroptions "github.com/kcp-dev/kcp/pkg/server/options"
	"github.com/kcp-dev/kcp/pkg/server/options/batteries"
	"github.com/kcp-dev/kcp/pkg/server/virtualresources"
	"github.com/kcp-dev/kcp/pkg/shardlookup"

	_ "net/http/pprof"
)

type Config struct {
	Options kcpserveroptions.CompletedOptions

	EmbeddedEtcd *embeddedetcd.Config

	GenericConfig                  *genericapiserver.Config // the config embedded into MiniAggregator, the head of the delegation chain
	MiniAggregator                 *miniaggregator.MiniAggregatorConfig
	Apis                           *controlplaneapiserver.Config
	ApiExtensions                  *apiextensionsapiserver.Config
	VirtualResources               *virtualresources.Config
	AggregatingCRDVersionDiscovery *aggregatingcrdversiondiscovery.Config
	OptionalVirtual                *VirtualConfig

	ExtraConfig
}

type ExtraConfig struct {
	// resolveIdentities is to be called on server start until it succeeds. It injects the kcp
	// resource identities into the rest.Config used by the client. Only after it succeeds,
	// the clients can wildcard-list/watch most kcp resources.
	resolveIdentities func(ctx context.Context) error
	IdentityConfig    *rest.Config

	// authentication
	kcpAdminToken, shardAdminToken, userToken string
	shardAdminTokenHash                       []byte

	// clients
	DynamicClusterClient                kcpdynamic.ClusterInterface
	KubeClusterClient                   kcpkubernetesclientset.ClusterInterface
	DeepSARClient                       kcpkubernetesclientset.ClusterInterface
	ApiExtensionsClusterClient          kcpapiextensionsclientset.ClusterInterface
	CacheApiExtensionsClusterClient     kcpapiextensionsclientset.ClusterInterface
	KcpClusterClient                    kcpclientset.ClusterInterface
	KcpCacheClusterClient               kcpclientset.ClusterInterface
	RootShardKcpClusterClient           kcpclientset.ClusterInterface
	BootstrapDynamicClusterClient       kcpdynamic.ClusterInterface
	BootstrapApiExtensionsClusterClient kcpapiextensionsclientset.ClusterInterface

	CacheDynamicClient kcpdynamic.ClusterInterface

	LogicalClusterAdminConfig         *rest.Config // client config connecting directly to shards, skipping the front proxy
	ExternalLogicalClusterAdminConfig *rest.Config // client config connecting to the front proxy

	// misc
	preHandlerChainMux       *handlerChainMuxes
	quotaAdmissionStopCh     chan struct{}
	ClusterContextManager    *contextmanager.Manager[logicalcluster.Path]
	MigratingLogicalClusters *logicalclustermigration.MigratingLogicalClusters
	MigrationDumpHandler     *migrationdump.Handler
	openAPIv3Controller      *openapiv3.Controller
	openAPIv3ServiceCache    *openapiv3.ServiceCache

	// ObjectCountRegistry tracks the total object count per logical cluster on
	// this shard, fed by a periodic etcd scan and admission-time deltas. It is
	// shared between the objectcountlimit admission plugin and the scanner.
	ObjectCountRegistry *objectcount.Registry

	// URL getters depending on genericspiserver.ExternalAddress which is initialized on server run
	ShardBaseURL             func() string
	ShardExternalURL         func() string
	ShardVirtualWorkspaceURL func() string

	// informers
	KcpSharedInformerFactory                kcpinformers.SharedInformerFactory
	KubeSharedInformerFactory               kcpkubernetesinformers.SharedInformerFactory
	ApiExtensionsSharedInformerFactory      kcpapiextensionsinformers.SharedInformerFactory
	PartialMetadataDDSIF                    *informer.DiscoveringDynamicSharedInformerFactory
	CachePartialMetadataDDSIF               *informer.DiscoveringDynamicSharedInformerFactory
	CacheApiExtensionsSharedInformerFactory kcpapiextensionsinformers.SharedInformerFactory
	CacheKcpSharedInformerFactory           kcpinformers.SharedInformerFactory
	CacheKubeSharedInformerFactory          kcpkubernetesinformers.SharedInformerFactory

	// DynRESTMapper is a workspace-aware REST mapper, backed by a reconciler,
	// which dynamically loads all bound resources through every type associated
	// with an APIBinding in the workspace into the mapper. Another controller can
	// use this to resolve the Kind/Resource of the objects.
	DynamicRESTMapper *dynamicrestmapper.DynamicRESTMapper
}

type completedConfig struct {
	Options kcpserveroptions.CompletedOptions

	GenericConfig                  genericapiserver.CompletedConfig
	EmbeddedEtcd                   embeddedetcd.CompletedConfig
	MiniAggregator                 miniaggregator.CompletedMiniAggregatorConfig
	Apis                           controlplaneapiserver.CompletedConfig
	ApiExtensions                  apiextensionsapiserver.CompletedConfig
	VirtualResources               virtualresources.CompletedConfig
	AggregatingCRDVersionDiscovery aggregatingcrdversiondiscovery.CompletedConfig
	OptionalVirtual                CompletedVirtualConfig

	ExtraConfig
}

type CompletedConfig struct {
	// Embed a private pointer that cannot be instantiated outside of this package.
	*completedConfig
}

// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (c *Config) Complete() (CompletedConfig, error) {
	miniAggregator := c.MiniAggregator.Complete()

	return CompletedConfig{&completedConfig{
		Options: c.Options,

		GenericConfig:                  c.GenericConfig.Complete(informerfactoryhack.Wrap(c.KubeSharedInformerFactory)),
		EmbeddedEtcd:                   c.EmbeddedEtcd.Complete(),
		MiniAggregator:                 miniAggregator,
		Apis:                           c.Apis.Complete(),
		ApiExtensions:                  c.ApiExtensions.Complete(),
		AggregatingCRDVersionDiscovery: c.AggregatingCRDVersionDiscovery.Complete(),
		VirtualResources:               c.VirtualResources.Complete(),
		OptionalVirtual: c.OptionalVirtual.Complete(
			miniAggregator.GenericConfig.Authentication,
			miniAggregator.GenericConfig.AuditPolicyRuleEvaluator,
			miniAggregator.GenericConfig.AuditBackend,
			c.GenericConfig.ExternalAddress,
		),

		ExtraConfig: c.ExtraConfig,
	}}, nil
}

const KcpBootstrapperUserName = "system:kcp:bootstrapper"

func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Config, error) {
	c := &Config{
		Options: opts,
	}

	if opts.Extra.ProfilerAddress != "" {
		//nolint:errcheck,gosec
		go http.ListenAndServe(opts.Extra.ProfilerAddress, nil)
	}

	if opts.EmbeddedEtcd.Enabled {
		var err error
		c.EmbeddedEtcd, err = embeddedetcd.NewConfig(opts.EmbeddedEtcd, opts.GenericControlPlane.Etcd.EnableWatchCache)
		if err != nil {
			return nil, err
		}
		c.EmbeddedEtcd.LogLevel = "error"
	}

	var err error
	var storageFactory *serverstorage.DefaultStorageFactory
	c.GenericConfig, c.KubeSharedInformerFactory, storageFactory, err = controlplaneapiserver.BuildGenericConfig(
		opts.GenericControlPlane,
		[]*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme},
		controlplane.DefaultAPIResourceConfigSource(),
		generatedopenapi.GetOpenAPIDefinitions,
	)
	if err != nil {
		return nil, err
	}

	// break connections on the tcp layer. Setting the client timeout would
	// also apply to watches, which we don't want.
	// To prevent data races when wrapping the default transport in
	// multiple goroutines the wrapping is done in a once.
	var wrapDefaultTransportWrapper = sync.Once{}
	c.GenericConfig.LoopbackClientConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper {
		wrapDefaultTransportWrapper.Do(func() {
			rt = network.DefaultTransportWrapper(rt)
		})
		return rt
	})
	// Re-install the in-process dialer after the network wrapper swapped
	// DialContext. This is a no-op for native builds that never set one.
	c.GenericConfig.LoopbackClientConfig.Wrap(wrapLoopbackDial)

	// Set effective version to the default kube version of the vendored libs.
	c.GenericConfig.EffectiveVersion = compatibility.DefaultBuildEffectiveVersion()

	c.KubeClusterClient, err = kcpkubernetesclientset.NewForConfig(rest.CopyConfig(c.GenericConfig.LoopbackClientConfig))
	if err != nil {
		return nil, err
	}
	cacheClientConfig, err := c.Options.Cache.Client.RestConfig(rest.CopyConfig(c.GenericConfig.LoopbackClientConfig))
	if err != nil {
		return nil, err
	}

	cacheKcpClusterClient, err := kcpclientset.NewForConfig(cacheClientConfig)
	if err != nil {
		return nil, err
	}
	c.KcpCacheClusterClient = cacheKcpClusterClient
	cacheKubeClusterClient, err := kcpkubernetesclientset.NewForConfig(cacheClientConfig)
	if err != nil {
		return nil, err
	}
	c.CacheKcpSharedInformerFactory = kcpinformers.NewSharedInformerFactoryWithOptions(
		cacheKcpClusterClient,
		resyncPeriod,
	)
	c.CacheKubeSharedInformerFactory = kcpkubernetesinformers.NewSharedInformerFactoryWithOptions(
		cacheKubeClusterClient,
		resyncPeriod,
	)
	c.CacheDynamicClient, err = kcpdynamic.NewForConfig(cacheClientConfig)
	if err != nil {
		return nil, err
	}

	// Setup kcp * informers, but those will need the identities for the APIExports used to make the APIs available.
	// The identities are not known before we can get them from the APIExports via the loopback client or from the root shard in case this is a non-root shard,
	// hence we postpone this to getOrCreateKcpIdentities() in the kcp-start-informers post-start hook.
	// The informers here are not used before the informers are actually started (i.e. no race).
	if len(c.Options.Extra.RootShardKubeconfigFile) > 0 {
		// TODO(p0lyn0mial): use kcp-admin instead of system:admin
		nonIdentityRootKcpShardBaseConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: c.Options.Extra.RootShardKubeconfigFile}, &clientcmd.ConfigOverrides{CurrentContext: "shard-base"}).ClientConfig()
		if err != nil {
			return nil, fmt.Errorf("failed to load the kubeconfig from: %s, for the root shard, err: %w", c.Options.Extra.RootShardKubeconfigFile, err)
		}

		var kcpShardIdentityRoundTripper func(rt http.RoundTripper) http.RoundTripper
		kcpShardIdentityRoundTripper, c.resolveIdentities = bootstrap.NewWildcardIdentitiesWrappingRoundTripper(bootstrap.KcpRootGroupExportNames, bootstrap.KcpRootGroupResourceExportNames, nonIdentityRootKcpShardBaseConfig, c.KubeClusterClient)
		rootKcpShardIdentityConfig := rest.CopyConfig(nonIdentityRootKcpShardBaseConfig)
		rootKcpShardIdentityConfig.Wrap(kcpShardIdentityRoundTripper)
		c.RootShardKcpClusterClient, err = kcpclientset.NewForConfig(rootKcpShardIdentityConfig)
		if err != nil {
			return nil, err
		}

		c.IdentityConfig = rest.CopyConfig(c.GenericConfig.LoopbackClientConfig)
		c.IdentityConfig.Wrap(kcpShardIdentityRoundTripper)
		c.KcpClusterClient, err = kcpclientset.NewForConfig(c.IdentityConfig) // this is now generic to be used for all kcp API groups
		if err != nil {
			return nil, err
		}
	} else {
		// The informers here are not used before the informers are actually started (i.e. no race).

		c.IdentityConfig, c.resolveIdentities = bootstrap.NewConfigWithWildcardIdentities(c.GenericConfig.LoopbackClientConfig, bootstrap.KcpRootGroupExportNames, bootstrap.KcpRootGroupResourceExportNames, nil)
		c.KcpClusterClient, err = kcpclientset.NewForConfig(c.IdentityConfig) // this is now generic to be used for all kcp API groups
		if err != nil {
			return nil, err
		}
		c.RootShardKcpClusterClient = c.KcpClusterClient
	}

	informerConfig := rest.CopyConfig(c.IdentityConfig)
	informerConfig.UserAgent = "kcp-informers"
	informerKcpClient, err := kcpclientset.NewForConfig(informerConfig)
	if err != nil {
		return nil, err
	}
	c.KcpSharedInformerFactory = kcpinformers.NewSharedInformerFactoryWithOptions(
		informerKcpClient,
		resyncPeriod,
	)
	c.DeepSARClient, err = kcpkubernetesclientset.NewForConfig(authorization.WithDeepSARConfig(rest.CopyConfig(c.GenericConfig.LoopbackClientConfig)))
	if err != nil {
		return nil, err
	}

	c.LogicalClusterAdminConfig = rest.CopyConfig(c.GenericConfig.LoopbackClientConfig)
	if len(c.Options.Extra.LogicalClusterAdminKubeconfig) > 0 {
		c.LogicalClusterAdminConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: c.Options.Extra.LogicalClusterAdminKubeconfig}, nil).ClientConfig()
		if err != nil {
			return nil, fmt.Errorf("failed to load the logical cluster admin kubeconfig from %q: %w", c.Options.Extra.LogicalClusterAdminKubeconfig, err)
		}
	}

	c.ExternalLogicalClusterAdminConfig = rest.CopyConfig(c.GenericConfig.LoopbackClientConfig)
	if len(c.Options.Extra.ExternalLogicalClusterAdminKubeconfig) > 0 {
		c.ExternalLogicalClusterAdminConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: c.Options.Extra.ExternalLogicalClusterAdminKubeconfig}, nil).ClientConfig()
		if err != nil {
			return nil, fmt.Errorf("failed to load the external logical cluster admin kubeconfig from %q: %w", c.Options.Extra.ExternalLogicalClusterAdminKubeconfig, err)
		}
	}

	externalKubeClient, err := kcpkubernetesclientset.NewForConfig(c.ExternalLogicalClusterAdminConfig)
	if err != nil {
		return nil, err
	}
	opts.Extra.ServiceAccountCache.SetKubeShardClient(externalKubeClient)
	opts.Extra.ServiceAccountCache.SetInformers(c.KcpSharedInformerFactory)

	// Setup apiextensions * informers
	c.ApiExtensionsClusterClient, err = kcpapiextensionsclientset.NewForConfig(c.GenericConfig.LoopbackClientConfig)
	if err != nil {
		return nil, err
	}
	c.ApiExtensionsSharedInformerFactory = kcpapiextensionsinformers.NewSharedInformerFactoryWithOptions(
		c.ApiExtensionsClusterClient,
		resyncPeriod,
	)

	c.CacheApiExtensionsClusterClient, err = kcpapiextensionsclientset.NewForConfig(cacheClientConfig)
	if err != nil {
		return nil, err
	}
	c.CacheApiExtensionsSharedInformerFactory = kcpapiextensionsinformers.NewSharedInformerFactoryWithOptions(
		c.CacheApiExtensionsClusterClient,
		resyncPeriod,
	)

	// Setup dynamic client
	c.DynamicClusterClient, err = kcpdynamic.NewForConfig(c.GenericConfig.LoopbackClientConfig)
	if err != nil {
		return nil, err
	}

	if err := opts.Authorization.ApplyTo(ctx, c.GenericConfig, c.KubeSharedInformerFactory, c.CacheKubeSharedInformerFactory, c.KcpSharedInformerFactory, c.CacheKcpSharedInformerFactory); err != nil {
		return nil, err
	}
	var userToken string
	if sets.New[string](opts.Extra.BatteriesIncluded...).Has(batteries.Admin) {
		c.kcpAdminToken, c.shardAdminToken, userToken, c.shardAdminTokenHash, err = opts.AdminAuthentication.ApplyTo(c.GenericConfig)
		if err != nil {
			return nil, err
		}
		if sets.New[string](opts.Extra.BatteriesIncluded...).Has(batteries.User) {
			c.userToken = userToken
		}
	}

	bootstrapConfig := rest.CopyConfig(c.GenericConfig.LoopbackClientConfig)
	bootstrapConfig.Impersonate.UserName = KcpBootstrapperUserName
	bootstrapConfig.Impersonate.Groups = []string{bootstrappolicy.SystemKcpWorkspaceBootstrapper}
	bootstrapConfig = rest.AddUserAgent(bootstrapConfig, "kcp-bootstrapper")

	c.BootstrapApiExtensionsClusterClient, err = kcpapiextensionsclientset.NewForConfig(bootstrapConfig)
	if err != nil {
		return nil, err
	}

	c.BootstrapDynamicClusterClient, err = kcpdynamic.NewForConfig(bootstrapConfig)
	if err != nil {
		return nil, err
	}

	if err := opts.GenericControlPlane.Audit.ApplyTo(c.GenericConfig); err != nil {
		return nil, err
	}

	var shardVirtualWorkspaceURL *url.URL
	if !opts.Virtual.Enabled && opts.Extra.ShardVirtualWorkspaceURL != "" {
		shardVirtualWorkspaceURL, err = url.Parse(opts.Extra.ShardVirtualWorkspaceURL)
		if err != nil {
			return nil, err
		}
	}

	var virtualWorkspaceServerProxyTransport http.RoundTripper
	var virtualWorkspaceTranposportTLSClientConfig *tls.Config
	if opts.Extra.ShardClientCertFile != "" && opts.Extra.ShardClientKeyFile != "" && opts.Extra.ShardVirtualWorkspaceCAFile != "" {
		caCert, err := os.ReadFile(opts.Extra.ShardVirtualWorkspaceCAFile)
		if err != nil {
			return nil, fmt.Errorf("failed to read CA file %q: %w", opts.Extra.ShardVirtualWorkspaceCAFile, err)
		}

		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)

		cert, err := tls.LoadX509KeyPair(opts.Extra.ShardClientCertFile, opts.Extra.ShardClientKeyFile)
		if err != nil {
			return nil, fmt.Errorf("failed to load client certificate %q or key %q: %w", opts.Extra.ShardClientCertFile, opts.Extra.ShardClientKeyFile, err)
		}

		transport := http.DefaultTransport.(*http.Transport).Clone()
		virtualWorkspaceTranposportTLSClientConfig = &tls.Config{
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
		transport.TLSClientConfig = virtualWorkspaceTranposportTLSClientConfig
		virtualWorkspaceServerProxyTransport = transport
	}

	// Transport used by the local proxy when it forwards a request for a mounted
	// workspace back to the front-proxy. It presents a requestheader-CA-signed client
	// certificate so the front-proxy authenticates the forwarded identity headers
	// (via its requestheader authenticator) instead of clearing them.
	var mountProxyTransport http.RoundTripper
	if opts.Extra.MountProxyClientCertFile != "" && opts.Extra.MountProxyClientKeyFile != "" {
		cert, err := tls.LoadX509KeyPair(opts.Extra.MountProxyClientCertFile, opts.Extra.MountProxyClientKeyFile)
		if err != nil {
			return nil, fmt.Errorf("failed to load mount proxy client certificate %q or key %q: %w", opts.Extra.MountProxyClientCertFile, opts.Extra.MountProxyClientKeyFile, err)
		}

		transport := http.DefaultTransport.(*http.Transport).Clone()
		transport.TLSClientConfig = &tls.Config{
			Certificates: []tls.Certificate{cert},
			// TODO: verify the front-proxy serving cert here instead of skipping, once a CA is wired through in dev mode.
			InsecureSkipVerify: true,
		}
		mountProxyTransport = transport
	}

	// Prepare a local cluster index that can be used both by the LocalProxy, as well as the authenticator
	// (the authenticator cannot always use the data the localproxy puts into the context because the
	// authentication rest storage provider calls the authenticator outside of the handler chain).
	clusterIndex := newLocalClusterIndex(
		opts.Extra.ShardName,
		opts.Extra.ShardBaseURL,
		c.KcpSharedInformerFactory.Tenancy().V1alpha1().Workspaces(),
		c.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters(),
	)

	// Prepare an authentication index to be used later by a middleware. We start it early
	// because it can potentially fail and the BuildHandlerChainFunc() has no way to return
	// an error.
	var authIndex authentication.AuthenticatorIndex
	if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.WorkspaceAuthentication) {
		localWSTIndexer := c.KcpSharedInformerFactory.Tenancy().V1alpha1().WorkspaceTypes().Informer().GetIndexer()
		_ = localWSTIndexer.AddIndexers(cache.Indexers{
			indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName,
		})
		cacheWSTIndexer := c.CacheKcpSharedInformerFactory.Tenancy().V1alpha1().WorkspaceTypes().Informer().GetIndexer()
		_ = cacheWSTIndexer.AddIndexers(cache.Indexers{
			indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName,
		})

		logicalClusterLister := c.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters().Lister()
		wacLister := c.KcpSharedInformerFactory.Tenancy().V1alpha1().WorkspaceAuthenticationConfigurations().Lister()
		externalKcpClient, err := kcpclientset.NewForConfig(c.ExternalLogicalClusterAdminConfig)
		if err != nil {
			return nil, fmt.Errorf("failed to create external kcp client for workspace authentication: %w", err)
		}

		wacCache := shardlookup.NewTTLCache[*tenancyv1alpha1.WorkspaceAuthenticationConfiguration]()
		wacCache.StartWithContext(ctx)
		wacLookup := shardlookup.NewLookup(
			wacCache,
			func(clusterName logicalcluster.Name, _, _ string) bool {
				_, err := logicalClusterLister.Cluster(clusterName).Get(corev1alpha1.LogicalClusterName)
				return err == nil
			},
			func(clusterName logicalcluster.Name, _, name string) (*tenancyv1alpha1.WorkspaceAuthenticationConfiguration, error) {
				return wacLister.Cluster(clusterName).Get(name)
			},
			func(clusterName logicalcluster.Name, _, name string) (*tenancyv1alpha1.WorkspaceAuthenticationConfiguration, error) {
				// Enforce a timeout in case the front-proxy or targeted shard do not answer.
				ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
				defer cancel()
				return externalKcpClient.TenancyV1alpha1().WorkspaceAuthenticationConfigurations().Cluster(clusterName.Path()).Get(ctx, name, metav1.GetOptions{})
			},
		)

		authIndex = authentication.NewLazyIndex(
			ctx,
			c.GenericConfig.Authentication.APIAudiences,
			localWSTIndexer, cacheWSTIndexer,
			func(clusterName logicalcluster.Name, name string) (*tenancyv1alpha1.WorkspaceAuthenticationConfiguration, error) {
				return wacLookup.Get(clusterName, "", name)
			},
		)
	}

	c.ExtraConfig.ClusterContextManager = contextmanager.New[logicalcluster.Path](ctx)

	// preHandlerChainMux is called before the actual handler chain. Note that BuildHandlerChainFunc below
	// is called multiple times, but only one of the handler chain will actually be used. Hence, we wrap it
	// to give handlers below one mux.Handle func to call.
	//
	// NOTE: kcp fully replaces BuildHandlerChainFunc here, which means any handler chain customization
	// done earlier in the setup pipeline (e.g. genericConfig.BuildHandlerChainFunc =
	// genericapiserver.BuildHandlerChainWithStorageVersionPrecondition in CreateAggregatorConfig when
	// StorageVersionAPI + APIServerIdentity feature gates are enabled) is overwritten and never applied.
	// WithStorageVersionPrecondition is therefore applied explicitly below, gated behind the
	// StorageVersionAPI feature gate (alpha, off by default) to match upstream k8s behaviour.
	// See: https://github.com/kcp-dev/kubernetes/pull/185
	c.preHandlerChainMux = &handlerChainMuxes{}
	c.GenericConfig.BuildHandlerChainFunc = func(apiHandler http.Handler, genericConfig *genericapiserver.Config) (secure http.Handler) {
		apiHandler = openapiv3.WithOpenAPIv3(apiHandler, c.openAPIv3ServiceCache) // will be initialized further down after apiextensions-apiserver
		apiHandler = kcpfilters.WithWildcardListWatchGuard(apiHandler)
		apiHandler = kcpfilters.WithResourceIdentity(apiHandler)
		apiHandler = authorization.WithSubjectAccessReviewAuditAnnotations(apiHandler)
		apiHandler = authorization.WithDeepSubjectAccessReview(apiHandler)

		// WithStorageVersionPrecondition blocks write requests to resources whose storage versions
		// have not yet converged across all kcp shards during a rolling upgrade. It must wrap the
		// API handler before the authz chain runs, and relies on WithRequestInfo
		// (in DefaultBuildHandlerChainFromStartToBeforeImpersonation below) being present in the
		// outer chain at request time.
		// Gated behind StorageVersionAPI (alpha, off by default) to match upstream k8s behaviour.
		if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.StorageVersionAPI) {
			apiHandler = filters.WithStorageVersionPrecondition(apiHandler, genericConfig.StorageVersionManager, genericConfig.Serializer)
		}

		// The following ensures that only the default main api handler chain executes authorizers which log audit messages.
		// All other invocations of the same authorizer chain still work but do not produce audit log entries.
		// This compromises audit log size and information overflow vs. having audit reasons for the main api handler only.
		// First, remember authorizer chain with audit logging disabled.
		authorizerWithoutAudit := genericConfig.Authorization.Authorizer
		// configure audit logging enabled authorizer chain and build the apiHandler using this configuration.
		genericConfig.Authorization.Authorizer = authorization.WithAuditLogging("request.auth.kcp.io", genericConfig.Authorization.Authorizer)
		apiHandler = genericapiserver.DefaultBuildHandlerChainFromAuthzToCompletion(apiHandler, genericConfig)
		// reset authorizer chain with audit logging disabled.
		genericConfig.Authorization.Authorizer = authorizerWithoutAudit

		if opts.HomeWorkspaces.Enabled {
			apiHandler, err = WithHomeWorkspaces(
				apiHandler,
				genericConfig.Authorization.Authorizer,
				c.KubeClusterClient,
				c.KcpClusterClient,
				c.KubeSharedInformerFactory,
				c.KcpSharedInformerFactory,
				c.GenericConfig.ExternalAddress,
			)
			if err != nil {
				panic(err) // shouldn't happen due to flag validation
			}
		}

		// Only include this when the virtual workspace server is external
		if shardVirtualWorkspaceURL != nil && virtualWorkspaceServerProxyTransport != nil {
			proxy := &httputil.ReverseProxy{
				Director: func(r *http.Request) {
					r.URL.Scheme = shardVirtualWorkspaceURL.Scheme
					r.URL.Host = shardVirtualWorkspaceURL.Host
					delete(r.Header, "X-Forwarded-For")
				},
			}
			// Note: this has to come after DefaultBuildHandlerChainBeforeAuthz because it needs the user info, which
			// is only available after DefaultBuildHandlerChainBeforeAuthz.
			apiHandler = WithVirtualWorkspacesProxy(apiHandler, shardVirtualWorkspaceURL, virtualWorkspaceServerProxyTransport, proxy)
		}

		// Wrap authenticator with a per-workspace authenticator if desired. This authenticator
		// requires the WorkspaceAuth middleware to have looked up and injected the relevant
		// authenticator into the request context already.
		if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.WorkspaceAuthentication) {
			genericConfig.Authentication.Authenticator = authenticatorunion.New(
				genericConfig.Authentication.Authenticator,
				authentication.NewStandaloneWorkspaceAuthenticator(clusterIndex, authIndex),
			)
		}

		// There is ordering here in play:
		// 1. Default handlers up to impersonation gatekeeper preventing impersonation of the privileged user.
		// 2. Rest of the handlers up to Authz
		// 3. Scoping handlers to ensure that the request is scoped to the user's clusters before authz is done.
		// 4. Rest of the handlers.
		if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.LogicalClusterMigration) {
			apiHandler = kcpfilters.WithMigrationDumpHandler(apiHandler, c.MigrationDumpHandler)
			apiHandler = kcpfilters.WithBlockMigratingLogicalClusters(apiHandler, c.MigratingLogicalClusters.IsMigrating)
		}
		apiHandler = kcpfilters.WithImpersonationScoping(apiHandler)
		apiHandler = genericapiserver.DefaultBuildHandlerChainFromImpersonationToAuthz(apiHandler, genericConfig)
		apiHandler = kcpfilters.WithImpersonationGatekeeper(apiHandler)
		apiHandler = genericapiserver.DefaultBuildHandlerChainFromStartToBeforeImpersonation(apiHandler, genericConfig)

		// When workspace auth is enabled, it depends on the target cluster whether
		// a custom authenticator exists or not. This needs to be determined before
		// the authentication middleware can run, as it needs to know about the
		// workspace authenticator.
		if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.WorkspaceAuthentication) {
			apiHandler = authentication.WithWorkspaceAuthResolver(apiHandler, authIndex)
		}

		// this will be replaced in DefaultBuildHandlerChain. So at worst we get twice as many warning.
		// But this is not harmful as the kcp warnings are not many.
		apiHandler = filters.WithWarningRecorder(apiHandler)

		apiHandler = kcpfilters.WithAuditEventClusterAnnotation(apiHandler, c.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters())
		apiHandler = kcpfilters.WithPerClusterContext(apiHandler, c.ClusterContextManager)
		apiHandler = kcpfilters.WithBlockInactiveLogicalClusters(apiHandler, c.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters())

		// Add a mux before the chain, for other handlers with their own handler chain to hook in. For example, when
		// the virtual workspace server is running as part of kcp, it registers /services with the mux so it can handle
		// that path itself, instead of the rest of the handler chain above handling it.
		mux := http.NewServeMux()
		mux.Handle("/", apiHandler)
		*c.preHandlerChainMux = []*http.ServeMux{mux}
		apiHandler = mux

		apiHandler = filters.WithAuditInit(apiHandler) // Must run before any audit annotation is made
		// Defense-in-depth: after WithLocalProxy has resolved the workspace path to
		// a logical cluster name, reject any request whose cluster.Name is still
		// path-shaped. Prevents path strings from leaking into etcd keys if the
		// localproxy logic ever regresses.
		apiHandler = kcpfilters.WithClusterNameShapeInvariant(apiHandler)
		// After WithLocalProxy has set (or left empty) the cluster context, decide
		// whether shard-wide URLs like /metrics may be served. Workspace-scoped
		// requests are 501'd; top-level requests are evaluated against root RBAC.
		apiHandler = kcpfilters.WithShardLevelPaths(apiHandler)
		// The local proxy forwards requests for mounted workspaces before the
		// authentication filter runs, so it authenticates those itself and only
		// forwards the resulting identity, never the inbound identity headers.
		mountAuthenticator := authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
			if genericConfig.Authentication.Authenticator == nil {
				return nil, false, nil
			}
			if auds := genericConfig.Authentication.APIAudiences; len(auds) > 0 {
				req = req.WithContext(authenticator.WithAudiences(req.Context(), auds))
			}
			return genericConfig.Authentication.Authenticator.AuthenticateRequest(req)
		})
		apiHandler, err = WithLocalProxy(apiHandler, opts.Extra.ShardName, opts.Extra.AdditionalMappingsFile, clusterIndex, mountProxyTransport, mountAuthenticator)
		if err != nil {
			panic(err) // shouldn't happen due to flag validation
		}
		apiHandler = kcpfilters.WithInClusterServiceAccountRequestRewrite(apiHandler)
		apiHandler = kcpfilters.WithAcceptHeader(apiHandler)
		apiHandler = kcpfilters.WithUserAgent(apiHandler)

		return apiHandler
	}

	// TODO(ncdc): find a way to support the default configuration. For now, don't use it, because it is difficult
	// to get support for the special evaluators for pods/services/pvcs.
	// quotaConfiguration := quotainstall.NewQuotaConfigurationForAdmission()
	quotaConfiguration := generic.NewConfiguration(nil, quotainstall.DefaultIgnoredResources())

	c.ExtraConfig.quotaAdmissionStopCh = make(chan struct{})
	c.ExtraConfig.MigratingLogicalClusters = logicalclustermigration.NewMigratingLogicalClusters()
	c.ExtraConfig.ObjectCountRegistry = objectcount.NewRegistry(opts.Extra.LogicalClusterTotalObjectLimit)

	// DynamicRESTMapper is initialized here, but it starts to be populated only once its controller starts.
	c.DynamicRESTMapper = dynamicrestmapper.NewDynamicRESTMapper()

	admissionPluginInitializers := []admission.PluginInitializer{ //nolint:prealloc
		kcpadmissioninitializers.NewKcpInformersInitializer(c.KcpSharedInformerFactory, c.CacheKcpSharedInformerFactory),
		kcpadmissioninitializers.NewObjectCountRegistryInitializer(c.ObjectCountRegistry),
		kcpadmissioninitializers.NewKubeInformersInitializer(c.KubeSharedInformerFactory, c.CacheKubeSharedInformerFactory),
		kcpadmissioninitializers.NewKubeClusterClientInitializer(c.KubeClusterClient),
		kcpadmissioninitializers.NewKcpClusterClientInitializer(c.KcpClusterClient),
		kcpadmissioninitializers.NewDeepSARClientInitializer(c.DeepSARClient),
		// The external address is provided as a function, as its value may be updated
		// with the default secure port, when the config is later completed.
		kcpadmissioninitializers.NewKubeQuotaConfigurationInitializer(quotaConfiguration),
		kcpadmissioninitializers.NewServerShutdownInitializer(c.quotaAdmissionStopCh),
		kcpadmissioninitializers.NewDynamicClusterClientInitializer(c.DynamicClusterClient),
		kcpadmissioninitializers.NewDynamicRESTMapperInitializer(c.DynamicRESTMapper),
		kcpadmissioninitializers.NewClusterContextManagerInitializer(c.ClusterContextManager),
	}

	c.ShardBaseURL = func() string {
		if opts.Extra.ShardBaseURL != "" {
			return opts.Extra.ShardBaseURL
		}
		return "https://" + c.GenericConfig.ExternalAddress
	}
	c.ShardExternalURL = func() string {
		if opts.Extra.ShardExternalURL != "" {
			return opts.Extra.ShardExternalURL
		}
		return c.ShardBaseURL()
	}
	// thisShardLabels reports the labels of this shard's own Shard object, which
	// is what an endpoint's shard selector is matched against. Every shard
	// labels itself with its name when it registers, so a selector can name a
	// shard without the installation having to label anything.
	thisShardLabels := func() labels.Set {
		shard, err := indexers.ShardByName(
			c.CacheKcpSharedInformerFactory.Core().V1alpha1().Shards().Lister(),
			opts.Extra.ShardName,
		)
		if err != nil {
			// Before the Shard object exists or syncs there is nothing to match
			// on. Selective endpoints will not match, which fails loudly, and
			// endpoints without a selector are unaffected.
			return nil
		}
		return labels.Set(shard.Labels)
	}

	c.ShardVirtualWorkspaceURL = func() string {
		if opts.Extra.ShardVirtualWorkspaceURL != "" {
			return opts.Extra.ShardVirtualWorkspaceURL
		}
		return c.ShardBaseURL()
	}

	serviceResolver := webhook.NewDefaultServiceResolver()
	kubeControlPlane, kubePluginInitializer, err := controlplaneapiserver.CreateConfig(
		opts.GenericControlPlane,
		c.GenericConfig,
		c.KubeSharedInformerFactory,
		storageFactory,
		serviceResolver,
		admissionPluginInitializers,
	)
	if err != nil {
		return nil, err
	}
	c.Apis = kubeControlPlane
	admissionPluginInitializers = append(admissionPluginInitializers, kubePluginInitializer...)

	authInfoResolver := webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeControlPlane.ProxyTransport, kubeControlPlane.Generic.EgressSelector, kubeControlPlane.Generic.LoopbackClientConfig, kubeControlPlane.Generic.TracerProvider)
	conversionFactory, err := NewCRConverterFactory(serviceResolver, authInfoResolver)
	if err != nil {
		return nil, err
	}
	apiextGenericConfig := *c.GenericConfig
	apiextGenericConfig.SkipOpenAPIInstallation = true // we run our own OpenAPI service
	c.ApiExtensions, err = controlplaneapiserver.CreateAPIExtensionsConfig(
		apiextGenericConfig,
		informerfactoryhack.Wrap(c.KubeSharedInformerFactory),
		admissionPluginInitializers,
		opts.GenericControlPlane,
		opts.Extra.APIExtensionsMasterCount(),
		conversionFactory)
	if err != nil {
		return nil, fmt.Errorf("error configuring api extensions: %w", err)
	}

	// make sure the informer gets started, otherwise conversions will not work!
	_ = c.KcpSharedInformerFactory.Apis().V1alpha1().APIConversions().Informer()

	_ = c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions().Informer().GetIndexer().AddIndexers(cache.Indexers{byGroupResourceName: indexCRDByGroupResourceName})
	_ = c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Informer().GetIndexer().AddIndexers(cache.Indexers{
		indexers.APIBindingByIdentityAndGroupResource: indexers.IndexAPIBindingByIdentityGroupResource,
		indexers.APIBindingByBoundResources:           indexers.IndexAPIBindingByBoundResources,
	})
	_ = c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer().AddIndexers(cache.Indexers{
		indexers.APIExportByVirtualResourceFingerprint: indexers.IndexAPIExportByVirtualResourceFingerprint,
	})
	_ = c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer().AddIndexers(cache.Indexers{
		indexers.APIExportByVirtualResourceFingerprint: indexers.IndexAPIExportByVirtualResourceFingerprint,
	})

	apiBindingAwareCRDClusterLister := &apiBindingAwareCRDClusterLister{
		kcpClusterClient:  c.KcpClusterClient,
		crdLister:         c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions().Lister(),
		crdIndexer:        c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions().Informer().GetIndexer(),
		workspaceLister:   c.KcpSharedInformerFactory.Tenancy().V1alpha1().Workspaces().Lister(),
		apiBindingLister:  c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Lister(),
		apiBindingIndexer: c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Informer().GetIndexer(),
		getAPIExportByPath: func(clusterPath logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) {
			return indexers.ByPathAndNameWithFallback[*apisv1alpha2.APIExport](
				apisv1alpha2.Resource("apiexports"),
				c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer(),
				c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer(),
				clusterPath,
				name,
			)
		},
	}
	c.ApiExtensions.ExtraConfig.ClusterAwareCRDLister = apiBindingAwareCRDClusterLister
	c.ApiExtensions.ExtraConfig.Client = c.ApiExtensionsClusterClient
	c.ApiExtensions.ExtraConfig.Informers = c.ApiExtensionsSharedInformerFactory
	c.ApiExtensions.ExtraConfig.TableConverterProvider = NewTableConverterProvider()

	if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.CacheAPIs) {
		// vwClientConfig is used by the aggregating CRD-version discovery server and the virtual
		// resources server to forward requests to the virtual workspace endpoint URLs advertised
		// in ClusterCachedResourceEndpointSlices.
		vwClientConfig := rest.CopyConfig(c.GenericConfig.LoopbackClientConfig)
		if opts.Extra.ShardClientCertFile != "" && opts.Extra.ShardClientKeyFile != "" {
			// On a real, multi-component deployment the advertised endpoint URL is the serving
			// shard's own external virtual-workspace URL (Shard.Spec.VirtualWorkspaceURL, e.g.
			// https://shard.example:8443/services/replication/...), which is reached through the
			// cluster ingress that routes by TLS SNI directly to that shard — not the in-cluster
			// Service address and not loopback, even when the virtual workspace runs embedded in
			// this shard.
			//
			// The loopback config therefore cannot be used: its bearer token is only valid on the
			// loopback listener (the forwarded request would authenticate as anonymous and be
			// denied) and its ServerName is the loopback name (so the ingress cannot SNI-route the
			// connection and resets it). Authenticate with the shard client certificate against a
			// CA that trusts the shard serving certificate, drop the loopback bearer token, and
			// leave ServerName empty so the SNI is derived from the target host.
			virtualWorkspaceCAFile := opts.Extra.ShardVirtualWorkspaceCAFile
			if virtualWorkspaceCAFile == "" {
				// With embedded virtual workspaces --shard-virtual-workspace-ca-file is typically
				// not set; fall back to the root CA, which signs the shard serving certificate.
				virtualWorkspaceCAFile = opts.Controllers.SAController.RootCAFile
			}
			vwClientConfig.BearerToken = ""
			vwClientConfig.BearerTokenFile = ""
			vwClientConfig.TLSClientConfig = rest.TLSClientConfig{
				CAFile:   virtualWorkspaceCAFile,
				CertFile: opts.Extra.ShardClientCertFile,
				KeyFile:  opts.Extra.ShardClientKeyFile,
			}
		}
		// Otherwise (no shard client certificate, e.g. an in-process test server) the endpoint URL
		// points back at this same server, so the loopback config is correct and is kept as-is.

		// We need an aggregating version discovery for CRDs that is RESTstorage-aware.
		// The apiextensions apiserver sources its data from the apiBindingAwareCRDClusterLister.
		// With the onset of virtual resources, not all bound CRDs use CRD storage, and as a consequence,
		// the verbs such a resource supports may also be different -- compared to what apiextensions
		// thinks a CRD should support. This special version discovery server is aware of the storage
		// defined in the APIExport of the bound CRD, and will advertise correct set of verbs in
		// APIResourceList for CRD- or virtual-backed resources.
		aggregatingVersionDiscoveryConfig := *c.GenericConfig
		c.AggregatingCRDVersionDiscovery, err = aggregatingcrdversiondiscovery.NewConfig(
			&aggregatingVersionDiscoveryConfig,
			c.DynamicRESTMapper,
			c.CacheDynamicClient,
			vwClientConfig,
			c.Options.Extra.ShardName,
			c.ShardVirtualWorkspaceURL,
			thisShardLabels,
			c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions(),
			apiBindingAwareCRDClusterLister,
			c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings(),
			c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports(),
			c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports(),
			c.CacheKcpSharedInformerFactory.Core().V1alpha1().Shards(),
		)
		if err != nil {
			return nil, fmt.Errorf("failed to create config for aggregating version discovery server: %v", err)
		}

		// The virtual resources apiserver serves resources from a virtual workspace.
		virtualResourcesConfig := *c.GenericConfig
		virtualResourcesConfig.SkipOpenAPIInstallation = true
		c.VirtualResources, err = virtualresources.NewConfig(
			&virtualResourcesConfig,
			vwClientConfig,
			c.CacheDynamicClient,
			c.Options.Extra.ShardName,
			c.ShardVirtualWorkspaceURL,
			thisShardLabels,
			c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions(),
			c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings(),
			c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports(),
			c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports(),
		)
		if err != nil {
			return nil, fmt.Errorf("failed to create config for virtual resources server: %v", err)
		}
	}

	if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.LogicalClusterMigration) &&
		!isMemoryStorageConfig(c.Options.GenericControlPlane.Etcd.StorageConfig.Transport) {
		etcdClient, err := newEtcdClient(c.Options.GenericControlPlane.Etcd.StorageConfig.Transport)
		if err != nil {
			return nil, fmt.Errorf("failed to create etcd client for migration dump handler: %w", err)
		}
		c.ExtraConfig.MigrationDumpHandler = migrationdump.NewHandler(
			etcdClient,
			c.Options.GenericControlPlane.Etcd.StorageConfig.Prefix,
			c.ExtraConfig.MigratingLogicalClusters,
		)
	}

	c.openAPIv3Controller = openapiv3.NewController(c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions())
	c.openAPIv3ServiceCache = openapiv3.NewServiceCache(c.GenericConfig.OpenAPIV3Config, c.ApiExtensions.ExtraConfig.ClusterAwareCRDLister, c.openAPIv3Controller, openapiv3.DefaultServiceCacheSize)

	c.MiniAggregator = &miniaggregator.MiniAggregatorConfig{
		GenericConfig: *c.GenericConfig,
	}
	// same as in kube-aggregator, remove hooks and OpenAPI
	c.MiniAggregator.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{}
	c.MiniAggregator.GenericConfig.SkipOpenAPIInstallation = true

	if opts.Virtual.Enabled {
		virtualWorkspacesConfig := rest.CopyConfig(c.GenericConfig.LoopbackClientConfig)
		virtualWorkspacesConfig = rest.AddUserAgent(virtualWorkspacesConfig, "virtual-workspaces")

		// externalLogicalClusterAdminConfig (front-proxy) is used by VWs that
		// need to reach clusters hosted on other shards. Defaults to the
		// loopback when no dedicated kubeconfig was supplied; the SAR will
		// fall back to local-only resolution which works in single-shard
		// deployments.
		externalLogicalClusterAdminConfig := rest.CopyConfig(c.ExternalLogicalClusterAdminConfig)
		externalLogicalClusterAdminConfig = rest.AddUserAgent(externalLogicalClusterAdminConfig, "virtual-workspaces-external")

		c.OptionalVirtual, err = newVirtualConfig(
			opts,
			virtualWorkspacesConfig,
			cacheClientConfig,
			externalLogicalClusterAdminConfig,
			c.KubeSharedInformerFactory,
			c.KcpSharedInformerFactory,
			c.CacheKcpSharedInformerFactory,
		)
		if err != nil {
			return nil, err
		}
	}

	return c, nil
}