/* Copyright 2021 The kcp Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package server import ( "crypto/tls" "fmt" "net/http" "net/http/httputil" "net/url" "os" "sigs.k8s.io/yaml" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/client-go/tools/cache" "k8s.io/component-base/metrics/legacyregistry" "k8s.io/klog/v2" "github.com/kcp-dev/logicalcluster/v3" corev1alpha1 "github.com/kcp-dev/sdk/apis/core/v1alpha1" tenancyv1alpha1 "github.com/kcp-dev/sdk/apis/tenancy/v1alpha1" corev1alpha1informers "github.com/kcp-dev/sdk/client/informers/externalversions/core/v1alpha1" tenancyv1alpha1informers "github.com/kcp-dev/sdk/client/informers/externalversions/tenancy/v1alpha1" "github.com/kcp-dev/kcp/pkg/index" indexrewriters "github.com/kcp-dev/kcp/pkg/index/rewriters" "github.com/kcp-dev/kcp/pkg/proxy/authheaders" "github.com/kcp-dev/kcp/pkg/proxy/lookup" "github.com/kcp-dev/kcp/pkg/server/filters" "github.com/kcp-dev/kcp/pkg/server/proxy" "github.com/kcp-dev/kcp/pkg/server/proxy/types" ) func newLocalClusterIndex( shardName, shardBaseURL string, workspaceInformer tenancyv1alpha1informers.WorkspaceClusterInformer, logicalClusterInformer corev1alpha1informers.LogicalClusterClusterInformer, ) *index.State { indexState := index.New([]index.PathRewriter{ indexrewriters.UserRewriter, }) indexState.UpsertShard(shardName, shardBaseURL) _, _ = workspaceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { ws := obj.(*tenancyv1alpha1.Workspace) indexState.UpsertWorkspace(shardName, ws) }, UpdateFunc: func(old, obj interface{}) { ws := obj.(*tenancyv1alpha1.Workspace) indexState.UpsertWorkspace(shardName, ws) }, DeleteFunc: func(obj interface{}) { if final, ok := obj.(cache.DeletedFinalStateUnknown); ok { obj = final.Obj } ws := obj.(*tenancyv1alpha1.Workspace) indexState.DeleteWorkspace(shardName, ws) }, }) _, _ = logicalClusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { logicalCluster := obj.(*corev1alpha1.LogicalCluster) indexState.UpsertLogicalCluster(shardName, logicalCluster) }, UpdateFunc: func(old, obj interface{}) { logicalCluster := obj.(*corev1alpha1.LogicalCluster) indexState.UpsertLogicalCluster(shardName, logicalCluster) }, DeleteFunc: func(obj interface{}) { if final, ok := obj.(cache.DeletedFinalStateUnknown); ok { obj = final.Obj } logicalCluster := obj.(*corev1alpha1.LogicalCluster) indexState.DeleteLogicalCluster(shardName, logicalCluster) }, }) return indexState } // WithLocalProxy returns a handler with a local-only mini-front-proxy. It is // able to translate logical clusters with the data on the local shard. This is // mainly interesting for standalone mode, without a real front-proxy in-front. // // mountAuthenticator is used to authenticate requests for mounted workspaces // before they are forwarded, as this handler runs in front of the shard's // authentication filter. func WithLocalProxy( handler http.Handler, shardName, additionalMappingsFile string, clusterIndex *index.State, mountProxyTransport http.RoundTripper, mountAuthenticator authenticator.Request, ) (http.Handler, error) { defaultHandlerFunc := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { ctx := req.Context() logger := klog.FromContext(ctx) path, newURL, foundInIndex, err := filters.ClusterPathFromAndStrip(req) if err != nil { responsewriters.ErrorNegotiated( apierrors.NewBadRequest(err.Error()), errorCodecs, schema.GroupVersion{}, w, req, ) return } if foundInIndex { req.URL = newURL } cluster := request.Cluster{ PartialMetadataRequest: filters.IsPartialMetadataRequest(req.Context()), Wildcard: path == logicalcluster.Wildcard, } if path.Empty() || cluster.Wildcard { ctx := request.WithCluster(req.Context(), cluster) handler.ServeHTTP(w, req.WithContext(ctx)) return } if !path.IsValid() { responsewriters.ErrorNegotiated( apierrors.NewBadRequest(fmt.Sprintf("invalid cluster: %q does not match the regex", path)), errorCodecs, schema.GroupVersion{}, w, req) return } // here, we have a real path we have to translate to get a logical cluster name // first pure names if name, isName := path.Name(); isName { cluster.Name = name handler.ServeHTTP(w, req.WithContext(request.WithCluster(ctx, cluster))) return } // lookup in our local, potentially partial index r, found := clusterIndex.Lookup(path) if found && r.ErrorCode != 0 { // return code if set. // TODO(mjudeikis): Would be nice to have a way to return a custom error message. http.Error(w, "Not available.", r.ErrorCode) return } if found && r.Shard != shardName && r.URL == "" { logger.WithValues("cluster", cluster.Name, "requestedShard", r.Shard, "actualShard", shardName).Info("cluster is not on this shard, but on another") w.Header().Set("Retry-After", fmt.Sprintf("%d", 1)) http.Error(w, "Not found on this shard", http.StatusTooManyRequests) return } if r.URL != "" { u, err := url.Parse(r.URL) if err != nil { logger.WithValues("cluster", cluster.Name, "url", r.URL).Error(err, "invalid url") http.Error(w, "invalid url", http.StatusInternalServerError) return } logger.WithValues("from", path, "to", r.URL).V(4).Info("mounting cluster") proxy := httputil.NewSingleHostReverseProxy(u) if mountProxyTransport != nil { // Present a requestheader-CA-signed client certificate so the front-proxy // authenticates the forwarded identity headers instead of clearing them. proxy.Transport = mountProxyTransport } else { // TODO(mjudeikis): remove this once we have a real cert wired in dev mode proxy.Transport = &http.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, } } setMountProxyAuthHeaders(req, mountAuthenticator) proxy.ServeHTTP(w, req) return } // if this is a non-path request, there is nothing to lookup clusterName, isName := path.Name() if !isName && !found { // The request targets a workspace path (e.g. root:internal-cluster) // that we cannot resolve to a logical cluster on this shard. Do NOT // forward with the raw path stuffed into cluster.Name: the storage // key builder (NoNamespaceKeyRootFunc) would serialize it into the // etcd key as if it were a real logical cluster name, producing // orphaned rows that are invisible to the normal read path but // still consume space and can leak via wildcard partial-metadata // lists. Reject the request here instead. logger.WithValues("path", path).V(4).Info("cluster path not found on this shard") responsewriters.ErrorNegotiated( apierrors.NewNotFound( schema.GroupResource{Group: tenancyv1alpha1.SchemeGroupVersion.Group, Resource: "workspaces"}, path.String(), ), errorCodecs, schema.GroupVersion{}, w, req, ) return } if found { if r.Cluster.Path() != path { logger.WithValues("from", path, "to", r.Cluster).V(4).Info("rewriting cluster") } cluster.Name = r.Cluster } else { // we check "!isName && !foundInIndex" above. So, here it is a name. cluster.Name = clusterName } ctx = request.WithCluster(ctx, cluster) ctx = lookup.WithClusterName(ctx, cluster.Name) ctx = lookup.WithWorkspaceType(ctx, r.Type) handler.ServeHTTP(w, req.WithContext(ctx)) }) // if no additional mappings are provided, do not start local-only mini-front-proxy. if additionalMappingsFile == "" { return defaultHandlerFunc, nil } // If additional mappings file is provided, read it and add the mappings to the handler handlers, err := NewLocalProxyHandler(defaultHandlerFunc, additionalMappingsFile) if err != nil { return nil, fmt.Errorf("failed to create local proxy handler: %w", err) } return handlers, nil } // NewLocalProxyHandler returns a handler with a local-only mini-front-proxy. // This function is very similar to proxy/mapping.go.NewHandler. // If we want to re-use that code, we basically would be merging proxy with server packages. // Which is not desirable at the point of writing (2024-10-26), but might be in the future. func NewLocalProxyHandler(defaultHandler http.Handler, additionalMappingsFile string) (http.Handler, error) { mapping := []types.PathMapping{} if additionalMappingsFile != "" { mappingData, err := os.ReadFile(additionalMappingsFile) if err != nil { return nil, fmt.Errorf("failed to read mapping file %q: %w", additionalMappingsFile, err) } if err = yaml.Unmarshal(mappingData, &mapping); err != nil { return nil, fmt.Errorf("failed to unmarshal mapping file %q: %w", additionalMappingsFile, err) } } handlers := proxy.HttpHandler{ Mappings: types.HttpHandlerMappings{ { Weight: 0, Path: "/metrics", Handler: legacyregistry.Handler(), }, }, DefaultHandler: defaultHandler, } for _, m := range mapping { u, err := url.Parse(m.Backend) if err != nil { return nil, fmt.Errorf("failed to create path mapping for path %q: failed to parse URL %q: %w", m.Path, m.Backend, err) } // This is insecure, but just don't tell it to anyone :) transport, err := newInsecureTransport() if err != nil { return nil, fmt.Errorf("failed to create path mapping for path %q: %w", m.Path, err) } var handler http.Handler p := httputil.NewSingleHostReverseProxy(u) p.Transport = transport handler = p userHeader := "X-Remote-User" groupHeader := "X-Remote-Group" extraHeaderPrefix := "X-Remote-Extra-" if m.UserHeader != "" { userHeader = m.UserHeader } if m.GroupHeader != "" { groupHeader = m.GroupHeader } if m.ExtraHeaderPrefix != "" { extraHeaderPrefix = m.ExtraHeaderPrefix } handler = withProxyAuthHeaders(handler, userHeader, groupHeader, extraHeaderPrefix) handlers.Mappings = append(handlers.Mappings, types.HttpHandlerMapping{ Weight: len(m.Path), Path: m.Path, Handler: handler, }) } handlers.Mappings.Sort() return &handlers, nil } func newInsecureTransport() (*http.Transport, error) { transport := http.DefaultTransport.(*http.Transport).Clone() transport.TLSClientConfig = &tls.Config{ InsecureSkipVerify: true, } return transport, nil } // setMountProxyAuthHeaders replaces any inbound identity headers on a request // for a mounted workspace with the identity the shard itself authenticated. // // The mount branch of WithLocalProxy runs before the shard's authentication // filter, so inbound X-Remote-* headers are attacker-controlled until proven // otherwise. Forwarding them verbatim over the requestheader-CA-signed mount // transport would let a client that reaches the shard directly assert any // identity (including a forged warrant) to the front-proxy. Identity headers // forwarded by the front-proxy itself survive, because the shard's // requestheader authenticator verifies the front-proxy's client certificate. // // Anonymous and unauthenticated requests are forwarded without identity // headers, so that credentials only the mount target can verify (e.g. a // bearer token) are still evaluated there. func setMountProxyAuthHeaders(req *http.Request, authn authenticator.Request) { if authn != nil { // Authenticate a clone: authenticators mutate the request (e.g. the bearer // token authenticator deletes the Authorization header on success), but the // mount target may need those credentials to authenticate the request itself. resp, ok, err := authn.AuthenticateRequest(req.Clone(req.Context())) if err == nil && ok && resp.User != nil && resp.User.GetName() != user.Anonymous { authheaders.SetAuthHeaders(req.Header, resp.User, authheaders.DefaultUserHeader, authheaders.DefaultGroupHeader, authheaders.DefaultExtraHeaderPrefix) return } } authheaders.ClearAuthHeaders(req.Header, authheaders.DefaultUserHeader, authheaders.DefaultGroupHeader, authheaders.DefaultExtraHeaderPrefix) } // withProxyAuthHeaders does client cert termination by extracting the user and groups and // passing them through access headers to the shard. func withProxyAuthHeaders(delegate http.Handler, userHeader, groupHeader string, extraHeaderPrefix string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if u, ok := request.UserFrom(r.Context()); ok { authheaders.SetAuthHeaders(r.Header, u, userHeader, groupHeader, extraHeaderPrefix) } else { authheaders.ClearAuthHeaders(r.Header, userHeader, groupHeader, extraHeaderPrefix) } delegate.ServeHTTP(w, r) } }