~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/socialweb-computer-console

console.socialweb.computer: kcp, kubectl and ssh compiled to wasm in a browser tab with a terminal; a kcp fork

0
0
main
382 lines · 13.0 KB Raw History
/*
Copyright 2022 The kcp Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/abiosoft/lineprefix"
	"github.com/fatih/color"
	"sigs.k8s.io/yaml"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
	"k8s.io/klog/v2"

	kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster"
	kcptestingserver "github.com/kcp-dev/sdk/testing/server"
	"github.com/kcp-dev/sdk/testing/third_party/library-go/crypto"

	"github.com/kcp-dev/kcp/cmd/test-server/helpers"
	"github.com/kcp-dev/kcp/pkg/server/proxy/types"
)

func startFrontProxy(
	ctx context.Context,
	args []string,
	servingCA *crypto.CA,
	hostIP string,
	logDirPath, workDirPath string,
	vwPort string,
	quiet bool,
) error {
	blue := color.New(color.BgGreen, color.FgBlack).SprintFunc()
	inverse := color.New(color.BgHiWhite, color.FgGreen).SprintFunc()
	out := lineprefix.New(
		lineprefix.Prefix(blue(" PROXY ")),
		lineprefix.Color(color.New(color.FgHiGreen)),
	)
	successOut := lineprefix.New(
		lineprefix.Prefix(inverse(" PROXY ")),
		lineprefix.Color(color.New(color.FgHiWhite)),
	)

	logger := klog.FromContext(ctx)

	mappings := []types.PathMapping{
		{
			Path:            "/e2e/clusters/{cluster}/",
			Backend:         "https://localhost:2443",
			BackendServerCA: filepath.Join(workDirPath, ".kcp", "serving-ca.crt"),
			// in the existing testcases, these two do not matter, but have to be non-empty
			ProxyClientCert: filepath.Join(workDirPath, ".kcp-front-proxy", "requestheader.crt"),
			ProxyClientKey:  filepath.Join(workDirPath, ".kcp-front-proxy", "requestheader.key"),
		},
		{
			Path: "/clusters/",
			// this path is not actually used, since shard URLs are determined based on the Shard
			Backend:         "https://localhost:6444",
			BackendServerCA: filepath.Join(workDirPath, ".kcp", "serving-ca.crt"),
			ProxyClientCert: filepath.Join(workDirPath, ".kcp-front-proxy", "requestheader.crt"),
			ProxyClientKey:  filepath.Join(workDirPath, ".kcp-front-proxy", "requestheader.key"),
		},
	}

	mappingsYAML, err := yaml.Marshal(mappings)
	if err != nil {
		return fmt.Errorf("error marshaling mappings yaml: %w", err)
	}

	if err := os.WriteFile(filepath.Join(workDirPath, ".kcp-front-proxy", "mapping.yaml"), mappingsYAML, 0644); err != nil {
		return fmt.Errorf("failed to create front-proxy mapping.yaml: %w", err)
	}

	// write root shard kubeconfig
	configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(workDirPath, ".kcp-0", "admin.kubeconfig")},
		&clientcmd.ConfigOverrides{CurrentContext: "shard-base"},
	)
	raw, err := configLoader.RawConfig()
	if err != nil {
		return err
	}
	raw.CurrentContext = "shard-base"
	if err := clientcmdapi.MinifyConfig(&raw); err != nil {
		return err
	}
	if err := clientcmd.WriteToFile(raw, filepath.Join(workDirPath, ".kcp", "root.kubeconfig")); err != nil {
		return err
	}

	// create serving cert
	hostnames := sets.New("localhost", hostIP)
	logger.Info("creating kcp-front-proxy serving cert with hostnames", "hostnames", hostnames)
	cert, err := servingCA.MakeServerCert(hostnames, 365)
	if err != nil {
		return fmt.Errorf("failed to create server cert: %w", err)
	}
	if err := cert.WriteCertConfigFile(filepath.Join(workDirPath, ".kcp-front-proxy", "apiserver.crt"), filepath.Join(workDirPath, ".kcp-front-proxy", "apiserver.key")); err != nil {
		return fmt.Errorf("failed to write server cert: %w", err)
	}

	// run front-proxy command
	workdir, commandLine := kcptestingserver.Command("kcp-front-proxy", "front-proxy")
	commandLine = append(commandLine,
		"--bind-address="+hostIP,
		fmt.Sprintf("--mapping-file=%s", filepath.Join(workDirPath, ".kcp-front-proxy", "mapping.yaml")),
		fmt.Sprintf("--root-directory=%s", filepath.Join(workDirPath, ".kcp-front-proxy")),
		fmt.Sprintf("--root-kubeconfig=%s", filepath.Join(workDirPath, ".kcp", "root.kubeconfig")),
		fmt.Sprintf("--shards-kubeconfig=%s", filepath.Join(workDirPath, ".kcp-front-proxy", "shards.kubeconfig")),
		fmt.Sprintf("--client-ca-file=%s", filepath.Join(workDirPath, ".kcp", "client-ca.crt")),
		// Trust requestheader-CA-signed clients (e.g. a shard's local proxy re-entering
		// the front-proxy while forwarding a mounted workspace) to assert identity via
		// X-Remote-* headers. Without this the front-proxy cannot authenticate the
		// forwarded identity on that hop and clears the headers, breaking mounts.
		fmt.Sprintf("--requestheader-client-ca-file=%s", filepath.Join(workDirPath, ".kcp", "requestheader-ca.crt")),
		"--requestheader-username-headers=X-Remote-User",
		"--requestheader-group-headers=X-Remote-Group",
		"--requestheader-extra-headers-prefix=X-Remote-Extra-",
		// Restrict which requestheader-CA-signed client certs may assert identity to the
		// known infra CNs (the front-proxy itself and the shards' local proxy). This keeps
		// the front-proxy stricter than the shards: only these certs can forward headers,
		// not any cert the requestheader CA happens to sign.
		"--requestheader-allowed-names=kcp-front-proxy,kcp-mounts-proxy",
		fmt.Sprintf("--tls-cert-file=%s", filepath.Join(workDirPath, ".kcp-front-proxy", "apiserver.crt")),
		fmt.Sprintf("--tls-private-key-file=%s", filepath.Join(workDirPath, ".kcp-front-proxy", "apiserver.key")),
		"--secure-port=6443",
		"--v=4",
	)
	commandLine = append(commandLine, args...)
	fmt.Fprintf(out, "running: %v\n", strings.Join(commandLine, " "))

	cmd := exec.CommandContext(ctx, commandLine[0], commandLine[1:]...) //nolint:gosec
	cmd.Dir = workdir

	logFilePath := filepath.Join(workDirPath, ".kcp-front-proxy", "proxy.log")
	if logDirPath != "" {
		logFilePath = filepath.Join(logDirPath, "kcp-front-proxy.log")
	}

	logFile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		return err
	}

	writer := helpers.NewHeadWriter(logFile, out)
	cmd.Stdout = writer
	cmd.Stdin = os.Stdin
	cmd.Stderr = writer

	if quiet {
		writer.StopOut()
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	go func() {
		<-ctx.Done()
		if err := cmd.Process.Kill(); err != nil {
			logger.Error(err, "failed to kill process")
		}
	}()

	terminatedCh := make(chan int, 1)
	go func() {
		if err := cmd.Wait(); err != nil {
			var exitErr *exec.ExitError
			if errors.As(err, &exitErr) {
				terminatedCh <- exitErr.ExitCode()
			}
		} else {
			terminatedCh <- 0
		}
	}()

	// wait for readiness
	logger.Info("waiting for kcp-front-proxy to be up")
	for {
		time.Sleep(time.Second)

		select {
		case <-ctx.Done():
			return fmt.Errorf("context canceled")
		case rc := <-terminatedCh:
			return fmt.Errorf("kcp-front-proxy terminated with exit code %d", rc)
		default:
		}

		// intentionally load again every iteration because it can change
		configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(workDirPath, ".kcp", "admin.kubeconfig")},
			&clientcmd.ConfigOverrides{CurrentContext: "base"},
		)
		config, err := configLoader.ClientConfig()
		if err != nil {
			continue
		}
		kcpClient, err := kcpclientset.NewForConfig(config)
		if err != nil {
			logger.Error(err, "failed to create kcp client")
			continue
		}

		res := kcpClient.RESTClient().Get().AbsPath("/readyz").Do(ctx)
		if err := res.Error(); err != nil {
			logger.V(3).Info("kcp-front-proxy not ready", "err", err)
		} else {
			var rc int
			res.StatusCode(&rc)
			if rc == http.StatusOK {
				break
			}
			if bs, err := res.Raw(); err != nil {
				logger.V(3).Info("kcp-front-proxy not ready", "err", err)
			} else {
				logger.V(3).WithValues("rc", rc, "raw", string(bs)).Info("kcp-front-proxy not ready: http")
			}
		}
	}
	if !logger.V(3).Enabled() {
		writer.StopOut()
	}
	fmt.Fprintf(successOut, "kcp-front-proxy is ready\n")
	cfg, err := configLoader.ClientConfig()
	if err != nil {
		return err
	}
	return scrapeMetrics(ctx, cfg, workDirPath)
}

func scrapeMetrics(ctx context.Context, cfg *rest.Config, workDir string) error {
	promUrl, set := os.LookupEnv("PROMETHEUS_URL")
	if !set || promUrl == "" {
		return nil
	}
	return kcptestingserver.ScrapeMetrics(ctx, cfg, promUrl, workDir, "kcp-front-proxy", filepath.Join(workDir, ".kcp-front-proxy", "apiserver.crt"), map[string]string{
		"server": "kcp-front-proxy",
	})
}

func writeAdminKubeConfig(hostIP string, workDirPath string) error {
	baseHost := "https://" + net.JoinHostPort(hostIP, "6443")

	var kubeConfig clientcmdapi.Config
	kubeConfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{
		"kcp-admin": {
			ClientKey:         filepath.Join(workDirPath, ".kcp", "kcp-admin.key"),
			ClientCertificate: filepath.Join(workDirPath, ".kcp", "kcp-admin.crt"),
		},
	}
	kubeConfig.Clusters = map[string]*clientcmdapi.Cluster{
		"root": {
			Server:               baseHost + "/clusters/root",
			CertificateAuthority: filepath.Join(workDirPath, ".kcp", "serving-ca.crt"),
		},
		"base": {
			Server:               baseHost,
			CertificateAuthority: filepath.Join(workDirPath, ".kcp", "serving-ca.crt"),
		},
	}
	kubeConfig.Contexts = map[string]*clientcmdapi.Context{
		"root": {Cluster: "root", AuthInfo: "kcp-admin"},
		"base": {Cluster: "base", AuthInfo: "kcp-admin"},
	}
	kubeConfig.CurrentContext = "root"

	if err := clientcmdapi.FlattenConfig(&kubeConfig); err != nil {
		return err
	}

	return clientcmd.WriteToFile(kubeConfig, filepath.Join(workDirPath, ".kcp", "admin.kubeconfig"))
}

func writeShardKubeConfig(workDirPath string) error {
	var kubeConfig clientcmdapi.Config
	kubeConfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{
		"shard-admin": {
			ClientKey:         filepath.Join(workDirPath, ".kcp-front-proxy", "shard-admin.key"),
			ClientCertificate: filepath.Join(workDirPath, ".kcp-front-proxy", "shard-admin.crt"),
		},
	}
	kubeConfig.Clusters = map[string]*clientcmdapi.Cluster{
		"base": {
			CertificateAuthority: filepath.Join(workDirPath, ".kcp", "serving-ca.crt"),
		},
	}
	kubeConfig.Contexts = map[string]*clientcmdapi.Context{
		"base": {Cluster: "base", AuthInfo: "shard-admin"},
	}
	kubeConfig.CurrentContext = "base"

	if err := clientcmdapi.FlattenConfig(&kubeConfig); err != nil {
		return err
	}

	return clientcmd.WriteToFile(kubeConfig, filepath.Join(workDirPath, ".kcp-front-proxy", "shards.kubeconfig"))
}

func writeLogicalClusterAdminKubeConfig(hostIP, workDirPath string) error {
	// The logical-cluster-admin-kubeconfig references the root shard, it's
	// also used to access other shards directly; also see the external
	// config below which is used for connections via the front-proxy
	baseHost := fmt.Sprintf("https://%s", net.JoinHostPort(hostIP, "6444"))

	var kubeConfig clientcmdapi.Config
	kubeConfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{
		"logical-cluster-admin": {
			ClientKey:         filepath.Join(workDirPath, ".kcp", "logical-cluster-admin.key"),
			ClientCertificate: filepath.Join(workDirPath, ".kcp", "logical-cluster-admin.crt"),
		},
	}
	kubeConfig.Clusters = map[string]*clientcmdapi.Cluster{
		"base": {
			Server:               baseHost,
			CertificateAuthority: filepath.Join(workDirPath, ".kcp", "serving-ca.crt"),
		},
	}
	kubeConfig.Contexts = map[string]*clientcmdapi.Context{
		"base": {Cluster: "base", AuthInfo: "logical-cluster-admin"},
	}
	kubeConfig.CurrentContext = "base"

	if err := clientcmdapi.FlattenConfig(&kubeConfig); err != nil {
		return err
	}

	return clientcmd.WriteToFile(kubeConfig, filepath.Join(workDirPath, ".kcp", "logical-cluster-admin.kubeconfig"))
}

func writeExternalLogicalClusterAdminKubeConfig(hostIP, workDirPath string) error {
	// The external config references the front-proxy endpoint
	baseHost := fmt.Sprintf("https://%s", net.JoinHostPort(hostIP, "6443"))

	var kubeConfig clientcmdapi.Config
	kubeConfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{
		"external-logical-cluster-admin": {
			ClientKey:         filepath.Join(workDirPath, ".kcp", "external-logical-cluster-admin.key"),
			ClientCertificate: filepath.Join(workDirPath, ".kcp", "external-logical-cluster-admin.crt"),
		},
	}
	kubeConfig.Clusters = map[string]*clientcmdapi.Cluster{
		"base": {
			Server:               baseHost,
			CertificateAuthority: filepath.Join(workDirPath, ".kcp", "serving-ca.crt"),
		},
	}
	kubeConfig.Contexts = map[string]*clientcmdapi.Context{
		"base": {Cluster: "base", AuthInfo: "external-logical-cluster-admin"},
	}
	kubeConfig.CurrentContext = "base"

	if err := clientcmdapi.FlattenConfig(&kubeConfig); err != nil {
		return err
	}

	return clientcmd.WriteToFile(kubeConfig, filepath.Join(workDirPath, ".kcp", "external-logical-cluster-admin.kubeconfig"))
}