0
0
package clusterprovider
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"net"
"net/http"
"os"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/johnandersen777/socialweb-computer-kcp/internal/clusterstack"
"github.com/johnandersen777/socialweb-computer-kcp/internal/materialiser"
"github.com/johnandersen777/socialweb-computer-kcp/internal/portallocator"
"github.com/johnandersen777/socialweb-computer-kcp/internal/procgroup"
)
const externalSecretsReadyMark = "Starting workers"
const externalSecretsReadyController = `"controller":"externalsecret"`
var ExternalSecretsComponents = []procgroup.Component{
procgroup.ComponentExternalSecretsWebhook, procgroup.ComponentExternalSecrets,
}
var ExternalSecretsPurposes = []portallocator.Purpose{
portallocator.PurposeExternalSecretsHealth,
portallocator.PurposeExternalSecretsWebhook,
portallocator.PurposeExternalSecretsWebhookHealth,
}
var ErrNoExternalSecrets = errors.New(
"clusterprovider: this Stack runs the External Secrets controller beside the control plane and " +
"no ExternalSecrets config names the binary and the pinned bundle. The controller's CRDs, its " +
"ClusterRole and its webhook registration come from that bundle, and a controller started " +
"against a tenant that serves none of its kinds fails its own start")
var ErrNoExternalSecretsSeam = errors.New(
"clusterprovider: the External Secrets pair is in this Stack and k3s is not. The pair dials the " +
"tenant apiserver with a credential minted from the cluster's CAs, so with no apiserver in " +
"the group there is nothing for it to reconcile against")
var ErrExternalSecretsKubeconfigUnwritable = errors.New(
"clusterprovider: the External Secrets controller's minted tenant kubeconfig could not be put on " +
"disk before its step ran. The controller and its webhook take that file as their only " +
"credential, and controller-runtime exits at start without one")
type ExternalSecrets struct {
Binary string
Bundle string
LegacyNamespace string
WebhookCAName string
Install func(ctx context.Context, o ExternalSecretsInstall) error
}
type ExternalSecretsInstall struct {
StateDir string
Cluster string
APIServer string
DataDir string
WebhookPort int
Bundle string
LegacyNamespace string
}
func (cfg StackConfig) runsExternalSecrets() bool {
return slices.ContainsFunc(ExternalSecretsComponents, cfg.Stack.Has)
}
func (cfg StackConfig) WithoutUnleasedExternalSecrets(l materialiser.Leases) StackConfig {
if !cfg.runsExternalSecrets() {
return cfg
}
for _, p := range ExternalSecretsPurposes {
if _, ok := l[p]; ok {
continue
}
kept := make(Stack, 0, len(cfg.Stack))
for _, c := range cfg.Stack {
if c == procgroup.ComponentExternalSecrets || c == procgroup.ComponentExternalSecretsWebhook {
continue
}
kept = append(kept, c)
}
cfg.Stack = kept
return cfg
}
return cfg
}
func (cfg StackConfig) WithoutUnleased(l materialiser.Leases) StackConfig {
return cfg.WithoutUnleasedExternalSecrets(l)
}
func externalSecretsSpec(cfg StackConfig, l materialiser.Leases, cluster string) (clusterstack.ExternalSecrets, error) {
if cfg.ExternalSecrets == nil || cfg.ExternalSecrets.Binary == "" || cfg.ExternalSecrets.Bundle == "" {
return clusterstack.ExternalSecrets{}, fmt.Errorf("%w (cluster %s)", ErrNoExternalSecrets, cluster)
}
health, err := leasePort(l, portallocator.PurposeExternalSecretsHealth, cluster)
if err != nil {
return clusterstack.ExternalSecrets{}, err
}
webhook, err := leasePort(l, portallocator.PurposeExternalSecretsWebhook, cluster)
if err != nil {
return clusterstack.ExternalSecrets{}, err
}
webhookHealth, err := leasePort(l, portallocator.PurposeExternalSecretsWebhookHealth, cluster)
if err != nil {
return clusterstack.ExternalSecrets{}, err
}
return clusterstack.ExternalSecrets{
Binary: cfg.ExternalSecrets.Binary,
HealthPort: health,
WebhookPort: webhook,
WebhookHealth: webhookHealth,
}, nil
}
func ExternalSecretsWebhookURL(port int) string {
return "https://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
}
func prepareExternalSecrets(steps []procgroup.Step, cfg StackConfig, spec clusterstack.Spec, cluster, apiserver string) ([]procgroup.Step, error) {
if !cfg.Stack.Has(procgroup.ComponentK3s) {
return nil, fmt.Errorf("%w (stack %v, cluster %s)", ErrNoExternalSecretsSeam, cfg.Stack, cluster)
}
idx, ctlIdx := -1, -1
for i, s := range steps {
switch s.Name {
case procgroup.ComponentExternalSecretsWebhook:
idx = i
case procgroup.ComponentExternalSecrets:
ctlIdx = i
}
}
if idx < 0 {
return nil, fmt.Errorf("clusterprovider: the External Secrets controller of %s is in the Stack "+
"without its webhook, and the webhook step is the one that installs the tenant half both "+
"read: the CRDs, the ClusterRole binding and the webhook registration", cluster)
}
es := *cfg.ExternalSecrets
o := ExternalSecretsInstall{StateDir: cfg.StateDir, Cluster: cluster, APIServer: apiserver, DataDir: spec.DataDir,
WebhookPort: spec.ExternalSecrets.WebhookPort, Bundle: es.Bundle, LegacyNamespace: es.LegacyNamespace}
install := func(ctx context.Context) error {
if es.Install == nil {
return fmt.Errorf("%w (cluster %s)", ErrNoExternalSecrets, cluster)
}
return es.Install(ctx, o)
}
var once sync.Once
var installErr error
shared := func(ctx context.Context, ps procgroup.Spec) (procgroup.Spec, error) {
once.Do(func() { installErr = install(ctx) })
return ps, installErr
}
steps[idx].Prepare = shared
if ctlIdx >= 0 {
steps[ctlIdx].Prepare = shared
}
return steps, nil
}
func externalSecretsEnv(spec clusterstack.Spec) []string {
return []string{"KUBECONFIG=" + clusterstack.ExternalSecretsKubeconfig(spec.DataDir)}
}
func httpReady(ctx context.Context, url string) (bool, string) {
client := &http.Client{Timeout: 2 * time.Second}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return false, err.Error()
}
resp, err := client.Do(req)
if err != nil {
return false, err.Error()
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false, fmt.Sprintf("%s answered HTTP %d", url, resp.StatusCode)
}
return true, ""
}
func (cfg StackConfig) waitExternalSecretsWebhook(ctx context.Context, spec clusterstack.Spec, p procgroup.Member) error {
caName := ""
if cfg.ExternalSecrets != nil {
caName = cfg.ExternalSecrets.WebhookCAName
}
readyz := "http://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(spec.ExternalSecrets.WebhookHealth)) + "/readyz"
serving := net.JoinHostPort("127.0.0.1", strconv.Itoa(spec.ExternalSecrets.WebhookPort))
last := "no answer"
return poll(ctx, func(ctx context.Context) (bool, error) {
ok, why := httpReady(ctx, readyz)
if !ok {
last = why
return false, nil
}
if err := dialWebhookAsTheApiserverWould(spec, caName, serving); err != nil {
last = err.Error()
return false, nil
}
return true, nil
}, func() error {
return &errNotReady{
what: "the External Secrets webhook. Its readyz answers once its serving pair verifies, and " +
"the gate then dials " + serving + " verifying the minted CA for 127.0.0.1, which is " +
"exactly what the tenant apiserver does when it calls the registered URL",
last: last + "; output tail: " + tail(p.Output()),
}
})
}
func dialWebhookAsTheApiserverWould(spec clusterstack.Spec, caName, addr string) error {
caPEM, err := os.ReadFile(clusterstack.ExternalSecretsCertDir(spec.DataDir) + "/" + caName)
if err != nil {
return err
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caPEM) {
return errors.New("the webhook CA on disk is not PEM")
}
conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 2 * time.Second}, "tcp", addr,
&tls.Config{RootCAs: pool, ServerName: "127.0.0.1", MinVersion: tls.VersionTLS12})
if err != nil {
return err
}
return conn.Close()
}
func waitExternalSecretsController(ctx context.Context, spec clusterstack.Spec, p procgroup.Member) error {
readyz := "http://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(spec.ExternalSecrets.HealthPort)) + "/readyz"
last := "no answer"
return poll(ctx, func(ctx context.Context) (bool, error) {
ok, why := httpReady(ctx, readyz)
if !ok {
last = why
return false, nil
}
if !externalSecretsWorkersStarted(p.Output()) {
last = "readyz answers and the externalsecret controller has not started its workers"
return false, nil
}
return true, nil
}, func() error {
return &errNotReady{
what: "the External Secrets controller. Its readyz is a ping that answers before any cache " +
"has synced, so the gate also waits for the externalsecret controller's own " +
"\"Starting workers\" line, which controller-runtime logs only after that kind's " +
"informer has synced against the tenant apiserver",
last: last + "; output tail: " + tail(p.Output()),
}
})
}
func externalSecretsWorkersStarted(out string) bool {
for line := range strings.SplitSeq(out, "\n") {
if strings.Contains(line, externalSecretsReadyMark) && strings.Contains(line, externalSecretsReadyController) {
return true
}
}
return false
}