package clusterprovider import ( "context" "crypto/tls" "crypto/x509" "errors" "fmt" "net" "net/http" "os" "path/filepath" "slices" "strconv" "strings" "time" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/johnandersen777/socialweb-computer-kcp/internal/apiserverfront" "github.com/johnandersen777/socialweb-computer-kcp/internal/clusterstack" "github.com/johnandersen777/socialweb-computer-kcp/internal/materialiser" "github.com/johnandersen777/socialweb-computer-kcp/internal/portallocator" "github.com/johnandersen777/socialweb-computer-kcp/internal/procgroup" ) const GateTimeout = 60 * time.Second const PollInterval = 250 * time.Millisecond const ( EnvCloudControllerManagerKubeconfig = "SOCIALWEB_CLOUDCONTROLLERMANAGER_TENANT_KUBECONFIG" EnvCloudControllerManagerCluster = "SOCIALWEB_CLOUDCONTROLLERMANAGER_CLUSTER" EnvCloudControllerManagerInstances = "SOCIALWEB_CLOUDCONTROLLERMANAGER_INSTANCES_FILE" ) const cloudControllerManagerReadyMark = `"phase":"Ready"` var ErrNoLease = errors.New("clusterprovider: the cluster holds no lease for this component") var ErrNoTenantClient = errors.New("clusterprovider: no client for the cluster's own apiserver was supplied") var ErrNoVaultStore = errors.New( "clusterprovider: the vault is in this Stack and no per-cluster postgres DSN was supplied " + "for it. The vault's store IS the cluster's own embedded-postgres, so without the DSN " + "the vault cannot be rendered and the Stack would lease a vault port and start nothing") type Stack []procgroup.Component var DesignedStack = Stack{procgroup.ComponentOpenBao, procgroup.ComponentKine, procgroup.ComponentK3s} var ProvenStack = Stack{procgroup.ComponentK3s} func (s Stack) Has(c procgroup.Component) bool { return slices.Contains(s, c) } func (s Stack) NeedsLocalPostgres(localStore bool) bool { return localStore || s.Has(procgroup.ComponentOpenBao) } func componentLog(dataDir string, c procgroup.Component) (string, error) { if err := os.MkdirAll(LogDir(dataDir), 0o700); err != nil { return "", fmt.Errorf("clusterprovider: the log directory for this cluster's components: %w", err) } return filepath.Join(LogDir(dataDir), string(c)+".log"), nil } func LogDir(dataDir string) string { return filepath.Join(dataDir, "logs") } func (cfg StackConfig) StepsFrom(spec clusterstack.Spec, leases materialiser.Leases) ([]procgroup.Step, error) { stack := cfg.Stack if err := spec.Validate(); err != nil { return nil, err } if len(stack) == 0 { return nil, errors.New("clusterprovider: the Stack is empty; a group with no components is not a cluster") } in := make(map[procgroup.Component]bool, len(stack)) for _, c := range stack { if c == procgroup.ComponentPostgres { return nil, errors.New( "clusterprovider: ComponentPostgres cannot be a Step (clusterstack.ErrNotAProcess): " + "embedded-postgres is a library and is started by the component that links it") } in[c] = true } var bao portallocator.Lease if in[procgroup.ComponentOpenBao] { var ok bool if bao, ok = leases[portallocator.PurposeOpenBao]; !ok { return nil, fmt.Errorf("%w: openbao", ErrNoLease) } } api := leases[portallocator.PurposeK3sSecure] if in[procgroup.ComponentK3s] && !in[procgroup.ComponentApiserverFront] { return nil, fmt.Errorf("%w: this Stack runs k3s and no apiserver front (data dir %s)", clusterstack.ErrMissingFront, spec.DataDir) } steps := make([]procgroup.Step, 0, len(stack)) for _, c := range procgroup.StackOrder { if !in[c] { continue } if c == procgroup.ComponentApiserverFront { st, err := frontStep(spec) if err != nil { return nil, err } steps = append(steps, st) continue } argv, err := spec.Argv(c) if err != nil { return nil, err } if len(argv) == 0 { return nil, fmt.Errorf("clusterprovider: %s has an empty argv", c) } log, err := componentLog(spec.DataDir, c) if err != nil { return nil, err } slot := "" if c == procgroup.ComponentKine || c == procgroup.ComponentK3s { slot = spec.Slot } if slot != "" && slot != ActiveSlot { log = strings.TrimSuffix(log, ".log") + "@" + slot + ".log" } ps := procgroup.Spec{Argv: argv, Env: stepEnv(spec, c), Log: log} if c == procgroup.ComponentOpenBao { ps.Listen = net.JoinHostPort(portallocator.LoopbackAdvertiseAddress, strconv.Itoa(bao.Port)) } st := procgroup.Step{ Name: c, Slot: slot, Spec: ps, Ready: cfg.gate(c, spec, bao, api), } if c == procgroup.ComponentK3s { st.Prepare = repointApiserverClients(spec) } steps = append(steps, st) } return steps, nil } func stepEnv(spec clusterstack.Spec, c procgroup.Component) []string { if c == procgroup.ComponentExternalSecrets || c == procgroup.ComponentExternalSecretsWebhook { return externalSecretsEnv(spec) } if c != procgroup.ComponentCloudControllerManager { return nil } return []string{ EnvCloudControllerManagerKubeconfig + "=" + clusterstack.CloudControllerManagerKubeconfig(spec.DataDir), EnvCloudControllerManagerCluster + "=" + spec.Cluster, EnvCloudControllerManagerInstances + "=" + clusterstack.NodeSnapshotPath(spec.DataDir), } } func (cfg StackConfig) gate(c procgroup.Component, spec clusterstack.Spec, bao, api portallocator.Lease) func(context.Context, procgroup.Member) error { return func(ctx context.Context, p procgroup.Member) error { ctx, cancel := context.WithTimeout(ctx, GateTimeout) defer cancel() switch c { case procgroup.ComponentOpenBao: return waitOpenBao(ctx, spec.DataDir, bao.Port) case procgroup.ComponentKine: return waitDial(ctx, spec.KineListen) case procgroup.ComponentApiserverFront: return waitFront(ctx, spec.FrontControl) case procgroup.ComponentK3s: if api.Purpose != portallocator.PurposeK3sSecure { return fmt.Errorf("%w: the apiserver front, so this k3s step has no leased port to hold it to (data dir %s)", ErrNoLease, spec.DataDir) } if err := portallocator.AssertMarketAdvertiseInvariant(portallocator.Advertisement{Address: spec.AdvertiseAddress, Port: spec.AdvertisePort, LinkTarget: spec.AdvertisePort, Front: api, SlotListen: []int{spec.K3sSecurePort}}); err != nil { return err } if err := cfg.waitSlotUp(ctx, spec); err != nil { return err } if _, err := (apiserverfront.Client{Path: spec.FrontControl}).Set(ctx, SlotBackend(spec)); err != nil { return fmt.Errorf("clusterprovider: pointing the front of %s at slot %s: %w", spec.Cluster, spec.Slot, err) } if err := waitK3s(ctx, spec.BindAddress, spec.AdvertisePort); err != nil { return err } return cfg.CheckAdvertisedEndpoints(ctx, spec) case procgroup.ComponentCloudControllerManager: return waitCloudControllerManager(ctx, p) case procgroup.ComponentExternalSecretsWebhook: return cfg.waitExternalSecretsWebhook(ctx, spec, p) case procgroup.ComponentExternalSecrets: return waitExternalSecretsController(ctx, spec, p) } return fmt.Errorf("clusterprovider: no readiness gate for %q", c) } } type errNotReady struct { what string last string } func (e *errNotReady) Error() string { return fmt.Sprintf("%s never became ready (last observation: %s)", e.what, e.last) } func waitOpenBao(ctx context.Context, dataDir string, port int) error { pool, err := vaultCA(dataDir) if err != nil { return err } return waitOpenBaoTLS(ctx, VaultAddr(port), pool) } func waitOpenBaoTLS(ctx context.Context, addr string, pool *x509.CertPool) error { url := addr + "/v1/sys/health" client := &http.Client{ Timeout: 5 * time.Second, Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}, } last := "no answer" return poll(ctx, func(ctx context.Context) (bool, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return false, err } resp, err := client.Do(req) if err != nil { last = err.Error() return false, nil } defer resp.Body.Close() switch resp.StatusCode { case http.StatusOK, http.StatusTooManyRequests, 473: return true, nil } last = fmt.Sprintf("HTTP %d", resp.StatusCode) return false, nil }, func() error { return &errNotReady{what: "openbao", last: last} }) } func waitDial(ctx context.Context, addr string) error { last := "no answer" return poll(ctx, func(context.Context) (bool, error) { conn, err := net.DialTimeout("tcp", addr, 2*time.Second) if err != nil { last = err.Error() return false, nil } _ = conn.Close() return true, nil }, func() error { return &errNotReady{what: "kine at " + addr, last: last} }) } func waitK3s(ctx context.Context, host string, port int) error { client := &http.Client{ Timeout: 5 * time.Second, Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, } url := fmt.Sprintf("https://%s:%d/readyz", host, port) last := "no answer" return poll(ctx, func(ctx context.Context) (bool, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return false, err } resp, err := client.Do(req) if err != nil { last = err.Error() return false, nil } defer resp.Body.Close() if apiserverIsAnswering(resp.StatusCode) { return true, nil } last = fmt.Sprintf("HTTP %d", resp.StatusCode) return false, nil }, func() error { return &errNotReady{what: "k3s server at " + url, last: last} }) } const readyzStreak = 5 func (cfg StackConfig) open(kubeconfig string, o OpenOptions) (Tenant, error) { if cfg.Open == nil { return nil, ErrNoTenantClient } return cfg.Open(kubeconfig, o) } func (cfg StackConfig) waitK3sReadyz(ctx context.Context, kubeconfig string) error { last := "no answer" streak := 0 return poll(ctx, func(ctx context.Context) (bool, error) { core, err := cfg.open(kubeconfig, OpenOptions{Timeout: 3 * time.Second}) if err != nil { last = err.Error() return false, nil } if err := core.Readyz(ctx); err != nil { streak = 0 last = err.Error() return false, nil } streak++ last = fmt.Sprintf("/readyz answered 200 %d of %d times in a row", streak, readyzStreak) return streak >= readyzStreak, nil }, func() error { return &errNotReady{ what: "the k3s server whose admin kubeconfig is " + kubeconfig + " answering /readyz 200 " + strconv.Itoa(readyzStreak) + " times in a row", last: last, } }) } func (cfg StackConfig) adminClient(ctx context.Context, kubeconfig string, last *string) (Tenant, error) { var core Tenant err := untilOutput(ctx, unitMember{path: kubeconfig}, func(string) (bool, error) { var err error if core, err = cfg.open(kubeconfig, OpenOptions{}); err != nil { *last = err.Error() return false, nil } return true, nil }, ctx.Err) return core, err } func (cfg StackConfig) waitK3sControllers(ctx context.Context, kubeconfig string) error { last := "no answer" notReady := func() error { return &errNotReady{ what: "the k3s controllers of the server whose admin kubeconfig is " + kubeconfig + ". The apiserver answers /readyz before they start, and k3s registers the " + "node-password validator that answers /v1-k3s/*.crt for a REMOTE node on the line " + "before it sets Runtime.Core: a node given its guest in that window asks for its " + "serving certificate, is refused with 503 Service Unavailable, and retries on a " + "backoff of five seconds jittered by the same five, which is the difference between " + "a node that joins at five seconds and one that joins at twelve to twenty. The " + "controllers' own first write is an addon Deployment in kube-system and that is what " + "this waits for: /v1-k3s/readyz is the endpoint that flips exactly with them, and it " + "takes a node's credentials, which this process does not hold", last: last, } } core, err := cfg.adminClient(ctx, kubeconfig, &last) if err != nil { if ctx.Err() != nil { return notReady() } return err } last = "the apiserver answered and " + metav1.NamespaceSystem + " holds no Deployment yet" err = core.AwaitDeployments(ctx, metav1.NamespaceSystem, 0, func(addons []*appsv1.Deployment) (bool, error) { return len(addons) > 0, nil }) if err != nil && ctx.Err() != nil { return notReady() } return err } func waitCloudControllerManager(ctx context.Context, p procgroup.Member) error { last := "no output yet" return untilOutput(ctx, p, func(out string) (bool, error) { if strings.Contains(out, cloudControllerManagerReadyMark) { return true, nil } last = fmt.Sprintf("the process has written %d bytes and no readiness line", len(out)) return false, nil }, func() error { return &errNotReady{ what: "the cloud controller manager's informer sync. It logs phase Ready once its " + "Node informer has synced, which is the first moment it can act; before that it has " + "observed nothing and would write nothing. It listens on no port, so this line is the " + "only readiness signal there is", last: last, } }) } func apiserverIsAnswering(code int) bool { switch code { case http.StatusOK, http.StatusUnauthorized: return true default: return false } } func poll(ctx context.Context, up func(context.Context) (bool, error), timeout func() error) error { for { ok, err := up(ctx) if err != nil { return err } if ok { return nil } select { case <-ctx.Done(): return timeout() case <-time.After(PollInterval): } } } func Probe(ctx context.Context, url string) (code int, reachable bool) { client := &http.Client{ Timeout: 5 * time.Second, Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url+"/readyz", nil) if err != nil { return 0, false } resp, err := client.Do(req) if err != nil { return 0, false } defer resp.Body.Close() code = resp.StatusCode return code, apiserverIsAnswering(code) }