package eso import ( "bytes" "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "net/url" "os" "path/filepath" "sort" "strings" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" utilyaml "k8s.io/apimachinery/pkg/util/yaml" "github.com/johnandersen777/socialweb-computer-kcp/api/kcpgvr" "github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/kube" ) const FieldManager = "socialweb-clusterprovider" const ( DefaultControllerNamespace = "default" ClientSecretName = "openbao-eso-client-cert" StoreName = "openbao" CertKey = "tls.crt" KeyKey = "tls.key" CAKey = "ca.crt" OperatorGroupVersion = "external-secrets.io/v1" ) var ErrNoOperator = errors.New( "eso: the cluster serves no " + OperatorGroupVersion + ", so the OpenBao ClusterSecretStore " + "and ExternalSecret have no kind to be applied as. That group is served by the External " + "Secrets Operator, and no operator is installed in this cluster") func Installed(ctx context.Context, c *kube.Client) (bool, error) { _, err := kube.List[kube.Doc](ctx, c.Resource(externalSecretsResources["ClusterSecretStore"]), metav1.ListOptions{Limit: 1}) if err == nil { return true, nil } if apierrors.IsNotFound(err) { return false, nil } return false, fmt.Errorf("eso: asking the cluster whether it serves %s: %w", OperatorGroupVersion, err) } func InstallOperator(ctx context.Context, c *kube.Client, install func(context.Context) error) error { ok, err := Installed(ctx, c) if err != nil { return err } if ok { return nil } if install == nil { return fmt.Errorf("%w (this provider configures no installer for it)", ErrNoOperator) } if err := install(ctx); err != nil { return fmt.Errorf("eso: installing the External Secrets Operator, whose %s the store "+ "needs: %w", OperatorGroupVersion, err) } ok, err = Installed(ctx, c) if err != nil { return err } if !ok { return fmt.Errorf("%w (the installer ran and the group is still not served)", ErrNoOperator) } return nil } var ErrIncomplete = errors.New( "eso: the OpenBao-backed External Secrets install needs a manifest directory, the tenant " + "namespace to write into, the cluster vault's tenant-facing https URL, the vault server CA " + "and the ESO client certificate and key") var secretResource = kcpgvr.Secrets var externalSecretsResources = map[string]schema.GroupVersionResource{ "ClusterSecretStore": kcpgvr.ClusterSecretStores, "ExternalSecret": kcpgvr.ExternalSecrets, "SecretStore": kcpgvr.SecretStores, } type Options struct { ManifestDir string Namespace string VaultServer string ServerCAPEM []byte ClientCertPEM []byte ClientKeyPEM []byte } func (o Options) validate() error { var problems []error for _, f := range []struct { name string empty bool }{ {"manifest directory", o.ManifestDir == ""}, {"tenant namespace", o.Namespace == ""}, {"vault server URL", o.VaultServer == ""}, {"vault server CA", len(o.ServerCAPEM) == 0}, {"client certificate", len(o.ClientCertPEM) == 0}, {"client key", len(o.ClientKeyPEM) == 0}, } { if f.empty { problems = append(problems, fmt.Errorf("%w: it names no %s", ErrIncomplete, f.name)) } } if err := errors.Join(problems...); err != nil { return err } u, err := url.Parse(o.VaultServer) if err != nil { return fmt.Errorf("%w: the vault server %q does not parse: %v", ErrIncomplete, o.VaultServer, err) } if u.Scheme != "https" || u.Host == "" { return fmt.Errorf( "%w: the vault server %q is not https with a host, and the ESO vault provider carries "+ "the client certificate inside the TLS handshake, so a plaintext or hostless URL is "+ "one on which auth/cert can never answer", ErrIncomplete, o.VaultServer) } return nil } func Load(dir string) ([]kube.Doc, error) { entries, err := os.ReadDir(dir) if err != nil { return nil, fmt.Errorf("eso: reading the manifest directory %s: %w", dir, err) } var names []string for _, e := range entries { if e.IsDir() { continue } if ext := strings.ToLower(filepath.Ext(e.Name())); ext != ".yaml" && ext != ".yml" { continue } names = append(names, e.Name()) } sort.Strings(names) var out []kube.Doc for _, name := range names { path := filepath.Join(dir, name) raw, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("eso: reading %s: %w", path, err) } objs, err := decodeObjects(raw, path) if err != nil { return nil, err } out = append(out, objs...) } if len(out) == 0 { return nil, fmt.Errorf("eso: the manifest directory %s holds no objects", dir) } return out, nil } func decodeObjects(raw []byte, source string) ([]kube.Doc, error) { var out []kube.Doc dec := utilyaml.NewYAMLOrJSONDecoder(bytes.NewReader(raw), 4096) for { var obj map[string]any err := dec.Decode(&obj) if errors.Is(err, io.EOF) { break } if err != nil { return nil, fmt.Errorf("eso: %s: %w", source, err) } if len(obj) == 0 { continue } u := kube.Doc(obj) if kindOf(u) == "" || docString(u, "apiVersion") == "" { return nil, fmt.Errorf("eso: %s holds an object with no apiVersion or kind", source) } if nameOf(u) == "" { return nil, fmt.Errorf("eso: %s holds a %s with no name", source, kindOf(u)) } out = append(out, u) } return out, nil } func Resolve(u kube.Doc, o Options) error { switch kindOf(u) { case "ClusterSecretStore", "SecretStore": if err := setField(u, o.VaultServer, "spec", "provider", "vault", "server"); err != nil { return err } if err := setField(u, base64.StdEncoding.EncodeToString(o.ServerCAPEM), "spec", "provider", "vault", "caBundle"); err != nil { return err } cert := map[string]any{"name": ClientSecretName, "namespace": o.Namespace, "key": CertKey} key := map[string]any{"name": ClientSecretName, "namespace": o.Namespace, "key": KeyKey} if err := setField(u, cert, "spec", "provider", "vault", "auth", "cert", "clientCert"); err != nil { return err } if err := setField(u, key, "spec", "provider", "vault", "auth", "cert", "secretRef"); err != nil { return err } case "ExternalSecret": setNamespace(u, o.Namespace) default: return fmt.Errorf("eso: %s %s is not a kind this installer applies", kindOf(u), nameOf(u)) } return nil } func Apply(ctx context.Context, c *kube.Client, o Options) error { if err := o.validate(); err != nil { return err } objs, err := Load(o.ManifestDir) if err != nil { return err } for _, u := range objs { if err := Resolve(u, o); err != nil { return err } } if err := applyClientSecret(ctx, c, o); err != nil { return err } for _, u := range objs { gvr, ok := externalSecretsResources[kindOf(u)] if !ok { return fmt.Errorf("eso: no resource is known for the %s %s this installer loaded", kindOf(u), nameOf(u)) } if err := applyObject(ctx, c.Resource(gvr).In(namespaceOf(u)), u); err != nil { return err } } return nil } func Remove(ctx context.Context, c *kube.Client, o Options) error { if o.ManifestDir == "" { return fmt.Errorf("%w: the manifest directory, which is what names the objects to remove", ErrIncomplete) } if o.Namespace == "" { return fmt.Errorf("%w: the tenant namespace the ExternalSecret was written into", ErrIncomplete) } objs, err := Load(o.ManifestDir) if err != nil { return err } var problems []error for _, wantKind := range []string{"ExternalSecret", "ClusterSecretStore", "SecretStore"} { for _, u := range objs { if kindOf(u) != wantKind { continue } gvr, ok := externalSecretsResources[kindOf(u)] if !ok { problems = append(problems, fmt.Errorf( "eso: %s %s is not a kind this installer writes, so it is not one it may remove", kindOf(u), nameOf(u))) continue } r := c.Resource(gvr) if kindOf(u) == "ExternalSecret" { r = r.In(o.Namespace) } if err := r.Delete(ctx, nameOf(u), metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { problems = append(problems, fmt.Errorf("eso: removing the %s %s: %w -- an object this "+ "installer can no longer stand behind is one a reader keeps taking at face value, "+ "and a store whose vault address nothing answers reads as a broken ExternalSecret "+ "rather than as an address nothing binds", kindOf(u), nameOf(u), err)) } if kindOf(u) == "ExternalSecret" { if err := releaseExternalSecret(ctx, c, o, nameOf(u)); err != nil { problems = append(problems, err) } } } } if err := c.Resource(secretResource).In(o.Namespace). Delete(ctx, ClientSecretName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { problems = append(problems, fmt.Errorf("eso: removing the client certificate Secret %s/%s: %w", o.Namespace, ClientSecretName, err)) } return errors.Join(problems...) } func releaseExternalSecret(ctx context.Context, c *kube.Client, o Options, name string) error { r := c.Resource(externalSecretsResources["ExternalSecret"]).In(o.Namespace) live, err := kube.Get[metav1.PartialObjectMetadata](ctx, r, name) if apierrors.IsNotFound(err) { return nil } if err != nil { return fmt.Errorf("eso: reading the ExternalSecret %s/%s back after its deletion: %w", o.Namespace, name, err) } if live.GetDeletionTimestamp() == nil { return nil } kept := make([]string, 0, len(live.GetFinalizers())) for _, f := range live.GetFinalizers() { if !strings.HasPrefix(f, "externalsecrets.external-secrets.io/") { kept = append(kept, f) } } if len(kept) == len(live.GetFinalizers()) { return nil } finalizers, err := json.Marshal(kept) if err != nil { return fmt.Errorf("eso: encoding the finalizers kept on %s/%s: %w", o.Namespace, name, err) } body := []byte(`{"metadata":{"finalizers":` + string(finalizers) + `}}`) if _, err := kube.Patch[kube.Doc](ctx, r, name, types.MergePatchType, body, metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("eso: releasing the External Secrets controller's finalizer on %s/%s, which "+ "is what lets the object go: the controller removes it after cleaning up the Secret it "+ "owns, and on a cluster with no node the controller is Pending and never does -- a "+ "cluster at rest has no node, so the object would stay Terminating for as long as that "+ "lasts: %w", o.Namespace, name, err) } return nil } func applyClientSecret(ctx context.Context, c *kube.Client, o Options) error { secret := &corev1.Secret{ TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, Type: corev1.SecretTypeTLS, ObjectMeta: metav1.ObjectMeta{ Name: ClientSecretName, Namespace: o.Namespace, }, Data: map[string][]byte{ CertKey: o.ClientCertPEM, KeyKey: o.ClientKeyPEM, CAKey: o.ServerCAPEM, }, } if _, err := kube.Apply[kube.Doc](ctx, c.Resource(secretResource).In(o.Namespace), secret, kube.ApplyOptions{FieldManager: FieldManager, Force: true}); err != nil { return fmt.Errorf("eso: applying the client certificate Secret %s/%s: %w", o.Namespace, ClientSecretName, err) } return nil } func CurrentClientCertificate(ctx context.Context, c *kube.Client, namespace string) (certPEM, keyPEM []byte, found bool, err error) { if namespace == "" { return nil, nil, false, fmt.Errorf("%w: it names no tenant namespace to read the client certificate Secret from", ErrIncomplete) } if c == nil { return nil, nil, false, errors.New("eso: no dynamic client to read the client certificate Secret") } secret, err := kube.Get[kube.Doc](ctx, c.Resource(secretResource).In(namespace), ClientSecretName) if apierrors.IsNotFound(err) { return nil, nil, false, nil } if err != nil { return nil, nil, false, fmt.Errorf("eso: reading the client certificate Secret %s/%s: %w", namespace, ClientSecretName, err) } certB64 := docString(*secret, "data", CertKey) keyB64 := docString(*secret, "data", KeyKey) if certB64 == "" || keyB64 == "" { return nil, nil, true, nil } certPEM, _ = base64.StdEncoding.DecodeString(certB64) keyPEM, _ = base64.StdEncoding.DecodeString(keyB64) return certPEM, keyPEM, true, nil } func applyObject(ctx context.Context, r kube.Resource, u kube.Doc) error { if _, err := kube.Apply[kube.Doc](ctx, r, u, kube.ApplyOptions{FieldManager: FieldManager, Force: true}); err != nil { return fmt.Errorf("eso: applying the %s %s: %w", kindOf(u), nameOf(u), err) } return nil } func field(d kube.Doc, fields ...string) (any, bool, error) { var val any = map[string]any(d) for i, f := range fields { if val == nil { return nil, false, nil } m, ok := val.(map[string]any) if !ok { return nil, false, fmt.Errorf("%v accessor error: %v is of the type %T, expected map[string]interface{}", "."+strings.Join(fields[:i+1], "."), val, val) } if val, ok = m[f]; !ok { return nil, false, nil } } return val, true, nil } func docString(d kube.Doc, fields ...string) string { v, _, _ := field(d, fields...) s, _ := v.(string) return s } func docSlice(d kube.Doc, fields ...string) ([]any, bool, error) { v, found, err := field(d, fields...) if !found || err != nil { return nil, found, err } s, ok := v.([]any) if !ok { return nil, false, fmt.Errorf("%v accessor error: %v is of the type %T, expected []interface{}", "."+strings.Join(fields, "."), v, v) } return s, true, nil } func setField(d kube.Doc, value any, fields ...string) error { m := map[string]any(d) for i, f := range fields[:len(fields)-1] { v, ok := m[f] if !ok { next := map[string]any{} m[f] = next m = next continue } if m, ok = v.(map[string]any); !ok { return fmt.Errorf("value cannot be set because %v is not a map[string]interface{}", "."+strings.Join(fields[:i+1], ".")) } } m[fields[len(fields)-1]] = value return nil } func setNamespace(d kube.Doc, namespace string) { if namespace == "" { if meta, ok := d["metadata"].(map[string]any); ok { delete(meta, "namespace") } return } _ = setField(d, namespace, "metadata", "namespace") } func kindOf(d kube.Doc) string { return docString(d, "kind") } func nameOf(d kube.Doc) string { return docString(d, "metadata", "name") } func namespaceOf(d kube.Doc) string { return docString(d, "metadata", "namespace") } //go:fix inline