0
0
package clusterprovider
import (
"bytes"
"context"
"crypto"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net/url"
"os"
"path/filepath"
"slices"
"time"
"github.com/johnandersen777/socialweb-computer-kcp/internal/clusterstack"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/opt"
"github.com/johnandersen777/socialweb-computer-kcp/internal/vault/openbao"
)
const (
clientCACertName = "client-ca.crt"
clientCAKeyName = "client-ca.key"
serverCACertName = "server-ca.crt"
)
const SuperuserName = "system:admin"
const SuperuserOrganization = "system:masters"
const DefaultCredentialValidity = 10 * 365 * 24 * time.Hour
const credentialSerialBits = 128
var ErrNoCredentialDataDir = errors.New(
"clusterprovider: the tenant credential is minted from the cluster's own ClientCA, which " +
"lives in that cluster's data dir; with no data dir the CA paths resolve against the " +
"working directory, and the only CA that could be signed there belongs to some other " +
"cluster or to no cluster at all")
var ErrNoCredentialCommonName = errors.New(
"clusterprovider: a client certificate with an empty CommonName authenticates to the " +
"apiserver as the empty username, which no RBAC rule and no ValidatingAdmissionPolicy " +
"carve-out admits; the credential would be refused at the first apply with a 403 that " +
"reads as a broken grant rather than as a malformed certificate")
var ErrNoCredentialOrganization = errors.New(
"clusterprovider: a client certificate with no organization carries no group. The apiserver " +
"reads the username off the CommonName and the groups off the Organizations, so a " +
"credential with no organization is an unprivileged user, and every object the tenant " +
"grant exists to apply is refused")
var ErrNoCredentialServer = errors.New(
"clusterprovider: the tenant credential needs the https URL of the apiserver it will " +
"authenticate to. That URL's host is the name the apiserver's own certificate must carry, " +
"so an empty or hostless one is a credential whose server identity nothing checks")
var ErrCredentialServerNotHTTPS = errors.New(
"clusterprovider: the tenant credential's server URL must be https. The apiserver serves " +
"nothing else, and a plaintext URL is either a dial that cannot connect or an identity " +
"crossing the network in the clear")
var ErrNoClientCA = errors.New(
"clusterprovider: the cluster's ClientCA files are not readable under <DataDir>/server/tls, " +
"so no credential can be minted from them. This is a refusal and NOT a reason to reach " +
"for a wider credential: the only other usable credential on this host is k3s's own admin " +
"kubeconfig, which is shared state and is not this provider's to hand out")
var ErrClientCAUnusable = errors.New(
"clusterprovider: the files under <DataDir>/server/tls are present and are not a usable " +
"signing pair. A re-seed that wrote something else here leaves a cluster whose every " +
"minted credential the apiserver refuses as an unknown authority")
var ErrClientCAKeyMismatch = errors.New(
"clusterprovider: the ClientCA key is not the key of the ClientCA certificate. A re-seed " +
"that replaced one and not the other still signs, and the certificate it produces is one " +
"no apiserver trusts, so the failure would surface as an authentication error at the " +
"first apply, hundreds of lines from the pair that caused it")
var ErrNoServerCA = errors.New(
"clusterprovider: the cluster's server CA is absent or holds no CA certificate, so nothing " +
"the apiserver presents for this URL could ever verify. A credential built without it " +
"could only be used with server verification switched off, and that is not a state this " +
"provider will construct")
var ErrCredentialSign = errors.New(
"clusterprovider: the tenant credential could not be signed")
var ErrCredentialDoesNotChain = errors.New(
"clusterprovider: the freshly signed certificate does not verify against the very CA it was " +
"signed with, so the credential is refused here rather than by the apiserver as an " +
"authentication error at the first apply")
var ErrCredentialSourceUnknown = errors.New(
"clusterprovider: the tenant credential's source is one of cluster-ca, which signs it from " +
"the cluster's own ClientCA on disk, or vault-if-sealed, which reaches for the cluster's " +
"vault when that cluster has left unseal material beside it. A caller that names neither " +
"has left the choice to a default, and the default is what this seam exists to remove: on " +
"a market node the vault is not between the provider and the tenant apiserver at all, so " +
"the seal's mere presence would put a second TLS dependency on that path and its failure " +
"would read as a broken CA rather than as a source nothing should have consulted")
type CredentialSource string
const (
CredentialSourceClusterCA CredentialSource = "cluster-ca"
CredentialSourceVaultIfSealed CredentialSource = "vault-if-sealed"
)
type Credential struct {
Host string
CAData []byte
CertData []byte
KeyData []byte
}
type CredentialOptions struct {
DataDir string
Server string
CommonName string
Organizations []string
ValidFor time.Duration
}
func MintTenantCredential(o CredentialOptions) (*Credential, error) {
if o.DataDir == "" {
return nil, ErrNoCredentialDataDir
}
if o.CommonName == "" {
return nil, ErrNoCredentialCommonName
}
if len(o.Organizations) == 0 {
return nil, fmt.Errorf("%w (none were given)", ErrNoCredentialOrganization)
}
if slices.Contains(o.Organizations, "") {
return nil, fmt.Errorf("%w (one of the %d given is empty)",
ErrNoCredentialOrganization, len(o.Organizations))
}
server, err := credentialServer(o.Server)
if err != nil {
return nil, err
}
validFor := o.ValidFor
validFor = opt.Positive(validFor, DefaultCredentialValidity)
tlsDir := clusterstack.TLSDir(o.DataDir)
caCert, caKey, caPool, err := loadClientCA(
filepath.Join(tlsDir, clientCACertName),
filepath.Join(tlsDir, clientCAKeyName),
)
if err != nil {
return nil, err
}
serverCA, err := loadServerCA(filepath.Join(tlsDir, serverCACertName))
if err != nil {
return nil, err
}
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, fmt.Errorf("clusterprovider: generating the tenant credential's key: %w", err)
}
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), credentialSerialBits))
if err != nil {
return nil, fmt.Errorf("clusterprovider: drawing the tenant credential's serial: %w", err)
}
serial.Add(serial, big.NewInt(1))
now := time.Now()
template := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{
CommonName: o.CommonName,
Organization: o.Organizations,
},
NotBefore: now,
NotAfter: now.Add(validFor),
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
BasicConstraintsValid: true,
IsCA: false,
}
der, err := x509.CreateCertificate(rand.Reader, template, caCert, key.Public(), caKey)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrCredentialSign, err)
}
leaf, err := x509.ParseCertificate(der)
if err != nil {
return nil, fmt.Errorf("%w: the certificate it produced does not parse: %w", ErrCredentialSign, err)
}
if _, err := leaf.Verify(x509.VerifyOptions{
Roots: caPool,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}); err != nil {
return nil, fmt.Errorf("%w: %w", ErrCredentialDoesNotChain, err)
}
keyDER, err := x509.MarshalPKCS8PrivateKey(key)
if err != nil {
return nil, fmt.Errorf("clusterprovider: marshalling the tenant credential's key: %w", err)
}
return &Credential{
Host: server.String(),
CAData: serverCA,
CertData: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
KeyData: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}),
}, nil
}
func MintTenantCredentialForCluster(ctx context.Context, logins *openbao.Logins, stateDir, cluster, server string, source CredentialSource) (*Credential, error) {
dataDir, err := ClusterDataDir(stateDir, cluster)
if err != nil {
return nil, err
}
return MintTenantCredentialForDataDir(ctx, logins, dataDir, server, source)
}
func MintTenantCredentialForDataDir(ctx context.Context, logins *openbao.Logins, dataDir, server string, source CredentialSource) (*Credential, error) {
o := CredentialOptions{
DataDir: dataDir,
Server: server,
CommonName: SuperuserName,
Organizations: []string{SuperuserOrganization},
}
switch source {
case CredentialSourceClusterCA:
return MintTenantCredential(o)
case CredentialSourceVaultIfSealed:
client, a, err := clusterVault(ctx, logins, dataDir)
if err == nil {
return mintTenantCredentialFromVault(ctx, o, client, a.Address)
}
return MintTenantCredential(o)
}
return nil, fmt.Errorf("%w (got %q)", ErrCredentialSourceUnknown, source)
}
func MintTenantCredentialForClusterAs(ctx context.Context, stateDir, cluster, server, commonName string, organizations []string, validFor time.Duration) (*Credential, error) {
dataDir, err := ClusterDataDir(stateDir, cluster)
if err != nil {
return nil, err
}
return MintTenantCredentialForDataDirAs(dataDir, server, commonName, organizations, validFor)
}
func MintTenantCredentialForDataDirAs(dataDir, server, commonName string, organizations []string, validFor time.Duration) (*Credential, error) {
return MintTenantCredential(CredentialOptions{
DataDir: dataDir,
Server: server,
CommonName: commonName,
Organizations: organizations,
ValidFor: validFor,
})
}
func mintTenantCredentialFromVault(ctx context.Context, o CredentialOptions, client *openbao.Client, addr string) (*Credential, error) {
if o.DataDir == "" {
return nil, ErrNoCredentialDataDir
}
if o.CommonName == "" {
return nil, ErrNoCredentialCommonName
}
if len(o.Organizations) == 0 {
return nil, fmt.Errorf("%w (none were given)", ErrNoCredentialOrganization)
}
if slices.Contains(o.Organizations, "") {
return nil, fmt.Errorf("%w (one of the %d given is empty)",
ErrNoCredentialOrganization, len(o.Organizations))
}
server, err := credentialServer(o.Server)
if err != nil {
return nil, err
}
validFor := o.ValidFor
validFor = opt.Positive(validFor, DefaultCredentialValidity)
tlsDir := clusterstack.TLSDir(o.DataDir)
serverCA, err := loadServerCA(filepath.Join(tlsDir, serverCACertName))
if err != nil {
return nil, err
}
caPath := filepath.Join(tlsDir, clientCACertName)
caCertPEM, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("%w: %s: %w", ErrNoClientCA, caPath, err)
}
caCert, err := parseCertificate(caCertPEM)
if err != nil {
return nil, fmt.Errorf("%w: %s: %w", ErrClientCAUnusable, caPath, err)
}
if !caCert.IsCA {
return nil, fmt.Errorf("%w: %s is not a CA certificate", ErrClientCAUnusable, caPath)
}
if remaining := time.Until(caCert.NotAfter) - time.Hour; remaining < validFor {
validFor = remaining
}
if validFor <= 0 {
return nil, fmt.Errorf("%w: the client CA at %s expires at %s, so the vault can issue no "+
"credential that outlives it", ErrCredentialSign, caPath, caCert.NotAfter.UTC())
}
issued, err := client.IssueCert(ctx, VaultPKIPath, VaultPKIRoleTenant, openbao.CertRequest{
CommonName: o.CommonName,
TTL: vaultTTL(validFor),
})
if err != nil {
return nil, fmt.Errorf("%w: issuing the tenant credential from the vault at %s: %w",
ErrCredentialSign, addr, err)
}
leaf, err := parseCertificate([]byte(issued.Certificate))
if err != nil {
return nil, fmt.Errorf("%w: the vault's certificate does not parse: %w", ErrCredentialSign, err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caCertPEM) {
return nil, fmt.Errorf("%w: %s holds no CERTIFICATE PEM block", ErrClientCAUnusable, caPath)
}
if _, err := leaf.Verify(x509.VerifyOptions{
Roots: pool,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}); err != nil {
return nil, fmt.Errorf("%w: %w", ErrCredentialDoesNotChain, err)
}
return &Credential{
Host: server.String(),
CAData: serverCA,
CertData: []byte(issued.Certificate),
KeyData: []byte(issued.PrivateKey),
}, nil
}
func credentialServer(server string) (*url.URL, error) {
if server == "" {
return nil, fmt.Errorf("%w (it is empty)", ErrNoCredentialServer)
}
u, err := url.Parse(server)
if err != nil {
return nil, fmt.Errorf("%w (%q does not parse): %w", ErrNoCredentialServer, server, err)
}
if u.Scheme != "https" {
return nil, fmt.Errorf("%w (got %q)", ErrCredentialServerNotHTTPS, u.Scheme)
}
if u.Hostname() == "" {
return nil, fmt.Errorf("%w (%q carries no host)", ErrNoCredentialServer, server)
}
return u, nil
}
func loadClientCA(certPath, keyPath string) (*x509.Certificate, crypto.Signer, *x509.CertPool, error) {
certPEM, err := os.ReadFile(certPath)
if err != nil {
return nil, nil, nil, fmt.Errorf("%w: %s: %w", ErrNoClientCA, certPath, err)
}
keyPEM, err := os.ReadFile(keyPath)
if err != nil {
return nil, nil, nil, fmt.Errorf("%w: %s: %w", ErrNoClientCA, keyPath, err)
}
cert, err := parseCertificate(certPEM)
if err != nil {
return nil, nil, nil, fmt.Errorf("%w: %s: %w", ErrClientCAUnusable, certPath, err)
}
if !cert.IsCA {
return nil, nil, nil, fmt.Errorf(
"%w: %s is not a CA certificate, so anything it signs is a certificate no apiserver "+
"resolves to a group", ErrClientCAUnusable, certPath)
}
key, err := parseSigner(keyPEM)
if err != nil {
return nil, nil, nil, fmt.Errorf("%w: %s: %w", ErrClientCAUnusable, keyPath, err)
}
if err := sameKey(certPath, keyPath, cert, key); err != nil {
return nil, nil, nil, err
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(certPEM) {
return nil, nil, nil, fmt.Errorf("%w: %s holds no CERTIFICATE PEM block", ErrClientCAUnusable, certPath)
}
return cert, key, pool, nil
}
func loadServerCA(path string) ([]byte, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("%w: %s: %w", ErrNoServerCA, path, err)
}
certs, err := parseCertificates(raw)
if err != nil {
return nil, fmt.Errorf("%w: %s: %w", ErrNoServerCA, path, err)
}
anyCA := false
for _, c := range certs {
if c.IsCA {
anyCA = true
}
}
if !anyCA {
return nil, fmt.Errorf("%w: %s holds %d certificate(s) and not one is a CA",
ErrNoServerCA, path, len(certs))
}
return raw, nil
}
func sameKey(certPath, keyPath string, cert *x509.Certificate, key crypto.Signer) error {
want, err := x509.MarshalPKIXPublicKey(cert.PublicKey)
if err != nil {
return fmt.Errorf("%w: the public key of %s: %w", ErrClientCAUnusable, certPath, err)
}
got, err := x509.MarshalPKIXPublicKey(key.Public())
if err != nil {
return fmt.Errorf("%w: the public half of %s: %w", ErrClientCAUnusable, keyPath, err)
}
if !bytes.Equal(want, got) {
return fmt.Errorf(
"%w: %s is the certificate %q and %s holds the key of another certificate, so the "+
"minted credential would be signed by a key that CA never published",
ErrClientCAKeyMismatch, certPath, cert.Subject.CommonName, keyPath)
}
return nil
}
func parseSigner(keyPEM []byte) (crypto.Signer, error) {
block, _ := pem.Decode(keyPEM)
if block == nil {
return nil, errors.New("the file holds no PEM block")
}
if parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes); err == nil {
signer, ok := parsed.(crypto.Signer)
if !ok {
return nil, fmt.Errorf("the PEM block holds a %T, which cannot sign a certificate", parsed)
}
return signer, nil
}
if parsed, err := x509.ParseECPrivateKey(block.Bytes); err == nil {
return parsed, nil
}
if parsed, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil {
return parsed, nil
}
return nil, errors.New("the PEM block is not a PKCS#8, SEC 1 or PKCS#1 private key")
}
func parseCertificate(certPEM []byte) (*x509.Certificate, error) {
certs, err := parseCertificates(certPEM)
if err != nil {
return nil, err
}
if len(certs) == 0 {
return nil, errors.New("the file holds no CERTIFICATE PEM block")
}
return certs[0], nil
}
func parseCertificates(certPEM []byte) ([]*x509.Certificate, error) {
var out []*x509.Certificate
for block, rest := pem.Decode(certPEM); block != nil; block, rest = pem.Decode(rest) {
if block.Type != "CERTIFICATE" {
continue
}
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil, err
}
out = append(out, cert)
}
return out, nil
}