0
0
package kcpserver
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"io"
"math/big"
"net/http"
"net/url"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"syscall"
"time"
"k8s.io/client-go/tools/clientcmd"
"github.com/johnandersen777/socialweb-computer-kcp/internal/hostsvc"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/opt"
)
const WorkspaceMountsGate = "--feature-gates=WorkspaceMounts=true"
const LeaderElection = "--enable-leader-election"
const SlotStopTimeout = 30 * time.Second
const EventTTL = "1h"
const EventTTLFlag = "--event-ttl=" + EventTTL
const ShardAdminUser = "shard-admin"
const ShardBaseURLFlag = "--shard-base-url="
const MountProxyCertFlag = "--mount-proxy-client-cert-file="
const (
MountProxyCertName = "mount-proxy-client.crt"
MountProxyKeyName = "mount-proxy-client.key"
)
const AuditPolicy = `apiVersion: audit.k8s.io/v1
kind: Policy
omitStages:
- RequestReceived
rules:
- level: Metadata
`
type SlotConfig struct {
Unit string
Slice string
Exec string
Root string
EtcdServers string
FrontURL string
Wants []string
Shape Shape
}
func SlotArgv(c SlotConfig) ([]string, error) {
if !filepath.IsAbs(c.Exec) {
return nil, fmt.Errorf("kcpserver: the kcp executable must be an absolute path, got %q (dev builds take it from kcp-path, never PATH)", c.Exec)
}
if !filepath.IsAbs(c.Root) {
return nil, errors.New("kcpserver: the slot root directory must be absolute")
}
if c.EtcdServers == "" {
return nil, errors.New("kcpserver: a slot needs --etcd-servers (its kine); there is no embedded etcd")
}
if c.Shape != ShapeSingle && c.Shape != ShapeBlueGreen {
return nil, fmt.Errorf("kcpserver: slot shape %q: want %s (a bidder's one slot) or %s (the gateway's slots)", c.Shape, ShapeSingle, ShapeBlueGreen)
}
argv := []string{c.Exec, "start",
"--root-directory=" + c.Root,
"--etcd-servers=" + c.EtcdServers,
"--bind-address=127.0.0.1",
"--secure-port=0",
"--experimental-bind-free-port",
"--shutdown-delay-duration=2s",
WorkspaceMountsGate,
EventTTLFlag,
"--audit-policy-file=" + filepath.Join(c.Root, "audit-policy.yaml"),
"--audit-log-path=" + filepath.Join(c.Root, "audit.log"),
"--audit-log-maxsize=100", "--audit-log-maxage=1", "--audit-log-maxbackup=4",
MountProxyCertFlag + filepath.Join(c.Root, MountProxyCertName),
"--mount-proxy-client-key-file=" + filepath.Join(c.Root, MountProxyKeyName),
}
if c.Shape == ShapeBlueGreen {
argv = append(argv, LeaderElection)
}
if c.FrontURL != "" {
argv = append(argv, ShardBaseURLFlag+c.FrontURL)
}
return argv, CheckArgv(argv, c.Shape)
}
func CheckArgv(argv []string, shape Shape) error {
has := func(want string) bool {
return slices.Contains(argv, want)
}
hasPrefix := func(want string) bool {
for _, a := range argv {
if strings.HasPrefix(a, want) && len(a) > len(want) {
return true
}
}
return false
}
if !has(WorkspaceMountsGate) {
return fmt.Errorf("kcpserver: kcp refused without %s: every tenant's mounted API silently stops being served (versions.json kcp invariant)", WorkspaceMountsGate)
}
if !has(EventTTLFlag) {
return fmt.Errorf("kcpserver: kcp refused without %s: the Event objects of a busy network would stay in the store for the apiserver's default", EventTTLFlag)
}
if !has("--secure-port=0") {
return errors.New("kcpserver: kcp refused on a fixed port: behind the edge every listener binds port 0")
}
if !hasPrefix(ShardBaseURLFlag) {
return fmt.Errorf("kcpserver: kcp refused without %s<front>: a slot that advertises its own port is cut by the next swap", ShardBaseURLFlag)
}
switch shape {
case ShapeBlueGreen:
if !has(LeaderElection) {
return fmt.Errorf("kcpserver: a blue/green slot refused without %s: two slots would both run every controller", LeaderElection)
}
case ShapeSingle:
if has(LeaderElection) {
return fmt.Errorf("kcpserver: a single slot refused with %s: no second slot exists and a restart would strand the controllers for the 60 s lease", LeaderElection)
}
default:
return fmt.Errorf("kcpserver: unknown slot shape %q", shape)
}
return nil
}
func WriteSlotFiles(root string) error {
if err := os.MkdirAll(root, 0o700); err != nil {
return err
}
if err := os.WriteFile(filepath.Join(root, "audit-policy.yaml"), []byte(AuditPolicy), 0o600); err != nil {
return err
}
return writeMountProxyClient(root)
}
func writeMountProxyClient(root string) error {
certPath, keyPath := filepath.Join(root, MountProxyCertName), filepath.Join(root, MountProxyKeyName)
if _, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
return nil
}
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return err
}
now := time.Now()
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(now.UnixNano()),
Subject: pkix.Name{CommonName: "kcp-mount-proxy"},
NotBefore: now.Add(-time.Hour),
NotAfter: now.AddDate(20, 0, 0),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
return err
}
keyDER, err := x509.MarshalPKCS8PrivateKey(key)
if err != nil {
return err
}
if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
return err
}
return os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600)
}
const SlotMemoryMax = 3 << 30
func SlotSpec(c SlotConfig) (hostsvc.UnitSpec, error) {
argv, err := SlotArgv(c)
if err != nil {
return hostsvc.UnitSpec{}, err
}
return hostsvc.UnitSpec{
Name: c.Unit,
Description: "kcp slot of one application instance (" + string(c.Shape) + ")",
Exec: argv,
WorkDir: c.Root,
Slice: c.Slice,
Restart: hostsvc.RestartOnFailure,
KillSignal: syscall.SIGTERM,
StopTimeout: SlotStopTimeout,
Limits: hostsvc.Limits{MemoryMax: SlotMemoryMax},
After: c.Wants,
Wants: c.Wants,
}, nil
}
type Admin struct {
Token string
CA []byte
Server string
}
func ReadAdmin(kubeconfig string) (Admin, error) {
cfg, err := clientcmd.LoadFromFile(kubeconfig)
if err != nil {
return Admin{}, err
}
var a Admin
if u, ok := cfg.AuthInfos[ShardAdminUser]; ok && u.Token != "" {
a.Token = u.Token
}
for name, ctx := range cfg.Contexts {
if ctx.AuthInfo != ShardAdminUser && !(a.Token == "" && name == cfg.CurrentContext) {
continue
}
if cl, ok := cfg.Clusters[ctx.Cluster]; ok {
a.CA, a.Server = cl.CertificateAuthorityData, cl.Server
}
if a.Token == "" {
if u, ok := cfg.AuthInfos[ctx.AuthInfo]; ok {
a.Token = u.Token
}
}
break
}
if a.Token == "" || len(a.CA) == 0 {
return Admin{}, fmt.Errorf("kcpserver: %s holds no %s token and CA yet", kubeconfig, ShardAdminUser)
}
return a, nil
}
func PortOf(server string) (int, error) {
u, err := url.Parse(server)
if err != nil {
return 0, err
}
return strconv.Atoi(u.Port())
}
func Readyz(ctx context.Context, port int, a Admin) (int, string, error) {
c, err := loopbackClient(a.CA, 5*time.Second)
if err != nil {
return 0, "", errors.New("kcpserver: the kubeconfig CA does not parse")
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://127.0.0.1:"+strconv.Itoa(port)+"/readyz", nil)
if err != nil {
return 0, "", err
}
req.Header.Set("Authorization", "Bearer "+a.Token)
resp, err := c.Do(req)
if err != nil {
return 0, "", err
}
defer resp.Body.Close()
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
return resp.StatusCode, strings.TrimSpace(string(b)), nil
}
func WaitReady(ctx context.Context, port int, kubeconfig string, notBefore time.Time, every time.Duration,
progress ...func(attempt int, last error)) (time.Time, error) {
every = opt.Positive(every, ProbePoll)
var last error
for attempt := 1; ; attempt++ {
fi, err := os.Stat(kubeconfig)
switch {
case err != nil:
last = err
case fi.ModTime().Before(notBefore.Add(-time.Second)):
last = fmt.Errorf("%s predates this start", kubeconfig)
default:
a, err := ReadAdmin(kubeconfig)
if err != nil {
last = err
break
}
code, body, err := Readyz(ctx, port, a)
if err == nil && code == http.StatusOK {
return time.Now().UTC(), nil
}
last = fmt.Errorf("readyz on 127.0.0.1:%d: %d %q %v", port, code, body, err)
}
report(progress, attempt, last)
select {
case <-ctx.Done():
return time.Time{}, fmt.Errorf("kcpserver: not ready: %v (%w)", last, ctx.Err())
case <-time.After(every):
}
}
}
func report(progress []func(attempt int, last error), attempt int, last error) {
for _, p := range progress {
if p != nil {
p(attempt, last)
}
}
}