0
0
package componentcore
import (
"cmp"
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"hash/fnv"
"log/slog"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"k8s.io/client-go/rest"
"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/app"
"github.com/johnandersen777/socialweb-computer-kcp/internal/component"
"github.com/johnandersen777/socialweb-computer-kcp/internal/component/kcpclients"
"github.com/johnandersen777/socialweb-computer-kcp/internal/computevm"
"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/tunnel"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/atircnode"
nodeclaimctl "github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/nodeclaim"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/kube"
"github.com/johnandersen777/socialweb-computer-kcp/internal/exposure"
"github.com/johnandersen777/socialweb-computer-kcp/internal/hostlifecycle"
"github.com/johnandersen777/socialweb-computer-kcp/internal/ingress/webrtc"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kcpclient"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kcplibs/impl/kcpstore"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kcpserver"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/supervise"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/tlsreload"
"github.com/johnandersen777/socialweb-computer-kcp/internal/oauthcallback"
"github.com/johnandersen777/socialweb-computer-kcp/internal/openbaoctl"
"github.com/johnandersen777/socialweb-computer-kcp/internal/opsread/hostread"
"github.com/johnandersen777/socialweb-computer-kcp/internal/policyaction"
policycontroller "github.com/johnandersen777/socialweb-computer-kcp/internal/policyengine/controller"
"github.com/johnandersen777/socialweb-computer-kcp/internal/postgresctl"
"github.com/johnandersen777/socialweb-computer-kcp/internal/services/bidder"
"github.com/johnandersen777/socialweb-computer-kcp/internal/services/dispatchertls"
"github.com/johnandersen777/socialweb-computer-kcp/internal/services/gatewayapi"
"github.com/johnandersen777/socialweb-computer-kcp/internal/services/guestcontrolplane"
"github.com/johnandersen777/socialweb-computer-kcp/internal/services/ingressproxy"
opsreadsvc "github.com/johnandersen777/socialweb-computer-kcp/internal/services/opsread"
"github.com/johnandersen777/socialweb-computer-kcp/internal/services/pds"
"github.com/johnandersen777/socialweb-computer-kcp/internal/services/plc"
"github.com/johnandersen777/socialweb-computer-kcp/internal/services/relay"
"github.com/johnandersen777/socialweb-computer-kcp/internal/services/requester"
"github.com/johnandersen777/socialweb-computer-kcp/internal/services/sshserver"
"github.com/johnandersen777/socialweb-computer-kcp/internal/tlsfront"
"github.com/johnandersen777/socialweb-computer-kcp/internal/vault/openbao"
)
const (
DefaultWorkspace = "root"
ComponentWorkspace = "app"
RetryAfter = 2 * time.Second
PortsDir = "ports"
)
var ErrNoKubeconfig = errors.New("componentcore: a kubeconfig or kcp clients are required")
type Config struct {
Kubeconfig string
CoreGeneration int64
Workspaces []string
WorkspacesFunc func(ctx context.Context) []string
StateDir string
Log *slog.Logger
TLSCertFile string
TLSKeyFile string
CredentialServer string
CredentialClient string
KCP component.KCPClients
Store *kcpstore.Store
DynamicFactory DynamicFactory
Ports *component.PortRegistry
Handlers *component.HandlerRegistry
DisablePolicy bool
ExposureIngressIdentity exposure.IngressIdentity
ExposureIngressRegistrar exposure.IngressRegistrar
DumbpipePath string
SlotsFile string
FrontControl string
HostAuthority hostlifecycle.Authority
InstallStateRoot string
InstallAccount string
PLCBaseURL string
Relays []string
Network *v1alpha1.MarketNetworkValue
NodeClaimAgent *kcpclient.ServiceAccountRef
OAuthScopes []string
Compute *computevm.Config
Postgres *postgresctl.Host
AtircNodes *atircnode.Options
OpenBao *openbaoctl.Servers
TTLScope func(workspace string) bool
OpsReadSecretFile string
OpsReadLogFile string
OnTTL func(report func() *v1alpha1.TTLStatus)
Apps *app.Config
}
type Core struct {
cfg Config
tls *tls.Config
egressTLS *tls.Config
hosts component.NetworkHosts
runtime *component.Runtime
scope *workspaceScope
registry *component.Registry
ports *component.PortRegistry
store *kcpstore.Store
handlers *component.HandlerRegistry
linkPolicy *exposure.LinkPolicy
ingressRegistrar exposure.IngressRegistrar
nodeClaims *nodeclaimctl.Controller
logins *openbao.Logins
oauthCallbacks *oauthcallback.Registry
closers []func()
}
func AppWorkspaces(base string) []string {
base = strings.TrimSpace(base)
base = cmp.Or(base, DefaultWorkspace)
out := []string{base}
app := base + ":" + ComponentWorkspace
if app != base {
out = append(out, app)
}
return out
}
func NetworkWorkspaces(base string) []string {
base = cmp.Or(strings.TrimSpace(base), DefaultWorkspace)
if base == DefaultWorkspace {
return []string{base}
}
return AppWorkspaces(base)
}
func New(ctx context.Context, cfg Config) (*Core, error) {
if cfg.Log == nil {
cfg.Log = slog.New(slog.DiscardHandler)
}
if cfg.StateDir == "" {
return nil, errors.New("componentcore: a state dir is required")
}
if len(cfg.Workspaces) == 0 {
cfg.Workspaces = AppWorkspaces(DefaultWorkspace)
}
if cfg.KCP == nil {
set, err := kcpclients.FromKubeconfig(cfg.Kubeconfig)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrNoKubeconfig, err)
}
cfg.KCP = set
}
base := cfg.Workspaces[0]
store := cfg.Store
if store == nil {
client, err := cfg.KCP.Client(base)
if err != nil {
return nil, fmt.Errorf("componentcore: the kcp client for %s: %w", base, err)
}
restCfg := client.RESTConfig()
if restCfg == nil {
return nil, errors.New("componentcore: the kcp client has no rest config")
}
store, err = kcpstore.New(kcpstore.Options{Host: restCfg.Host, RestConfig: restCfg})
if err != nil {
return nil, fmt.Errorf("componentcore: the kcp store: %w", err)
}
}
factory := cfg.DynamicFactory
if factory == nil {
if cfg.Kubeconfig != "" {
f, err := NewDynamicFactory(cfg.Kubeconfig)
if err != nil {
return nil, err
}
factory = f
} else {
f, err := DynamicFromKCP(cfg.KCP)
if err != nil {
return nil, err
}
factory = f
}
}
ports := cfg.Ports
if ports == nil {
var err error
first, last := portBlock(cfg.StateDir)
ports, err = component.OpenPortRegistry(component.PortRegistryOptions{StateDir: filepath.Join(cfg.StateDir, PortsDir), First: first, Last: last})
if err != nil {
return nil, fmt.Errorf("componentcore: the port registry: %w", err)
}
}
cfg.DynamicFactory = factory
if cfg.SlotsFile == "" {
cfg.SlotsFile = filepath.Join(cfg.StateDir, "kcp", "kcp-slots.json")
}
if cfg.FrontControl == "" {
cfg.FrontControl = filepath.Join(cfg.StateDir, "run", "front.sock")
}
tlsConfig, err := frontTLS(cfg)
if err != nil {
return nil, err
}
registry := component.NewRegistry()
handlers := cfg.Handlers
if handlers == nil {
handlers = component.NewHandlerRegistry(ctx)
}
handlers.SetLogger(cfg.Log.With("component", "bus"))
registrar := cfg.ExposureIngressRegistrar
fallbackName := "configured-registrar"
if registrar == nil {
registrar = exposure.NewSubscriberRegistrar(registry, handlers, cfg.ExposureIngressIdentity)
fallbackName = "xrpc-subscription"
}
dispatcher := newIngressDispatcher(registry, registrar, fallbackName, cfg.Log)
runtime := component.NewRuntime(component.RuntimeConfig{Log: cfg.Log, StateDir: cfg.StateDir, Registry: registry, Bus: handlers}, nil)
runtime.SetStateDirMaker(func(dir string) error { return os.MkdirAll(dir, 0o700) })
core := &Core{cfg: cfg, tls: tlsConfig, egressTLS: egressTLS(tlsConfig, cfg.Network), hosts: registryHosts{registry: registry}, runtime: runtime,
logins: openbao.NewLogins(nil), oauthCallbacks: oauthcallback.NewRegistry(),
registry: registry, ports: ports, store: store, handlers: handlers, linkPolicy: exposure.NewLinkPolicy(handlers),
ingressRegistrar: dispatcher, scope: newWorkspaceScope(cfg.Workspaces, cfg.WorkspacesFunc), nodeClaims: newNodeClaims(handlers, cfg.Log)}
if err := core.register(); err != nil {
return nil, err
}
return core, nil
}
func (c *Core) register() error {
c.provideComponents()
kcp, tlsConfig, hosts, registry, identity := c.cfg.KCP, c.tls, c.hosts, c.registry, c.cfg.ExposureIngressIdentity
egress := func(in component.Instance) component.Egress {
return component.Egress{Hosts: hosts, TLS: c.egressTLS, Log: in.Log}
}
opsSecret, err := opsreadsvc.LoadSecret(cmp.Or(c.cfg.OpsReadSecretFile, filepath.Join(c.cfg.StateDir, opsreadsvc.SecretFileName)))
if err != nil {
c.cfg.Log.Warn("the ops read routes are not served", "err", err.Error())
}
opsHost := hostread.New()
opsHost.LogFile = c.cfg.OpsReadLogFile
for _, entry := range []struct {
kind string
factory component.ServiceFactory
}{
{plc.Kind, component.Typed(func(_ context.Context, in component.Instance, spec v1alpha1.PlcSpec) (*plc.Service, error) {
return plc.New(spec, in), nil
})},
{relay.Kind, component.Typed(func(_ context.Context, in component.Instance, spec v1alpha1.RelaySpec) (*relay.Service, error) {
return relay.New(spec, relay.Deps{Instance: in, Registry: registry, TLS: tlsConfig, Egress: egress(in)})
})},
{dispatchertls.Kind, component.Typed(func(_ context.Context, in component.Instance, spec v1alpha1.DispatcherTLSSpec) (*dispatchertls.Service, error) {
return dispatchertls.New(spec, dispatchertls.Deps{Instance: in, Registry: registry, TLS: tlsConfig}), nil
})},
{ingressproxy.Kind, component.Typed(func(_ context.Context, in component.Instance, spec v1alpha1.IngressProxySpec) (*ingressproxy.Service, error) {
return ingressproxy.New(spec, ingressproxy.Deps{Instance: in, Registry: registry, TLS: tlsConfig, Identity: identity, Dumbpipe: c.cfg.DumbpipePath})
})},
{gatewayapi.Kind, component.Typed(func(_ context.Context, in component.Instance, spec v1alpha1.GatewayAPISpec) (*gatewayapi.Service, error) {
return gatewayapi.New(spec, gatewayapi.Deps{Instance: in, KCP: kcp, TLS: tlsConfig, NodeClaimAgent: c.cfg.NodeClaimAgent, NodeClaims: c.nodeClaims, Logins: c.logins})
})},
{bidder.Kind, component.Typed(func(ctx context.Context, in component.Instance, spec v1alpha1.BidderSpec) (*bidder.Service, error) {
deps := bidder.Deps{Instance: in, Egress: egress(in), OAuthCallbacks: c.oauthCallbacks, Sandbox: newSandbox(), OpsSecret: opsSecret, OpsHost: opsHost}
if c.cfg.Compute != nil && c.cfg.Compute.Runtime != nil {
caps := c.cfg.Compute.Runtime.Capabilities()
deps.NestedVirt, deps.ImageTarget, deps.ImageReady = caps.NestedVirt, caps.ImageTarget, c.cfg.Compute.Runtime.EnsureImage
}
return bidder.New(ctx, spec, deps)
})},
{guestcontrolplane.Kind, component.Typed(func(_ context.Context, in component.Instance, spec v1alpha1.GuestControlPlaneSpec) (*guestcontrolplane.Service, error) {
return guestcontrolplane.New(spec, guestcontrolplane.Deps{Instance: in, Hosts: hosts})
})},
{pds.Kind, component.Typed(func(_ context.Context, in component.Instance, spec v1alpha1.PdsSpec) (*pds.Service, error) {
return pds.New(spec, pds.Deps{Instance: in, Egress: egress(in)})
})},
{requester.Kind, component.Typed(func(ctx context.Context, in component.Instance, spec v1alpha1.RequesterSpec) (*requester.Service, error) {
return requester.New(ctx, spec, requester.Deps{
Instance: in,
Egress: egress(in),
OAuthCallbacks: c.oauthCallbacks,
OpenIngress: func(handler http.Handler, options tunnel.ServiceOptions) (tunnel.Service, error) {
return webrtc.NewService(webrtc.ServiceConfig{Handler: handler, Log: in.Log, Options: options})
},
})
})},
{sshserver.Kind, component.Typed(func(_ context.Context, in component.Instance, spec v1alpha1.SshServerSpec) (*sshserver.Service, error) {
return sshserver.New(spec, sshserver.Deps{Instance: in, Hosts: hosts, KCP: kcp, TLS: tlsConfig, OpsSecret: opsSecret, OpsHost: opsHost})
})},
} {
if err := c.runtime.Register(entry.kind, entry.factory); err != nil {
return err
}
}
return nil
}
func policyActionHandlers(store *kcpstore.Store, workspace string) (map[string]policycontroller.ActionHandler, error) {
handler, err := policyaction.NewComputeVMHandler(store, workspace)
if err != nil {
return nil, err
}
registry := &policycontroller.TriggerActionRegistry{}
if err := registry.RegisterComputeVM(handler); err != nil {
return nil, fmt.Errorf("componentcore: the ComputeVM policy action: %w", err)
}
return registry.Handlers(), nil
}
func (c *Core) Run(ctx context.Context) error {
if err := c.runtime.Start(ctx); err != nil {
return err
}
err := c.runControllers(ctx)
for _, closer := range c.closers {
closer()
}
stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), component.DefaultStopTimeout)
defer cancel()
err = errors.Join(err, c.runtime.Stop(stopCtx))
// a caller's bus (cfg.Handlers) outlives this core: a restarted core and the app controllers keep using it
if c.cfg.Handlers == nil {
err = errors.Join(err, c.handlers.Stop(stopCtx))
}
return err
}
func (c *Core) Registry() *component.Registry {
return c.registry
}
func (c *Core) Ports() *component.PortRegistry {
return c.ports
}
func (c *Core) Runtime() *component.Runtime {
return c.runtime
}
func (c *Core) pdsEndpoint(workspace, kind, name string) (string, bool) {
if c.registry == nil {
return "", false
}
if root, ok := strings.CutSuffix(workspace, ":"+ComponentWorkspace); ok && !c.registry.Has(workspace, kind, name) {
workspace = root
}
if endpoint, ok := c.registry.Endpoint(workspace, kind, name, "http"); ok && endpoint.URL != "" {
return endpoint.URL, true
}
endpoint, ok := component.PrimaryEndpoint(c.registry.Endpoints(workspace, kind, name))
if !ok {
return "", false
}
if endpoint.URL != "" {
return endpoint.URL, true
}
if endpoint.Address != "" {
return "http://" + endpoint.Address, true
}
return "", false
}
func DynamicFromKCP(kcp component.KCPClients) (DynamicFactory, error) {
if kcp == nil {
return nil, ErrNoKubeconfig
}
return func(workspace string) (*kube.Client, error) {
client, err := kcp.Client(workspace)
if err != nil {
return nil, err
}
return dynamicForRestConfig(client.RESTConfig(), workspace)
}, nil
}
func NewDynamicFactory(kubeconfig string) (DynamicFactory, error) {
if kubeconfig == "" {
return nil, ErrNoKubeconfig
}
cfg, err := kcpserver.FrontRestConfig(kubeconfig)
if err != nil {
return nil, fmt.Errorf("componentcore: the kcp front rest config: %w", err)
}
return func(workspace string) (*kube.Client, error) {
return dynamicForRestConfig(cfg, workspace)
}, nil
}
func egressTLS(front *tls.Config, n *v1alpha1.MarketNetworkValue) *tls.Config {
if n == nil || strings.TrimSpace(n.CABundle) == "" {
return front
}
pool, err := x509.SystemCertPool()
if err != nil || !pool.AppendCertsFromPEM([]byte(n.CABundle)) {
return front
}
out := front.Clone()
out.RootCAs = pool
return out
}
func frontTLS(cfg Config) (*tls.Config, error) {
certFile, keyFile := cfg.TLSCertFile, cfg.TLSKeyFile
if certFile == "" && keyFile == "" {
var err error
if certFile, keyFile, err = tlsfront.EnsureSelfSigned(filepath.Join(cfg.StateDir, "tls")); err != nil {
return nil, fmt.Errorf("componentcore: the self-signed front certificate: %w", err)
}
} else if certFile == "" || keyFile == "" {
return nil, errors.New("componentcore: a TLS certificate and key are both required")
}
reloader, err := tlsreload.New(certFile, []string{keyFile}, cfg.Log)
if err != nil {
return nil, fmt.Errorf("componentcore: the TLS certificate and key: %w", err)
}
return reloader.TLSConfig(&tls.Config{MinVersion: tls.VersionTLS12}), nil
}
type registryHosts struct {
registry *component.Registry
}
func (h registryHosts) Address(hostport string) (string, bool) {
m, err := component.NewNetworkHostMapFromRegistry(h.registry)
if err != nil {
return "", false
}
return m.Address(hostport)
}
func (h registryHosts) Hosts() map[string]string {
m, err := component.NewNetworkHostMapFromRegistry(h.registry)
if err != nil {
return nil
}
return m.Hosts()
}
func Supervise(ctx context.Context, cfg Config) {
log := cfg.Log
if log == nil {
log = slog.New(slog.DiscardHandler)
}
log = log.With("component", "component-controller")
supervise.Restart(ctx, "componentcore", supervise.Backoff{Min: RetryAfter, Max: RetryAfter, OnError: func(err error, _ time.Duration) {
if err != nil {
log.Warn("the component controller stopped; starting it again", "err", fmt.Sprint(err))
}
}}, func(ctx context.Context) error {
core, err := New(ctx, cfg)
if err != nil {
return err
}
log.Info("the component controller reconciles the application workspaces", "workspaces", strings.Join(cfg.Workspaces, ", "))
return core.Run(ctx)
})
}
func dynamicForRestConfig(cfg *rest.Config, workspace string) (*kube.Client, error) {
if cfg == nil {
return nil, errors.New("componentcore: the kcp client has no rest config")
}
workspace = strings.TrimSpace(workspace)
workspace = cmp.Or(workspace, DefaultWorkspace)
out := rest.CopyConfig(cfg)
host := strings.TrimSuffix(out.Host, "/")
if !strings.Contains(host, "/clusters/") {
out.Host = host + "/clusters/" + workspace
}
dc, err := kube.New(out)
if err != nil {
return nil, fmt.Errorf("componentcore: the dynamic client for %s: %w", workspace, err)
}
return dc, nil
}
const (
portRangeFirst = 20000
portRangeLast = 29999
portBlockSize = 64
)
func portBlock(stateDir string) (int, int) {
h := fnv.New32a()
_, _ = h.Write([]byte(filepath.Clean(stateDir)))
blocks := uint32((portRangeLast - portRangeFirst + 1) / portBlockSize)
first := portRangeFirst + int(h.Sum32()%blocks)*portBlockSize
return first, first + portBlockSize - 1
}