0
0
package exposure
import (
"cmp"
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"net/url"
"strings"
"sync"
"syscall"
"time"
exposurev1alpha1 "github.com/johnandersen777/socialweb-computer-kcp/api/exposure/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/atproto/ingress"
"github.com/johnandersen777/socialweb-computer-kcp/internal/bus"
"github.com/johnandersen777/socialweb-computer-kcp/internal/component"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/servetls"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/strs"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/tlsreload"
)
type PublisherConfig struct {
Handlers *component.Handlers
Ports *component.PortRegistry
TLS *tls.Config
Registrar IngressRegistrar
Log *slog.Logger
}
type Binding struct {
Workspace string
Name string
Spec exposurev1alpha1.ExposureSpec
Service exposurev1alpha1.ServiceSpec
ServiceGeneration int64
Target component.Key
SubscriberChanged func()
}
type Published struct {
Address string
Endpoint string
Provider string
Ingress *exposurev1alpha1.ExposureIngress
}
const addrInUseRelease = 12
type Publisher struct {
handlers *component.Handlers
ports *component.PortRegistry
tls *tls.Config
registrar IngressRegistrar
log *slog.Logger
mu sync.Mutex
running map[string]*publication
busy map[string]int
subscribers map[string]exposurev1alpha1.ExposureIngressSubscriber
}
type publication struct {
fingerprint string
ingress string
published Published
caPool *tlsreload.Pool
caBundle string
once sync.Once
close func()
}
func (p *publication) shutdown() {
if p == nil {
return
}
p.once.Do(func() {
if p.close != nil {
p.close()
}
})
}
func NewPublisher(cfg PublisherConfig) *Publisher {
log := cfg.Log
if log == nil {
log = slog.New(slog.DiscardHandler)
}
return &Publisher{handlers: cfg.Handlers, ports: cfg.Ports, tls: cfg.TLS, registrar: cfg.Registrar,
log: log.With("component", "exposure-publisher"), running: map[string]*publication{}, busy: map[string]int{},
subscribers: map[string]exposurev1alpha1.ExposureIngressSubscriber{}}
}
func (p *Publisher) Handler(ctx context.Context, target component.Key, protocol exposurev1alpha1.ServiceProtocol, name string) (http.Handler, bool, error) {
return FindHandler(ctx, p.handlers, target, protocol, name)
}
func (p *Publisher) Publish(ctx context.Context, b Binding) (Published, error) {
key := publicationKey(b.Workspace, b.Name)
ingress, served := p.ingressInstance(b)
fingerprint, err := p.fingerprint(b, ingress, served)
if err != nil {
return Published{}, err
}
current := p.current(key)
if current != nil && current.fingerprint == fingerprint {
err := p.rotateCA(b.Name, current, b.Spec.TLS.CABundle)
if err == nil {
return current.published, nil
}
p.log.Warn("rebuilding the Exposure after a bad CA bundle", "workspace", b.Workspace, "exposure", b.Name, "err", err)
}
if current != nil && current.ingress != ingress {
p.log.Info("exposure_ingress_instance_changed", "workspace", b.Workspace, "exposure", b.Name,
"ingress", b.Spec.IngressRef, "from", current.ingress, "to", ingress)
}
p.stop(key)
var pub *publication
switch b.Spec.Mode {
case exposurev1alpha1.ExposureModeLocal:
pub, err = p.startLocal(ctx, b)
case exposurev1alpha1.ExposureModeIngress:
pub, err = p.startIngress(ctx, b)
default:
err = fmt.Errorf("unknown mode %q", b.Spec.Mode)
}
if err != nil {
if errors.Is(err, syscall.EADDRINUSE) && p.addrInUse(key) {
p.log.Warn("the kept port of the Exposure stays taken; it takes a new port", "workspace", b.Workspace, "exposure", b.Name)
p.release(b.Workspace, b.Name)
}
return Published{}, err
}
p.mu.Lock()
delete(p.busy, key)
p.mu.Unlock()
pub.fingerprint = fingerprint
pub.ingress = ingress
pub.caBundle = strings.TrimSpace(b.Spec.TLS.CABundle)
p.mu.Lock()
p.running[key] = pub
p.mu.Unlock()
return pub.published, nil
}
func (p *Publisher) addrInUse(key string) bool {
p.mu.Lock()
defer p.mu.Unlock()
p.busy[key]++
if p.busy[key] < addrInUseRelease {
return false
}
delete(p.busy, key)
return true
}
func (p *Publisher) Withdraw(workspace, name string) {
key := publicationKey(workspace, name)
p.stop(key)
p.release(workspace, name)
p.mu.Lock()
delete(p.subscribers, key)
p.mu.Unlock()
}
func (p *Publisher) Subscriber(workspace, name string) (exposurev1alpha1.ExposureIngressSubscriber, bool) {
p.mu.Lock()
defer p.mu.Unlock()
state, ok := p.subscribers[publicationKey(workspace, name)]
return state, ok
}
func (p *Publisher) subscriberReporter(key string, changed func()) func(ingress.SubscriberStatus) {
return func(status ingress.SubscriberStatus) {
if status.State == ingress.SubscriberReplaced {
p.log.Error("the ingress subscriber was replaced by a newer subscription of the same key and stopped; this exposure has no ingress until the process restarts", "publication", key, "closeCode", status.CloseCode)
}
p.mu.Lock()
p.subscribers[key] = SubscriberStateOf(status)
p.mu.Unlock()
if changed != nil {
changed()
}
}
}
func (p *Publisher) Pause(workspace, name string) {
p.stop(publicationKey(workspace, name))
}
func (p *Publisher) stop(key string) {
p.mu.Lock()
pub := p.running[key]
delete(p.running, key)
p.mu.Unlock()
pub.shutdown()
}
func (p *Publisher) Close() {
p.mu.Lock()
running := p.running
p.running = map[string]*publication{}
p.mu.Unlock()
for _, pub := range running {
pub.shutdown()
}
}
func publicationKey(workspace, name string) string {
return workspace + "/" + name
}
func (p *Publisher) fingerprint(b Binding, ingress string, served bool) (string, error) {
spec := b.Spec
spec.TLS.CABundle = caBundleKey(spec.TLS.CABundle)
value := map[string]any{
"exposure": map[string]any{"spec": spec},
"service": map[string]any{"spec": b.Service, "generation": b.ServiceGeneration},
"target": b.Target,
}
if served {
value["ingress"] = ingress
}
raw, err := json.Marshal(value)
if err != nil {
return "", err
}
revision := uint64(0)
if p.handlers != nil {
revision = p.handlers.HandlerRevision(b.Target, b.Service.Handler)
}
return fmt.Sprintf("%s|handler-%d", raw, revision), nil
}
func (p *Publisher) ingressInstance(b Binding) (string, bool) {
instances, ok := p.registrar.(IngressInstances)
if !ok || b.Spec.Mode != exposurev1alpha1.ExposureModeIngress {
return "", false
}
return instances.IngressInstance(b.Workspace, b.Spec.IngressRef), true
}
func caBundleKey(bundle string) string {
bundle = strings.TrimSpace(bundle)
if isPEMBundle(bundle) {
return "pem"
}
return bundle
}
func isPEMBundle(bundle string) bool {
return strings.Contains(bundle, "-----BEGIN")
}
func (p *Publisher) rotateCA(name string, pub *publication, bundle string) error {
bundle = strings.TrimSpace(bundle)
p.mu.Lock()
defer p.mu.Unlock()
if bundle == pub.caBundle {
return nil
}
if pub.caPool != nil && isPEMBundle(bundle) {
if err := pub.caPool.Set([]byte(bundle)); err != nil {
return err
}
p.log.Info("rotated the Exposure client CA bundle", "exposure", name)
}
pub.caBundle = bundle
return nil
}
func (p *Publisher) current(key string) *publication {
p.mu.Lock()
defer p.mu.Unlock()
return p.running[key]
}
func (p *Publisher) release(workspace, name string) {
if p.ports == nil {
return
}
_ = p.ports.Release(leaseName(workspace, name))
}
func leaseName(workspace, name string) string {
return "Exposure/" + workspace + "/" + name
}
func (p *Publisher) startLocal(ctx context.Context, b Binding) (*publication, error) {
if p.ports == nil {
return nil, errors.New("the port registry is not configured")
}
listener, lease, err := p.ports.Listen(leaseName(b.Workspace, b.Name))
if err != nil {
return nil, err
}
tlsConfig, caPool, err := p.localTLS(b.Spec.TLS)
if err != nil {
_ = listener.Close()
return nil, err
}
if tlsConfig != nil {
listener = tls.NewListener(listener, tlsConfig)
}
protocol := protocolKind(b.Service.Protocol)
path := normalizePath(b.Spec.Path)
target := b.Target
pub := &publication{caPool: caPool}
switch protocol {
case component.ProtocolHTTP, component.ProtocolWS:
if _, ok := p.handlers.Handler(target, b.Service.Handler); !ok {
_ = listener.Close()
return nil, ErrHandlerMissing
}
name := HandlerName(b.Service.Handler)
current := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h, ok := p.handlers.HandlerNamed(target, name)
if !ok {
http.Error(w, "the target handler is not registered", http.StatusServiceUnavailable)
return
}
h.ServeHTTP(w, r.WithContext(component.WithCaller(r.Context(), target)))
})
server := &http.Server{Handler: current, ReadHeaderTimeout: 10 * time.Second}
done := make(chan struct{})
go func() {
defer close(done)
_ = server.Serve(listener)
}()
pub.close = func() {
_ = server.Close()
<-done
}
scheme := httpScheme(protocol, tlsConfig != nil)
pub.published = Published{Address: scheme + "://" + lease.Address() + path, Endpoint: lease.Address()}
return pub, nil
case component.ProtocolTCP, component.ProtocolSSH:
runCtx, cancel := context.WithCancel(bus.WithCaller(context.WithoutCancel(ctx), ExposureCaller(b.Workspace, b.Name)))
done := make(chan struct{})
var conns sync.WaitGroup
go func() {
defer close(done)
for {
conn, err := listener.Accept()
if err != nil {
return
}
conns.Go(func() { bridge(runCtx, p.handlers, target, b.Service.Handler, conn) })
}
}()
pub.close = func() {
cancel()
_ = listener.Close()
<-done
conns.Wait()
}
scheme := cmp.Or(protocol, component.ProtocolTCP)
pub.published = Published{Address: scheme + "://" + lease.Address(), Endpoint: lease.Address()}
return pub, nil
default:
_ = listener.Close()
return nil, fmt.Errorf("unknown protocol %q", b.Service.Protocol)
}
}
func (p *Publisher) startIngress(ctx context.Context, b Binding) (*publication, error) {
if p.registrar == nil {
return nil, errors.New("no ingress registrar is configured")
}
life, cancel := context.WithCancel(context.WithoutCancel(ctx))
stop := context.AfterFunc(ctx, cancel)
registration, err := p.registrar.Register(life, IngressSpec{
Workspace: b.Workspace,
Exposure: b.Name,
Service: strs.LastSegment(b.Spec.ServiceRef),
Protocol: b.Service.Protocol,
Handler: b.Service.Handler,
Target: b.Target,
IngressRef: b.Spec.IngressRef,
Hostname: b.Spec.Hostname,
IngressAddress: b.Spec.IngressAddress,
IngressDial: b.Spec.IngressDial,
PublicHost: b.Spec.PublicHost,
Path: normalizePath(b.Spec.Path),
TLS: b.Spec.TLS.Enabled,
AuthRequired: b.Spec.Auth.Required,
AllowedDIDs: append([]string(nil), b.Spec.Auth.AllowedDIDs...),
OnStatus: p.subscriberReporter(publicationKey(b.Workspace, b.Name), b.SubscriberChanged),
})
if !stop() || err != nil {
cancel()
if err == nil {
_ = registration.Close()
err = ctx.Err()
}
return nil, err
}
address := registration.URL()
provider := ""
if described, ok := registration.(IngressRegistrationProvider); ok {
provider = strings.TrimSpace(described.Provider())
}
if path := normalizePath(b.Spec.Path); path == "" || path == "/" {
address = originURL(address)
}
var route *exposurev1alpha1.ExposureIngress
if routed, ok := registration.(IngressRegistrationRoute); ok {
value := routed.Route()
route = &value
}
return &publication{published: Published{Address: address, Endpoint: address, Provider: provider, Ingress: route},
close: func() {
_ = registration.Close()
cancel()
}}, nil
}
func originURL(raw string) string {
parsed, err := url.Parse(strings.TrimSpace(raw))
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
return raw
}
return parsed.Scheme + "://" + parsed.Host
}
func (p *Publisher) localTLS(spec exposurev1alpha1.ExposureTLS) (*tls.Config, *tlsreload.Pool, error) {
if !spec.Enabled && strings.TrimSpace(spec.CertRef) == "" {
return nil, nil, nil
}
var config *tls.Config
switch {
case strings.TrimSpace(spec.CertRef) != "":
certFile := strings.TrimSpace(spec.CertRef)
reloader, err := tlsreload.New(certFile, []string{servetls.KeyFileFor(certFile)}, p.log)
if err != nil {
return nil, nil, fmt.Errorf("loading the certificate %s: %w", certFile, err)
}
config = reloader.TLSConfig(&tls.Config{MinVersion: tls.VersionTLS12})
case p.tls != nil:
config = p.tls.Clone()
config.MinVersion = cmp.Or(config.MinVersion, tls.VersionTLS12)
default:
return nil, nil, errors.New("TLS is enabled but no certificate is available")
}
bundle := strings.TrimSpace(spec.CABundle)
if bundle == "" {
return config, nil, nil
}
pool, err := tlsreload.PoolFromBundle(bundle, p.log)
if err != nil {
return nil, nil, err
}
return pool.ServerConfig(config, tls.VerifyClientCertIfGiven), pool, nil
}
func bridge(ctx context.Context, handlers *component.Handlers, target component.Key, name string, conn net.Conn) {
defer conn.Close()
if handlers == nil {
return
}
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
defer stop()
stream, err := handlers.DialStream(ctx, target, name)
if err != nil {
return
}
defer stream.Close()
done := make(chan struct{}, 2)
go func() {
_, _ = io.Copy(stream, conn)
done <- struct{}{}
}()
go func() {
_, _ = io.Copy(conn, stream)
done <- struct{}{}
}()
<-done
_ = conn.Close()
_ = stream.Close()
<-done
}
func normalizePath(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
if !strings.HasPrefix(value, "/") {
value = "/" + value
}
return value
}
func httpScheme(protocol string, secure bool) string {
switch protocol {
case component.ProtocolWS:
if secure {
return "wss"
}
return "ws"
default:
if secure {
return "https"
}
return "http"
}
}