0
0
package xrpcserver
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"strconv"
"sync"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/gorilla/websocket"
"github.com/johnandersen777/socialweb-computer-kcp/internal/atprotoauth"
"github.com/johnandersen777/socialweb-computer-kcp/internal/gateway"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/httpjson"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/supervise"
)
const (
nsidSubscribeClusters = NSIDPrefix + "subscribeClusters"
DefaultStreamLifetime = time.Hour
DefaultStreamsPerCaller = 8
streamQueue = 512
streamReadLimit = 1 << 12
pingEvery = 30 * time.Second
pongWait = 75 * time.Second
frameWriteLimit = 10 * time.Second
)
var drisl = func() cbor.EncMode {
m, err := cbor.EncOptions{Sort: cbor.SortLengthFirst}.EncMode()
if err != nil {
panic("xrpcserver: the DRISL-CBOR encoder options were refused: " + err.Error())
}
return m
}()
type frameHeader struct {
Op int64 `cbor:"op"`
T string `cbor:"t,omitempty"`
}
type errorFrame struct {
Error string `cbor:"error"`
Message string `cbor:"message,omitempty"`
}
func encodeFrame(h frameHeader, payload any) ([]byte, error) {
head, err := drisl.Marshal(h)
if err != nil {
return nil, err
}
body, err := drisl.Marshal(payload)
if err != nil {
return nil, err
}
return append(head, body...), nil
}
func messageFrame(ev gateway.Event) ([]byte, error) {
return encodeFrame(frameHeader{Op: 1, T: "#" + ev.Type}, ev.Payload)
}
func errFrame(xe *gateway.XRPCError) []byte {
b, err := encodeFrame(frameHeader{Op: -1}, errorFrame{Error: xe.Name, Message: xe.Message})
if err != nil {
panic("xrpcserver: an error frame did not encode: " + err.Error())
}
return b
}
type streams struct {
wg sync.WaitGroup
mu sync.Mutex
open map[gateway.Caller]int
limit int
ctx context.Context
shutdown context.CancelFunc
}
func newStreams(ctx context.Context, limit int) *streams {
ctx, cancel := context.WithCancel(ctx)
return &streams{open: map[gateway.Caller]int{}, limit: limit, ctx: ctx, shutdown: cancel}
}
func (s *streams) acquire(c gateway.Caller) bool {
s.mu.Lock()
defer s.mu.Unlock()
if s.open[c] >= s.limit {
return false
}
s.open[c]++
return true
}
func (s *streams) release(c gateway.Caller) {
s.mu.Lock()
defer s.mu.Unlock()
if s.open[c]--; s.open[c] <= 0 {
delete(s.open, c)
}
}
const closeWait = 3 * time.Second
func (s *Server) Close() {
s.streams.shutdown()
ctx, cancel := context.WithTimeout(context.WithoutCancel(s.streams.ctx), closeWait)
defer cancel()
_ = supervise.Join(ctx, s.streams.wg.Wait)
}
var upgrader = websocket.Upgrader{
ReadBufferSize: 1 << 10,
WriteBufferSize: 1 << 12,
CheckOrigin: func(*http.Request) bool { return true },
}
func (s *Server) subscribeClusters(w http.ResponseWriter, req *http.Request) {
if req.Method != http.MethodGet {
w.Header().Set("Allow", http.MethodGet)
httpjson.Write(w, http.StatusMethodNotAllowed, errorBody{Error: "MethodNotAllowed", Message: "a subscription is opened with GET and a WebSocket upgrade"})
return
}
if !websocket.IsWebSocketUpgrade(req) {
w.Header().Set("Upgrade", "websocket")
w.Header().Set("Connection", "Upgrade")
httpjson.Write(w, http.StatusUpgradeRequired, errorBody{Error: "UpgradeRequired", Message: "this is an event stream: connect with a WebSocket upgrade"})
return
}
caller, refused := s.verifySubscriber(req)
name := req.URL.Query().Get("name")
if refused == nil {
refused = s.checkSubscribeParams(req)
}
if refused == nil {
if !s.streams.acquire(caller) {
w.Header().Set("Retry-After", strconv.Itoa(int(s.opts.RetryAfter.Seconds())))
httpjson.Write(w, http.StatusTooManyRequests, errorBody{Error: "RateLimitExceeded",
Message: fmt.Sprintf("this account already has %d open streams", s.streams.limit)})
return
}
defer s.streams.release(caller)
}
s.streams.wg.Add(1)
defer s.streams.wg.Done()
conn, err := upgrader.Upgrade(w, req, nil)
if err != nil {
slog.Warn("xrpc: subscription upgrade failed", "nsid", nsidSubscribeClusters, "err", err)
return
}
defer conn.Close()
if refused != nil {
closeWithError(conn, refused)
return
}
s.stream(conn, caller, name)
}
func (s *Server) verifySubscriber(req *http.Request) (gateway.Caller, *gateway.XRPCError) {
token, ok := bearer(req.Header.Get("Authorization"))
if !ok {
return "", gateway.ErrUnauthorized
}
did, err := atprotoauth.Verify(req.Context(), token, nsidSubscribeClusters, s.opts.Audiences, s.opts.Directory)
switch {
case err == nil:
return gateway.Caller(did), nil
case errors.Is(err, atprotoauth.ErrExpired):
return "", gateway.ErrTokenExpired
case errors.Is(err, atprotoauth.ErrNoKey):
return "", gateway.ErrClusterUnavailable
default:
return "", gateway.ErrUnauthorized
}
}
func (s *Server) checkSubscribeParams(req *http.Request) *gateway.XRPCError {
def := s.params[nsidSubscribeClusters]
params, err := queryParams(req.URL.Query(), def)
if err != nil {
return gateway.ErrInvalidRequest
}
if err := validate(nsidSubscribeClusters+" parameters", s.catalog, def, params); err != nil {
return gateway.ErrInvalidRequest
}
if name, ok := params["name"].(string); ok && labelOrInvalidRequest(name) != nil {
return gateway.ErrInvalidRequest
}
return nil
}
func closeWithError(conn *websocket.Conn, xe *gateway.XRPCError) {
_ = conn.SetWriteDeadline(time.Now().Add(frameWriteLimit))
if err := conn.WriteMessage(websocket.BinaryMessage, errFrame(xe)); err != nil {
return
}
_ = conn.WriteControl(websocket.CloseMessage,
websocket.FormatCloseMessage(websocket.CloseNormalClosure, xe.Name), time.Now().Add(frameWriteLimit))
}
func (s *Server) stream(conn *websocket.Conn, caller gateway.Caller, name string) {
lifetime := s.opts.StreamLifetime
ctx, cancel := context.WithTimeout(s.streams.ctx, lifetime)
defer cancel()
frames := make(chan []byte, streamQueue)
emit := func(ev gateway.Event) error {
b, err := messageFrame(ev)
if err != nil {
return err
}
select {
case frames <- b:
return nil
default:
return gateway.ErrConsumerTooSlow
}
}
clientGone := make(chan struct{})
go func() {
defer close(clientGone)
defer cancel()
conn.SetReadLimit(streamReadLimit)
_ = conn.SetReadDeadline(time.Now().Add(pongWait))
conn.SetPongHandler(func(string) error { return conn.SetReadDeadline(time.Now().Add(pongWait)) })
for {
if _, _, err := conn.NextReader(); err != nil {
return
}
}
}()
defer func() {
_ = conn.Close()
<-clientGone
}()
done := make(chan error, 1)
go func() { done <- s.gateway.SubscribeClusters(ctx, caller, name, emit) }()
ping := time.NewTicker(pingEvery)
defer ping.Stop()
write := func(b []byte) bool {
_ = conn.SetWriteDeadline(time.Now().Add(frameWriteLimit))
return conn.WriteMessage(websocket.BinaryMessage, b) == nil
}
slog.Info("xrpc: subscription open", "nsid", nsidSubscribeClusters, "caller", string(caller), "name", name)
for {
select {
case b := <-frames:
if !write(b) {
cancel()
<-done
return
}
case <-ping.C:
if err := conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(frameWriteLimit)); err != nil {
cancel()
<-done
return
}
case err := <-done:
for drained := errors.Is(err, gateway.ErrConsumerTooSlow); !drained; {
select {
case b := <-frames:
if !write(b) {
return
}
default:
drained = true
}
}
s.finish(conn, caller, name, ctx, clientGone, err)
return
}
}
}
func (s *Server) finish(conn *websocket.Conn, caller gateway.Caller, name string, ctx context.Context, clientGone <-chan struct{}, err error) {
select {
case <-clientGone:
slog.Info("xrpc: subscription closed by the client", "caller", string(caller), "name", name)
return
default:
}
if s.streams.ctx.Err() != nil {
_ = conn.WriteControl(websocket.CloseMessage,
websocket.FormatCloseMessage(websocket.CloseGoingAway, "the gateway is shutting down; reconnect"),
time.Now().Add(frameWriteLimit))
return
}
var xe *gateway.XRPCError
switch {
case errors.Is(ctx.Err(), context.DeadlineExceeded):
xe = &gateway.XRPCError{Name: gateway.ErrTokenExpired.Name, Status: 401,
Message: fmt.Sprintf("the stream reached its %s lifetime bound; get a fresh token and reconnect", s.opts.StreamLifetime)}
case errors.As(err, &xe):
case err == nil:
xe = &gateway.XRPCError{Name: "ClusterUnavailable", Status: 503, Message: "the cluster feed ended"}
default:
slog.Error("xrpc: subscription failed", "caller", string(caller), "name", name, "err", err)
xe = &gateway.XRPCError{Name: "ClusterUnavailable", Status: 503, Message: "the cluster feed stopped"}
}
slog.Info("xrpc: subscription ended", "caller", string(caller), "name", name, "error", xe.Name, "message", xe.Message)
closeWithError(conn, xe)
}