0
0
package queue
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"github.com/johnandersen777/socialweb-computer-kcp/internal/contract/guestcp"
)
const (
Dir = "guest-events"
MaxPending = 256
MaxRecordBytes = 32 << 10
fileSuffix = ".ndjson"
)
var computeVMName = regexp.MustCompile(`^[a-z0-9]([a-z0-9.-]{0,251}[a-z0-9])?$`)
type Hub struct {
mu sync.Mutex
dir string
vms map[string]*vmQueue
seen map[string]bool
retained bool
publish guestcp.Publish
publisher int
}
type vmQueue struct {
path string
entries []guestcp.QueuedEvent
keys map[string]bool
last int64
wake chan struct{}
gone bool
}
var (
_ guestcp.Events = (*Hub)(nil)
_ guestcp.Queue = (*Hub)(nil)
)
func New() *Hub {
return &Hub{vms: map[string]*vmQueue{}, seen: map[string]bool{}}
}
func (h *Hub) Open(dir string) error {
dir = strings.TrimSpace(dir)
if dir == "" {
return errors.New("guestcp: the guest event queue names no state dir")
}
dir = filepath.Join(dir, Dir)
if err := os.MkdirAll(dir, 0o700); err != nil {
return fmt.Errorf("guestcp: creating the guest event queue dir: %w", err)
}
h.mu.Lock()
defer h.mu.Unlock()
if h.dir == dir {
return nil
}
h.dir, h.vms, h.seen, h.retained = dir, map[string]*vmQueue{}, map[string]bool{}, false
return nil
}
func (h *Hub) Append(_ context.Context, computeVM string, event guestcp.QueuedEvent) (bool, error) {
if event.URI == "" || event.CID == "" || len(event.Record) == 0 {
return false, errors.New("guestcp: a queued guest event needs a uri, a cid and a record")
}
if len(event.Record) > MaxRecordBytes {
return false, fmt.Errorf("guestcp: the guest event record is larger than %d bytes", MaxRecordBytes)
}
var compact bytes.Buffer
if err := json.Compact(&compact, event.Record); err != nil {
return false, fmt.Errorf("guestcp: the guest event record is not JSON: %w", err)
}
h.mu.Lock()
defer h.mu.Unlock()
q, err := h.queueLocked(computeVM)
if err != nil {
return false, err
}
key := event.URI + "#" + event.CID
if q.keys[key] {
return false, nil
}
if q.pending() >= MaxPending {
return false, guestcp.ErrQueueFull
}
entry := guestcp.QueuedEvent{Seq: q.last + 1, URI: event.URI, CID: event.CID, Record: compact.Bytes()}
line, err := json.Marshal(entry)
if err != nil {
return false, err
}
if err := appendLine(q.path, line); err != nil {
return false, err
}
q.entries = append(q.entries, entry)
q.keys[key] = true
q.last = entry.Seq
q.notify()
return true, nil
}
func (h *Hub) Serve(publish guestcp.Publish) func() {
h.mu.Lock()
h.publisher++
id := h.publisher
h.publish = publish
h.mu.Unlock()
return func() {
h.mu.Lock()
if h.publisher == id {
h.publish = nil
}
h.mu.Unlock()
}
}
func (h *Hub) Publish(ctx context.Context, event guestcp.GuestEvent) error {
h.mu.Lock()
publish := h.publish
h.mu.Unlock()
if publish == nil {
return guestcp.ErrNoPublisher
}
return publish(ctx, event)
}
func (h *Hub) Subscribe(computeVM string, cursor *int64) (guestcp.EventStream, error) {
h.mu.Lock()
defer h.mu.Unlock()
q, err := h.queueLocked(computeVM)
if err != nil {
return nil, err
}
after := int64(0)
if cursor != nil {
if *cursor > q.last {
return nil, fmt.Errorf("%w: cursor %d, last seq %d", guestcp.ErrFutureCursor, *cursor, q.last)
}
if err := q.prune(*cursor); err != nil {
return nil, err
}
after = *cursor
}
return &stream{hub: h, q: q, after: after, closed: make(chan struct{})}, nil
}
type stream struct {
hub *Hub
q *vmQueue
after int64
once sync.Once
closed chan struct{}
}
func (s *stream) Next(ctx context.Context) (guestcp.QueuedEvent, error) {
for {
s.hub.mu.Lock()
if s.q.gone {
s.hub.mu.Unlock()
return guestcp.QueuedEvent{}, guestcp.ErrQueueGone
}
for _, entry := range s.q.entries {
if entry.Seq > s.after && len(entry.Record) > 0 {
s.after = entry.Seq
s.hub.mu.Unlock()
return entry, nil
}
}
wake := s.q.wake
s.hub.mu.Unlock()
select {
case <-ctx.Done():
return guestcp.QueuedEvent{}, ctx.Err()
case <-s.closed:
return guestcp.QueuedEvent{}, guestcp.ErrQueueGone
case <-wake:
}
}
}
func (s *stream) Close() {
s.once.Do(func() { close(s.closed) })
}
func (q *vmQueue) prune(through int64) error {
changed := false
for i := range q.entries {
if q.entries[i].Seq <= through && len(q.entries[i].Record) > 0 {
q.entries[i].Record = nil
changed = true
}
}
if !changed {
return nil
}
return q.rewrite()
}
func (h *Hub) Retain(computeVMs []string) {
live := make(map[string]bool, len(computeVMs))
for _, name := range computeVMs {
live[name] = true
}
h.mu.Lock()
defer h.mu.Unlock()
if h.dir == "" {
return
}
stale := map[string]bool{}
if !h.retained {
for _, name := range h.onDisk() {
stale[name] = true
}
h.retained = true
}
for name := range h.seen {
stale[name] = true
}
for name := range stale {
if !live[name] {
h.dropLocked(name)
}
}
h.seen = live
}
func (h *Hub) onDisk() []string {
entries, err := os.ReadDir(h.dir)
if err != nil {
return nil
}
var out []string
for _, entry := range entries {
if name, ok := strings.CutSuffix(entry.Name(), fileSuffix); ok && !entry.IsDir() {
out = append(out, name)
}
}
return out
}
func (h *Hub) dropLocked(name string) {
if q := h.vms[name]; q != nil {
q.gone = true
q.notify()
delete(h.vms, name)
}
if computeVMName.MatchString(name) {
_ = os.Remove(filepath.Join(h.dir, name+fileSuffix))
syncDir(h.dir)
}
}
func (h *Hub) queueLocked(computeVM string) (*vmQueue, error) {
if h.dir == "" {
return nil, guestcp.ErrQueueNotOpen
}
if !computeVMName.MatchString(computeVM) {
return nil, fmt.Errorf("guestcp: %q is not a ComputeVM name", computeVM)
}
if q := h.vms[computeVM]; q != nil {
return q, nil
}
q, err := load(filepath.Join(h.dir, computeVM+fileSuffix))
if err != nil {
return nil, err
}
h.vms[computeVM] = q
return q, nil
}
func load(path string) (*vmQueue, error) {
q := &vmQueue{path: path, keys: map[string]bool{}, wake: make(chan struct{})}
f, err := os.Open(path)
if errors.Is(err, os.ErrNotExist) {
return q, nil
}
if err != nil {
return nil, fmt.Errorf("guestcp: reading the guest event queue: %w", err)
}
defer f.Close()
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 0, 64<<10), 2*MaxRecordBytes)
torn := false
for scanner.Scan() {
var entry guestcp.QueuedEvent
if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil || entry.Seq <= q.last || entry.URI == "" {
torn = true
continue
}
q.entries = append(q.entries, entry)
q.keys[entry.URI+"#"+entry.CID] = true
q.last = entry.Seq
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("guestcp: reading the guest event queue: %w", err)
}
if torn {
if err := q.rewrite(); err != nil {
return nil, err
}
}
return q, nil
}
func (q *vmQueue) pending() int {
n := 0
for _, entry := range q.entries {
if len(entry.Record) > 0 {
n++
}
}
return n
}
func (q *vmQueue) notify() {
close(q.wake)
q.wake = make(chan struct{})
}
func (q *vmQueue) rewrite() error {
var buf bytes.Buffer
for _, entry := range q.entries {
line, err := json.Marshal(entry)
if err != nil {
return err
}
buf.Write(line)
buf.WriteByte('\n')
}
tmp := q.path + ".tmp"
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
if err != nil {
return fmt.Errorf("guestcp: rewriting the guest event queue: %w", err)
}
if _, err := f.Write(buf.Bytes()); err != nil {
f.Close()
return fmt.Errorf("guestcp: rewriting the guest event queue: %w", err)
}
if err := f.Sync(); err != nil {
f.Close()
return fmt.Errorf("guestcp: rewriting the guest event queue: %w", err)
}
if err := f.Close(); err != nil {
return err
}
if err := os.Rename(tmp, q.path); err != nil {
return fmt.Errorf("guestcp: rewriting the guest event queue: %w", err)
}
syncDir(filepath.Dir(q.path))
return nil
}
func appendLine(path string, line []byte) error {
_, statErr := os.Stat(path)
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
if err != nil {
return fmt.Errorf("guestcp: appending to the guest event queue: %w", err)
}
if _, err := f.Write(append(line, '\n')); err != nil {
f.Close()
return fmt.Errorf("guestcp: appending to the guest event queue: %w", err)
}
if err := f.Sync(); err != nil {
f.Close()
return fmt.Errorf("guestcp: syncing the guest event queue: %w", err)
}
if err := f.Close(); err != nil {
return err
}
if errors.Is(statErr, os.ErrNotExist) {
syncDir(filepath.Dir(path))
}
return nil
}
func syncDir(dir string) {
d, err := os.Open(dir)
if err != nil {
return
}
_ = d.Sync()
_ = d.Close()
}