0
0
package kcpserver
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"maps"
"net"
"net/http"
"strconv"
"time"
"github.com/johnandersen777/socialweb-computer-kcp/internal/apiserverfront"
"github.com/johnandersen777/socialweb-computer-kcp/internal/hostsvc"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/clock"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/opt"
)
const (
DefaultRollReadyTimeout = time.Minute
DefaultLeaseTimeout = 30 * time.Second
DefaultProbeTimeout = 10 * time.Second
DefaultDrainTimeout = 5 * time.Second
DefaultRollPoll = 200 * time.Millisecond
PortPoll = 20 * time.Millisecond
)
type FrontSwitch interface {
Set(ctx context.Context, backend string) (apiserverfront.Status, error)
Status(ctx context.Context) (apiserverfront.Status, error)
Drain(ctx context.Context, backend string) (int, error)
}
type ShardAdmin interface {
Readyz(ctx context.Context, port int) (int, error)
Lease(ctx context.Context) (Lease, error)
AwaitLease(ctx context.Context, timeout time.Duration, done func(Lease) bool) (Lease, error)
DeleteLease(ctx context.Context) error
ProbeWorkspace(ctx context.Context, name string) error
}
type RollConfig struct {
SlotsFile string
Shared string
MemberToken string
SpecFor func(slot string) (hostsvc.UnitSpec, string, error)
KineFor func(slot string) string
Backup func(ctx context.Context) (string, error)
Start func(ctx context.Context, slot string, spec hostsvc.UnitSpec) (hostsvc.UnitStatus, error)
Retire func(ctx context.Context, old, current string) error
Abort func(ctx context.Context) error
Describe func(slot string) (sha256, version string)
Outage bool
ListenPort func(ctx context.Context, pid int) (int, error)
OnListen func(name string, addr net.Addr)
Poll time.Duration
ReadyTimeout time.Duration
LeaseTimeout time.Duration
ProbeTimeout time.Duration
DrainTimeout time.Duration
Log *slog.Logger
}
type RollReport struct {
From string `json:"from"`
To string `json:"to"`
PlannedOutage bool `json:"plannedOutage,omitempty"`
OutageSeconds float64 `json:"outageSeconds,omitempty"`
OldUnit string `json:"oldUnit"`
StoreBackup string `json:"storeBackup,omitempty"`
BackupSeconds float64 `json:"backupSeconds,omitempty"`
NewUnit string `json:"newUnit"`
OldPort int `json:"oldPort"`
NewPort int `json:"newPort"`
NewPID int `json:"newPID"`
NewInvocationID string `json:"newInvocationID"`
OldHolder string `json:"oldHolder"`
NewHolder string `json:"newHolder"`
StartedAt time.Time `json:"startedAt"`
PortAt time.Time `json:"portAt"`
ReadyAt time.Time `json:"readyAt"`
PersistedAt time.Time `json:"persistedAt"`
FlippedAt time.Time `json:"flippedAt"`
MemberTokenAt time.Time `json:"memberTokenAt,omitzero"`
SigtermAt time.Time `json:"sigtermAt"`
OldExitedAt time.Time `json:"oldExitedAt"`
LeaseDeletedAt time.Time `json:"leaseDeletedAt"`
LeaseHeldAt time.Time `json:"leaseHeldAt"`
ProbeReadyAt time.Time `json:"probeReadyAt"`
DoneAt time.Time `json:"doneAt"`
StartToReadySeconds float64 `json:"startToReadySeconds"`
OldExitSeconds float64 `json:"oldExitSeconds"`
LeaseHandoverSeconds float64 `json:"leaseHandoverSeconds"`
ProbeSeconds float64 `json:"probeSeconds"`
TotalSeconds float64 `json:"totalSeconds"`
OpenOnOldAtExit int `json:"openOnOldAtExit"`
Drained int `json:"drained"`
}
func (c *RollConfig) defaults() {
c.Poll = opt.Positive(c.Poll, DefaultRollPoll)
c.ReadyTimeout = opt.Positive(c.ReadyTimeout, DefaultRollReadyTimeout)
c.LeaseTimeout = opt.Positive(c.LeaseTimeout, DefaultLeaseTimeout)
c.ProbeTimeout = opt.Positive(c.ProbeTimeout, DefaultProbeTimeout)
c.DrainTimeout = opt.Positive(c.DrainTimeout, DefaultDrainTimeout)
if c.ListenPort == nil {
c.ListenPort = func(ctx context.Context, pid int) (int, error) { return hostsvc.WaitListenPort(ctx, pid, PortPoll) }
}
if c.Log == nil {
c.Log = slog.New(slog.NewJSONHandler(io.Discard, nil))
}
}
func backendOf(port int) string { return "127.0.0.1:" + strconv.Itoa(port) }
func WaitOwnPortReady(ctx context.Context, admin ShardAdmin, port int, every time.Duration,
progress ...func(attempt int, last error)) error {
every = opt.Positive(every, DefaultRollPoll)
var last error
for attempt := 1; ; attempt++ {
code, err := admin.Readyz(ctx, port)
if err == nil && code == http.StatusOK {
return nil
}
last = fmt.Errorf("readyz on 127.0.0.1:%d: %d %v", port, code, err)
report(progress, attempt, last)
select {
case <-ctx.Done():
return fmt.Errorf("kcpserver: not ready: %v (%w)", last, ctx.Err())
case <-time.After(every):
}
}
}
func WaitNewHolder(ctx context.Context, admin ShardAdmin, old string) (Lease, error) {
l, err := admin.AwaitLease(ctx, 0, func(l Lease) bool { return l.Found && l.Holder != "" && l.Holder != old })
if err != nil {
return Lease{}, fmt.Errorf("kcpserver: no new holder of %s/%s: holder %q (%w)", ControllerLeaseNamespace, ControllerLeaseName, l.Holder, err)
}
return l, nil
}
func awaitDrained(ctx context.Context, front FrontSwitch, backend string, timeout, every time.Duration) {
every = opt.Positive(every, DefaultRollPoll)
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
for {
fst, err := front.Status(ctx)
if err != nil || fst.Open[backend] == 0 || !clock.Sleep(ctx, every) {
return
}
}
}
func Roll(ctx context.Context, m hostsvc.Manager, front FrontSwitch, admin ShardAdmin, cfg RollConfig) (RollReport, Slots, error) {
cfg.defaults()
log := cfg.Log.With("component", "kcp-roll")
s, err := ReadSlots(cfg.SlotsFile)
if err != nil {
return RollReport{}, Slots{}, fmt.Errorf("kcpserver: reading %s: %w", cfg.SlotsFile, err)
}
from := s.Active
old, ok := s.Slots[from]
if !ok || old.Unit == "" {
return RollReport{}, s, fmt.Errorf("kcpserver: %s names active slot %q, which it does not record", cfg.SlotsFile, from)
}
single := s.Shape != ShapeBlueGreen
to := Other(from)
if single {
to = from
}
outage := cfg.Outage || single
start := cfg.Start
if start == nil {
start = func(ctx context.Context, _ string, spec hostsvc.UnitSpec) (hostsvc.UnitStatus, error) {
return m.Start(ctx, spec)
}
}
rep := RollReport{From: from, To: to, PlannedOutage: outage, OldUnit: old.Unit, OldPort: old.Port, StartedAt: time.Now().UTC()}
if l, err := admin.Lease(ctx); err == nil {
rep.OldHolder = l.Holder
}
if cfg.Backup != nil {
path, err := cfg.Backup(ctx)
if err != nil {
return rep, s, fmt.Errorf("kcpserver: the store backup before the roll: %w", err)
}
rep.StoreBackup, rep.BackupSeconds = path, clock.SecondsBetween(rep.StartedAt, time.Now().UTC())
log.Info("store backed up before the roll", "path", path, "seconds", rep.BackupSeconds)
}
spec, root, err := cfg.SpecFor(to)
if err != nil {
return rep, s, err
}
rep.NewUnit = spec.Name
if err := SeedSlot(cfg.Shared, root); err != nil {
return rep, s, err
}
stopOld := func(ctx context.Context) error {
rep.SigtermAt = time.Now().UTC()
if err := m.Stop(ctx, old.Unit); err != nil {
return fmt.Errorf("kcpserver: stopping the old slot %s: %w", old.Unit, err)
}
if ost, err := m.Inspect(ctx, old.Unit); err == nil && (ost.State == hostsvc.Active || ost.State == hostsvc.Deactivating) {
return fmt.Errorf("kcpserver: the old slot %s is still %s after its stop", old.Unit, ost.State)
}
m.ResetFailed(ctx, old.Unit)
rep.OldExitedAt = time.Now().UTC()
rep.OldExitSeconds = clock.SecondsBetween(rep.SigtermAt, rep.OldExitedAt)
return nil
}
if outage {
log.Info("planned outage: stopping the old slot before the new one starts", "slot", from, "unit", old.Unit)
if err := stopOld(ctx); err != nil {
return rep, s, err
}
}
m.Stop(ctx, spec.Name)
m.ResetFailed(ctx, spec.Name)
restoreOld := func(ctx context.Context) error {
ospec, oroot, err := cfg.SpecFor(from)
if err != nil {
return err
}
st, err := start(ctx, from, ospec)
if err != nil {
return err
}
port, err := cfg.ListenPort(ctx, st.MainPID)
if err != nil {
return err
}
if err := WaitOwnPortReady(ctx, admin, port, cfg.Poll); err != nil {
return err
}
rec := old
rec.Port, rec.PID, rec.InvocationID = port, st.MainPID, st.InvocationID
s.Slots[from], s.Active = rec, from
if err := WriteSlots(cfg.SlotsFile, s); err != nil {
return err
}
if _, err := front.Set(ctx, backendOf(port)); err != nil {
return err
}
if cfg.MemberToken != "" {
RefreshMemberTokenIfPresent(oroot, cfg.MemberToken)
}
log.Warn("planned outage aborted; the old slot runs again behind the front", "slot", from, "port", port)
return nil
}
fail := func(err error) (RollReport, Slots, error) {
log.Error("roll aborted before the flip; stopping the new slot", "slot", to, "err", err.Error())
sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), SlotStopTimeout+cfg.ReadyTimeout)
defer cancel()
m.Stop(sctx, spec.Name)
m.ResetFailed(sctx, spec.Name)
if cfg.Abort != nil {
if aerr := cfg.Abort(sctx); aerr != nil {
err = errors.Join(err, aerr)
}
}
if outage {
if rerr := restoreOld(sctx); rerr != nil {
err = errors.Join(err, fmt.Errorf("kcpserver: restarting the old slot %s: %w", old.Unit, rerr))
}
}
return rep, s, err
}
st, err := start(ctx, to, spec)
if err != nil {
return fail(err)
}
if st.Adopted {
return fail(fmt.Errorf("kcpserver: %s was already running after its stop; refusing to adopt a stale slot", spec.Name))
}
rep.NewPID, rep.NewInvocationID = st.MainPID, st.InvocationID
pctx, cancel := context.WithTimeout(ctx, cfg.ReadyTimeout)
defer cancel()
port, err := cfg.ListenPort(pctx, st.MainPID)
if err != nil {
return fail(fmt.Errorf("kcpserver: reading the bound port of %s (pid %d): %w", spec.Name, st.MainPID, err))
}
rep.NewPort, rep.PortAt = port, time.Now().UTC()
if cfg.OnListen != nil {
cfg.OnListen("kcp-"+to, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port})
}
log.Info("new slot listening", "slot", to, "port", port, "pid", st.MainPID)
if err := WaitOwnPortReady(pctx, admin, port, cfg.Poll); err != nil {
return fail(err)
}
rep.ReadyAt = time.Now().UTC()
rep.StartToReadySeconds = clock.SecondsBetween(rep.StartedAt, rep.ReadyAt)
prev := s
prev.Slots = map[string]SlotRecord{}
maps.Copy(prev.Slots, s.Slots)
next := s
next.Slots = map[string]SlotRecord{}
maps.Copy(next.Slots, s.Slots)
next.Slots[to] = SlotRecord{Unit: spec.Name, Root: root, Port: port, PID: st.MainPID, InvocationID: st.InvocationID,
Exec: spec.Exec[0], StartedAt: rep.StartedAt, ReadyAt: rep.ReadyAt}
if cfg.KineFor != nil {
rec := next.Slots[to]
rec.Kine = cfg.KineFor(to)
next.Slots[to] = rec
next.KineURL = rec.Kine
}
if cfg.Describe != nil {
rec := next.Slots[to]
rec.SHA256, rec.Version = cfg.Describe(to)
next.Slots[to] = rec
}
next.Active = to
if err := WriteSlots(cfg.SlotsFile, next); err != nil {
return fail(err)
}
s = next
rep.PersistedAt = time.Now().UTC()
if _, err := front.Set(ctx, backendOf(port)); err != nil {
WriteSlots(cfg.SlotsFile, prev)
s = prev
return fail(fmt.Errorf("kcpserver: flipping the front: %w", err))
}
rep.FlippedAt = time.Now().UTC()
if cfg.MemberToken != "" {
if changed, err := RefreshMemberTokenIfPresent(root, cfg.MemberToken); err != nil {
log.Warn("the members' kcp-admin token was not rewritten at the flip; the instance's own check follows within seconds", "slot", to, "tokenFile", cfg.MemberToken, "err", err.Error())
} else if changed {
rep.MemberTokenAt = time.Now().UTC()
}
}
log.Info("front flipped", "from", backendOf(old.Port), "to", backendOf(port), "memberTokenAt", rep.MemberTokenAt)
if outage {
rep.OutageSeconds = clock.SecondsBetween(rep.SigtermAt, rep.FlippedAt)
} else {
if err := stopOld(ctx); err != nil {
return rep, s, err
}
if fst, err := front.Status(ctx); err == nil {
rep.OpenOnOldAtExit = fst.Open[backendOf(old.Port)]
}
}
if single {
rep.LeaseHeldAt = rep.FlippedAt
} else {
if err := admin.DeleteLease(ctx); err != nil {
return rep, s, err
}
rep.LeaseDeletedAt = time.Now().UTC()
lctx, lcancel := context.WithTimeout(ctx, cfg.LeaseTimeout)
defer lcancel()
l, err := WaitNewHolder(lctx, admin, rep.OldHolder)
if err != nil {
return rep, s, err
}
rep.NewHolder, rep.LeaseHeldAt = l.Holder, time.Now().UTC()
rep.LeaseHandoverSeconds = clock.SecondsBetween(rep.OldExitedAt, rep.LeaseHeldAt)
log.Info("controller lease handed over", "from", rep.OldHolder, "to", rep.NewHolder, "seconds", rep.LeaseHandoverSeconds)
}
probe := "roll-probe-" + strconv.FormatInt(time.Now().UnixMilli(), 36)
prctx, prcancel := context.WithTimeout(ctx, cfg.ProbeTimeout)
defer prcancel()
if err := admin.ProbeWorkspace(prctx, probe); err != nil {
return rep, s, fmt.Errorf("kcpserver: the probe workspace did not reconcile after the roll: %w", err)
}
rep.ProbeReadyAt = time.Now().UTC()
rep.ProbeSeconds = clock.SecondsBetween(rep.LeaseHeldAt, rep.ProbeReadyAt)
if !outage {
awaitDrained(ctx, front, backendOf(old.Port), cfg.DrainTimeout, cfg.Poll)
}
if n, err := front.Drain(ctx, backendOf(old.Port)); err == nil {
rep.Drained = n
} else if !errors.Is(err, apiserverfront.ErrCurrentBackend) {
log.Warn("draining the old backend", "err", err.Error())
}
if to != from {
delete(s.Slots, from)
if cfg.Retire != nil {
if err := cfg.Retire(ctx, from, to); err != nil {
return rep, s, fmt.Errorf("kcpserver: retiring slot %s: %w", from, err)
}
}
}
if err := WriteSlots(cfg.SlotsFile, s); err != nil {
return rep, s, err
}
rep.DoneAt = time.Now().UTC()
rep.TotalSeconds = clock.SecondsBetween(rep.StartedAt, rep.DoneAt)
return rep, s, nil
}