0
0
package hostprocess
import (
"cmp"
"context"
"fmt"
"log/slog"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/fsnotify/fsnotify"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/rest"
"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/kcpapply"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/kube"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/manifests"
"github.com/johnandersen777/socialweb-computer-kcp/internal/hostsvc"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kcpserver"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/conds"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/fswatch"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/supervise"
"github.com/johnandersen777/socialweb-computer-kcp/internal/logcap"
)
const (
ManifestSet = "hostprocesses"
ProviderSet = "provider"
DefaultWorkspace = "root"
RetryAfter = 2 * time.Second
MaxRetryAfter = time.Minute
MirrorPrefix = "bootstrap-"
MirrorLabel = "socialweb.computer/placement"
ConditionReady = "Ready"
ConditionLimits = "LimitsEnforced"
)
var GVR = v1alpha1.HostProcessGVR
type Mirror struct {
Role string
Unit string
Exec []string
Slice string
After []string
Wants []string
Port int
}
type Config struct {
Kubeconfig string
Workspace string
Manager hostsvc.Manager
Prefix string
Prefixes []string
Bound bool
Resolve func(v1alpha1.HostProcessAsset) (string, error)
Mirrors func() []Mirror
ApplyTimeout time.Duration
Selector string
KeepOrphans bool
Name string
Log *slog.Logger
}
type Supervisor struct {
c Config
client kube.Resource
placed map[string]bool
listed bool
backend hostsvc.Backend
mu sync.Mutex
locks sync.Map
placedAt map[string]time.Time
keys map[string]string
rekeyed map[string]string
slices map[string]bool
units map[string]string
backoff map[string]time.Duration
pending map[string]bool
pids map[string]int64
outputs map[string]follower
follows sync.WaitGroup
files *fswatch.Watcher
changed chan string
}
type follower struct {
key string
stop context.CancelFunc
}
func (s *Supervisor) track(name, unit string, gone bool) {
s.mu.Lock()
defer s.mu.Unlock()
if gone {
for u, n := range s.units {
if n == name {
delete(s.units, u)
}
}
return
}
if unit == "" {
return
}
if s.units == nil {
s.units = map[string]string{}
}
s.units[unit] = name
}
func (s *Supervisor) objectOf(unit string) (string, bool) {
s.mu.Lock()
defer s.mu.Unlock()
name, ok := s.units[unit]
return name, ok
}
func (s *Supervisor) settle(ctx context.Context, name string, retry bool, pid int64) {
s.mu.Lock()
defer s.mu.Unlock()
if !retry {
delete(s.backoff, name)
delete(s.pids, name)
return
}
if s.backoff == nil {
s.backoff, s.pending, s.pids = map[string]time.Duration{}, map[string]bool{}, map[string]int64{}
}
if s.pids[name] != pid {
delete(s.backoff, name)
s.pids[name] = pid
}
if s.changed == nil || s.pending[name] {
return
}
d := min(max(2*s.backoff[name], RetryAfter), MaxRetryAfter)
s.backoff[name], s.pending[name] = d, true
ch := s.changed
time.AfterFunc(d, func() {
s.mu.Lock()
delete(s.pending, name)
s.mu.Unlock()
select {
case ch <- name:
case <-ctx.Done():
}
})
}
func (s *Supervisor) setPlaced(unit string, on bool) {
s.mu.Lock()
defer s.mu.Unlock()
if on {
s.placed[unit] = true
if s.placedAt == nil {
s.placedAt = map[string]time.Time{}
}
s.placedAt[unit] = time.Now()
} else {
delete(s.placed, unit)
delete(s.placedAt, unit)
delete(s.keys, unit)
delete(s.rekeyed, unit)
}
}
func (s *Supervisor) setStartedKey(unit, key string) {
s.mu.Lock()
defer s.mu.Unlock()
if s.keys == nil {
s.keys = map[string]string{}
}
s.keys[unit] = key
}
func (s *Supervisor) setRekeyed(unit, key string) {
s.mu.Lock()
defer s.mu.Unlock()
if s.rekeyed == nil {
s.rekeyed = map[string]string{}
}
if key == "" {
delete(s.rekeyed, unit)
return
}
s.rekeyed[unit] = key
}
func (s *Supervisor) rekeyedFor(unit string) string {
s.mu.Lock()
defer s.mu.Unlock()
return s.rekeyed[unit]
}
func (s *Supervisor) startedKey(unit string) (string, bool) {
s.mu.Lock()
defer s.mu.Unlock()
key, ok := s.keys[unit]
return key, ok
}
func (s *Supervisor) placedAfter(unit string, t time.Time) bool {
s.mu.Lock()
defer s.mu.Unlock()
at, ok := s.placedAt[unit]
return ok && at.After(t)
}
func (s *Supervisor) isPlaced(unit string) bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.placed[unit]
}
func (s *Supervisor) lockObject(name string) func() {
v, _ := s.locks.LoadOrStore(name, &sync.Mutex{})
m := v.(*sync.Mutex)
m.Lock()
return m.Unlock
}
func (s *Supervisor) markAdopted(o v1alpha1.HostProcess, obs Observed) {
if o.Spec.Placement == v1alpha1.HostProcessPlacementBootstrap || obs.Inspect != nil || s.isPlaced(o.Spec.Unit) {
return
}
if obs.Unit.State == hostsvc.Active || obs.Unit.State == hostsvc.Activating {
s.setPlaced(o.Spec.Unit, true)
s.c.log().Info("hostprocess_unit_adopted", "hostprocess", o.Name, "unit", o.Spec.Unit, "pid", obs.Unit.MainPID,
"invocation", obs.Unit.InvocationID, "activeSince", obs.Unit.ActiveSince.UTC().Format(time.RFC3339Nano), "state", string(obs.Unit.State))
}
}
func (s *Supervisor) reconcileObject(ctx context.Context, o v1alpha1.HostProcess) (v1alpha1.HostProcess, bool) {
unlock := s.lockObject(o.Name)
defer unlock()
cur := o.DeepCopy()
s.track(o.Name, o.Spec.Unit, false)
s.capOutput(o)
obs := s.observe(ctx, o)
s.markAdopted(o, obs)
r := Reconcile(obs)
next, failed := s.apply(ctx, o, r)
err := s.writeStatus(ctx, cur, next)
if err != nil && !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) {
s.c.log().Warn("writing status", "name", o.Name, "err", err.Error())
}
s.followOutput(ctx, next)
rt := next.Status.Runtime
listening := next.Status.Phase == v1alpha1.HostProcessPhaseRunning && rt != nil && rt.Pid > 0 && len(next.Status.Listeners) == 0
var pid int64
if listening {
pid = rt.Pid
}
s.settle(ctx, o.Name, failed || r.Unknown || err != nil || listening, pid)
return o, true
}
func (s *Supervisor) reconcileName(ctx context.Context, name string) {
cur, err := kube.Get[v1alpha1.HostProcess](ctx, s.client, name)
if err != nil {
if !apierrors.IsNotFound(err) {
s.c.log().Warn("reading a changed HostProcess", "name", name, "err", err.Error())
}
return
}
s.reconcileObject(ctx, *cur)
}
func (c Config) log() *slog.Logger {
if c.Log == nil {
return slog.New(slog.DiscardHandler)
}
if c.Name != "" {
return c.Log.With("component", "hostprocess", "supervisor", c.Name)
}
return c.Log.With("component", "hostprocess")
}
func (c Config) prefixes() []string {
if c.Prefix == "" {
return c.Prefixes
}
return append([]string{c.Prefix}, c.Prefixes...)
}
func (c Config) prefixOf(unit string) string {
ps := c.prefixes()
for _, p := range ps {
if strings.HasPrefix(unit, p) {
return p
}
}
if len(ps) == 0 {
return ""
}
return ps[0]
}
func (c Config) workspace() string {
if c.Workspace == "" {
return DefaultWorkspace
}
return c.Workspace
}
func applySets(ctx context.Context, c Config, restCfg *rest.Config) error {
ap, err := kcpapply.New(restCfg, kcpapply.Options{WaitTimeout: c.ApplyTimeout, Base: c.workspace(), Log: c.log()})
if err != nil {
return err
}
ix, err := manifests.Load()
if err != nil {
return err
}
for _, name := range []string{ProviderSet, ManifestSet} {
set, ok := ix.Set(name)
if !ok {
return fmt.Errorf("hostprocess: deploy/sets.yaml names no %s set", name)
}
values, err := manifests.DecodeValues(set.Values, nil)
if err != nil {
return err
}
if _, err := ap.Apply(ctx, set, values); err != nil {
return fmt.Errorf("hostprocess: applying the %s set: %w", name, err)
}
}
return nil
}
func New(ctx context.Context, c Config) (*Supervisor, error) {
restCfg, err := kcpserver.FrontRestConfig(c.Kubeconfig)
if err != nil {
return nil, err
}
if !c.Bound {
if err := applySets(ctx, c, restCfg); err != nil {
return nil, err
}
}
cfg := rest.CopyConfig(restCfg)
cfg.Host = strings.TrimSuffix(cfg.Host, "/") + "/clusters/" + c.workspace()
kc, err := kube.New(cfg)
if err != nil {
return nil, err
}
return &Supervisor{c: c, client: kc.Resource(GVR), placed: map[string]bool{}, backend: c.Manager.Capabilities().Backend,
files: fswatch.New()}, nil
}
func MirrorName(role string) string { return MirrorPrefix + role }
func (s *Supervisor) syncMirrors(ctx context.Context) {
if s.c.Mirrors == nil {
return
}
for _, m := range s.c.Mirrors() {
name := MirrorName(m.Role)
want := v1alpha1.HostProcessSpec{Unit: m.Unit, Placement: v1alpha1.HostProcessPlacementBootstrap, Slice: m.Slice,
After: m.After, Wants: m.Wants}
if len(m.Exec) > 0 {
want.Exec, want.Argv = m.Exec[0], m.Exec[1:]
}
cur, err := kube.Get[v1alpha1.HostProcess](ctx, s.client, name)
if apierrors.IsNotFound(err) {
o := v1alpha1.HostProcess{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{MirrorLabel: string(v1alpha1.HostProcessPlacementBootstrap)}}, Spec: want}
if _, err := kube.Create[v1alpha1.HostProcess](ctx, s.client, &o, metav1.CreateOptions{}); err != nil {
s.c.log().Warn("writing a bootstrap mirror", "name", name, "err", err.Error())
}
continue
}
if err != nil {
s.c.log().Warn("reading a bootstrap mirror", "name", name, "err", err.Error())
continue
}
o := *cur
if o.Spec.Unit != want.Unit || o.Spec.Exec != want.Exec || !slices.Equal(o.Spec.Argv, want.Argv) || o.Spec.Placement != want.Placement {
o.Spec = want
if _, err := kube.Update[v1alpha1.HostProcess](ctx, s.client, &o, metav1.UpdateOptions{}); err != nil {
s.c.log().Warn("updating a bootstrap mirror", "name", name, "err", err.Error())
}
}
}
}
func (s *Supervisor) observe(ctx context.Context, o v1alpha1.HostProcess) Observed {
m := s.c.Manager
obs := Observed{Object: o, Backend: s.backend, Prefix: s.c.prefixOf(o.Spec.Unit), Deleting: o.DeletionTimestamp != nil}
if o.Spec.Placement == v1alpha1.HostProcessPlacementBootstrap {
obs.Prefix = ""
}
obs.Unit, obs.Inspect = m.Inspect(ctx, o.Spec.Unit)
if obs.Inspect != nil {
return obs
}
if o.Spec.Asset != nil && s.c.Resolve != nil {
if p, err := s.c.Resolve(*o.Spec.Asset); err == nil {
obs.Exec = p
}
}
rt := o.Status.Runtime
if rt != nil && rt.InvocationID != "" && rt.Backend == string(s.backend) && o.Spec.Placement != v1alpha1.HostProcessPlacementBootstrap &&
o.Status.Phase != v1alpha1.HostProcessPhaseLost {
obs.StartedKey = s.rekeyedFor(o.Spec.Unit)
if RestartedInPlace(o, obs.Unit) {
obs.Restarted = true
return obs
}
want := hostsvc.Identity{Unit: o.Spec.Unit, InvocationID: rt.InvocationID, PID: int(rt.Pid), StartTicks: uint64(rt.StartTicks),
BootID: rt.HostBootID, Backend: hostsvc.Backend(rt.Backend), Manager: rt.Manager}
c, err := m.Check(ctx, want)
if err != nil {
obs.Inspect = err
return obs
}
obs.Check = &c
}
return obs
}
func (s *Supervisor) apply(ctx context.Context, o v1alpha1.HostProcess, r Result) (v1alpha1.HostProcess, bool) {
m := s.c.Manager
log := s.c.log().With("hostprocess", o.Name, "unit", o.Spec.Unit)
started, stopFailed, failed := false, false, false
stopped := map[string]bool{}
for _, op := range r.Ops {
if stopFailed {
break
}
switch op.Kind {
case OpStart:
wasPlaced := s.isPlaced(op.Unit)
s.setPlaced(op.Unit, true)
st, err := m.Start(ctx, op.Spec)
if err == nil && st.Adopted && st.State != hostsvc.Active && st.State != hostsvc.Activating {
log.Info("a loaded unit of this name is not running; resetting it before the start", "state", string(st.State), "result", st.Result)
if err = m.Stop(ctx, op.Unit); err == nil {
m.ResetFailed(ctx, op.Unit)
st, err = m.Start(ctx, op.Spec)
}
}
if err != nil {
r.Phase, r.Message = v1alpha1.HostProcessPhaseFailed, err.Error()
failed = true
log.Warn("start", "err", err.Error())
if !wasPlaced {
s.setPlaced(op.Unit, false)
}
continue
}
s.setPlaced(op.Unit, true)
s.setStartedKey(op.Unit, o.Spec.RestartKey)
if stopped[op.Unit] {
s.setRekeyed(op.Unit, o.Spec.RestartKey)
} else {
s.setRekeyed(op.Unit, "")
}
started = true
log.Info("placed", "state", string(st.State), "adopted", st.Adopted, "restartKey", o.Spec.RestartKey)
case OpStop:
if err := m.Stop(ctx, op.Unit); err != nil {
log.Warn("stop", "err", err.Error())
stopFailed, failed = true, true
continue
}
s.setPlaced(op.Unit, false)
stopped[op.Unit] = true
log.Info("stopped")
}
}
next := o
if r.Unknown {
next.Status.Message = r.Message
return next, failed
}
next.Status.Phase, next.Status.LostReason, next.Status.Message = r.Phase, r.LostReason, r.Message
next.Status.ObservedGeneration = o.Generation
if started {
next.Status.RestartKey = o.Spec.RestartKey
} else if key, ok := s.startedKey(o.Spec.Unit); ok {
next.Status.RestartKey = key
}
if r.Identify {
if id, err := m.Identify(ctx, o.Spec.Unit, 0); err == nil {
st, _ := m.Inspect(ctx, o.Spec.Unit)
next.Status.Runtime = &v1alpha1.HostProcessRuntime{Backend: string(id.Backend), Unit: id.Unit, InvocationID: id.InvocationID,
Pid: int64(id.PID), StartTicks: int64(id.StartTicks), HostBootID: id.BootID, Manager: id.Manager}
if !st.ActiveSince.IsZero() {
next.Status.Runtime.ActiveSince = st.ActiveSince.UTC().Format(time.RFC3339Nano)
}
if o.Spec.Placement == v1alpha1.HostProcessPlacementSupervisor {
s.setPlaced(o.Spec.Unit, true)
}
}
}
if next.Status.Runtime != nil && next.Status.Runtime.Pid > 0 && next.Status.Phase == v1alpha1.HostProcessPhaseRunning {
next.Status.Listeners = listenersOf(int(next.Status.Runtime.Pid))
} else {
next.Status.Listeners = nil
}
if r.Phase == v1alpha1.HostProcessPhaseStopped || r.Phase == v1alpha1.HostProcessPhaseFailed || r.Phase == v1alpha1.HostProcessPhaseLost {
if st, err := m.Inspect(ctx, o.Spec.Unit); err == nil && st.State != hostsvc.NotFound {
next.Status.LastResult = &v1alpha1.HostProcessResult{Result: st.Result, ExitCode: st.ExecMainStatus,
At: time.Now().UTC().Format(time.RFC3339)}
}
}
conds.SetOn(&next, ConditionReady, next.Status.Phase == v1alpha1.HostProcessPhaseRunning, string(next.Status.Phase), next.Status.Message)
caps := m.Capabilities()
conds.SetOn(&next, ConditionLimits, caps.EnforcesLimits, string(caps.Backend), "limits are enforced only where the backend has cgroups")
return next, failed
}
func (s *Supervisor) writeStatus(ctx context.Context, cur *v1alpha1.HostProcess, next v1alpha1.HostProcess) error {
next.ResourceVersion = cur.ResourceVersion
was, err := kube.Encode(cur)
if err != nil {
return err
}
u, err := kube.Encode(&next)
if err != nil {
return err
}
if equalStatus(was["status"], u["status"]) {
return nil
}
_, err = kube.UpdateStatus[v1alpha1.HostProcess](ctx, s.client, u, metav1.UpdateOptions{})
return err
}
func equalStatus(a, b any) bool { return fmt.Sprint(a) == fmt.Sprint(b) }
func (s *Supervisor) Pass(ctx context.Context) error {
s.syncMirrors(ctx)
listedAt := time.Now()
list, err := kube.List[v1alpha1.HostProcess](ctx, s.client, metav1.ListOptions{LabelSelector: s.c.Selector})
if err != nil {
return err
}
seen := map[string]bool{}
var liveSlices []string
for i := range list {
if o, ok := s.reconcileObject(ctx, list[i]); ok {
seen[o.Spec.Unit] = true
liveSlices = append(liveSlices, o.Spec.Slice)
}
}
for _, prefix := range s.c.prefixes() {
if s.c.KeepOrphans {
break
}
units, err := s.c.Manager.List(ctx, prefix)
if err == nil {
for _, u := range units {
if !seen[u.Name] && u.State == hostsvc.Failed && !s.placedAfter(u.Name, listedAt) {
s.c.log().Info("the HostProcess of a failed unit is gone; resetting the unit", "unit", u.Name)
if err := s.c.Manager.ResetFailed(ctx, u.Name); err != nil {
s.c.log().Warn("resetting the failed unit of a gone HostProcess failed; the next pass retries", "unit", u.Name, "err", err.Error())
} else {
s.setPlaced(u.Name, false)
}
continue
}
if !s.isPlaced(u.Name) && s.listed {
continue
}
if !seen[u.Name] && s.placedAfter(u.Name, listedAt) {
continue
}
if !seen[u.Name] && (u.State == hostsvc.Active || u.State == hostsvc.Activating) {
s.c.log().Info("the HostProcess of a placed unit is gone; stopping the unit", "unit", u.Name)
if err := s.c.Manager.Stop(ctx, u.Name); err != nil {
s.c.log().Warn("stopping the unit of a gone HostProcess failed; the next pass retries", "unit", u.Name, "err", err.Error())
} else {
s.setPlaced(u.Name, false)
}
}
}
}
}
s.listed = true
if !s.c.KeepOrphans {
s.sweepSlices(ctx, liveSlices)
}
return nil
}
const ClusterSlicePart = "cluster-"
func (s *Supervisor) sweepable(slice string) bool {
if hostsvc.ValidateSlice(slice) != nil {
return false
}
for _, p := range s.c.prefixes() {
if strings.HasPrefix(slice, p+ClusterSlicePart) {
return true
}
}
return false
}
func (s *Supervisor) sweepSlices(ctx context.Context, live []string) {
s.mu.Lock()
if s.slices == nil {
s.slices = map[string]bool{}
}
for _, sl := range live {
if s.sweepable(sl) {
s.slices[sl] = true
}
}
var empty []string
for sl := range s.slices {
held := false
for _, l := range live {
if hostsvc.SliceContains(sl, l) {
held = true
break
}
}
if !held {
empty = append(empty, sl)
}
}
s.mu.Unlock()
for _, sl := range empty {
if err := s.c.Manager.StopSlice(ctx, sl); err != nil {
s.c.log().Warn("stopping the empty slice of gone HostProcesses failed; the next pass retries", "slice", sl, "err", err.Error())
continue
}
s.mu.Lock()
delete(s.slices, sl)
s.mu.Unlock()
s.c.log().Info("every HostProcess of the slice is gone; the empty slice is stopped", "slice", sl)
}
}
func outputFiles(o v1alpha1.HostProcess) ([]string, int64) {
out := o.Spec.Output
if out == nil {
return nil, 0
}
var files []string
for _, f := range []string{out.Stdout, out.Stderr} {
if f != "" && !slices.Contains(files, f) {
files = append(files, f)
}
}
return files, cmp.Or(out.CapBytes, logcap.DefaultCap)
}
const OutputDebounce = time.Second
func (s *Supervisor) followOutput(ctx context.Context, o v1alpha1.HostProcess) {
files, capBytes := outputFiles(o)
key := ""
if o.DeletionTimestamp == nil && o.Status.Phase == v1alpha1.HostProcessPhaseRunning && len(files) > 0 {
key = strings.Join(files, "\n") + "\n" + strconv.FormatInt(capBytes, 10)
}
s.mu.Lock()
defer s.mu.Unlock()
if cur, ok := s.outputs[o.Name]; ok {
if cur.key == key {
return
}
cur.stop()
delete(s.outputs, o.Name)
}
if key == "" {
return
}
w, err := s.files.Subscribe(files...)
if err != nil {
s.c.log().Warn("output files are capped only at reconcile: watching an output file failed", "name", o.Name, "files", strings.Join(files, ", "), "err", err.Error())
return
}
fctx, cancel := context.WithCancel(ctx)
if s.outputs == nil {
s.outputs = map[string]follower{}
}
s.outputs[o.Name] = follower{key: key, stop: cancel}
s.follows.Go(func() { s.rotateOnWrite(fctx, w, o, key) })
}
func (s *Supervisor) unfollowOutput(name string) {
s.mu.Lock()
defer s.mu.Unlock()
if cur, ok := s.outputs[name]; ok {
cur.stop()
delete(s.outputs, name)
}
}
func (s *Supervisor) rotateOnWrite(ctx context.Context, w *fswatch.Sub, o v1alpha1.HostProcess, key string) {
defer w.Close()
defer func() {
s.mu.Lock()
defer s.mu.Unlock()
if cur, ok := s.outputs[o.Name]; ok && cur.key == key && ctx.Err() == nil {
cur.stop()
delete(s.outputs, o.Name)
}
}()
for {
select {
case <-ctx.Done():
return
case _, ok := <-w.Errors():
if !ok {
return
}
case ev, ok := <-w.Events():
if !ok || ev.Has(fsnotify.Remove) || ev.Has(fsnotify.Rename) {
return
}
}
unlock := s.lockObject(o.Name)
s.capOutput(o)
unlock()
select {
case <-ctx.Done():
return
case <-time.After(OutputDebounce):
}
}
}
func (s *Supervisor) capOutput(o v1alpha1.HostProcess) {
files, capBytes := outputFiles(o)
for _, f := range files {
ok, err := logcap.Rotate(f, capBytes)
if err != nil {
s.c.log().Warn("an output file over its cap was not rotated", "name", o.Name, "file", f, "err", err.Error())
}
if ok {
s.c.log().Info("an output file over its cap was rotated; it keeps its newest bytes in a .1 copy", "name", o.Name, "file", f, "cap", capBytes)
}
}
}
func (s *Supervisor) Run(ctx context.Context) error {
s.c.log().Info("the HostProcess supervisor runs in the core", "workspace", s.c.workspace(), "backend", string(s.backend),
"prefixes", s.c.prefixes())
defer s.files.Close()
defer s.follows.Wait()
wake := make(chan struct{}, 1)
changed := make(chan string, 1024)
s.mu.Lock()
s.changed = changed
s.mu.Unlock()
var wg sync.WaitGroup
defer wg.Wait()
for range EventWorkers {
wg.Go(func() {
for {
select {
case <-ctx.Done():
return
case name := <-changed:
s.reconcileName(ctx, name)
}
}
})
}
wg.Go(func() { s.watch(ctx, wake, changed) })
for _, prefix := range s.c.prefixes() {
wg.Go(func() { s.watchUnits(ctx, prefix, wake, changed) })
}
for {
if err := s.Pass(ctx); err != nil {
s.c.log().Warn("a reconcile pass failed; units keep running and the pass retries", "err", err.Error(), "after", RetryAfter.String())
time.AfterFunc(RetryAfter, func() { poke(wake) })
}
select {
case <-ctx.Done():
return nil
case <-wake:
select {
case <-ctx.Done():
return nil
case <-time.After(WakeDebounce):
}
select {
case <-wake:
default:
}
}
}
}
const WakeDebounce = 100 * time.Millisecond
const EventWorkers = 4
func poke(wake chan<- struct{}) {
select {
case wake <- struct{}{}:
default:
}
}
func (s *Supervisor) watch(ctx context.Context, wake chan<- struct{}, changedCh chan<- string) {
generations := map[string]int64{}
failing := false
for ctx.Err() == nil {
events, stop, err := kube.Watch[v1alpha1.HostProcess](ctx, s.client, metav1.ListOptions{LabelSelector: s.c.Selector})
if err != nil {
if !failing {
s.c.log().Warn("watching HostProcess objects failed; the supervisor retries the watch", "err", err.Error(), "after", RetryAfter.String())
failing = true
}
select {
case <-ctx.Done():
return
case <-time.After(RetryAfter):
}
continue
}
if failing {
s.c.log().Info("watching HostProcess objects again")
failing = false
}
poke(wake)
for ev := range events {
if ev.Object == nil {
continue
}
name, gen, unit := ev.Object.Name, ev.Object.Generation, ev.Object.Spec.Unit
s.track(name, unit, ev.Type == watch.Deleted)
if ev.Type == watch.Deleted {
s.unfollowOutput(name)
}
changed := ev.Type != watch.Modified || generations[name] != gen
if ev.Type == watch.Deleted {
delete(generations, name)
} else {
generations[name] = gen
}
if !changed {
continue
}
if ev.Type != watch.Deleted {
select {
case changedCh <- name:
continue
default:
}
}
poke(wake)
}
stop()
select {
case <-ctx.Done():
return
case <-time.After(RetryAfter):
}
}
}
func (s *Supervisor) watchUnits(ctx context.Context, prefix string, wake chan<- struct{}, changed chan<- string) {
failing := false
for ctx.Err() == nil {
events, err := s.c.Manager.Watch(ctx, prefix)
if err != nil {
if !failing {
s.c.log().Warn("watching units failed; the supervisor retries the watch", "prefix", prefix, "err", err.Error(), "after", RetryAfter.String())
failing = true
}
select {
case <-ctx.Done():
return
case <-time.After(RetryAfter):
}
continue
}
if failing {
s.c.log().Info("watching units again", "prefix", prefix)
failing = false
}
poke(wake)
for ev := range events {
name, ok := s.objectOf(ev.Name)
if !ok {
if !s.c.KeepOrphans {
poke(wake)
}
continue
}
select {
case changed <- name:
default:
poke(wake)
}
}
select {
case <-ctx.Done():
return
case <-time.After(RetryAfter):
}
}
}
func Supervise(ctx context.Context, c Config) {
superviseRuns(ctx, c, func(ctx context.Context, up func()) error {
s, err := New(ctx, c)
if err != nil {
return err
}
up()
return s.Run(ctx)
})
}
func superviseRuns(ctx context.Context, c Config, run func(ctx context.Context, up func()) error) {
log := c.log()
bound := cmp.Or(c.ApplyTimeout, kcpapply.DefaultWaitTimeout)
var failingSince, upAt time.Time
supervise.Restart(ctx, "hostprocess", supervise.Backoff{Min: RetryAfter, Max: RetryAfter, OnError: func(err error, after time.Duration) {
now := time.Now()
if !upAt.IsZero() && now.Sub(upAt) >= bound {
failingSince = time.Time{}
}
upAt = time.Time{}
if failingSince.IsZero() {
failingSince = now
}
failing := now.Sub(failingSince)
level := slog.LevelWarn
if failing >= bound {
level = slog.LevelError
}
attrs := []any{"after", after.String(), "failingFor", failing.Round(time.Second).String(), "errorAfter", bound.String()}
if err != nil {
attrs = append(attrs, "err", err.Error())
}
log.Log(ctx, level, "the HostProcess supervisor stopped; starting it again", attrs...)
}}, func(ctx context.Context) error {
return run(ctx, func() { upAt = time.Now() })
})
}
func listenersOf(pid int) []v1alpha1.HostProcessListener {
ports, err := hostsvc.ListenPorts(pid)
if err != nil || len(ports) == 0 {
return nil
}
if len(ports) == 1 {
return []v1alpha1.HostProcessListener{{Name: "tcp", Address: "127.0.0.1:" + strconv.Itoa(ports[0])}}
}
out := make([]v1alpha1.HostProcessListener, 0, len(ports))
for _, p := range ports {
out = append(out, v1alpha1.HostProcessListener{Name: "tcp-" + strconv.Itoa(p), Address: "127.0.0.1:" + strconv.Itoa(p)})
}
return out
}