0
0
package materialiser
import (
"context"
"errors"
"fmt"
"time"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/johnandersen777/socialweb-computer-kcp/api/v1alpha1"
"github.com/johnandersen777/socialweb-computer-kcp/internal/identitybundle"
"github.com/johnandersen777/socialweb-computer-kcp/internal/portallocator"
"github.com/johnandersen777/socialweb-computer-kcp/internal/procgroup"
"github.com/johnandersen777/socialweb-computer-kcp/internal/revocation"
"github.com/johnandersen777/socialweb-computer-kcp/internal/subnetallocator"
)
const ConditionReady = "Ready"
const PathGateGrace = 10 * time.Second
const (
ReasonInitializing = "Initializing"
ReasonInitTimeout = "InitTimeout"
ReasonDraining = "Draining"
ReasonBundleFailed = "IdentityBundleVerifyFailed"
ReasonBundleHalted = "IdentityVerificationFailed"
ReasonAwaitPathShut = "AwaitTenantPathShut"
ReasonStopped = "Stopped"
ReasonTeardownStuck = "TeardownGateTimeout"
ReasonSessionCap = "SessionCapRefuses"
ReasonUp = "ProcessGroupServing"
ReasonSettling = "AwaitRevocationSettled"
ReasonUnhealthy = "MembersUnhealthy"
)
const ProbeRetry = 2 * time.Second
const GateTimeout = 10 * time.Second
const DefaultInitTimeout = 3 * time.Minute
type Op string
const (
OpReap Op = "reap"
OpAllocatePorts Op = "allocate-ports"
OpSpawn Op = "spawn"
OpIdentityHalted Op = "identity-halted"
OpSpawnFailed Op = "spawn-failed"
OpParked Op = "park-unavailable"
OpShutPath Op = "shut-tenant-path"
OpAwaitPathShut Op = "await-tenant-path-shut"
OpPathShut Op = "tenant-path-shut"
OpReleasePorts Op = "release-ports"
OpReleaseSubnet Op = "release-subnet"
OpDropFinalizer Op = "remove-finalizer"
OpBlocked Op = "teardown-blocked-nothing-destroyed"
)
type PathProbe struct {
Answered bool
Status int
Err string
}
func (p PathProbe) String() string {
if p.Answered {
return fmt.Sprintf("the tenant front answered %d", p.Status)
}
if p.Err == "" {
return "the tenant front was not probed"
}
return "the tenant front did not answer: " + p.Err
}
type Observed struct {
Cluster v1alpha1.Cluster
Path PathProbe
Reachable bool
IdentityState v1alpha1.IdentityState
ServiceAccountPresent bool
Now time.Time
}
type Result struct {
Phase v1alpha1.ClusterPhase
URL string
Instance string
Conditions []metav1.Condition
Alert string
Ops []Op
RequeueAfter time.Duration
Again bool
RemoveFinalizer bool
Halted bool
}
type Leases map[portallocator.Purpose]portallocator.Lease
type StepBuilder func(o Observed, l Leases) ([]procgroup.Step, error)
type IdentityGuard func(ctx context.Context, o Observed) error
type Host interface {
Live(ctx context.Context, cluster string) bool
Spawn(ctx context.Context, cluster string, steps []procgroup.Step) error
Reap(ctx context.Context, cluster string) error
Unhealthy(ctx context.Context, cluster string) ([]string, error)
}
type Options struct {
Host Host
Ports *portallocator.Allocator
Networks *subnetallocator.Allocator
Steps StepBuilder
Front func(ctx context.Context, cluster string) error
Identity IdentityGuard
Alert func(cluster, message string)
NewInstance func() string
Now func() time.Time
InitTimeout time.Duration
GateTimeout time.Duration
Purposes []portallocator.Purpose
}
type Reconciler struct {
opts Options
}
var DefaultPurposes = []portallocator.Purpose{
portallocator.PurposePostgres,
portallocator.PurposeKine,
portallocator.PurposeK3sSecure,
portallocator.PurposeK3sSlotA,
portallocator.PurposeK3sControllerManager,
portallocator.PurposeK3sScheduler,
portallocator.PurposeK3sCloudControllerManager,
}
func New(opts Options) (*Reconciler, error) {
if opts.Host == nil {
return nil, errors.New("materialiser: Host is required")
}
if opts.Ports == nil {
return nil, errors.New("materialiser: Ports is required; the lease file is the authority")
}
if opts.Networks == nil {
return nil, errors.New("materialiser: Networks is required. A cluster's range is leased " +
"for as long as the cluster is, exactly as its ports are -- the range holds the " +
"gateway its guests dial the apiserver through, which is baked into k3s's " +
"advertise-address and its certificate -- so a teardown that dropped the ports and " +
"kept the range would hold a range per deleted cluster until the pool ran out, and one " +
"that dropped the range would renumber a cluster whose CR was recreated")
}
if opts.Identity == nil {
return nil, errors.New("materialiser: Identity is required; the sha256 pre-check has no substitute")
}
if opts.NewInstance == nil {
return nil, errors.New("materialiser: NewInstance is required; status.instance is the only discriminator there is")
}
if opts.Steps == nil {
return nil, errors.New("materialiser: Steps is required")
}
if opts.Now == nil {
opts.Now = time.Now
}
if opts.InitTimeout == 0 {
opts.InitTimeout = DefaultInitTimeout
}
if opts.GateTimeout == 0 {
opts.GateTimeout = GateTimeout
}
if opts.Purposes == nil {
opts.Purposes = DefaultPurposes
}
return &Reconciler{opts: opts}, nil
}
func (r *Reconciler) Reconcile(ctx context.Context, o Observed) (Result, error) {
if o.Cluster.Name == "" {
return Result{}, errors.New("materialiser: observed Cluster has no name")
}
if o.Now.IsZero() {
o.Now = r.opts.Now()
}
if o.Cluster.DeletionTimestamp != nil {
return r.teardown(ctx, o)
}
if !o.Cluster.Spec.Running {
return r.off(ctx, o)
}
if halted(o.Cluster.Status.Conditions) {
return Result{
Phase: v1alpha1.ClusterUnavailable,
URL: o.Cluster.Status.URL,
Instance: o.Cluster.Status.Instance,
Conditions: carry(o.Cluster.Status.Conditions),
Ops: []Op{OpIdentityHalted},
}, nil
}
return r.on(ctx, o)
}
func StartupOwns(c v1alpha1.Cluster) bool {
return c.Spec.Running && c.DeletionTimestamp == nil && !halted(c.Status.Conditions)
}
func (r *Reconciler) off(ctx context.Context, o Observed) (Result, error) {
res := Result{
Phase: v1alpha1.ClusterUnavailable,
URL: o.Cluster.Status.URL,
Instance: o.Cluster.Status.Instance,
Conditions: carry(o.Cluster.Status.Conditions),
}
if !shut(o) {
res.Ops = []Op{OpShutPath, OpAwaitPathShut}
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonAwaitPathShut,
Message: "phase is Unavailable; waiting for the front to stop admitting tenant requests before reaping",
ObservedGeneration: o.Cluster.Generation,
})
res.RequeueAfter = shutIn(o)
return res, nil
}
res.Ops = []Op{OpPathShut, OpReap}
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: v1alpha1.ConditionTenantPathShut,
Status: metav1.ConditionTrue,
Reason: ReasonAwaitPathShut,
Message: "the front admits no tenant request; " + o.Path.String(),
ObservedGeneration: o.Cluster.Generation,
})
if err := r.opts.Host.Reap(ctx, o.Cluster.Name); err != nil {
return res, err
}
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonStopped,
Message: "spec.running is false; the front admits no tenant request and the members are reaped",
ObservedGeneration: o.Cluster.Generation,
})
return res, nil
}
func (r *Reconciler) teardown(ctx context.Context, o Observed) (Result, error) {
res := Result{
Phase: v1alpha1.ClusterDraining,
URL: o.Cluster.Status.URL,
Instance: o.Cluster.Status.Instance,
Conditions: carry(o.Cluster.Status.Conditions),
}
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonDraining,
Message: "teardown in progress; the tenant path is shut before anything is destroyed",
ObservedGeneration: o.Cluster.Generation,
})
if refuse, why := r.sessionCapRefuses(o); refuse {
res.Ops = append(res.Ops, OpBlocked)
res.Alert = why
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: v1alpha1.ConditionTeardownBlocked,
Status: metav1.ConditionTrue,
Reason: ReasonSessionCap,
Message: why,
ObservedGeneration: o.Cluster.Generation,
})
return res, nil
}
if !shut(o) {
res.Ops = append(res.Ops, OpShutPath, OpAwaitPathShut)
if o.Now.Sub(drainingSince(o, r.opts.Now())) >= r.opts.GateTimeout {
res.Ops = append(res.Ops, OpBlocked)
res.Alert = "teardown gate timed out; nothing is destroyed. Stuck-and-billing beats silent loss"
res.RequeueAfter = shutIn(o)
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: v1alpha1.ConditionTeardownBlocked,
Status: metav1.ConditionTrue,
Reason: ReasonTeardownStuck,
Message: fmt.Sprintf("the tenant path gate is not shut after %s; %s", r.opts.GateTimeout, o.Path),
ObservedGeneration: o.Cluster.Generation,
})
return res, nil
}
res.RequeueAfter = shutIn(o)
return res, nil
}
res.Ops = append(res.Ops, OpPathShut, OpReap)
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: v1alpha1.ConditionTenantPathShut,
Status: metav1.ConditionTrue,
Reason: ReasonAwaitPathShut,
Message: "the front admits no tenant request; " + o.Path.String(),
ObservedGeneration: o.Cluster.Generation,
})
if err := r.opts.Host.Reap(ctx, o.Cluster.Name); err != nil {
return res, err
}
res.Ops = append(res.Ops, OpReleasePorts)
if err := r.opts.Ports.Release(o.Cluster.Name); err != nil {
return res, err
}
res.Ops = append(res.Ops, OpReleaseSubnet)
if err := r.opts.Networks.Release(o.Cluster.Name); err != nil {
return res, err
}
res.Ops = append(res.Ops, OpDropFinalizer)
res.RemoveFinalizer = true
return res, nil
}
func (r *Reconciler) on(ctx context.Context, o Observed) (Result, error) {
live := r.opts.Host.Live(ctx, o.Cluster.Name)
if live {
sick, err := r.opts.Host.Unhealthy(ctx, o.Cluster.Name)
if err != nil {
return r.pending(o, "the members of this cluster could not be read: "+err.Error())
}
if len(sick) > 0 {
if readyTrue(o) {
return r.degraded(o, sick)
}
return r.bringUp(ctx, o)
}
}
if live && o.Reachable {
res := Result{
Phase: o.Cluster.Status.Phase,
URL: o.Cluster.Status.URL,
Instance: o.Cluster.Status.Instance,
Conditions: carry(o.Cluster.Status.Conditions),
}
if o.Cluster.Status.Phase != settledReadyPhase {
return r.claimReady(res, o)
}
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionTrue,
Reason: ReasonUp,
Message: "the apiserver answered",
ObservedGeneration: o.Cluster.Generation,
})
return res, nil
}
if r.opts.Host.Live(ctx, o.Cluster.Name) {
return r.pending(o, "the process group is up and the apiserver has not answered yet")
}
return r.bringUp(ctx, o)
}
func (r *Reconciler) claimReady(res Result, o Observed) (Result, error) {
settled, err := revocation.WakeSettled(wakeEffect(o))
if err != nil {
res.Phase = v1alpha1.ClusterInitializing
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonSettling,
Message: err.Error(),
ObservedGeneration: o.Cluster.Generation,
})
return res, nil
}
phase, err := revocation.Ready(settled)
if err != nil {
res.Phase = v1alpha1.ClusterInitializing
return res, nil
}
res.Phase = phase
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionTrue,
Reason: ReasonUp,
Message: "the apiserver answered and the revocation reconcile is settled",
ObservedGeneration: o.Cluster.Generation,
})
return res, nil
}
func wakeEffect(o Observed) revocation.Effect {
if revocation.Revoked(o.IdentityState) && o.ServiceAccountPresent {
return revocation.Effect{Cluster: o.Cluster.Name, State: v1alpha1.IdentityRevocationPending, DeleteSA: true}
}
return revocation.Effect{Cluster: o.Cluster.Name, State: v1alpha1.IdentityRevoked}
}
var settledReadyPhase = func() v1alpha1.ClusterPhase {
settled, err := revocation.WakeSettled(revocation.Effect{State: v1alpha1.IdentityRevoked})
if err != nil {
panic("materialiser: revocation.WakeSettled refused a terminal effect: " + err.Error())
}
phase, err := revocation.Ready(settled)
if err != nil {
panic("materialiser: revocation.Ready refused a Settled: " + err.Error())
}
return phase
}()
func (r *Reconciler) bringUp(ctx context.Context, o Observed) (Result, error) {
res := Result{
Phase: v1alpha1.ClusterInitializing,
URL: o.Cluster.Status.URL,
Instance: o.Cluster.Status.Instance,
Conditions: carry(o.Cluster.Status.Conditions),
}
fresh := newInitWindow(o)
if fresh {
meta.RemoveStatusCondition(&res.Conditions, ConditionReady)
}
meta.RemoveStatusCondition(&res.Conditions, v1alpha1.ConditionTenantPathShut)
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonInitializing,
Message: "bringing up embedded-postgres -> openbao -> kine -> k3s",
LastTransitionTime: metav1.NewTime(o.Now),
ObservedGeneration: o.Cluster.Generation,
})
if !fresh && o.Cluster.Status.Phase == v1alpha1.ClusterInitializing {
if since := o.Now.Sub(readySince(o, o.Now)); since >= r.opts.InitTimeout {
return r.park(res, o, fmt.Sprintf("still Initializing after %s; parking in Unavailable", since))
}
}
instance := res.Instance
if instance == "" {
instance = r.opts.NewInstance()
res.Instance = instance
}
if r.opts.Front != nil {
if err := r.opts.Front(ctx, o.Cluster.Name); err != nil {
return res, fmt.Errorf("materialiser: the apiserver front of %s holds the cluster's ports and did not come up: %w", o.Cluster.Name, err)
}
}
leases := Leases{}
for _, p := range r.opts.Purposes {
l, err := r.opts.Ports.Allocate(ctx, portallocator.Request{
Cluster: o.Cluster.Name,
Account: o.Cluster.Spec.Account,
Instance: instance,
Purpose: p,
})
if err != nil {
return res, err
}
leases[p] = l
}
res.Ops = append(res.Ops, OpAllocatePorts)
if api, ok := leases[portallocator.PurposeK3sSecure]; ok {
res.URL = mountURL(api.Port)
}
steps, err := r.opts.Steps(o, leases)
if err != nil {
return res, err
}
steps = guardBeforeK3s(steps, r.opts.Identity, o)
if err := r.opts.Host.Spawn(ctx, o.Cluster.Name, steps); err != nil {
if isBundleFailure(err) {
res.Phase = v1alpha1.ClusterUnavailable
res.Halted = true
res.Ops = append(res.Ops, OpIdentityHalted)
res.Alert = fmt.Sprintf("identity bundle failed verification: %v. Start is halted, not retried", err)
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: v1alpha1.ConditionIdentityVerificationFailed,
Status: metav1.ConditionTrue,
Reason: ReasonBundleHalted,
Message: err.Error(),
LastTransitionTime: metav1.NewTime(o.Now),
ObservedGeneration: o.Cluster.Generation,
})
return res, nil
}
res.Ops = append(res.Ops, OpSpawnFailed)
res.Alert = fmt.Sprintf("the process group did not come up: %v. A spawn failure that "+
"reports only that it happened leaves the readiness gate's own reason -- and the "+
"component's log under the cluster's data dir -- as the only account of it", err)
return res, nil
}
res.Ops = append(res.Ops, OpSpawn)
if !o.Reachable {
res.RequeueAfter = r.probeIn(o)
return res, nil
}
return r.claimReady(res, o)
}
func readyTrue(o Observed) bool {
c := meta.FindStatusCondition(o.Cluster.Status.Conditions, ConditionReady)
return c != nil && c.Status == metav1.ConditionTrue
}
func (r *Reconciler) degraded(o Observed, sick []string) (Result, error) {
res := Result{
Phase: v1alpha1.ClusterInitializing,
URL: o.Cluster.Status.URL,
Instance: o.Cluster.Status.Instance,
Conditions: carry(o.Cluster.Status.Conditions),
Again: true,
}
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonUnhealthy,
Message: fmt.Sprintf("not every member of the cluster's slice is running and verified; the next pass re-places them: %v", sick),
LastTransitionTime: metav1.NewTime(o.Now),
ObservedGeneration: o.Cluster.Generation,
})
return res, nil
}
func (r *Reconciler) pending(o Observed, msg string) (Result, error) {
res := Result{
Phase: v1alpha1.ClusterInitializing,
URL: o.Cluster.Status.URL,
Instance: o.Cluster.Status.Instance,
Conditions: carry(o.Cluster.Status.Conditions),
}
fresh := newInitWindow(o)
if fresh {
meta.RemoveStatusCondition(&res.Conditions, ConditionReady)
} else if o.Cluster.Status.Phase == v1alpha1.ClusterInitializing {
if since := o.Now.Sub(readySince(o, o.Now)); since >= r.opts.InitTimeout {
return r.park(res, o, fmt.Sprintf("still Initializing after %s; parking in Unavailable", since))
}
}
res.RequeueAfter = r.probeIn(o)
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonInitializing,
Message: msg,
LastTransitionTime: metav1.NewTime(o.Now),
ObservedGeneration: o.Cluster.Generation,
})
return res, nil
}
func (r *Reconciler) park(res Result, o Observed, msg string) (Result, error) {
res.Phase = v1alpha1.ClusterUnavailable
res.Ops = append(res.Ops, OpParked)
res.Alert = msg
res.RequeueAfter = 0
meta.SetStatusCondition(&res.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonInitTimeout,
Message: msg,
ObservedGeneration: o.Cluster.Generation,
})
return res, nil
}
func (r *Reconciler) sessionCapRefuses(o Observed) (bool, string) {
e := o.Cluster.Status.NodeExpiry
if e == nil {
if c := meta.FindStatusCondition(o.Cluster.Status.Conditions, v1alpha1.ConditionReachable); c == nil || c.Status != metav1.ConditionTrue {
return false, ""
}
return true, "status.nodeExpiry is unknown and the apiserver has answered; refusing to tear down against a gate that cannot compare"
}
if *e == 0 {
return false, ""
}
if o.Now.Add(r.opts.GateTimeout).Unix() >= *e {
return true, fmt.Sprintf("the gate would outlive status.nodeExpiry (%d); refusing and re-provisioning", *e)
}
return false, ""
}
func shutIn(o Observed) time.Duration {
since, closed := gateClosedSince(o)
if !closed {
return PathGateGrace
}
if left := since.Add(PathGateGrace).Sub(o.Now); left > 0 {
return left
}
return PathGateGrace
}
func (r *Reconciler) probeIn(o Observed) time.Duration {
left := readySince(o, o.Now).Add(r.opts.InitTimeout).Sub(o.Now)
if left > 0 && left < ProbeRetry {
return left
}
return ProbeRetry
}
func shut(o Observed) bool {
since, closed := gateClosedSince(o)
return closed && o.Now.Sub(since) >= PathGateGrace
}
func gateClosedSince(o Observed) (time.Time, bool) {
var since time.Time
closed := false
if d := o.Cluster.DeletionTimestamp; d != nil {
since, closed = d.Time, true
}
if o.Cluster.Status.Phase == v1alpha1.ClusterReady {
return since, closed
}
c := meta.FindStatusCondition(o.Cluster.Status.Conditions, ConditionReady)
switch {
case c == nil:
return time.Time{}, true
case c.Status == metav1.ConditionTrue:
return since, closed
case c.LastTransitionTime.IsZero():
return time.Time{}, true
case !closed || c.LastTransitionTime.Time.Before(since):
return c.LastTransitionTime.Time, true
}
return since, closed
}
func newInitWindow(o Observed) bool {
c := meta.FindStatusCondition(o.Cluster.Status.Conditions, ConditionReady)
if c == nil || c.Status == metav1.ConditionTrue {
return false
}
return c.Reason == ReasonStopped || c.ObservedGeneration != o.Cluster.Generation
}
func readySince(o Observed, fallback time.Time) time.Time {
c := meta.FindStatusCondition(o.Cluster.Status.Conditions, ConditionReady)
if c == nil || c.LastTransitionTime.IsZero() {
return fallback
}
return c.LastTransitionTime.Time
}
func drainingSince(o Observed, fallback time.Time) time.Time {
c := meta.FindStatusCondition(o.Cluster.Status.Conditions, ConditionReady)
if c == nil || c.Reason != ReasonDraining || c.LastTransitionTime.IsZero() {
return fallback
}
return c.LastTransitionTime.Time
}
func halted(conds []metav1.Condition) bool {
c := meta.FindStatusCondition(conds, v1alpha1.ConditionIdentityVerificationFailed)
return c != nil && c.Status == metav1.ConditionTrue
}
func carry(conds []metav1.Condition) []metav1.Condition {
out := make([]metav1.Condition, len(conds))
copy(out, conds)
return out
}
func isBundleFailure(err error) bool {
return errors.Is(err, identitybundle.ErrMismatch) || errors.Is(err, identitybundle.ErrIncomplete)
}
func mountURL(advertisePort int) string {
return fmt.Sprintf("https://127.0.0.1:%d", advertisePort)
}
func guardBeforeK3s(steps []procgroup.Step, guard IdentityGuard, o Observed) []procgroup.Step {
if guard == nil {
return steps
}
for i := range steps {
if steps[i].Name != procgroup.ComponentK3s {
continue
}
inner := steps[i].Prepare
steps[i].Prepare = func(ctx context.Context, ps procgroup.Spec) (procgroup.Spec, error) {
if err := guard(ctx, o); err != nil {
return ps, err
}
if inner == nil {
return ps, nil
}
return inner(ctx, ps)
}
}
return steps
}