0
0
package tenantnode
import (
"context"
"errors"
"fmt"
"sync"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"github.com/johnandersen777/socialweb-computer-kcp/internal/cloudprovider"
)
type Options struct {
Reconciler *cloudprovider.Reconciler
InstanceChanges <-chan struct{}
Queue workqueue.TypedRateLimitingInterface[string]
Log func(event string, fields map[string]any)
}
type Controller struct {
reconciler *cloudprovider.Reconciler
cluster string
changes <-chan struct{}
queue workqueue.TypedRateLimitingInterface[string]
log func(event string, fields map[string]any)
informer cache.SharedIndexInformer
}
func New(o Options, client kubernetes.Interface) (*Controller, error) {
if o.Reconciler == nil {
return nil, errors.New(
"tenantnode: the controller has no reconciler, so every Node event would be observed " +
"and dropped")
}
if o.InstanceChanges == nil {
return nil, errors.New(
"tenantnode: the controller has no instance change source, so a guest that is " +
"provisioned, drained or reaped would never reach the Node that it backs")
}
if client == nil {
return nil, errors.New(
"tenantnode: the controller has no client, so it cannot watch the cluster's Nodes")
}
queue := o.Queue
if queue == nil {
queue = workqueue.NewTypedRateLimitingQueue(
workqueue.DefaultTypedControllerRateLimiter[string]())
}
log := o.Log
if log == nil {
log = func(string, map[string]any) {}
}
lw := &cache.ListWatch{
ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) {
return client.CoreV1().Nodes().List(ctx, opts)
},
WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return client.CoreV1().Nodes().Watch(ctx, opts)
},
}
c := &Controller{
reconciler: o.Reconciler,
cluster: o.Reconciler.Cluster(),
changes: o.InstanceChanges,
queue: queue,
log: log,
informer: cache.NewSharedIndexInformer(lw, &corev1.Node{}, 0, cache.Indexers{}),
}
if _, err := c.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: c.enqueue,
UpdateFunc: func(_, obj any) { c.enqueue(obj) },
DeleteFunc: c.enqueue,
}); err != nil {
return nil, fmt.Errorf("tenantnode: attaching the Node event handler: %w", err)
}
return c, nil
}
func (c *Controller) Run(ctx context.Context) error {
var wg sync.WaitGroup
defer wg.Wait()
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
defer c.queue.ShutDown()
wg.Go(func() { c.informer.Run(runCtx.Done()) })
if !cache.WaitForCacheSync(ctx.Done(), c.informer.HasSynced) {
return errors.New(
"tenantnode: the Node informer did not sync before shutdown, so no Node was ever " +
"observed and nothing is written")
}
c.log("cloud controller manager: status", map[string]any{
"phase": "Ready",
"cluster": c.cluster,
"nodes": len(c.informer.GetStore().List()),
})
var watchErr error
wg.Go(func() {
defer c.queue.ShutDown()
for {
select {
case <-runCtx.Done():
return
case _, ok := <-c.changes:
if !ok {
if runCtx.Err() == nil {
watchErr = errors.New(
"tenantnode: the watch on the instance snapshot ended, so a change " +
"to the instances behind the Nodes would no longer reach them")
}
return
}
c.enqueueAll()
}
}
})
for {
nodeName, shutdown := c.queue.Get()
if shutdown {
cancel()
wg.Wait()
return watchErr
}
err := c.reconcile(ctx, nodeName)
if err != nil {
c.log("cloud controller manager: reconcile failed", map[string]any{
"phase": "Degraded",
"cluster": c.cluster,
"node": nodeName,
"error": err.Error(),
})
c.queue.AddRateLimited(nodeName)
} else {
c.queue.Forget(nodeName)
}
c.queue.Done(nodeName)
}
}
func (c *Controller) reconcile(ctx context.Context, nodeName string) error {
node, present, err := c.node(nodeName)
if err != nil {
return err
}
return c.reconciler.Reconcile(ctx, nodeName, node, present)
}
func (c *Controller) node(nodeName string) (corev1.Node, bool, error) {
obj, present, err := c.informer.GetStore().GetByKey(nodeName)
if err != nil {
return corev1.Node{}, false, fmt.Errorf(
"tenantnode: reading the Node %s out of the informer cache: %w", nodeName, err)
}
if !present {
return corev1.Node{}, false, nil
}
node, ok := obj.(*corev1.Node)
if !ok {
return corev1.Node{}, false, fmt.Errorf(
"tenantnode: the informer cache holds a %T under %s, not a Node", obj, nodeName)
}
return *node, true, nil
}
func (c *Controller) enqueueAll() {
for _, key := range c.informer.GetStore().ListKeys() {
c.queue.Add(key)
}
}
func (c *Controller) enqueue(obj any) {
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
if err != nil {
c.log("cloud controller manager: node event ignored", map[string]any{
"cluster": c.cluster,
"error": err.Error(),
})
return
}
c.queue.Add(key)
}