0
0
package runtime
import (
"context"
"maps"
"net/http"
"slices"
"strings"
"time"
apisv1alpha1 "github.com/kcp-dev/sdk/apis/apis/v1alpha1"
apisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2"
conditionsv1alpha1 "github.com/kcp-dev/sdk/apis/third_party/conditions/apis/conditions/v1alpha1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/johnandersen777/socialweb-computer-kcp/api/kcpgvr"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/kube"
"github.com/johnandersen777/socialweb-computer-kcp/internal/controllers/runtime/manifests"
)
const (
ExportKindsTimeout = time.Minute
exportEndpointPoll = 200 * time.Millisecond
)
func (m *Manager) addKind(export string, obj client.Object) {
gvks, _, err := m.scheme.ObjectKinds(obj)
if err != nil || len(gvks) == 0 {
return
}
m.mu.Lock()
defer m.mu.Unlock()
if m.kinds[export] == nil {
m.kinds[export] = map[schema.GroupKind]bool{}
}
m.kinds[export][gvks[0].GroupKind()] = true
}
func (m *Manager) awaitKinds(ctx context.Context, export string) {
m.mu.Lock()
want := maps.Clone(m.kinds[export])
m.mu.Unlock()
if len(want) == 0 {
return
}
c, err := kube.New(clusterConfig(m.base, m.exportWorkspace))
if err != nil {
m.log.Warn("the APIExport provider does not wait for its kinds", "export", export, "err", err.Error())
return
}
carried, err := manifests.ExportSchemas(export)
if err != nil {
m.log.Warn("the APIExport provider does not know the schemas it carries", "export", export, "err", err.Error())
}
schemas := c.Resource(kcpgvr.APIResourceSchemas)
known := map[string]schema.GroupKind{}
var resources []apisv1alpha2.ResourceSchema
var left []string
waiting := false
_, err = kube.AwaitList(ctx, c.Resource(kcpgvr.APIExports), metav1.ListOptions{}, ExportKindsTimeout, func(items []apisv1alpha2.APIExport) (bool, error) {
i := slices.IndexFunc(items, func(e apisv1alpha2.APIExport) bool { return e.Name == export })
served := map[schema.GroupKind]bool{}
if i >= 0 {
resources = items[i].Spec.Resources
for _, r := range resources {
if len(carried) > 0 && !carried[r.Schema] {
continue
}
gk, ok := known[r.Schema]
if !ok {
s, err := kube.Get[apisv1alpha1.APIResourceSchema](ctx, schemas, r.Schema)
if err != nil {
continue
}
gk = schema.GroupKind{Group: s.Spec.Group, Kind: s.Spec.Names.Kind}
known[r.Schema] = gk
}
served[gk] = true
}
}
left = left[:0]
for gk := range want {
if !served[gk] {
left = append(left, gk.String())
}
}
if len(left) == 0 {
return true, nil
}
if !waiting {
waiting = true
slices.Sort(left)
m.log.Info("the APIExport provider waits for its kinds at the schemas this binary carries; the schema apply wakes it", "export", export, "kinds", left)
}
return false, nil
})
if err != nil {
m.log.Warn("the APIExport provider starts without all its kinds", "export", export, "kinds", left, "err", err.Error())
return
}
for _, ws := range m.boundIn {
m.awaitBound(ctx, ws, export, resources)
}
if waiting {
m.log.Info("the APIExport serves its kinds", "export", export)
}
}
func (m *Manager) awaitBound(ctx context.Context, ws, export string, resources []apisv1alpha2.ResourceSchema) {
c, err := kube.New(clusterConfig(m.base, ws))
if err != nil {
m.log.Warn("the APIExport provider does not wait for its binding", "export", export, "workspace", ws, "err", err.Error())
return
}
bindings := c.Resource(kcpgvr.APIBindings)
boundAll := func(items []apisv1alpha2.APIBinding) (bool, error) {
for _, b := range items {
if ref := b.Spec.Reference.Export; ref == nil || ref.Name != export {
continue
}
bound := map[schema.GroupResource]string{}
for _, r := range b.Status.BoundResources {
bound[schema.GroupResource{Group: r.Group, Resource: r.Resource}] = r.Schema.Name
}
for _, r := range resources {
if bound[schema.GroupResource{Group: r.Group, Resource: r.Name}] != r.Schema {
return false, nil
}
}
}
return true, nil
}
items, err := kube.List[apisv1alpha2.APIBinding](ctx, bindings, metav1.ListOptions{})
if err != nil {
return
}
if done, _ := boundAll(items); done {
return
}
if _, err := kube.AwaitList(ctx, bindings, metav1.ListOptions{}, ExportKindsTimeout, boundAll); err != nil {
m.log.Warn("the APIExport provider starts before its binding serves every kind", "export", export, "workspace", ws, "err", err.Error())
}
}
func (m *Manager) awaitEndpoints(ctx context.Context, export string) {
c, err := kube.New(clusterConfig(m.base, m.exportWorkspace))
if err != nil {
return
}
hc, err := rest.HTTPClientFor(m.base)
if err != nil {
return
}
serves := func(url string) bool {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSuffix(url, "/")+"/clusters/*/apis", nil)
if err != nil {
return false
}
resp, err := hc.Do(req)
if err != nil {
return false
}
resp.Body.Close()
return resp.StatusCode == http.StatusOK
}
endpointSlices := c.Resource(kcpgvr.APIExportEndpointSlices)
deadline := time.Now().Add(ExportKindsTimeout)
waiting := false
for ctx.Err() == nil {
s, err := kube.Get[apisv1alpha1.APIExportEndpointSlice](ctx, endpointSlices, export)
if err != nil && !apierrors.IsNotFound(err) {
return
}
var ready bool
if err == nil {
urls := s.Status.APIExportEndpoints
ready = len(urls) > 0 && !slices.ContainsFunc(urls, func(e apisv1alpha1.APIExportEndpoint) bool { return !serves(e.URL) }) ||
len(urls) == 0 && slices.ContainsFunc(s.Status.Conditions, func(c conditionsv1alpha1.Condition) bool {
return c.Type == apisv1alpha1.APIExportEndpointSliceURLsReady && string(c.Status) == "True"
})
}
if ready {
if waiting {
m.log.Info("the APIExport's virtual workspace serves discovery", "export", export)
}
return
}
if !time.Now().Before(deadline) {
m.log.Warn("the APIExport provider starts before its virtual workspace serves discovery", "export", export, "timeout", ExportKindsTimeout.String())
return
}
if !waiting {
waiting = true
m.log.Info("the APIExport provider waits for its virtual workspace to serve discovery", "export", export)
}
select {
case <-ctx.Done():
case <-time.After(exportEndpointPoll):
}
}
}