Skip to content

Commit db4c30a

Browse files
authored
Add hearhbeat on secret, when bindings is not yet present. (#478)
On-behalf-of: @SAP mangirdas.judeikis@sap.com Signed-off-by: Mangirdas Judeikis <mangirdas@judeikis.lt>
1 parent d2211f6 commit db4c30a

4 files changed

Lines changed: 180 additions & 7 deletions

File tree

cli/pkg/kubectl/base/kubeconfig.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ import (
2929
"k8s.io/client-go/tools/clientcmd"
3030
"k8s.io/client-go/util/retry"
3131
"k8s.io/klog/v2"
32+
33+
kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2"
3234
)
3335

3436
func ParseRemoteKubeconfig(kubeconfig []byte) (host string, ns string, err error) {
@@ -96,6 +98,9 @@ func EnsureKubeconfigSecret(ctx context.Context, kubeconfig, name string, client
9698
ObjectMeta: v1.ObjectMeta{
9799
Namespace: "kube-bind",
98100
GenerateName: "kubeconfig-",
101+
Labels: map[string]string{
102+
kubebindv1alpha2.LabelProviderKubeconfig: "true",
103+
},
99104
},
100105
Data: map[string][]byte{
101106
"kubeconfig": []byte(kubeconfig),
@@ -129,6 +134,11 @@ func EnsureKubeconfigSecret(ctx context.Context, kubeconfig, name string, client
129134
return errors.NewAlreadyExists(corev1.Resource("secret"), secret.Name)
130135
}
131136
secret.Data["kubeconfig"] = []byte(kubeconfig)
137+
// Ensure label is set on existing secrets
138+
if secret.Labels == nil {
139+
secret.Labels = make(map[string]string)
140+
}
141+
secret.Labels[kubebindv1alpha2.LabelProviderKubeconfig] = "true"
132142
if _, err := client.CoreV1().Secrets("kube-bind").Update(ctx, secret, v1.UpdateOptions{}); err != nil {
133143
return err
134144
}

pkg/konnector/konnector_controller.go

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package konnector
1919
import (
2020
"context"
2121
"fmt"
22+
"strings"
2223
"time"
2324

2425
corev1 "k8s.io/api/core/v1"
@@ -231,6 +232,14 @@ func (c *Controller) enqueueSecret(logger klog.Logger, obj any) {
231232
return
232233
}
233234

235+
// Check if this is a provider kubeconfig secret (for early heartbeat)
236+
secret, ok := obj.(*corev1.Secret)
237+
if ok && secret.Labels[kubebindv1alpha2.LabelProviderKubeconfig] == "true" {
238+
// Queue it as a heartbeat secret with a special prefix
239+
logger.V(2).Info("queueing heartbeat secret", "key", key)
240+
c.queue.Add("__heartbeat_secret__" + key)
241+
}
242+
234243
bindings, err := c.serviceBindingIndexer.ByIndex(indexers.ByServiceBindingKubeconfigSecret, fmt.Sprintf("%s/%s", ns, name))
235244
if err != nil {
236245
runtime.HandleError(err)
@@ -302,7 +311,33 @@ func (c *Controller) processNextWorkItem(ctx context.Context) bool {
302311
return true
303312
}
304313

314+
const heartbeatSecretPrefix = "__heartbeat_secret__"
315+
305316
func (c *Controller) process(ctx context.Context, key string) error {
317+
logger := klog.FromContext(ctx)
318+
319+
// Handle heartbeat secret keys (for early heartbeat without APIServiceBinding)
320+
if strings.HasPrefix(key, heartbeatSecretPrefix) {
321+
secretKey := strings.TrimPrefix(key, heartbeatSecretPrefix)
322+
ns, name, err := cache.SplitMetaNamespaceKey(secretKey)
323+
if err != nil {
324+
runtime.HandleError(err)
325+
return nil
326+
}
327+
328+
secret, err := c.secretLister.Secrets(ns).Get(name)
329+
if err != nil {
330+
if errors.IsNotFound(err) {
331+
logger.V(2).Info("heartbeat secret not found, skipping", "secret", secretKey)
332+
return nil
333+
}
334+
return err
335+
}
336+
337+
// Start controller for this secret
338+
return c.reconciler.startClusterControllerForSecret(ctx, secret)
339+
}
340+
306341
_, name, err := cache.SplitMetaNamespaceKey(key)
307342
if err != nil {
308343
runtime.HandleError(err)

pkg/konnector/konnector_reconcile.go

Lines changed: 130 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,134 @@ type startable interface {
3434
Start(ctx context.Context)
3535
}
3636

37+
// reconciler manages cluster controllers for provider connections.
38+
//
39+
// Controller Sharing Model:
40+
// -------------------------
41+
// Multiple APIServiceBindings can share the same cluster controller if they use the same
42+
// kubeconfig (i.e., connect to the same provider namespace). This is tracked via controllerContext:
43+
//
44+
// r.controllers map:
45+
// "__heartbeat__kubeconfig-xyz" -> controllerContext{kubeconfig: "...", serviceBindings: ["__heartbeat__kubeconfig-xyz", "binding-a", "binding-b"]}
46+
// "binding-a" -> (same controllerContext as above)
47+
// "binding-b" -> (same controllerContext as above)
48+
//
49+
// Flow when APIServiceBinding arrives using same kubeconfig as existing heartbeat controller:
50+
// 1. reconcile() is called with the new binding
51+
// 2. It extracts kubeconfig from binding.Spec.KubeconfigSecretRef
52+
// 3. It loops through r.controllers looking for matching kubeconfig (lines 174-182)
53+
// 4. Finds the existing controllerContext (created by startClusterControllerForSecret)
54+
// 5. Points r.controllers[binding.Name] to the SAME controllerContext
55+
// 6. Adds binding.Name to controllerContext.serviceBindings set
56+
// 7. No new controller is started - the existing one handles everything
3757
type reconciler struct {
3858
lock sync.RWMutex
39-
controllers map[string]*controllerContext // by service binding name
59+
controllers map[string]*controllerContext // keyed by binding name or synthetic heartbeat key
4060

4161
newClusterController func(consumerSecretRefKey, providerNamespace string, reconcileServiceBinding func(binding *kubebindv1alpha2.APIServiceBinding) bool, providerConfig *rest.Config) (startable, error)
4262
getSecret func(ns, name string) (*corev1.Secret, error)
4363
}
4464

65+
// controllerContext tracks a running cluster controller and the bindings it serves.
66+
// Multiple map entries in reconciler.controllers can point to the same controllerContext
67+
// when they share the same kubeconfig.
4568
type controllerContext struct {
46-
kubeconfig string
47-
cancel func()
48-
serviceBindings sets.Set[string] // when this is empty, the Controller should be stopped by closing the context
69+
kubeconfig string // the kubeconfig content - used to match bindings to controllers
70+
cancel func() // cancels the controller's context when no bindings remain
71+
serviceBindings sets.Set[string] // all binding names (including synthetic heartbeat keys) using this controller
72+
}
73+
74+
// startClusterControllerForSecret starts a cluster controller for the given kubeconfig secret.
75+
// This enables heartbeat reporting immediately without requiring an APIServiceBinding.
76+
//
77+
// The controller is registered with a synthetic key "__heartbeat__<secret-name>".
78+
// When an APIServiceBinding later arrives using the same kubeconfig, reconcile() will:
79+
// 1. Find this existing controller by matching kubeconfig content
80+
// 2. Add the binding to the same controllerContext.serviceBindings set
81+
// 3. Point r.controllers[binding.Name] to the same controllerContext
82+
// 4. NOT start a new controller (reuses existing one)
83+
func (r *reconciler) startClusterControllerForSecret(ctx context.Context, secret *corev1.Secret) error {
84+
logger := klog.FromContext(ctx)
85+
86+
kubeconfig := string(secret.Data["kubeconfig"])
87+
if kubeconfig == "" {
88+
logger.V(2).Info("secret does not contain kubeconfig", "secret", secret.Namespace+"/"+secret.Name)
89+
return nil
90+
}
91+
92+
r.lock.Lock()
93+
defer r.lock.Unlock()
94+
95+
// Check if we already have a controller for this kubeconfig
96+
for _, ctrlContext := range r.controllers {
97+
if ctrlContext.kubeconfig == kubeconfig {
98+
logger.V(2).Info("cluster controller already exists for secret", "secret", secret.Namespace+"/"+secret.Name)
99+
return nil
100+
}
101+
}
102+
103+
// Extract which namespace this kubeconfig points to
104+
cfg, err := clientcmd.Load([]byte(kubeconfig))
105+
if err != nil {
106+
logger.Error(err, "invalid kubeconfig in secret", "namespace", secret.Namespace, "name", secret.Name)
107+
return nil // nothing we can do here
108+
}
109+
kubeContext, found := cfg.Contexts[cfg.CurrentContext]
110+
if !found {
111+
logger.Error(err, "kubeconfig in secret does not have a current context", "namespace", secret.Namespace, "name", secret.Name)
112+
return nil
113+
}
114+
if kubeContext.Namespace == "" {
115+
logger.Error(err, "kubeconfig in secret does not have a namespace set for the current context", "namespace", secret.Namespace, "name", secret.Name)
116+
return nil
117+
}
118+
providerNamespace := kubeContext.Namespace
119+
providerConfig, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeconfig))
120+
if err != nil {
121+
logger.Error(err, "invalid kubeconfig in secret", "namespace", secret.Namespace, "name", secret.Name)
122+
return nil
123+
}
124+
125+
// Use the secret name as a synthetic binding key for tracking.
126+
// This key is used to track the controller context, but the controller
127+
// will also handle real APIServiceBindings that use the same kubeconfig.
128+
syntheticKey := "__heartbeat__" + secret.Name
129+
130+
ctrlCtx, cancel := context.WithCancel(ctx)
131+
ctrlContext := &controllerContext{
132+
kubeconfig: kubeconfig,
133+
cancel: cancel,
134+
serviceBindings: sets.New(syntheticKey),
135+
}
136+
r.controllers[syntheticKey] = ctrlContext
137+
138+
// Create and start the cluster controller.
139+
// The reconcileServiceBinding function checks if a binding should be processed
140+
// by this controller. We check if the binding is in our serviceBindings set,
141+
// which will include both the synthetic heartbeat key and any real bindings
142+
// that get added later when they use the same kubeconfig.
143+
logger.Info("starting cluster controller for early heartbeat", "secret", secret.Namespace+"/"+secret.Name, "providerNamespace", providerNamespace)
144+
ctrl, err := r.newClusterController(
145+
secret.Namespace+"/"+secret.Name,
146+
providerNamespace,
147+
func(svcBinding *kubebindv1alpha2.APIServiceBinding) bool {
148+
r.lock.RLock()
149+
defer r.lock.RUnlock()
150+
// Check if this binding is registered with this controller context
151+
return ctrlContext.serviceBindings.Has(svcBinding.Name)
152+
},
153+
providerConfig,
154+
)
155+
if err != nil {
156+
logger.Error(err, "failed to start cluster controller for heartbeat")
157+
cancel()
158+
delete(r.controllers, syntheticKey)
159+
return err
160+
}
161+
162+
go ctrl.Start(ctrlCtx)
163+
164+
return nil
49165
}
50166

51167
func (r *reconciler) reconcile(ctx context.Context, binding *kubebindv1alpha2.APIServiceBinding) error {
@@ -83,11 +199,18 @@ func (r *reconciler) reconcile(ctx context.Context, binding *kubebindv1alpha2.AP
83199
return nil
84200
}
85201

86-
// find existing with new kubeconfig
202+
// Find existing controller with the same kubeconfig.
203+
// This handles the case where:
204+
// 1. A heartbeat controller was started from a labeled secret (via startClusterControllerForSecret)
205+
// 2. An APIServiceBinding now arrives that uses the same kubeconfig
206+
// 3. Instead of starting a duplicate controller, we reuse the existing one
207+
//
208+
// The binding is added to the existing controllerContext's serviceBindings set,
209+
// and r.controllers[binding.Name] points to the same controllerContext.
87210
for _, ctrlContext := range r.controllers {
88211
if ctrlContext.kubeconfig == kubeconfig {
89-
// add to it
90-
logger.V(2).Info("adding to existing Controller", "secret", ref.Namespace+"/"+ref.Name)
212+
// Reuse existing controller - no new controller started
213+
logger.V(2).Info("adding binding to existing Controller", "secret", ref.Namespace+"/"+ref.Name)
91214
r.controllers[binding.Name] = ctrlContext
92215
ctrlContext.serviceBindings.Insert(binding.Name)
93216
return nil

sdk/apis/kubebind/v1alpha2/types.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,9 @@ const (
2121
OIDCProviderTypeEmbedded = "embedded"
2222
// OIDCProviderTypeExternal represents an external OIDC provider managed outside of kube-bind.
2323
OIDCProviderTypeExternal = "external"
24+
25+
// LabelProviderKubeconfig identifies secrets containing provider kubeconfig.
26+
// Secrets with this label are used by konnector to start heartbeat reporting
27+
// immediately at startup, even without APIServiceBinding resources.
28+
LabelProviderKubeconfig = "kube-bind.io/provider-kubeconfig"
2429
)

0 commit comments

Comments
 (0)