fix: instance shared assistant objects (#10173)

(cherry picked from commit 237b1d0)

Author: leon-ape

controllers/apps/component/transformer_component_service.go

@@ -280,7 +280,7 @@ func (t *componentServiceTransformer) createOrUpdateService(ctx graph.TransformC
 	}
 
 	if podService && kind == multiClusterServicePlacementInUnique {
-		service.Annotations[constant.KBAppMultiClusterObjectProvisionPolicyKey] = "ordinal" // HACK
+		service.Annotations[constant.KBAppMultiClusterObjectProvisionPolicyKey] = constant.KBAppMultiClusterObjectProvisionOrdinal
 	}
 
 	createOrUpdateService := func(service *corev1.Service) error {

controllers/workloads/instance_controller.go

@@ -92,7 +92,7 @@ func (r *InstanceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	return kubebuilderx.NewController(ctx, r.Client, req, r.Recorder, logger).
 		Prepare(instance.NewTreeLoader()).
 		Do(instance.NewFixMetaReconciler()).
-		Do(instance.NewDeletionReconciler()).
+		Do(instance.NewDeletionReconciler(r.Client)).
 		Do(instance.NewRevisionUpdateReconciler()).
 		Do(instance.NewStatusReconciler()).
 		// Do(instance.NewRevisionUpdateReconciler()).

pkg/constant/annotations.go

@@ -73,6 +73,7 @@ const (
 	KBAppMultiClusterPlacementKey             = "apps.kubeblocks.io/multi-cluster-placement"
 	KBAppMultiClusterServicePlacementKey      = "apps.kubeblocks.io/multi-cluster-service-placement"
 	KBAppMultiClusterObjectProvisionPolicyKey = "apps.kubeblocks.io/multi-cluster-object-provision-policy"
+	KBAppMultiClusterObjectProvisionOrdinal   = "ordinal"
 )
 
 func InheritedAnnotations() []string {

pkg/controller/instance/reconciler_assistant_object.go

@@ -38,6 +38,10 @@ import (
 	"github.com/apecloud/kubeblocks/pkg/controller/model"
 )
 
+const (
+	assistantObjectAnnotationKey = "workloads.kubeblocks.io/assistant-object"
+)
+
 func NewAssistantObjectReconciler() kubebuilderx.Reconciler {
 	return &assistantObjectReconciler{}
 }
@@ -67,10 +71,29 @@ func (r *assistantObjectReconciler) Reconcile(tree *kubebuilderx.ObjectTree) (ku
 }
 
 func (r *assistantObjectReconciler) createOrUpdate(tree *kubebuilderx.ObjectTree, inst *workloads.Instance, assistantObj workloads.InstanceAssistantObject) error {
-	obj := r.checkObjectProvisionPolicy(inst, r.instanceAssistantObject(assistantObj))
-	if obj == nil {
-		return nil // skip the object
+	obj, ok := instanceAssistantObject(assistantObj)
+	if !ok {
+		return nil
+	}
+	if isOrdinalAssistantObject(obj) {
+		obj = r.checkObjectProvisionPolicy(inst, obj)
+		if obj == nil {
+			return nil // skip the object
+		}
+		return r.createOrUpdateOwned(tree, inst, assistantObj, obj)
+	}
+	return r.createOrUpdateShared(tree, inst, assistantObj, obj)
+}
+
+func (r *assistantObjectReconciler) checkObjectProvisionPolicy(inst *workloads.Instance, obj client.Object) client.Object {
+	if isCurrentInstanceOrdinalAssistantObject(inst, obj) {
+		return obj
 	}
+	return nil
+}
+
+func (r *assistantObjectReconciler) createOrUpdateOwned(tree *kubebuilderx.ObjectTree, inst *workloads.Instance,
+	assistantObj workloads.InstanceAssistantObject, obj client.Object) error {
 	robj, err := tree.Get(obj)
 	if err != nil && !errors.IsNotFound(err) {
 		return err
@@ -94,43 +117,95 @@ func (r *assistantObjectReconciler) createOrUpdate(tree *kubebuilderx.ObjectTree
 	return nil
 }
 
-func (r *assistantObjectReconciler) instanceAssistantObject(obj workloads.InstanceAssistantObject) client.Object {
+func (r *assistantObjectReconciler) createOrUpdateShared(tree *kubebuilderx.ObjectTree, inst *workloads.Instance,
+	assistantObj workloads.InstanceAssistantObject, obj client.Object) error {
+	robj, err := tree.Get(obj)
+	if err != nil {
+		return err
+	}
+	if robj == nil {
+		markSharedAssistantObject(inst, obj)
+		return tree.Add(obj)
+	}
+
+	desired := obj.DeepCopyObject().(client.Object)
+	markSharedAssistantObject(inst, desired)
+	if merged := r.copyAndMerge(assistantObj, robj, desired); merged != nil {
+		return tree.Update(merged)
+	}
+	return nil
+}
+
+func instanceAssistantObject(obj workloads.InstanceAssistantObject) (client.Object, bool) {
 	if obj.Service != nil {
-		return obj.Service
+		return obj.Service, true
 	}
 	if obj.ConfigMap != nil {
-		return obj.ConfigMap
+		return obj.ConfigMap, true
 	}
 	if obj.Secret != nil {
-		return obj.Secret
+		return obj.Secret, true
 	}
 	if obj.ServiceAccount != nil {
-		return obj.ServiceAccount
+		return obj.ServiceAccount, true
 	}
 	if obj.Role != nil {
-		return obj.Role
+		return obj.Role, true
+	}
+	if obj.RoleBinding != nil {
+		return obj.RoleBinding, true
 	}
-	return obj.RoleBinding
+	return nil, false
 }
 
-func (r *assistantObjectReconciler) checkObjectProvisionPolicy(inst *workloads.Instance, obj client.Object) client.Object {
-	var policy string
-	if obj.GetAnnotations() != nil {
-		policy = obj.GetAnnotations()[constant.KBAppMultiClusterObjectProvisionPolicyKey]
-	}
-	if policy != "ordinal" { // HACK
-		return obj
+func isOrdinalAssistantObject(obj client.Object) bool {
+	if obj.GetAnnotations() == nil {
+		return false
 	}
+	return obj.GetAnnotations()[constant.KBAppMultiClusterObjectProvisionPolicyKey] == constant.KBAppMultiClusterObjectProvisionOrdinal
+}
 
+func isCurrentInstanceOrdinalAssistantObject(inst *workloads.Instance, obj client.Object) bool {
 	ordinal := func() int {
 		subs := strings.Split(inst.GetName(), "-")
 		o, _ := strconv.Atoi(subs[len(subs)-1])
 		return o
 	}
-	if strings.HasSuffix(obj.GetName(), fmt.Sprintf("-%d", ordinal())) {
-		return obj
+	return strings.HasSuffix(obj.GetName(), fmt.Sprintf("-%d", ordinal()))
+}
+
+func markSharedAssistantObject(inst *workloads.Instance, obj client.Object) {
+	labels := obj.GetLabels()
+	if labels == nil {
+		labels = map[string]string{}
 	}
-	return nil
+	labels[constant.AppManagedByLabelKey] = constant.AppName
+	if clusterName := inst.Labels[constant.AppInstanceLabelKey]; clusterName != "" {
+		labels[constant.AppInstanceLabelKey] = clusterName
+	}
+	if compName := inst.Labels[constant.KBAppComponentLabelKey]; compName != "" {
+		labels[constant.KBAppComponentLabelKey] = compName
+	}
+	delete(labels, constant.KBAppInstanceNameLabelKey)
+	obj.SetLabels(labels)
+
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	annotations[assistantObjectAnnotationKey] = "true"
+	obj.SetAnnotations(annotations)
+}
+
+func isSharedAssistantObject(obj client.Object, inst *workloads.Instance) bool {
+	labels := obj.GetLabels()
+	annotations := obj.GetAnnotations()
+	return labels[constant.AppManagedByLabelKey] == constant.AppName &&
+		labels[constant.AppInstanceLabelKey] != "" &&
+		labels[constant.AppInstanceLabelKey] == inst.Labels[constant.AppInstanceLabelKey] &&
+		labels[constant.KBAppComponentLabelKey] != "" &&
+		labels[constant.KBAppComponentLabelKey] == inst.Labels[constant.KBAppComponentLabelKey] &&
+		annotations[assistantObjectAnnotationKey] == "true"
 }
 
 func (r *assistantObjectReconciler) copyAndMerge(obj workloads.InstanceAssistantObject, oldObj, newObj client.Object) client.Object {