[OSPRH-21020] Scrape all rabbitmq pods

RabbitMQ can be deployed with multiple replicas, while each replica
will export different metrics. Having a ScrapeConfig with a service
as a target means we're not scraping all metrics. This PR
uses the Endpoints object to retrieve IP addresses of all rabbitmq
related pods and creates a ScrapeConfig to scrape each pod through
its IP address. This keeps the pattern of using ScrapeConfigs for
configuring scraping.

Author: vyzigold

ci/increase-rabbitmq-replicas.yml

@@ -0,0 +1,32 @@
+---
+- name: "Create the kustomization for increasing rabbitmq replicas"
+  hosts: "{{ cifmw_target_hook_host | default('localhost')  }}"
+  gather_facts: false
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  tasks:
+    - name: Copy controlplane kustomization
+      ansible.builtin.copy:
+        dest: "{{ cifmw_basedir }}/artifacts/manifests/kustomizations/controlplane/91-kustomize-controlplane-increase-rabbit-replicas.yaml"
+        content: |-
+          apiVersion: kustomize.config.k8s.io/v1beta1
+          kind: Kustomization
+          namespace: openstack
+          patches:
+          - patch: |-
+              apiVersion: core.openstack.org/v1beta1
+              kind: OpenStackControlPlane
+              metadata:
+                name: unused
+              spec:
+                # By increasing the rabbitmq replicas, the CI jobs by default
+                # run out of PVs. This storage class will create PVs
+                # dynamically, but works only on CRC based environments.
+                storageClass: crc-csi-hostpath-provisioner
+                rabbitmq:
+                  templates:
+                    rabbitmq:
+                      replicas: 3
+            target:
+              kind: OpenStackControlPlane

ci/vars-autoscaling.yml

@@ -18,3 +18,7 @@ cifmw_openshift_obs_definition:
 pre_deploy_kustomize_autoscaling:
   source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-autoscaling.yml"
   type: playbook
+
+pre_deploy_kustomize_increase_rabbitmq_replicas:
+  source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/increase-rabbitmq-replicas.yml"
+  type: playbook

config/rbac/role.yaml

@@ -108,6 +108,14 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - heat.openstack.org
   resources:

internal/controller/metricstorage_controller.go

@@ -114,6 +114,7 @@ func (r *MetricStorageReconciler) GetLogger(ctx context.Context) logr.Logger {
 //+kubebuilder:rbac:groups=ovn.openstack.org,resources=ovnnorthds,verbs=get;list;watch
 //+kubebuilder:rbac:groups=observability.openshift.io,resources=uiplugins,verbs=get;list;watch;create;patch
 //+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch
 //+kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch
 
 // Reconcile reconciles MetricStorage
@@ -763,9 +764,6 @@ func (r *MetricStorageReconciler) createScrapeConfigs(
 	}
 
 	// ScrapeConfigs for RabbitMQ monitoring
-	// NOTE: We're watching Rabbits and reconciling with each of their change
-	//       that should keep the targets inside the ScrapeConfig always
-	//       up to date.
 	rabbitList := &rabbitmqv1.RabbitmqClusterList{}
 	listOpts := []client.ListOption{
 		client.InNamespace(instance.GetNamespace()),
@@ -774,22 +772,25 @@ func (r *MetricStorageReconciler) createScrapeConfigs(
 	if err != nil && !k8s_errors.IsNotFound(err) {
 		return ctrl.Result{}, err
 	}
-	rabbitTargets := []string{}
 	for _, rabbit := range rabbitList.Items {
-		rabbitServerName := fmt.Sprintf("%s.%s.svc", rabbit.Name, rabbit.Namespace)
-		rabbitTargets = append(rabbitTargets, net.JoinHostPort(rabbitServerName, strconv.Itoa(metricstorage.RabbitMQPrometheusPort)))
-	}
-	rabbitCfgName := fmt.Sprintf("%s-rabbitmq", telemetry.ServiceName)
-	desiredScrapeConfig = metricstorage.ScrapeConfig(
-		instance,
-		serviceLabels,
-		rabbitTargets,
-		instance.Spec.PrometheusTLS.Enabled(),
-	)
-	err = r.createServiceScrapeConfig(ctx, instance, Log, "RabbitMQ",
-		rabbitCfgName, desiredScrapeConfig)
-	if err != nil {
-		return ctrl.Result{}, err
+		desiredScrapeConfig, err = metricstorage.ScrapeConfigRabbitMQ(
+			ctx,
+			instance,
+			serviceLabels,
+			helper,
+			rabbit,
+			instance.Spec.PrometheusTLS.Enabled(),
+		)
+		if err != nil {
+			Log.Info(fmt.Sprintf("Cannot build rabbitMQ scrape config. Scrape configs not created. Error: %s", err))
+			return ctrl.Result{}, err
+		}
+		rabbitCfgName := fmt.Sprintf("%s-%s", telemetry.ServiceName, rabbit.Name)
+		err = r.createServiceScrapeConfig(ctx, instance, Log, "RabbitMQ",
+			rabbitCfgName, desiredScrapeConfig)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
 	}
 
 	// mysqld exporter
@@ -1594,10 +1595,6 @@ func (r *MetricStorageReconciler) SetupWithManager(ctx context.Context, mgr ctrl
 			handler.EnqueueRequestsFromMapFunc(r.nodeSetWatchFn),
 			builder.WithPredicates(inventoryPredicator),
 		).
-		Watches(
-			&rabbitmqv1.RabbitmqCluster{},
-			handler.EnqueueRequestsFromMapFunc(reconcileAllMetricStoragesWatchFn),
-		).
 		Watches(
 			&ovnv1.OVNNorthd{},
 			handler.EnqueueRequestsFromMapFunc(reconcileAllMetricStoragesWatchFn),
@@ -1608,6 +1605,18 @@ func (r *MetricStorageReconciler) SetupWithManager(ctx context.Context, mgr ctrl
 			handler.EnqueueRequestsFromMapFunc(reconcileAllMetricStoragesWatchFn),
 			builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
 		).
+		Watches(
+			&discoveryv1.EndpointSlice{},
+			handler.EnqueueRequestsFromMapFunc(reconcileAllMetricStoragesWatchFn),
+			builder.WithPredicates(
+				predicate.ResourceVersionChangedPredicate{},
+				// Don't call the WatchFn on unrelated Endpointslices changes
+				predicate.NewPredicateFuncs(func(o client.Object) bool {
+					labels := o.GetLabels()
+					return labels["app.kubernetes.io/component"] == "rabbitmq"
+				}),
+			),
+		).
 		Build(r)
 	r.Controller = control
 	return err