LOG-9356: Implement Collector ServiceMonitor Changes

Author: Clee2691

internal/controller/observability/collector.go

@@ -153,7 +153,7 @@ func ReconcileCollector(context internalcontext.ForwarderContext, pollInterval,
 		return err
 	}
 	metricsSelector := metrics.BuildSelector(constants.CollectorName, resourceNames.CommonName)
-	if err := metrics.ReconcileServiceMonitor(context.Client, context.Forwarder.Namespace, resourceNames.CommonName, ownerRef, metricsSelector, constants.MetricsPortName); err != nil {
+	if err := metrics.ReconcileServiceMonitor(context.Client, context.Forwarder.Namespace, resourceNames.CommonName, ownerRef, metricsSelector, constants.MetricsPortName, metrics.CollectorMetricRelabelConfigs); err != nil {
 		log.Error(err, "collector.ReconcileServiceMonitor")
 		return err
 	}

internal/metrics/logfilemetricexporter/metric_exporter.go

@@ -76,7 +76,7 @@ func Reconcile(lfmeInstance *loggingv1alpha1.LogFileMetricExporter,
 	}
 
 	metricsSelector := metrics.BuildSelector(constants.LogfilesmetricexporterName, lfmeInstance.Name)
-	if err := metrics.ReconcileServiceMonitor(requestClient, lfmeInstance.Namespace, resNames.CommonName, owner, metricsSelector, constants.MetricsPortName); err != nil {
+	if err := metrics.ReconcileServiceMonitor(requestClient, lfmeInstance.Namespace, resNames.CommonName, owner, metricsSelector, constants.MetricsPortName, nil); err != nil {
 		log.Error(err, "logfilemetricexporter.ReconcileServiceMonitor")
 		return err
 	}

internal/metrics/service_monitor.go

@@ -17,7 +17,26 @@ const (
 	prometheusBearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
 )
 
-func newServiceMonitor(namespace, name string, owner metav1.OwnerReference, selector map[string]string, portName string) *monitoringv1.ServiceMonitor {
+var CollectorMetricRelabelConfigs = []*monitoringv1.RelabelConfig{
+	{
+		SourceLabels: []monitoringv1.LabelName{"__name__"},
+		TargetLabel:  "__name__",
+		Regex:        "(.*)-(.*)",
+		Replacement:  "${1}_${2}",
+	},
+	{
+		Action:       "keep",
+		SourceLabels: []monitoringv1.LabelName{"__name__"},
+		Regex:        "logcollector_component_event_unmatched_count|vector_buffer_byte_size|vector_buffer_discarded_events_total|vector_buffer_events|vector_buffer_sent_events_total|vector_component_discarded_events_total|vector_component_errors_total|vector_component_received_bytes_total|vector_component_received_event_bytes_total|vector_component_received_events_total|vector_component_sent_bytes_total|vector_events_in_total|vector_http_client_errors_total|vector_http_client_requests_sent_total|vector_http_client_responses_total|vector_open_files",
+	},
+	{
+		Action:       "drop",
+		SourceLabels: []monitoringv1.LabelName{"component_kind", "__name__"},
+		Regex:        `transform;(vector_component_received_bytes_total|vector_component_received_event_bytes_total|vector_component_received_events_total|vector_component_sent_bytes_total)`,
+	},
+}
+
+func newServiceMonitor(namespace, name string, owner metav1.OwnerReference, selector map[string]string, portName string, metricRelabelConfigs []*monitoringv1.RelabelConfig) *monitoringv1.ServiceMonitor {
 	var endpoint = []monitoringv1.Endpoint{
 		{
 			Port:            portName,
@@ -30,19 +49,7 @@ func newServiceMonitor(namespace, name string, owner metav1.OwnerReference, sele
 					ServerName: fmt.Sprintf("%s.%s.svc", name, namespace),
 				},
 			},
-			// Replaces labels that have `-` with `_`
-			// Example:
-			// app_kubernetes_io_part-of -> app_kubernetes_io_part_of
-			MetricRelabelConfigs: []*monitoringv1.RelabelConfig{
-				{
-					SourceLabels: []monitoringv1.LabelName{
-						"__name__",
-					},
-					TargetLabel: "__name__",
-					Regex:       "(.*)-(.*)",
-					Replacement: "${1}_${2}",
-				},
-			},
+			MetricRelabelConfigs: metricRelabelConfigs,
 		},
 	}
 
@@ -77,7 +84,7 @@ func BuildSelector(component, instance string) map[string]string {
 	}
 }
 
-func ReconcileServiceMonitor(k8sClient client.Client, namespace, name string, owner metav1.OwnerReference, selector map[string]string, portName string) error {
-	desired := newServiceMonitor(namespace, name, owner, selector, portName)
+func ReconcileServiceMonitor(k8sClient client.Client, namespace, name string, owner metav1.OwnerReference, selector map[string]string, portName string, metricRelabelConfigs []*monitoringv1.RelabelConfig) error {
+	desired := newServiceMonitor(namespace, name, owner, selector, portName, metricRelabelConfigs)
 	return reconcile.ServiceMonitor(k8sClient, desired)
 }

internal/metrics/service_monitor_test.go

@@ -45,15 +45,15 @@ var _ = Describe("Reconcile ServiceMonitor", func() {
 		smInstance        = &monitoringv1.ServiceMonitor{}
 	)
 
-	It("should successfully reconcile the ServiceMonitor", func() {
-		// Reconcile the exporter daemonset
+	It("should successfully reconcile the ServiceMonitor with collector relabel configs", func() {
 		Expect(ReconcileServiceMonitor(
 			reqClient,
 			constants.OpenshiftNS,
 			serviceName,
 			owner,
 			selector,
 			portName,
+			CollectorMetricRelabelConfigs,
 		)).To(Succeed())
 
 		// Get and check the ServiceMonitor
@@ -72,5 +72,37 @@ var _ = Describe("Reconcile ServiceMonitor", func() {
 
 		Expect(smInstance.Spec.Endpoints[0].BearerTokenFile).
 			To(Equal("/var/run/secrets/kubernetes.io/serviceaccount/token"))
+
+		By("verifying MetricRelabelConfigs")
+		relabelConfigs := smInstance.Spec.Endpoints[0].MetricRelabelConfigs
+		Expect(relabelConfigs).To(HaveLen(3))
+
+		Expect(relabelConfigs[0].Regex).To(Equal("(.*)-(.*)"))
+		Expect(relabelConfigs[0].TargetLabel).To(Equal("__name__"))
+
+		Expect(string(relabelConfigs[1].Action)).To(Equal("keep"))
+		Expect(relabelConfigs[1].SourceLabels).To(Equal([]monitoringv1.LabelName{"__name__"}))
+
+		Expect(string(relabelConfigs[2].Action)).To(Equal("drop"))
+		Expect(relabelConfigs[2].SourceLabels).To(Equal([]monitoringv1.LabelName{"component_kind", "__name__"}))
+		Expect(relabelConfigs[2].Regex).To(Equal(`transform;(vector_component_received_bytes_total|vector_component_received_event_bytes_total|vector_component_received_events_total|vector_component_sent_bytes_total)`))
+	})
+
+	It("should create a ServiceMonitor with no metric relabel configs when nil is passed", func() {
+		smName := "test-no-relabel"
+		Expect(ReconcileServiceMonitor(
+			reqClient,
+			constants.OpenshiftNS,
+			smName,
+			owner,
+			selector,
+			portName,
+			nil,
+		)).To(Succeed())
+
+		sm := &monitoringv1.ServiceMonitor{}
+		Expect(reqClient.Get(context.TODO(), types.NamespacedName{Name: smName, Namespace: constants.OpenshiftNS}, sm)).Should(Succeed())
+		Expect(sm.Spec.Endpoints).ToNot(BeEmpty())
+		Expect(sm.Spec.Endpoints[0].MetricRelabelConfigs).To(BeNil())
 	})
 })