Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 14 additions & 12 deletions internal/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,18 +39,20 @@ const (
InjectTrustedCABundleLabel = "config.openshift.io/inject-trusted-cabundle"

//ServiceAccountSecretPath is the path to find the projected serviceAccount token and other SA secrets
ServiceAccountSecretPath = "/var/run/ocp-collector/serviceaccount"
TrustedCABundleMountFile = "tls-ca-bundle.pem"
TrustedCABundleMountDir = "/etc/pki/ca-trust/extracted/pem/"
ElasticsearchName = "elasticsearch"
VectorName = "vector"
KibanaName = "kibana"
LogfilesmetricexporterName = "logfilesmetricexporter"
LogfilesmetricexporterPort = int32(2112)
MetricsPortName = "metrics"
MetricsPort = int32(24231)
PodSecurityLabelEnforce = "pod-security.kubernetes.io/enforce"
PodSecurityLabelValue = "privileged"
ServiceAccountSecretPath = "/var/run/ocp-collector/serviceaccount"
TrustedCABundleMountFile = "tls-ca-bundle.pem"
TrustedCABundleMountDir = "/etc/pki/ca-trust/extracted/pem/"
ElasticsearchName = "elasticsearch"
VectorName = "vector"
KibanaName = "kibana"
LogfilesmetricexporterName = "logfilesmetricexporter"
LogfilesmetricexporterPort = int32(2112)
MetricsPortName = "metrics"
MetricsPort = int32(24231)
MetricsCollectionProfileFull = "full"
MetricsCollectionProfileMinimal = "minimal"
PodSecurityLabelEnforce = "pod-security.kubernetes.io/enforce"
PodSecurityLabelValue = "privileged"
// Disable gosec linter, complains "possible hard-coded secret"
CollectorSecretsDir = "/var/run/ocp-collector/secrets" //nolint:gosec
ConfigMapBaseDir = "/var/run/ocp-collector/config"
Expand Down
2 changes: 2 additions & 0 deletions internal/constants/labels.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ const (
LabelLoggingServiceType = "logging.observability.openshift.io/service-type"
LabelLoggingInputServiceType = "logging.observability.openshift.io/input-service-type"

LabelMetricsCollectionProfile = "monitoring.openshift.io/collection-profile"

ServiceTypeMetrics = "metrics"
ServiceTypeInput = "input"
)
8 changes: 6 additions & 2 deletions internal/controller/observability/collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,12 @@ func ReconcileCollector(context internalcontext.ForwarderContext, pollInterval,
return err
}
metricsSelector := metrics.BuildSelector(constants.CollectorName, resourceNames.CommonName)
if err := metrics.ReconcileServiceMonitor(context.Client, context.Forwarder.Namespace, resourceNames.CommonName, ownerRef, metricsSelector, constants.MetricsPortName); err != nil {
log.Error(err, "collector.ReconcileServiceMonitor")
if err := metrics.ReconcileServiceMonitor(context.Client, context.Forwarder.Namespace, resourceNames.CommonName, resourceNames.CommonName, ownerRef, metricsSelector, constants.MetricsPortName, metrics.FullRelabelConfigs, constants.MetricsCollectionProfileFull); err != nil {
log.Error(err, "collector.ReconcileServiceMonitor full")
return err
}
if err := metrics.ReconcileServiceMonitor(context.Client, context.Forwarder.Namespace, constants.MetricsCollectionProfileMinimal+"-"+resourceNames.CommonName, resourceNames.CommonName, ownerRef, metricsSelector, constants.MetricsPortName, metrics.CollectorMinimalRelabelConfigs, constants.MetricsCollectionProfileMinimal); err != nil {
log.Error(err, "collector.ReconcileServiceMonitor minimal")
return err
}

Expand Down
18 changes: 16 additions & 2 deletions internal/controller/observability/collector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,8 +225,22 @@ var _ = Describe("Reconciling the Collector", func() {
service := &corev1.Service{}
Expect(client.Get(context.TODO(), key, service)).Should(Succeed(), "Exp. to create a Service for metrics")

sm := &monitoringv1.ServiceMonitor{}
Expect(client.Get(context.TODO(), key, sm)).Should(Succeed(), "Exp. to create a ServiceMonitor for metrics")
By("verifying the full profile ServiceMonitor")
fullSM := &monitoringv1.ServiceMonitor{}
Expect(client.Get(context.TODO(), key, fullSM)).Should(Succeed(), "Exp. to create a full profile ServiceMonitor")
Expect(fullSM.Labels[constants.LabelMetricsCollectionProfile]).To(Equal(constants.MetricsCollectionProfileFull))
Expect(fullSM.Spec.Endpoints).ToNot(BeEmpty())
Expect(fullSM.Spec.Endpoints[0].MetricRelabelConfigs).To(HaveLen(1), "full profile should only have the rename rule")

By("verifying the minimal profile ServiceMonitor")
minimalKey := types.NamespacedName{Name: constants.MetricsCollectionProfileMinimal + "-" + clfName, Namespace: namespaceName}
minimalSM := &monitoringv1.ServiceMonitor{}
Expect(client.Get(context.TODO(), minimalKey, minimalSM)).Should(Succeed(), "Exp. to create a minimal profile ServiceMonitor")
Expect(minimalSM.Labels[constants.LabelMetricsCollectionProfile]).To(Equal(constants.MetricsCollectionProfileMinimal))
Expect(minimalSM.Spec.Endpoints).ToNot(BeEmpty())
Expect(minimalSM.Spec.Endpoints[0].MetricRelabelConfigs).To(HaveLen(3), "minimal profile should have rename + keep + drop")
Expect(string(minimalSM.Spec.Endpoints[0].MetricRelabelConfigs[1].Action)).To(Equal("keep"))
Expect(string(minimalSM.Spec.Endpoints[0].MetricRelabelConfigs[2].Action)).To(Equal("drop"))

},
Entry("when deployed as a DaemonSet", forwarder),
Expand Down
8 changes: 6 additions & 2 deletions internal/metrics/logfilemetricexporter/metric_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,12 @@ func Reconcile(lfmeInstance *loggingv1alpha1.LogFileMetricExporter,
}

metricsSelector := metrics.BuildSelector(constants.LogfilesmetricexporterName, lfmeInstance.Name)
if err := metrics.ReconcileServiceMonitor(requestClient, lfmeInstance.Namespace, resNames.CommonName, owner, metricsSelector, constants.MetricsPortName); err != nil {
log.Error(err, "logfilemetricexporter.ReconcileServiceMonitor")
if err := metrics.ReconcileServiceMonitor(requestClient, lfmeInstance.Namespace, resNames.CommonName, resNames.CommonName, owner, metricsSelector, constants.MetricsPortName, metrics.FullRelabelConfigs, constants.MetricsCollectionProfileFull); err != nil {
log.Error(err, "logfilemetricexporter.ReconcileServiceMonitor full")
return err
}
if err := metrics.ReconcileServiceMonitor(requestClient, lfmeInstance.Namespace, constants.MetricsCollectionProfileMinimal+"-"+resNames.CommonName, resNames.CommonName, owner, metricsSelector, constants.MetricsPortName, metrics.LFMEMinimalRelabelConfigs, constants.MetricsCollectionProfileMinimal); err != nil {
log.Error(err, "logfilemetricexporter.ReconcileServiceMonitor minimal")
return err
}

Expand Down
15 changes: 13 additions & 2 deletions internal/metrics/logfilemetricexporter/metric_exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,11 @@ var _ = Describe("Reconcile LogFileMetricExporter", func() {
Expect(serviceInstance.Annotations[constants.AnnotationServingCertSecretName]).
To(Equal(ExporterMetricsSecretName))

// ServiceMonitor
// Get and check the ServiceMonitor
// ServiceMonitor (full profile)
Expect(reqClient.Get(context.TODO(), serviceMonitorKey, smInstance)).Should(Succeed())

Expect(smInstance.Name).To(Equal(constants.LogfilesmetricexporterName))
Expect(smInstance.Labels[constants.LabelMetricsCollectionProfile]).To(Equal(constants.MetricsCollectionProfileFull))

expJobLabel := fmt.Sprintf("monitor-%s", constants.LogfilesmetricexporterName)
Expect(smInstance.Spec.JobLabel).To(Equal(expJobLabel))
Expand All @@ -132,6 +132,17 @@ var _ = Describe("Reconcile LogFileMetricExporter", func() {

Expect(smInstance.Spec.Endpoints[0].BearerTokenFile).
To(Equal("/var/run/secrets/kubernetes.io/serviceaccount/token"))
Expect(smInstance.Spec.Endpoints[0].MetricRelabelConfigs).To(HaveLen(1), "full profile should only have the rename rule")

// ServiceMonitor (minimal profile)
minimalName := constants.MetricsCollectionProfileMinimal + "-" + constants.LogfilesmetricexporterName
minimalSM := &monitoringv1.ServiceMonitor{}
Expect(reqClient.Get(context.TODO(), types.NamespacedName{Name: minimalName, Namespace: namespace.Name}, minimalSM)).Should(Succeed())
Expect(minimalSM.Labels[constants.LabelMetricsCollectionProfile]).To(Equal(constants.MetricsCollectionProfileMinimal))
Expect(minimalSM.Spec.Endpoints).ToNot(BeEmpty())
Expect(minimalSM.Spec.Endpoints[0].TLSConfig.SafeTLSConfig.ServerName).To(Equal(svcURL))
Expect(minimalSM.Spec.Endpoints[0].MetricRelabelConfigs).To(HaveLen(2), "LFME minimal profile should have rename + keep")
Expect(string(minimalSM.Spec.Endpoints[0].MetricRelabelConfigs[1].Action)).To(Equal("keep"))

// Metrics Auth RBAC
// Verify the metrics auth ClusterRoleBinding exists and references system:auth-delegator
Expand Down
98 changes: 98 additions & 0 deletions internal/metrics/relabel.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package metrics

import (
"strings"

monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
)

type metricAllowlistConfig struct {
allowedMetrics []string
}

type metricDropConfig struct {
labelName string
labelValue string
excludeMetrics []string
}

var collectorMinimalAllowlist = &metricAllowlistConfig{
allowedMetrics: []string{
// Metrics used in alerts (collector_alerts.yaml)
"logcollector_component_event_unmatched_count",
"vector_http_client_errors_total",
"vector_http_client_requests_sent_total",
"vector_http_client_responses_total",
"vector_buffer_byte_size",
"vector_component_errors_total",
"vector_component_received_events_total",

// Metrics used in recording rules (collector_alerts.yaml, telemetry_rules.yaml)
"vector_component_received_bytes_total",

// Metrics used in dashboards (openshift-logging-dashboard.json)
"vector_component_sent_bytes_total",
"vector_component_received_event_bytes_total",
"vector_open_files",
"vector_component_discarded_events_total",

// Additional buffer and event metrics
"vector_buffer_discarded_events_total",
"vector_buffer_events",
"vector_buffer_sent_events_total",
"vector_events_in_total",
},
}

var collectorMinimalDropConfigs = []metricDropConfig{
{
labelName: "component_kind",
labelValue: "transform",
excludeMetrics: []string{
"vector_component_received_bytes_total",
"vector_component_received_event_bytes_total",
"vector_component_received_events_total",
"vector_component_sent_bytes_total",
},
},
}

var lfmeMinimalAllowlist = &metricAllowlistConfig{
allowedMetrics: []string{
// Used in recording rule (collector_alerts.yaml) and dashboard
"log_logged_bytes_total",
},
}

var CollectorMinimalRelabelConfigs = buildRelabelConfigs(collectorMinimalAllowlist, collectorMinimalDropConfigs)
var LFMEMinimalRelabelConfigs = buildRelabelConfigs(lfmeMinimalAllowlist, nil)
var FullRelabelConfigs = buildRelabelConfigs(nil, nil)

func buildRelabelConfigs(allowlist *metricAllowlistConfig, dropConfigs []metricDropConfig) []*monitoringv1.RelabelConfig {
configs := []*monitoringv1.RelabelConfig{
{
SourceLabels: []monitoringv1.LabelName{"__name__"},
TargetLabel: "__name__",
Regex: "(.*)-(.*)",
Replacement: "${1}_${2}",
},
}

if allowlist != nil && len(allowlist.allowedMetrics) > 0 {
configs = append(configs, &monitoringv1.RelabelConfig{
Action: "keep",
SourceLabels: []monitoringv1.LabelName{"__name__"},
Regex: strings.Join(allowlist.allowedMetrics, "|"),
Comment thread
jcantrill marked this conversation as resolved.
})
}

for _, drop := range dropConfigs {
configs = append(configs, &monitoringv1.RelabelConfig{
Action: "drop",
SourceLabels: []monitoringv1.LabelName{monitoringv1.LabelName(drop.labelName), "__name__"},
Comment thread
jcantrill marked this conversation as resolved.
Regex: drop.labelValue + ";(" + strings.Join(drop.excludeMetrics, "|") + ")",
})
}

return configs
}
101 changes: 101 additions & 0 deletions internal/metrics/relabel_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package metrics

import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
)

var _ = Describe("buildRelabelConfigs", func() {
It("should return only the rename rule when no allowlist or drops are provided", func() {
configs := buildRelabelConfigs(nil, nil)
Expect(configs).To(HaveLen(1))
Expect(configs[0].TargetLabel).To(Equal("__name__"))
Expect(configs[0].Regex).To(Equal("(.*)-(.*)"))
Expect(configs[0].Replacement).To(Equal("${1}_${2}"))
})

It("should return rename + keep when an allowlist is provided", func() {
allowlist := &metricAllowlistConfig{
allowedMetrics: []string{"metric_a", "metric_b"},
}
configs := buildRelabelConfigs(allowlist, nil)
Expect(configs).To(HaveLen(2))

Expect(configs[0].Regex).To(Equal("(.*)-(.*)"))

Expect(string(configs[1].Action)).To(Equal("keep"))
Expect(configs[1].SourceLabels).To(Equal([]monitoringv1.LabelName{"__name__"}))
Expect(configs[1].Regex).To(Equal("metric_a|metric_b"))
})

It("should return rename + keep + drop when allowlist and drops are provided", func() {
allowlist := &metricAllowlistConfig{
allowedMetrics: []string{"metric_a", "metric_b", "metric_c"},
}
drops := []metricDropConfig{
{
labelName: "component_kind",
labelValue: "transform",
excludeMetrics: []string{"metric_a", "metric_b"},
},
}
configs := buildRelabelConfigs(allowlist, drops)
Expect(configs).To(HaveLen(3))

Expect(string(configs[1].Action)).To(Equal("keep"))
Expect(configs[1].Regex).To(Equal("metric_a|metric_b|metric_c"))

Expect(string(configs[2].Action)).To(Equal("drop"))
Expect(configs[2].SourceLabels).To(Equal([]monitoringv1.LabelName{"component_kind", "__name__"}))
Expect(configs[2].Regex).To(Equal("transform;(metric_a|metric_b)"))
})

It("should build valid CollectorMinimalRelabelConfigs", func() {
Expect(CollectorMinimalRelabelConfigs).To(HaveLen(3))
Expect(string(CollectorMinimalRelabelConfigs[1].Action)).To(Equal("keep"))
Expect(string(CollectorMinimalRelabelConfigs[2].Action)).To(Equal("drop"))

keepRegex := CollectorMinimalRelabelConfigs[1].Regex
for _, m := range collectorMinimalAllowlist.allowedMetrics {
Expect(keepRegex).To(ContainSubstring(m), "missing metric in keep regex: %s", m)
}
})

It("should build valid LFMEMinimalRelabelConfigs", func() {
Expect(LFMEMinimalRelabelConfigs).To(HaveLen(2))
Expect(string(LFMEMinimalRelabelConfigs[1].Action)).To(Equal("keep"))
Expect(LFMEMinimalRelabelConfigs[1].Regex).To(Equal("log_logged_bytes_total"))
})

It("should build FullRelabelConfigs with only the rename rule", func() {
Expect(FullRelabelConfigs).To(HaveLen(1))
Expect(FullRelabelConfigs[0].Regex).To(Equal("(.*)-(.*)"))
})

It("should return only the rename rule when allowlist has empty metrics", func() {
allowlist := &metricAllowlistConfig{
allowedMetrics: []string{},
}
configs := buildRelabelConfigs(allowlist, nil)
Expect(configs).To(HaveLen(1))
Expect(configs[0].TargetLabel).To(Equal("__name__"))
})

It("should return rename + drop when only drop configs are provided", func() {
drops := []metricDropConfig{
{
labelName: "component_kind",
labelValue: "transform",
excludeMetrics: []string{"metric_a"},
},
}
configs := buildRelabelConfigs(nil, drops)
Expect(configs).To(HaveLen(2))

Expect(configs[0].Regex).To(Equal("(.*)-(.*)"))
Expect(string(configs[1].Action)).To(Equal("drop"))
Expect(configs[1].SourceLabels).To(Equal([]monitoringv1.LabelName{"component_kind", "__name__"}))
Expect(configs[1].Regex).To(Equal("transform;(metric_a)"))
})
})
26 changes: 9 additions & 17 deletions internal/metrics/service_monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ const (
prometheusBearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
)

func newServiceMonitor(namespace, name string, owner metav1.OwnerReference, selector map[string]string, portName string) *monitoringv1.ServiceMonitor {
func newServiceMonitor(namespace, name, serviceName string, owner metav1.OwnerReference, selector map[string]string, portName string, metricRelabelConfigs []*monitoringv1.RelabelConfig, profile string) *monitoringv1.ServiceMonitor {
var endpoint = []monitoringv1.Endpoint{
{
Port: portName,
Expand All @@ -27,26 +27,18 @@ func newServiceMonitor(namespace, name string, owner metav1.OwnerReference, sele
TLSConfig: &monitoringv1.TLSConfig{
CAFile: prometheusCAFile,
SafeTLSConfig: monitoringv1.SafeTLSConfig{
ServerName: fmt.Sprintf("%s.%s.svc", name, namespace),
},
},
// Replaces labels that have `-` with `_`
// Example:
// app_kubernetes_io_part-of -> app_kubernetes_io_part_of
MetricRelabelConfigs: []*monitoringv1.RelabelConfig{
{
SourceLabels: []monitoringv1.LabelName{
"__name__",
},
TargetLabel: "__name__",
Regex: "(.*)-(.*)",
Replacement: "${1}_${2}",
ServerName: fmt.Sprintf("%s.%s.svc", serviceName, namespace),
},
},
MetricRelabelConfigs: metricRelabelConfigs,
},
}

desired := runtime.NewServiceMonitor(namespace, name)
if desired.Labels == nil {
desired.Labels = map[string]string{}
}
desired.Labels[constants.LabelMetricsCollectionProfile] = profile
desired.Spec = monitoringv1.ServiceMonitorSpec{
JobLabel: fmt.Sprintf("monitor-%s", name),
Endpoints: endpoint,
Expand Down Expand Up @@ -77,7 +69,7 @@ func BuildSelector(component, instance string) map[string]string {
}
}

func ReconcileServiceMonitor(k8sClient client.Client, namespace, name string, owner metav1.OwnerReference, selector map[string]string, portName string) error {
desired := newServiceMonitor(namespace, name, owner, selector, portName)
func ReconcileServiceMonitor(k8sClient client.Client, namespace, name, serviceName string, owner metav1.OwnerReference, selector map[string]string, portName string, metricRelabelConfigs []*monitoringv1.RelabelConfig, profile string) error {
desired := newServiceMonitor(namespace, name, serviceName, owner, selector, portName, metricRelabelConfigs, profile)
return reconcile.ServiceMonitor(k8sClient, desired)
}
Loading