COO-1687: feat: migrate to EndpointSlice service discovery (#1028)

* feat: migrate to EndpointSlice service discovery

Prometheus Operator defaults to watching the deprecated Endpoints API for
service discovery. Switch the operator's own ServiceMonitors to use
EndpointSlice explicitly, which eliminates the deprecation log noise from
the operator's internal components.

Changes:
- Set serviceDiscoveryRole: EndpointSlice on the ServiceMonitors we own
  (observability-operator, health-analyzer, thanos-querier) so that
  prometheus-operator uses the EndpointSlice role for these jobs.
- Add discovery.k8s.io/endpointslices to all Prometheus RBAC roles and
  ClusterRoles (alongside the existing endpoints permission) so that
  Prometheus can serve both kinds of ServiceMonitors simultaneously.
- Add discovery.k8s.io/endpointslices to the korrel8r ClusterRole so
  the correlation tool can read both endpoint representations.
- Add the corresponding kubebuilder markers and update the generated
  cluster role YAML and CSV.

The Prometheus CR's global serviceDiscoveryRole is intentionally left
unset (defaulting to Endpoints) so that user-created ServiceMonitors
continue to work without modification. Users can opt individual
ServiceMonitors into EndpointSlice by setting serviceDiscoveryRole:
EndpointSlice on them.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: Jan Fajerski <jan@fajerski.name>

* fix: revert serviceDiscoveryRole from monitoring.coreos.com ServiceMonitors

The operator's self-monitoring ServiceMonitor and the health-analyzer
ServiceMonitor are monitoring.coreos.com objects processed by the
platform prometheus-operator on OpenShift, which we don't control.
Setting serviceDiscoveryRole: EndpointSlice on them requires the
platform Prometheus to have endpointslices access and the platform
prometheus-operator to correctly generate TLS-aware scrape configs
for the endpointslice role — neither of which is guaranteed across
OCP versions.

The thanos-querier ServiceMonitor (monitoring.rhobs) is handled by
the obo-prometheus-operator we manage, so it retains the EndpointSlice
setting safely.

Fixes TestOperatorMetrics/metrics_ingested_in_Prometheus on OCP clusters.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>

---------

Signed-off-by: Jan Fajerski <jan@fajerski.name>
Co-authored-by: Jan Fajerski <jan@fajerski.name>
Co-authored-by: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>

Author: 3 people

bundle/manifests/observability-operator.clusterserviceversion.yaml

@@ -43,7 +43,7 @@ metadata:
     certified: "false"
     console.openshift.io/operator-monitoring-default: "true"
     containerImage: observability-operator:1.3.0
-    createdAt: "2026-03-03T14:08:32Z"
+    createdAt: "2026-03-09T07:53:46Z"
     description: A Go based Kubernetes operator to setup and manage highly available
       Monitoring Stack using Prometheus, Alertmanager and Thanos Querier.
     operatorframework.io/cluster-monitoring: "true"
@@ -436,6 +436,14 @@ spec:
           - patch
           - update
           - watch
+        - apiGroups:
+          - discovery.k8s.io
+          resources:
+          - endpointslices
+          verbs:
+          - get
+          - list
+          - watch
         - apiGroups:
           - extensions
           - networking.k8s.io

deploy/operator/observability-operator-cluster-role.yaml

@@ -119,6 +119,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - extensions
   - networking.k8s.io

pkg/controllers/monitoring/monitoring-stack/components.go

@@ -100,6 +100,10 @@ func newPrometheusClusterRole(rbacResourceName string, rbacVerbs []string) *rbac
 			APIGroups: []string{""},
 			Resources: []string{"services", "endpoints", "pods"},
 			Verbs:     rbacVerbs,
+		}, {
+			APIGroups: []string{"discovery.k8s.io"},
+			Resources: []string{"endpointslices"},
+			Verbs:     rbacVerbs,
 		}, {
 			APIGroups: []string{"extensions", "networking.k8s.io"},
 			Resources: []string{"ingresses"},

pkg/controllers/monitoring/monitoring-stack/controller.go

@@ -80,6 +80,7 @@ const finalizerName = "monitoring.observability.openshift.io/finalizer"
 
 // RBAC for delegating permissions to Prometheus
 //+kubebuilder:rbac:groups="",resources=pods;services;endpoints,verbs=get;list;watch
+//+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch
 //+kubebuilder:rbac:groups=extensions;networking.k8s.io,resources=ingresses,verbs=get;list;watch
 
 // RBAC for delegating the use of SCC nonroot-v2 (for OpenShift >= 4.11) and nonroot (for OpenShift < 4.11)

pkg/controllers/monitoring/thanos-querier/components.go

@@ -249,6 +249,7 @@ func newServiceMonitor(name string, namespace string, thanos *msoapi.ThanosQueri
 			Labels:    componentLabels(name),
 		},
 		Spec: monv1.ServiceMonitorSpec{
+			ServiceDiscoveryRole: ptr.To(monv1.EndpointSliceRole),
 			Endpoints: []monv1.Endpoint{
 				{
 					Port:   "http",

pkg/controllers/operator/components.go

@@ -86,6 +86,10 @@ func newPrometheusRole(namespace string) *rbacv1.Role {
 			APIGroups: []string{""},
 			Resources: []string{"services", "endpoints", "pods"},
 			Verbs:     []string{"get", "list", "watch"},
+		}, {
+			APIGroups: []string{"discovery.k8s.io"},
+			Resources: []string{"endpointslices"},
+			Verbs:     []string{"get", "list", "watch"},
 		}},
 	}
 }

pkg/controllers/uiplugin/controller.go

@@ -95,6 +95,7 @@ const (
 //+kubebuilder:rbac:groups=apps,resources=daemonsets;deployments;replicasets;statefulsets,verbs=get;list;watch
 //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings;clusterroles;clusterrolebindings,verbs=get;list;watch
 //+kubebuilder:rbac:groups="",resources=configmaps;endpoints;events;namespaces;nodes;persistentvolumeclaims;persistentvolumes;pods;replicationcontrollers;secrets;serviceaccounts;services,verbs=get;list;watch
+//+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch
 //+kubebuilder:rbac:groups=batch,resources=cronjobs;jobs,verbs=get;list;watch
 //+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch
 //+kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch

pkg/controllers/uiplugin/health_analyzer.go

@@ -39,6 +39,11 @@ func newHealthAnalyzerPrometheusRole(namespace string) *rbacv1.Role {
 				Resources: []string{"services", "endpoints", "pods"},
 				Verbs:     []string{"get", "list", "watch"},
 			},
+			{
+				APIGroups: []string{"discovery.k8s.io"},
+				Resources: []string{"endpointslices"},
+				Verbs:     []string{"get", "list", "watch"},
+			},
 		},
 	}
 	return role

pkg/controllers/uiplugin/troubleshooting_panel.go

@@ -194,6 +194,11 @@ func korrel8rClusterRole(name string) *rbacv1.ClusterRole {
 				Resources: []string{"configmaps", "endpoints", "events", "namespaces", "nodes", "pods", "persistentvolumeclaims", "persistentvolumes", "replicationcontrollers", "secrets", "serviceaccounts", "services"},
 				Verbs:     []string{"get", "list", "watch"},
 			},
+			{
+				APIGroups: []string{"discovery.k8s.io"},
+				Resources: []string{"endpointslices"},
+				Verbs:     []string{"get", "list", "watch"},
+			},
 			{
 				APIGroups: []string{"rbac.authorization.k8s.io"},
 				Resources: []string{"roles", "rolebindings", "clusterroles", "clusterrolebindings"},