improve: remove otel from metrics e2e because of stability issues

Signed-off-by: Attila Mészáros <a_meszaros@apple.com>

Author: csviri

docs/content/en/blog/releases/v5-3-release.md

@@ -98,7 +98,7 @@ A ready-to-use **Grafana dashboard** is included at
 
 The
 [`metrics-processing` sample operator](https://github.com/java-operator-sdk/java-operator-sdk/tree/main/sample-operators/metrics-processing)
-provides a complete end-to-end setup with Prometheus, Grafana, and an OpenTelemetry Collector,
+provides a complete end-to-end setup with Prometheus and Grafana,
 installable via `observability/install-observability.sh`. This is a good starting point for
 verifying metrics in a real cluster.
 

docs/content/en/docs/documentation/observability.md

@@ -128,9 +128,7 @@ All meters use `controller.name` as their primary tag. Counters optionally carry
 \* `namespace` tag is only included when `withNamespaceAsTag()` is enabled.
 
 The execution timer uses explicit boundaries (10ms, 50ms, 100ms, 250ms, 500ms, 1s, 2s, 5s, 10s, 30s) to ensure
-compatibility with `histogram_quantile()` queries in Prometheus. This is important when using the OpenTelemetry Protocol (OTLP) registry, where
-`publishPercentileHistogram()` would otherwise produce Base2 Exponential Histograms that are incompatible with classic
-`_bucket` queries.
+compatibility with `histogram_quantile()` queries in Prometheus.
 
 > **Note on Prometheus metric names**: The exact Prometheus metric name suffix depends on the `MeterRegistry` in use.
 > For `PrometheusMeterRegistry` the timer is exposed as `reconciliations_execution_duration_seconds_*`. For
@@ -144,8 +142,8 @@ A ready-to-use Grafana dashboard is available at
 It visualizes all of the metrics listed above, including reconciliation throughput, error rates, queue depth, active
 executions, resource counts, and execution duration histograms and heatmaps.
 
-The dashboard is designed to work with metrics exported via OpenTelemetry Collector to Prometheus, as set up by the
-observability sample (see below).
+The dashboard is designed to work with metrics scraped directly by Prometheus from the operator's `/metrics` endpoint,
+as set up by the observability sample (see below).
 
 #### Exploring metrics end-to-end
 
@@ -155,7 +153,7 @@ includes a full end-to-end test,
 [`MetricsHandlingE2E`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java),
 that:
 
-1. Installs a local observability stack (Prometheus, Grafana, OpenTelemetry Collector) via
+1. Installs a local observability stack (Prometheus, Grafana) via
    `observability/install-observability.sh`. That imports also the Grafana dashboards.
 2. Runs two reconcilers that produce both successful and failing reconciliations over a sustained period
 3. Verifies that the expected metrics appear in Prometheus

observability/install-observability.sh

@@ -25,7 +25,7 @@ NC='\033[0m' # No Color
 
 echo -e "${GREEN}========================================${NC}"
 echo -e "${GREEN}Installing Observability Stack${NC}"
-echo -e "${GREEN}OpenTelemetry + Prometheus + Grafana${NC}"
+echo -e "${GREEN}Prometheus + Grafana${NC}"
 echo -e "${GREEN}========================================${NC}"
 
 # Check if helm is installed, download locally if not
@@ -47,51 +47,15 @@ fi
 
 # Add Helm repositories
 echo -e "\n${YELLOW}Adding Helm repositories...${NC}"
-helm repo add jetstack https://charts.jetstack.io
-helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm repo update
 echo -e "${GREEN}✓ Helm repositories added${NC}"
 
-echo -e "\n${GREEN}========================================${NC}"
-echo -e "${GREEN}Installing Components (Parallel)${NC}"
-echo -e "${GREEN}========================================${NC}"
-echo -e "The following will be installed:"
-echo -e "  • cert-manager"
-echo -e "  • OpenTelemetry Operator"
-echo -e "  • Prometheus & Grafana"
-echo -e "  • OpenTelemetry Collector"
-echo -e "  • Service Monitors"
-echo -e "\n${YELLOW}All resources will be applied first, then we'll wait for them to become ready.${NC}\n"
-
-# Install cert-manager (required for OpenTelemetry Operator)
-echo -e "\n${YELLOW}Installing cert-manager...${NC}"
-helm upgrade --install cert-manager jetstack/cert-manager \
-    --namespace cert-manager \
-    --create-namespace \
-    --set crds.enabled=true
-echo -e "${GREEN}✓ cert-manager installation or upgrade started${NC}"
-
 # Create observability namespace
 echo -e "\n${YELLOW}Creating observability namespace...${NC}"
 kubectl create namespace observability --dry-run=client -o yaml | kubectl apply -f -
 echo -e "${GREEN}✓ observability namespace ready${NC}"
 
-# Install OpenTelemetry Operator
-echo -e "\n${YELLOW}Installing OpenTelemetry Operator...${NC}"
-
-if helm list -n observability | grep -q opentelemetry-operator; then
-    echo -e "${YELLOW}OpenTelemetry Operator already installed, upgrading...${NC}"
-    helm upgrade opentelemetry-operator open-telemetry/opentelemetry-operator \
-        --namespace observability \
-        --set "manager.collectorImage.repository=otel/opentelemetry-collector-contrib"
-else
-    helm install opentelemetry-operator open-telemetry/opentelemetry-operator \
-        --namespace observability \
-        --set "manager.collectorImage.repository=otel/opentelemetry-collector-contrib"
-fi
-echo -e "${GREEN}✓ OpenTelemetry Operator installation started${NC}"
-
 # Install kube-prometheus-stack (includes Prometheus + Grafana)
 echo -e "\n${YELLOW}Installing Prometheus and Grafana stack...${NC}"
 if helm list -n observability | grep -q kube-prometheus-stack; then
@@ -110,115 +74,12 @@ else
 fi
 echo -e "${GREEN}✓ Prometheus and Grafana installation started${NC}"
 
-# Create OpenTelemetry Collector instance
-echo -e "\n${YELLOW}Creating OpenTelemetry Collector...${NC}"
-cat <<EOF | kubectl apply -f -
-apiVersion: opentelemetry.io/v1beta1
-kind: OpenTelemetryCollector
-metadata:
-  name: otel-collector
-  namespace: observability
-spec:
-  mode: deployment
-  config:
-    receivers:
-      otlp:
-        protocols:
-          grpc:
-            endpoint: 0.0.0.0:4317
-          http:
-            endpoint: 0.0.0.0:4318
-      prometheus:
-        config:
-          scrape_configs:
-            - job_name: 'otel-collector'
-              scrape_interval: 10s
-              static_configs:
-                - targets: ['0.0.0.0:8888']
-
-    processors:
-      batch:
-        timeout: 10s
-      memory_limiter:
-        check_interval: 1s
-        limit_percentage: 75
-        spike_limit_percentage: 15
-
-    exporters:
-      prometheus:
-        endpoint: "0.0.0.0:8889"
-        namespace: ""
-        send_timestamps: true
-        metric_expiration: 5m
-        resource_to_telemetry_conversion:
-          enabled: true
-      debug:
-        verbosity: detailed
-        sampling_initial: 5
-        sampling_thereafter: 200
-
-    service:
-      pipelines:
-        metrics:
-          receivers: [otlp, prometheus]
-          processors: [memory_limiter, batch]
-          exporters: [prometheus, debug]
-        traces:
-          receivers: [otlp]
-          processors: [memory_limiter, batch]
-          exporters: [debug]
-EOF
-echo -e "${GREEN}✓ OpenTelemetry Collector created${NC}"
-
-# Create ServiceMonitor for OpenTelemetry Collector
-echo -e "\n${YELLOW}Creating ServiceMonitor for OpenTelemetry...${NC}"
-cat <<EOF | kubectl apply -f -
-apiVersion: v1
-kind: Service
-metadata:
-  name: otel-collector-prometheus
-  namespace: observability
-  labels:
-    app: otel-collector
-spec:
-  ports:
-  - name: prometheus
-    port: 8889
-    targetPort: 8889
-    protocol: TCP
-  selector:
-    app.kubernetes.io/name: otel-collector-collector
----
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: otel-collector
-  namespace: observability
-  labels:
-    app: otel-collector
-    release: kube-prometheus-stack
-spec:
-  jobLabel: app
-  selector:
-    matchLabels:
-      app: otel-collector
-  endpoints:
-  - port: prometheus
-    interval: 30s
-EOF
-echo -e "${GREEN}✓ ServiceMonitor created${NC}"
-
 # Wait for all pods to be ready
 echo -e "\n${GREEN}========================================${NC}"
 echo -e "${GREEN}All resources have been applied!${NC}"
 echo -e "${GREEN}========================================${NC}"
 echo -e "\n${YELLOW}Waiting for all pods to become ready (this may take 2-3 minutes)...${NC}"
 
-# Wait for cert-manager pods
-echo -e "${YELLOW}Checking cert-manager pods...${NC}"
-kubectl wait --for=condition=ready pod --all -n cert-manager --timeout=300s 2>/dev/null || echo -e "${YELLOW}cert-manager already running or skipped${NC}"
-
-# Wait for observability pods
 echo -e "${YELLOW}Checking observability pods...${NC}"
 kubectl wait --for=condition=ready pod --all -n observability --timeout=300s
 
@@ -276,18 +137,11 @@ echo -e "\n${YELLOW}Prometheus:${NC}"
 echo -e "  Access with: ${GREEN}kubectl port-forward -n observability svc/kube-prometheus-stack-prometheus 9090:9090${NC}"
 echo -e "  Then open: ${GREEN}http://localhost:9090${NC}"
 
-echo -e "\n${YELLOW}OpenTelemetry Collector:${NC}"
-echo -e "  OTLP gRPC endpoint: ${GREEN}otel-collector-collector.observability.svc.cluster.local:4317${NC}"
-echo -e "  OTLP HTTP endpoint: ${GREEN}otel-collector-collector.observability.svc.cluster.local:4318${NC}"
-echo -e "  Prometheus metrics: ${GREEN}http://otel-collector-prometheus.observability.svc.cluster.local:8889/metrics${NC}"
-
-echo -e "\n${YELLOW}Configure your Java Operator to use OpenTelemetry:${NC}"
-echo -e "  Add dependency: ${GREEN}io.javaoperatorsdk:operator-framework-opentelemetry-support${NC}"
-echo -e "  Set environment variables:"
-echo -e "    ${GREEN}OTEL_SERVICE_NAME=your-operator-name${NC}"
-echo -e "    ${GREEN}OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector-collector.observability.svc.cluster.local:4318${NC}"
-echo -e "    ${GREEN}OTEL_METRICS_EXPORTER=otlp${NC}"
-echo -e "    ${GREEN}OTEL_TRACES_EXPORTER=otlp${NC}"
+echo -e "\n${YELLOW}Configure your Java Operator metrics:${NC}"
+echo -e "  Add dependency: ${GREEN}io.javaoperatorsdk:micrometer-support${NC}"
+echo -e "  Add dependency: ${GREEN}io.micrometer:micrometer-registry-prometheus${NC}"
+echo -e "  Expose a ${GREEN}/metrics${NC} endpoint using PrometheusMeterRegistry"
+echo -e "  Create a ServiceMonitor to let Prometheus scrape your operator"
 
 echo -e "\n${GREEN}========================================${NC}"
 echo -e "${GREEN}Grafana Dashboards${NC}"
@@ -303,10 +157,7 @@ echo -e "\n${YELLOW}Note:${NC} Dashboards may take 30-60 seconds to appear in Gr
 
 echo -e "\n${YELLOW}To uninstall:${NC}"
 echo -e "  kubectl delete configmap -n observability jvm-metrics-dashboard josdk-operator-metrics-dashboard"
-echo -e "  kubectl delete -n observability OpenTelemetryCollector otel-collector"
 echo -e "  helm uninstall -n observability kube-prometheus-stack"
-echo -e "  helm uninstall -n observability opentelemetry-operator"
-echo -e "  helm uninstall -n cert-manager cert-manager"
-echo -e "  kubectl delete namespace observability cert-manager"
+echo -e "  kubectl delete namespace observability"
 
 echo -e "\n${GREEN}Done!${NC}"

sample-operators/metrics-processing/k8s/operator.yaml

@@ -39,6 +39,10 @@ spec:
       - name: operator
         image: metrics-processing-operator
         imagePullPolicy: Never
+        ports:
+        - name: metrics
+          containerPort: 8080
+          protocol: TCP
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

sample-operators/metrics-processing/pom.xml

@@ -60,13 +60,7 @@
     </dependency>
     <dependency>
       <groupId>io.micrometer</groupId>
-      <artifactId>micrometer-registry-otlp</artifactId>
-      <version>${micrometer-core.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.yaml</groupId>
-      <artifactId>snakeyaml</artifactId>
-      <version>2.6</version>
+      <artifactId>micrometer-registry-prometheus</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>