wip

Signed-off-by: Attila Mészáros <a_meszaros@apple.com>

Author: csviri

docs/content/en/docs/documentation/observability.md

@@ -87,22 +87,22 @@ The simplest way to create an instance:
 
 ```java
 MeterRegistry registry; // initialize your registry implementation
-Metrics metrics = MicrometerMetricsV2.newPerResourceCollectingMicrometerMetricsBuilder(registry).build();
+Metrics metrics = MicrometerMetricsV2.newMicrometerMetricsV2Builder(registry).build();
 ```
 
 Optionally, include a `namespace` tag on per-reconciliation counters (disabled by default to avoid unexpected
 cardinality increases in existing deployments):
 
 ```java
-Metrics metrics = MicrometerMetricsV2.newPerResourceCollectingMicrometerMetricsBuilder(registry)
+Metrics metrics = MicrometerMetricsV2.newMicrometerMetricsV2Builder(registry)
         .withNamespaceAsTag()
         .build();
 ```
 
 You can also supply a custom timer configuration for `reconciliations.execution.duration`:
 
 ```java
-Metrics metrics = MicrometerMetricsV2.newPerResourceCollectingMicrometerMetricsBuilder(registry)
+Metrics metrics = MicrometerMetricsV2.newMicrometerMetricsV2Builder(registry)
         .withExecutionTimerConfig(builder -> builder.publishPercentiles(0.5, 0.95, 0.99))
         .build();
 ```

micrometer-support/src/main/java/io/javaoperatorsdk/operator/monitoring/micrometer/MicrometerMetricsV2.java

@@ -85,8 +85,7 @@ public class MicrometerMetricsV2 implements Metrics {
    * @return a MicrometerMetricsV2 instance configured to not collect per-resource metrics
    * @see MicrometerMetricsV2Builder
    */
-  public static MicrometerMetricsV2Builder newPerResourceCollectingMicrometerMetricsBuilder(
-      MeterRegistry registry) {
+  public static MicrometerMetricsV2Builder newMicrometerMetricsV2Builder(MeterRegistry registry) {
     return new MicrometerMetricsV2Builder(registry);
   }
 
@@ -176,9 +175,6 @@ public void receivedEvent(Event event, Map<String, Object> metadata) {
       if (resourceEvent.getAction() == ResourceAction.ADDED) {
         gauges.get(numberOfResourcesRefName(getControllerName(metadata))).incrementAndGet();
       }
-      if (resourceEvent.getAction() == ResourceAction.DELETED) {
-        gauges.get(numberOfResourcesRefName(getControllerName(metadata))).decrementAndGet();
-      }
       var namespace = resourceEvent.getRelatedCustomResourceID().getNamespace().orElse(null);
       incrementCounter(
           EVENTS_RECEIVED,

observability/josdk-operator-metrics-dashboard.json

@@ -195,6 +195,21 @@
                 }
               }
             ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "Retries.*"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "orange",
+                  "mode": "fixed"
+                }
+              }
+            ]
           }
         ]
       },
@@ -239,9 +254,20 @@
           "legendFormat": "Failure - {{controller_name}}",
           "range": true,
           "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(reconciliations_retries_total{service_name=~\"$service_name\"}[5m])) by (controller_name)",
+          "legendFormat": "Retries - {{controller_name}}",
+          "range": true,
+          "refId": "C"
         }
       ],
-      "title": "Reconciliation Success vs Failure Rate",
+      "title": "Reconciliation Success / Failure / Retry Rate",
       "type": "timeseries"
     },
     {
@@ -609,7 +635,7 @@
               }
             ]
           },
-          "unit": "s"
+          "unit": "ms"
         },
         "overrides": []
       },

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/monitoring/Metrics.java

@@ -59,9 +59,7 @@ default void receivedEvent(Event event, Map<String, Object> metadata) {}
    */
   @Deprecated(forRemoval = true)
   default void reconcileCustomResource(
-      HasMetadata resource, RetryInfo retryInfo, Map<String, Object> metadata) {
-    submittedForReconciliation(resource, retryInfo, metadata);
-  }
+      HasMetadata resource, RetryInfo retryInfo, Map<String, Object> metadata) {}
 
   /**
    * Called right before a resource is submitted to the ExecutorService for reconciliation.
@@ -71,7 +69,9 @@ default void reconcileCustomResource(
    * @param metadata metadata associated with the resource being processed
    */
   default void submittedForReconciliation(
-      HasMetadata resource, RetryInfo retryInfo, Map<String, Object> metadata) {}
+      HasMetadata resource, RetryInfo retryInfo, Map<String, Object> metadata) {
+    reconcileCustomResource(resource, retryInfo, metadata);
+  }
 
   default void reconciliationExecutionStarted(HasMetadata resource, Map<String, Object> metadata) {}
 
@@ -100,14 +100,16 @@ default void failedReconciliation(
    * @param metadata metadata associated with the resource being processed
    */
   default void successfullyFinishedReconciliation(
-      HasMetadata resource, Map<String, Object> metadata) {}
+      HasMetadata resource, Map<String, Object> metadata) {
+    finishedReconciliation(resource, metadata);
+  }
 
   /**
    * Always called not only if successfully finished.
    *
    * @param resource the {@link ResourceID} associated with the resource being processed
-   * @param retryInfo note that this retry info is in state after {@link RetryExecution#nextDelay()} is
-   *     called in case of exception.
+   * @param retryInfo note that this retry info is in state after {@link RetryExecution#nextDelay()}
+   *     is called in case of exception.
    * @param metadata metadata associated with the resource being processed
    */
   default void reconciliationExecutionFinished(
@@ -128,9 +130,7 @@ default void cleanupDoneFor(ResourceID resourceID, Map<String, Object> metadata)
    * @param metadata metadata associated with the resource being processed
    */
   @Deprecated(forRemoval = true)
-  default void finishedReconciliation(HasMetadata resource, Map<String, Object> metadata) {
-    successfullyFinishedReconciliation(resource, metadata);
-  }
+  default void finishedReconciliation(HasMetadata resource, Map<String, Object> metadata) {}
 
   /**
    * Encapsulates the information about a controller execution i.e. a call to either {@link

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java

@@ -180,7 +180,7 @@ private void submitReconciliationExecution(ResourceState state) {
                 state.deleteEventPresent(),
                 state.isDeleteFinalStateUnknown());
         state.unMarkEventReceived(triggerOnAllEvents());
-        metrics.reconcileCustomResource(latest, state.getRetry(), metricsMetadata);
+        metrics.submittedForReconciliation(latest, state.getRetry(), metricsMetadata);
         log.debug("Executing events for custom resource. Scope: {}", executionScope);
         executor.execute(new ReconcilerExecutor(resourceID, executionScope));
       } else {
@@ -286,7 +286,7 @@ synchronized void eventProcessingFinished(
       return;
     }
     cleanupOnSuccessfulExecution(executionScope);
-    metrics.finishedReconciliation(executionScope.getResource(), metricsMetadata);
+    metrics.successfullyFinishedReconciliation(executionScope.getResource(), metricsMetadata);
     if ((triggerOnAllEvents() && executionScope.isDeleteEvent())
         || (!triggerOnAllEvents() && state.deleteEventPresent())) {
       cleanupForDeletedEvent(executionScope.getResourceID());

operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java

@@ -338,7 +338,8 @@ void startProcessedMarkedEventReceivedBefore() {
     eventProcessor.start();
 
     verify(reconciliationDispatcherMock, timeout(100).times(1)).handleExecution(any());
-    verify(metricsMock, times(1)).reconcileCustomResource(any(HasMetadata.class), isNull(), any());
+    verify(metricsMock, times(1))
+        .submittedForReconciliation(any(HasMetadata.class), isNull(), any());
   }
 
   @Test

sample-operators/metrics-processing/src/main/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingSampleOperator.java

@@ -123,8 +123,7 @@ public Duration step() {
     new ProcessorMetrics().bindTo(compositeRegistry);
     new UptimeMetrics().bindTo(compositeRegistry);
 
-    return MicrometerMetricsV2.newPerResourceCollectingMicrometerMetricsBuilder(compositeRegistry)
-        .build();
+    return MicrometerMetricsV2.newMicrometerMetricsV2Builder(compositeRegistry).build();
   }
 
   @SuppressWarnings("unchecked")