wip

Signed-off-by: Attila Mészáros <a_meszaros@apple.com>

Author: csviri

micrometer-support/src/main/java/io/javaoperatorsdk/operator/monitoring/micrometer/MicrometerMetrics.java

@@ -21,6 +21,7 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
 
 import org.jspecify.annotations.NonNull;
 
@@ -75,6 +76,7 @@ public class MicrometerMetrics implements Metrics {
   private final MeterRegistry registry;
   private final Map<String, AtomicInteger> gauges = new ConcurrentHashMap<>();
   private final Cleaner cleaner;
+  private final Consumer<Timer.Builder> timerConfig;
 
   /**
    * Creates a MicrometerMetrics instance configured to not collect per-resource metrics, just
@@ -84,7 +86,7 @@ public class MicrometerMetrics implements Metrics {
    * @return a MicrometerMetrics instance configured to not collect per-resource metrics
    */
   public static MicrometerMetrics withoutPerResourceMetrics(MeterRegistry registry) {
-    return new MicrometerMetrics(registry, Cleaner.NOOP, false);
+    return new MicrometerMetrics(registry, Cleaner.NOOP, false, null);
   }
 
   /**
@@ -108,7 +110,7 @@ public static MicrometerMetricsBuilder newMicrometerMetricsBuilder(MeterRegistry
    */
   public static PerResourceCollectingMicrometerMetricsBuilder
       newPerResourceCollectingMicrometerMetricsBuilder(MeterRegistry registry) {
-    return new PerResourceCollectingMicrometerMetricsBuilder(registry);
+    return new PerResourceCollectingMicrometerMetricsBuilder(registry, null);
   }
 
   /**
@@ -119,12 +121,21 @@ public static MicrometerMetricsBuilder newMicrometerMetricsBuilder(MeterRegistry
    * @param registry the {@link MeterRegistry} instance to use for metrics recording
    * @param cleaner the {@link Cleaner} to use
    * @param collectingPerResourceMetrics whether to collect per resource metrics
+   * @param timerConfig optional configuration for timers, defaults to publishing percentiles 0.5,
+   *     0.95, 0.99 and histogram
    */
   private MicrometerMetrics(
-      MeterRegistry registry, Cleaner cleaner, boolean collectingPerResourceMetrics) {
+      MeterRegistry registry,
+      Cleaner cleaner,
+      boolean collectingPerResourceMetrics,
+      Consumer<Timer.Builder> timerConfig) {
     this.registry = registry;
     this.cleaner = cleaner;
     this.collectPerResourceMetrics = collectingPerResourceMetrics;
+    this.timerConfig =
+        timerConfig != null
+            ? timerConfig
+            : builder -> builder.publishPercentiles(0.5, 0.95, 0.99).publishPercentileHistogram();
   }
 
   @Override
@@ -163,12 +174,9 @@ public <T> T timeControllerExecution(ControllerExecution<T> execution) {
     final var tags = new ArrayList<Tag>(16);
     tags.add(Tag.of(CONTROLLER, name));
     addMetadataTags(resourceID, metadata, tags, true);
-    final var timer =
-        Timer.builder(execName)
-            .tags(tags)
-            .publishPercentiles(0.3, 0.5, 0.95)
-            .publishPercentileHistogram()
-            .register(registry);
+    final var timerBuilder = Timer.builder(execName).tags(tags);
+    timerConfig.accept(timerBuilder);
+    final var timer = timerBuilder.register(registry);
     try {
       final var result =
           timer.record(
@@ -379,8 +387,27 @@ public static class PerResourceCollectingMicrometerMetricsBuilder
     private int cleaningThreadsNumber;
     private int cleanUpDelayInSeconds;
 
-    private PerResourceCollectingMicrometerMetricsBuilder(MeterRegistry registry) {
+    private PerResourceCollectingMicrometerMetricsBuilder(
+        MeterRegistry registry, Consumer<Timer.Builder> timerConfig) {
       super(registry);
+      this.executionTimerConfig = timerConfig;
+    }
+
+    /**
+     * Configures the Timer used for timing controller executions. By default, timers are configured
+     * to publish percentiles 0.5, 0.95, 0.99 and a percentile histogram. You can set: {@code
+     * .minimumExpectedValue(Duration.ofMillis(...)).maximumExpectedValue(Duration.ofSeconds(...)) }
+     * so micrometer can create the buckets for you.
+     *
+     * @param executionTimerConfig a consumer that will configure the Timer.Builder. The builder
+     *     will already have the metric name and tags set.
+     * @return this builder for method chaining
+     */
+    @Override
+    public PerResourceCollectingMicrometerMetricsBuilder withExecutionTimerConfig(
+        Consumer<Timer.Builder> executionTimerConfig) {
+      this.executionTimerConfig = executionTimerConfig;
+      return this;
     }
 
     /**
@@ -412,23 +439,38 @@ public PerResourceCollectingMicrometerMetricsBuilder withCleanUpDelayInSeconds(
     public MicrometerMetrics build() {
       final var cleaner =
           new DelayedCleaner(registry, cleanUpDelayInSeconds, cleaningThreadsNumber);
-      return new MicrometerMetrics(registry, cleaner, true);
+      return new MicrometerMetrics(registry, cleaner, true, executionTimerConfig);
     }
   }
 
   public static class MicrometerMetricsBuilder {
     protected final MeterRegistry registry;
     private boolean collectingPerResourceMetrics = true;
+    protected Consumer<Timer.Builder> executionTimerConfig = null;
 
     private MicrometerMetricsBuilder(MeterRegistry registry) {
       this.registry = registry;
     }
 
+    /**
+     * Configures the Timer used for timing controller executions. By default, timers are configured
+     * to publish percentiles 0.5, 0.95, 0.99 and a percentile histogram.
+     *
+     * @param executionTimerConfig a consumer that will configure the Timer.Builder. The builder
+     *     will already have the metric name and tags set.
+     * @return this builder for method chaining
+     */
+    public MicrometerMetricsBuilder withExecutionTimerConfig(
+        Consumer<Timer.Builder> executionTimerConfig) {
+      this.executionTimerConfig = executionTimerConfig;
+      return this;
+    }
+
     /** Configures the instance to collect metrics on a per-resource basis. */
     @SuppressWarnings("unused")
     public PerResourceCollectingMicrometerMetricsBuilder collectingMetricsPerResource() {
       collectingPerResourceMetrics = true;
-      return new PerResourceCollectingMicrometerMetricsBuilder(registry);
+      return new PerResourceCollectingMicrometerMetricsBuilder(registry, executionTimerConfig);
     }
 
     /**
@@ -442,7 +484,8 @@ public MicrometerMetricsBuilder notCollectingMetricsPerResource() {
     }
 
     public MicrometerMetrics build() {
-      return new MicrometerMetrics(registry, Cleaner.NOOP, collectingPerResourceMetrics);
+      return new MicrometerMetrics(
+          registry, Cleaner.NOOP, collectingPerResourceMetrics, executionTimerConfig);
     }
   }
 

observability/josdk-operator-metrics-dashboard.json

@@ -992,7 +992,7 @@
         "type": "prometheus",
         "uid": "prometheus"
       },
-      "description": "Reconciliation retry information",
+      "description": "Current retry attempt number for resources being retried",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -1018,7 +1018,7 @@
             "scaleDistribution": {
               "type": "linear"
             },
-            "showPoints": "never",
+            "showPoints": "auto",
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1035,10 +1035,18 @@
               {
                 "color": "green",
                 "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 3
               }
             ]
           },
-          "unit": "ops"
+          "unit": "short"
         },
         "overrides": []
       },
@@ -1051,7 +1059,7 @@
       "id": 12,
       "options": {
         "legend": {
-          "calcs": ["last", "mean"],
+          "calcs": ["last", "max"],
           "displayMode": "table",
           "placement": "bottom",
           "showLegend": true
@@ -1068,24 +1076,119 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(rate(operator_sdk_reconciliations_started_total{service_name=\"josdk\", operator_sdk_reconciliations_retries_last=\"true\"}[5m]))",
-          "legendFormat": "Last Retry Attempts",
+          "expr": "operator_sdk_reconciliations_retries_number{service_name=\"josdk\"}",
+          "legendFormat": "{{kind}}/{{name}} ({{namespace}})",
           "range": true,
           "refId": "A"
+        }
+      ],
+      "title": "Reconciliation Retry Attempts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "Resources currently on their last retry attempt (1 = last attempt, 0 = not last or no retry)",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "stepAfter",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "text": "No"
+                },
+                "1": {
+                  "text": "Yes"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "short"
         },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 40
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": ["last"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(rate(operator_sdk_reconciliations_started_total{service_name=\"josdk\", operator_sdk_reconciliations_retries_last=\"false\"}[5m]))",
-          "legendFormat": "Retries (Not Last)",
+          "expr": "operator_sdk_reconciliations_retries_last{service_name=\"josdk\"}",
+          "legendFormat": "{{kind}}/{{name}} ({{namespace}})",
           "range": true,
-          "refId": "B"
+          "refId": "A"
         }
       ],
-      "title": "Reconciliation Retry Rate",
+      "title": "Resources on Last Retry Attempt",
       "type": "timeseries"
     }
   ],