feat: added metrics and allow to set a low priority separate queue for thread pool

Author: teoincontatto

stackgres-k8s/src/cluster-controller/src/main/java/io/stackgres/cluster/configuration/ClusterControllerJmxCollectorRegistry.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright (C) 2019 OnGres, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package io.stackgres.cluster.configuration;
+
+import javax.management.MalformedObjectNameException;
+
+import io.stackgres.common.ClusterControllerProperty;
+import io.stackgres.common.metrics.AbstractJmxCollectorRegistry;
+import jakarta.inject.Singleton;
+
+@Singleton
+public class ClusterControllerJmxCollectorRegistry extends AbstractJmxCollectorRegistry {
+
+  public ClusterControllerJmxCollectorRegistry() throws MalformedObjectNameException {
+    super(ClusterControllerProperty.CLUSTER_CONTROLLER_JMX_COLLECTOR_YAML_CONFIG
+        .get().orElse(""));
+  }
+
+}

stackgres-k8s/src/cluster-controller/src/main/resources/application.properties

@@ -38,3 +38,6 @@ quarkus.http.auth.basic=true
 org.jooq.no-logo=true
 quarkus.log.category."org.jooq.Constants".level=OFF
 quarkus.log.category."org.jooq.tools.LoggerListener".level=OFF
+
+# Use custom micrometer producer
+quarkus.micrometer.export.prometheus.default-registry=false

stackgres-k8s/src/common/pom.xml

@@ -45,6 +45,14 @@
       <groupId>com.ongres</groupId>
       <artifactId>fluent-process</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.quarkus</groupId>
+      <artifactId>quarkus-micrometer-registry-prometheus</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.prometheus.jmx</groupId>
+      <artifactId>collector</artifactId>
+    </dependency>
 
     <dependency>
       <groupId>io.stackgres</groupId>

stackgres-k8s/src/common/src/main/java/io/stackgres/common/ClusterControllerProperty.java

@@ -26,7 +26,9 @@ public enum ClusterControllerProperty implements StackGresPropertyReader {
   CLUSTER_CONTROLLER_RECONCILE_MANAGED_SQL(
       "stackgres.clusterReconcileManagedSql"),
   CLUSTER_CONTROLLER_RECONCILE_PATRONI_AFTER_MAJOR_VERSION_UPGRADE(
-      "stackgres.clusterReconcilePatroniAfterMajorVersionUpgrade");
+      "stackgres.clusterReconcilePatroniAfterMajorVersionUpgrade"),
+  CLUSTER_CONTROLLER_JMX_COLLECTOR_YAML_CONFIG(
+      "stackgres.cluster.controllerJmxCollectorYamlConfig");
 
   private static final Properties APPLICATION_PROPERTIES =
       StackGresPropertyReader.readApplicationProperties(ClusterControllerProperty.class);

stackgres-k8s/src/common/src/main/java/io/stackgres/common/OperatorProperty.java

@@ -46,11 +46,14 @@ public enum OperatorProperty implements StackGresPropertyReader {
   FORCE_UNLOCK_OPERATOR("stackgres.forceUnlockOperator"),
   STOP_AFTER_BOOTSTRAP("stackgres.stopAfterBootstrap"),
   RECONCILIATION_ENABLE_THREAD_POOL("stackgres.reconciliationEnableThreadPool"),
+  RECONCILIATION_THREAD_POOL_METRICS_SCRAPE_INTERVAL("stackgres.reconciliationThreadPoolMetricsScrapeInterval"),
   RECONCILIATION_THREADS("stackgres.reconciliationThreads"),
+  RECONCILIATION_LOW_PRIORITY_THREADS("stackgres.reconciliationLowPriorityThreads"),
   RECONCILIATION_PRIORITY_TIMEOUT("stackgres.reconciliationPriorityTimeout"),
   RECONCILIATION_INITIAL_BACKOFF("stackgres.reconciliationInitialBackoff"),
   RECONCILIATION_MAX_BACKOFF("stackgres.reconciliationMaxBackoff"),
-  RECONCILIATION_BACKOFF_VARIATION("stackgres.reconciliationBackoffVariation");
+  RECONCILIATION_BACKOFF_VARIATION("stackgres.reconciliationBackoffVariation"),
+  JMX_COLLECTOR_YAML_CONFIG("stackgres.jmxCollectorYamlConfig");
 
   private static final Properties APPLICATION_PROPERTIES =
       StackGresPropertyReader.readApplicationProperties(OperatorProperty.class);

stackgres-k8s/src/common/src/main/java/io/stackgres/common/metrics/AbstractJmxCollectorRegistry.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2019 OnGres, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package io.stackgres.common.metrics;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+
+import javax.management.MalformedObjectNameException;
+
+import io.prometheus.jmx.JmxCollector;
+import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter;
+import io.prometheus.metrics.model.registry.PrometheusRegistry;
+import io.stackgres.common.CdiUtil;
+
+public abstract class AbstractJmxCollectorRegistry {
+  private final PrometheusRegistry collectorRegistry;
+  private final PrometheusTextFormatWriter textFormatter = new PrometheusTextFormatWriter(true);
+
+  public AbstractJmxCollectorRegistry(String yamlConfig) throws MalformedObjectNameException {
+    new JmxCollector(yamlConfig).register(PrometheusRegistry.defaultRegistry);
+    this.collectorRegistry = PrometheusRegistry.defaultRegistry;
+  }
+
+  public AbstractJmxCollectorRegistry() {
+    CdiUtil.checkPublicNoArgsConstructorIsCalledToCreateProxy(getClass());
+    this.collectorRegistry = null;
+  }
+
+  public String scrape(String contentType) {
+    try (
+        ByteArrayOutputStream stream = new ByteArrayOutputStream();
+        OutputStreamWriter writer = new OutputStreamWriter(stream, StandardCharsets.UTF_8)) {
+      textFormatter.write(stream, collectorRegistry.scrape());
+      return stream.toString(StandardCharsets.UTF_8);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+}

stackgres-k8s/src/common/src/main/java/io/stackgres/common/metrics/CustomPrometheusMeterRegistry.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2019 OnGres, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package io.stackgres.common.metrics;
+
+import io.micrometer.prometheus.PrometheusConfig;
+import io.micrometer.prometheus.PrometheusMeterRegistry;
+import jakarta.inject.Singleton;
+
+@Singleton
+public class CustomPrometheusMeterRegistry extends PrometheusMeterRegistry {
+
+  private final AbstractJmxCollectorRegistry jmxCollectorRegistry;
+
+  public CustomPrometheusMeterRegistry(
+      PrometheusConfig config,
+      AbstractJmxCollectorRegistry jmxCollectorRegistry) {
+    super(config);
+    this.jmxCollectorRegistry = jmxCollectorRegistry;
+  }
+
+  @Override
+  public String scrape(String contentType) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(super.scrape(contentType));
+    if (jmxCollectorRegistry != null) {
+      sb.append(jmxCollectorRegistry.scrape(contentType));
+    }
+    return sb.toString();
+  }
+}

stackgres-k8s/src/jobs/src/main/java/io/stackgres/jobs/Main.java

@@ -7,7 +7,7 @@
 
 import io.quarkus.runtime.QuarkusApplication;
 import io.quarkus.runtime.annotations.QuarkusMain;
-import io.stackgres.jobs.app.JobsProperty;
+import io.stackgres.jobs.configuration.JobsProperty;
 import io.stackgres.jobs.dbops.DbOpsLauncher;
 import jakarta.inject.Inject;
 

stackgres-k8s/src/jobs/src/main/java/io/stackgres/jobs/configuration/JobsJmxCollectorRegistry.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2019 OnGres, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package io.stackgres.jobs.configuration;
+
+import javax.management.MalformedObjectNameException;
+
+import io.stackgres.common.metrics.AbstractJmxCollectorRegistry;
+import jakarta.inject.Singleton;
+
+@Singleton
+public class JobsJmxCollectorRegistry extends AbstractJmxCollectorRegistry {
+
+  public JobsJmxCollectorRegistry() throws MalformedObjectNameException {
+    super(JobsProperty.JOBS_JMX_COLLECTOR_YAML_CONFIG.get().orElse(""));
+  }
+
+}

…/io/stackgres/jobs/app/JobsProperty.java …res/jobs/configuration/JobsProperty.javastackgres-k8s/src/jobs/src/main/java/io/stackgres/jobs/app/JobsProperty.java renamed to stackgres-k8s/src/jobs/src/main/java/io/stackgres/jobs/configuration/JobsProperty.java stackgres-k8s/src/jobs/src/main/java/io/stackgres/jobs/app/JobsProperty.java renamed to stackgres-k8s/src/jobs/src/main/java/io/stackgres/jobs/configuration/JobsProperty.java

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
-package io.stackgres.jobs.app;
+package io.stackgres.jobs.configuration;
 
 import java.util.Properties;
 
@@ -20,7 +20,8 @@ public enum JobsProperty implements StackGresPropertyReader {
   SERVICE_ACCOUNT("stackgres.dbops.serviceAccount"),
   POD_NAME("stackgres.dbops.podName"),
   DBOPS_LOCK_POLL_INTERVAL("stackgres.dbops.lockPollInterval"),
-  DBOPS_LOCK_DURATION("stackgres.dbops.lockDuration");
+  DBOPS_LOCK_DURATION("stackgres.dbops.lockDuration"),
+  JOBS_JMX_COLLECTOR_YAML_CONFIG("stackgres.jobs.jmxCollectorYamlConfig");
 
   private static final Properties APPLICATION_PROPERTIES =
       StackGresPropertyReader.readApplicationProperties(JobsProperty.class);