Merge branch '2922-configure-exporter-queries-and-configuration' into 'main'

Resolve "Configure Exporter Queries And Configuration"

Closes #2922

See merge request ongresinc/stackgres!1662

Author: teoincontatto

stackgres-k8s/src/common/src/main/java/io/stackgres/common/crd/sgcluster/StackGresClusterConfigurations.java

@@ -46,6 +46,9 @@ public class StackGresClusterConfigurations {
   @Valid
   private StackGresClusterObservability observability;
 
+  @Valid
+  private StackGresClusterPostgresExporter postgresExporter;
+
   @ReferencedField("sgPostgresConfig")
   interface SgPostgresConfig extends FieldReference {
   }
@@ -114,10 +117,18 @@ public void setObservability(StackGresClusterObservability observability) {
     this.observability = observability;
   }
 
+  public StackGresClusterPostgresExporter getPostgresExporter() {
+    return postgresExporter;
+  }
+
+  public void setPostgresExporter(StackGresClusterPostgresExporter postgresExporter) {
+    this.postgresExporter = postgresExporter;
+  }
+
   @Override
   public int hashCode() {
-    return Objects.hash(backups, binding, credentials, observability, patroni, sgPoolingConfig,
-        sgPostgresConfig);
+    return Objects.hash(backups, binding, credentials, observability, patroni, postgresExporter,
+        sgPoolingConfig, sgPostgresConfig);
   }
 
   @Override
@@ -133,6 +144,7 @@ public boolean equals(Object obj) {
         && Objects.equals(credentials, other.credentials)
         && Objects.equals(observability, other.observability)
         && Objects.equals(patroni, other.patroni)
+        && Objects.equals(postgresExporter, other.postgresExporter)
         && Objects.equals(sgPoolingConfig, other.sgPoolingConfig)
         && Objects.equals(sgPostgresConfig, other.sgPostgresConfig);
   }

stackgres-k8s/src/common/src/main/java/io/stackgres/common/crd/sgcluster/StackGresClusterPostgresExporter.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2019 OnGres, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package io.stackgres.common.crd.sgcluster;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import io.stackgres.common.StackGresUtil;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class StackGresClusterPostgresExporter {
+
+  private StackGresClusterPostgresExporterQueries queries;
+
+  public StackGresClusterPostgresExporterQueries getQueries() {
+    return queries;
+  }
+
+  public void setQueries(StackGresClusterPostgresExporterQueries queries) {
+    this.queries = queries;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(queries);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (!(obj instanceof StackGresClusterPostgresExporter)) {
+      return false;
+    }
+    StackGresClusterPostgresExporter other = (StackGresClusterPostgresExporter) obj;
+    return Objects.equals(queries, other.queries);
+  }
+
+  public String toString() {
+    return StackGresUtil.toPrettyYaml(this);
+  }
+}

stackgres-k8s/src/common/src/main/java/io/stackgres/common/crd/sgcluster/StackGresClusterPostgresExporterQueries.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2019 OnGres, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package io.stackgres.common.crd.sgcluster;
+
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import io.stackgres.common.StackGresUtil;
+import io.stackgres.common.crd.JsonObject;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class StackGresClusterPostgresExporterQueries extends JsonObject {
+
+  public StackGresClusterPostgresExporterQueries() {
+    super();
+  }
+
+  public StackGresClusterPostgresExporterQueries(Map<String, Object> m) {
+    super(m);
+  }
+
+  public String toString() {
+    return StackGresUtil.toPrettyYaml(this);
+  }
+}

stackgres-k8s/src/common/src/main/java/io/stackgres/common/crd/sgshardedcluster/StackGresShardedClusterConfigurations.java

@@ -14,6 +14,7 @@
 import io.stackgres.common.StackGresUtil;
 import io.stackgres.common.crd.sgcluster.StackGresClusterCredentials;
 import io.stackgres.common.crd.sgcluster.StackGresClusterObservability;
+import io.stackgres.common.crd.sgcluster.StackGresClusterPostgresExporter;
 import io.stackgres.common.crd.sgcluster.StackGresClusterServiceBinding;
 import io.sundr.builder.annotations.Buildable;
 import jakarta.validation.Valid;
@@ -38,6 +39,9 @@ public class StackGresShardedClusterConfigurations {
   @Valid
   private StackGresClusterObservability observability;
 
+  @Valid
+  private StackGresClusterPostgresExporter postgresExporter;
+
   public List<StackGresShardedClusterBackupConfiguration> getBackups() {
     return backups;
   }
@@ -70,9 +74,17 @@ public void setObservability(StackGresClusterObservability observability) {
     this.observability = observability;
   }
 
+  public StackGresClusterPostgresExporter getPostgresExporter() {
+    return postgresExporter;
+  }
+
+  public void setPostgresExporter(StackGresClusterPostgresExporter postgresExporter) {
+    this.postgresExporter = postgresExporter;
+  }
+
   @Override
   public int hashCode() {
-    return Objects.hash(backups, binding, credentials, observability);
+    return Objects.hash(backups, binding, credentials, observability, postgresExporter);
   }
 
   @Override
@@ -86,7 +98,8 @@ public boolean equals(Object obj) {
     StackGresShardedClusterConfigurations other = (StackGresShardedClusterConfigurations) obj;
     return Objects.equals(backups, other.backups) && Objects.equals(binding, other.binding)
         && Objects.equals(credentials, other.credentials)
-        && Objects.equals(observability, other.observability);
+        && Objects.equals(observability, other.observability)
+        && Objects.equals(postgresExporter, other.postgresExporter);
   }
 
   @Override

stackgres-k8s/src/common/src/main/resources/crds/SGCluster.yaml

@@ -1080,6 +1080,19 @@ spec:
                       If not set, a default configuration will be used. Disabling connection pooling altogether is possible if the disableConnectionPooling property of the pods object is set to true.
 
                       **Changing this field may require a restart.**
+                  postgresExporter:
+                    type: object
+                    description: Section to configure the [postgres_exporter](https://github.com/prometheus-community/postgres_exporter) component.
+                    properties:
+                      queries:
+                        type: object
+                        description: |
+                          Section to configure the [postgres_exporter](https://github.com/prometheus-community/postgres_exporter) queries.yaml configuration.
+                          
+                          The queries defined here will overwrite those created by the operator.
+                          
+                          **WARNING**: Changing this may beak some of the functionality that depend on the query overwritten.
+                        "x-kubernetes-preserve-unknown-fields": true
                   observability:
                     type: object
                     description: Allow to specify Observability configuration (related to logs, metrics and traces)

stackgres-k8s/src/common/src/main/resources/crds/SGShardedCluster.yaml

@@ -563,6 +563,19 @@ spec:
                       - sgObjectStorage: 'backupconf'
                   ```
                 properties:
+                  postgresExporter:
+                    type: object
+                    description: Section to configure the [postgres_exporter](https://github.com/prometheus-community/postgres_exporter) component.
+                    properties:
+                      queries:
+                        type: object
+                        description: |
+                          Section to configure the [postgres_exporter](https://github.com/prometheus-community/postgres_exporter) queries.yaml configuration.
+                          
+                          The queries defined here will overwrite those created by the operator.
+                          
+                          **WARNING**: Changing this may beak some of the functionality that depend on the query overwritten.
+                        "x-kubernetes-preserve-unknown-fields": true
                   observability:
                     type: object
                     description: Allow to specify Observability configuration (related to logs, metrics and traces)
@@ -1443,6 +1456,19 @@ spec:
                           If sharding type is `shardingsphere` then this field is ignored.
 
                           **Changing this field may require a restart.**
+                      postgresExporter:
+                        type: object
+                        description: Section to configure the [postgres_exporter](https://github.com/prometheus-community/postgres_exporter) component.
+                        properties:
+                          queries:
+                            type: object
+                            description: |
+                              Section to configure the [postgres_exporter](https://github.com/prometheus-community/postgres_exporter) queries.yaml configuration.
+                              
+                              The queries defined here will overwrite those created by the operator.
+                              
+                              **WARNING**: Changing this may beak some of the functionality that depend on the query overwritten.
+                            "x-kubernetes-preserve-unknown-fields": true
                       patroni: &patroni
                         type: object
                         description: |
@@ -2086,6 +2112,19 @@ spec:
                           If not set, a default configuration will be used. Disabling connection pooling altogether is possible if the disableConnectionPooling property of the pods object is set to true.
 
                           **Changing this field may require a restart.**
+                      postgresExporter:
+                        type: object
+                        description: Section to configure the [postgres_exporter](https://github.com/prometheus-community/postgres_exporter) component.
+                        properties:
+                          queries:
+                            type: object
+                            description: |
+                              Section to configure the [postgres_exporter](https://github.com/prometheus-community/postgres_exporter) queries.yaml configuration.
+                              
+                              The queries defined here will overwrite those created by the operator.
+                              
+                              **WARNING**: Changing this may beak some of the functionality that depend on the query overwritten.
+                            "x-kubernetes-preserve-unknown-fields": true
                       patroni: *patroni
                   replication: *replication
                   metadata:
@@ -2384,6 +2423,19 @@ spec:
                                 Name of the [SGPoolingConfig](https://stackgres.io/doc/latest/reference/crd/sgpoolconfig) used for this cluster. Each pod contains a sidecar with a connection pooler (currently: [PgBouncer](https://www.pgbouncer.org/)). The connection pooler is implemented as a sidecar.
         
                                 If not set, a default configuration will be used. Disabling connection pooling altogether is possible if the disableConnectionPooling property of the pods object is set to true.
+                            postgresExporter:
+                              type: object
+                              description: Section to configure the [postgres_exporter](https://github.com/prometheus-community/postgres_exporter) component.
+                              properties:
+                                queries:
+                                  type: object
+                                  description: |
+                                    Section to configure the [postgres_exporter](https://github.com/prometheus-community/postgres_exporter) queries.yaml configuration.
+                                    
+                                    The queries defined here will overwrite those created by the operator.
+                                    
+                                    **WARNING**: Changing this may beak some of the functionality that depend on the query overwritten.
+                                  "x-kubernetes-preserve-unknown-fields": true
                         replication: *replication
                         metadata:
                           type: object

stackgres-k8s/src/operator/src/main/java/io/stackgres/operator/conciliation/factory/cluster/sidecars/pgexporter/PostgresExporter.java

@@ -7,14 +7,12 @@
 
 import static io.stackgres.common.StackGresUtil.getDefaultPullPolicy;
 
-import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Stream;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
 import com.google.common.io.Resources;
@@ -37,7 +35,9 @@
 import io.stackgres.common.crd.VolumeBuilder;
 import io.stackgres.common.crd.VolumeMountBuilder;
 import io.stackgres.common.crd.sgcluster.StackGresCluster;
+import io.stackgres.common.crd.sgcluster.StackGresClusterConfigurations;
 import io.stackgres.common.crd.sgcluster.StackGresClusterPods;
+import io.stackgres.common.crd.sgcluster.StackGresClusterPostgresExporter;
 import io.stackgres.common.crd.sgcluster.StackGresClusterSpec;
 import io.stackgres.common.labels.LabelFactoryForCluster;
 import io.stackgres.operator.common.Sidecar;
@@ -218,33 +218,6 @@ private HasMetadata buildSource(StackGresClusterContext context) {
             .map(StackGresClusterPods::getDisableConnectionPooling)
             .orElse(false);
 
-    if (pgBouncerIsDisabled) {
-      final YAMLMapper yamlMapper = yamlMapperProvider.get();
-      final ObjectNode queries;
-      final String data;
-      try {
-        queries = (ObjectNode) yamlMapper
-                .readTree(PostgresExporter.class.getResource("/prometheus-postgres-exporter/queries.yaml"));
-
-        var fieldNames = Seq.seq(queries.fieldNames()).toList();
-        for (var fieldName : fieldNames) {
-          if (fieldName.startsWith(POSTGRES_EXPORTER_PGBOUNCER_QUERIES_PREFIX)) {
-            queries.remove(fieldName);
-          }
-        }
-
-        data = yamlMapper.writeValueAsString(queries);
-
-        builder.withData(Map.of(QUERIES_YAML, data));
-
-        return builder.build();
-      } catch (JsonProcessingException e) {
-        throw new RuntimeException("couldn't serialize prometheus postgres exporter queries to a string", e);
-      } catch (IOException e) {
-        throw new RuntimeException("couldn't read prometheus postgres exporter queries file", e);
-      }
-    }
-
     final String queriesResourcePath;
 
     final long versionAsNumber = StackGresVersion.getStackGresVersionAsNumber(context.getCluster());
@@ -254,13 +227,47 @@ private HasMetadata buildSource(StackGresClusterContext context) {
       queriesResourcePath = "/prometheus-postgres-exporter/queries.yaml";
     }
 
+    final YAMLMapper yamlMapper = yamlMapperProvider.get();
+    final ObjectNode queries;
+    final String data;
+    try {
+      queries = (ObjectNode) yamlMapper
+          .readTree(Unchecked.supplier(
+              () -> Resources.asCharSource(
+                  Objects.requireNonNull(PostgresExporter.class.getResource(queriesResourcePath)),
+                  StandardCharsets.UTF_8)
+              .read()).get());
+    } catch (Exception ex) {
+      throw new RuntimeException("Error while deserializing " + queriesResourcePath, ex);
+    }
+
+    var fieldNames = Seq.seq(queries.fieldNames()).toList();
+    if (pgBouncerIsDisabled) {
+      for (var fieldName : fieldNames) {
+        if (fieldName.startsWith(POSTGRES_EXPORTER_PGBOUNCER_QUERIES_PREFIX)) {
+          queries.remove(fieldName);
+        }
+      }
+    }
+
+    Optional<ObjectNode> customQueriesFound = Optional.of(context.getCluster())
+        .map(StackGresCluster::getSpec)
+        .map(StackGresClusterSpec::getConfigurations)
+        .map(StackGresClusterConfigurations::getPostgresExporter)
+        .map(StackGresClusterPostgresExporter::getQueries)
+        .map(yamlMapper::valueToTree);
+
+    customQueriesFound.ifPresent(customQueries -> Seq.seq(customQueries.fields())
+        .forEach(customQuery -> queries.set(customQuery.getKey(), customQuery.getValue())));
+
+    try {
+      data = yamlMapper.writeValueAsString(queries);
+    } catch (Exception ex) {
+      throw new RuntimeException("Error while serializing queries", ex);
+    }
+
     return builder
-        .withData(Map.of(QUERIES_YAML,
-            Unchecked.supplier(() ->
-            Resources
-            .asCharSource(
-                Objects.requireNonNull(PostgresExporter.class.getResource(queriesResourcePath)),
-                StandardCharsets.UTF_8).read()).get()))
+        .withData(Map.of(QUERIES_YAML, data))
         .build();
   }