Merge pull request #855 from MetaCell/feature/postgres_operator_params

feat: Add postgres operator parameters

Author: filippomc

deployment-configuration/helm/templates/auto-database-postgres-operator.yaml

@@ -24,6 +24,14 @@ metadata:
 spec:
   instances: {{ .app.harness.database.postgres.instances | default 1 }}
 
+  {{- with .app.harness.database.postgres.parameters }}
+  postgresql:
+    parameters:
+{{- range $key, $value := . }}
+      {{ $key }}: {{ $value | quote }}
+{{- end }}
+  {{- end }}
+
   inheritedMetadata:
     labels:
       app: {{ .app.harness.database.name | quote }}

deployment-configuration/value-template.yaml

@@ -115,6 +115,8 @@ harness:
       # -- CIDR(s) allowed for CNPG pods to reach the Kubernetes API server (port 443).
       # -- Resolved automatically at deploy time via cluster lookup. Set explicitly only as a fallback for helm-template or air-gapped use.
       apiServerCidr: []
+      # -- PostgreSQL configuration parameters for CloudNative-PG clusters (operator: true). Values must be strings.
+      parameters: {}
       ports:
         - name: http
           port: 5432

docs/applications/databases.md

@@ -87,6 +87,7 @@ harness
       operator: false
       instances: 1
       apiServerCidr: []
+      parameters: {}
       ports:
         - name: http
           port: 5432
@@ -109,6 +110,8 @@ helm install cnpg cloudnative-pg/cloudnative-pg
 
 `apiServerCidr`: List of CIDRs allowed for CNPG database pods to reach the Kubernetes API server on port 443. **Resolved automatically at deploy time** by looking up the `kubernetes` Service and Endpoints in the `default` namespace. The explicit list is only used as a fallback when lookup returns nothing (e.g. `helm template` dry-run). Leave empty (`[]`) for auto-detection; set explicitly only for air-gapped or restricted environments.
 
+`parameters`: Optional map of PostgreSQL configuration parameters rendered to CloudNative-PG as `spec.postgresql.parameters` when `operator: true`. Values must be strings, for example `max_connections: "200"`. CloudNative-PG rejects parameters that are fixed or managed by the operator.
+
 
 #### Neo4j
 
@@ -209,4 +212,3 @@ Further reading: [MongoDB archiving & compression](https://www.mongodb.com/blog/
 Further reading: [pg_dumpall docs](https://www.postgresql.org/docs/10/app-pg-dumpall.html)
 
 
-

docs/model/DatabaseConfig.md

@@ -12,6 +12,7 @@ Name | Type | Description | Notes
 **operator** | **bool** | Use the CloudNative-PG operator instead of a plain Deployment (postgres only) | [optional] 
 **instances** | **int** | Number of PostgreSQL instances managed by the CNPG operator (only used when operator is true) | [optional] 
 **api_server_cidr** | **List[str]** | CIDR(s) allowed for CNPG pods to reach the Kubernetes API server (port 443). Override with your cluster API-server or service CIDR. | [optional] 
+**parameters** | **Dict[str, str]** | PostgreSQL configuration parameters passed to CloudNative-PG as spec.postgresql.parameters (postgres operator only). Values must be strings. | [optional] 
 **initialdb** | **str** | Initial database name (postgres only) | [optional] 
 **args** | **List[str]** | Additional command-line arguments passed to the database server process (postgres only) | [optional] 
 

libraries/models/api/openapi.yaml

@@ -962,6 +962,11 @@ components:
                     type: array
                     items:
                         type: string
+                parameters:
+                    description: 'PostgreSQL configuration parameters passed to CloudNative-PG as spec.postgresql.parameters (postgres operator only). Values must be strings.'
+                    type: object
+                    additionalProperties:
+                        type: string
                 initialdb:
                     description: 'Initial database name (postgres only)'
                     type: string

libraries/models/cloudharness_model/models/database_config.py

@@ -2,7 +2,7 @@
 from os.path import join, dirname as dn, realpath
 import oyaml as yaml
 
-from cloudharness_model import HarnessMainConfig, ApplicationConfig, User, ApplicationHarnessConfig, CDCEvent, ApplicationTestConfig
+from cloudharness_model import HarnessMainConfig, ApplicationConfig, User, ApplicationHarnessConfig, CDCEvent, ApplicationTestConfig, DatabaseConfig
 
 HERE = dn(realpath(__file__))
 
@@ -37,6 +37,27 @@ def test_camelcase():
     assert u.last_name == "a"
     assert u["lastName"] == "a"
 
+
+def test_database_config_parameters_round_trip():
+    config = DatabaseConfig.from_dict({
+        "image": "postgres:17",
+        "operator": True,
+        "parameters": {
+            "max_connections": "200",
+            "shared_buffers": "1GB",
+        },
+    })
+
+    assert config.parameters == {
+        "max_connections": "200",
+        "shared_buffers": "1GB",
+    }
+    assert config.to_dict()["parameters"] == {
+        "max_connections": "200",
+        "shared_buffers": "1GB",
+    }
+
+
 def test_robustness():
     d = {'aliases': [], 'database': {'auto': True, 'mongo': {'image': 'mongo:5', 'ports': [{'name': 'http', 'port': 27017}]}, 'name': 'keycloak-postgres', 'neo4j': {'dbms_security_auth_enabled': 'false', 'image': 'neo4j:4.1.9', 'memory': {'heap': {'initial': '64M', 'max': '128M'}, 'pagecache': {'size': '64M'}, 'size': '256M'}, 'ports': [{'name': 'http', 'port': 7474}, {'name': 'bolt', 'port': 7687}]}, 'pass': 'password', 'postgres': {'image': 'postgres:10.4', 'initialdb': 'auth_db', 'ports': [{'name': 'http', 'port': 5432}]}, 'resources': {'limits': {'cpu': '1000m', 'memory': '2Gi'}, 'requests': {'cpu': '100m', 'memory': '512Mi'}}, 'size': '2Gi', 'type': 'postgres', 'user': 'user'}, 'dependencies': {'build': [], 'hard': [], 'soft': []}, 'deployment': {'auto': True, 'image': 'osb/accounts:3e02a15477b4696ed554e08cedf4109c67908cbe6b03331072b5b73e83b4fc2b', 'name': 'accounts', 'port': 8080, 'replicas': 1, 'resources': {'limits': {'cpu': '500m', 'memory': '1024Mi'}, 'requests': {'cpu': '10m', 'memory': '512Mi'}}}, 'domain': None, 'env': [{'name': 'KEYCLOAK_IMPORT', 'value': '/tmp/realm.json'},
             {'name': 'KEYCLOAK_USER', 'value': 'admin'}, {'name': 'KEYCLOAK_PASSWORD', 'value': 'metacell'}, {'name': 'PROXY_ADDRESS_FORWARDING', 'value': 'true'}, {'name': 'DB_VENDOR', 'value': 'POSTGRES'}, {'name': 'DB_ADDR', 'value': 'keycloak-postgres'}, {'name': 'DB_DATABASE', 'value': 'auth_db'}, {'name': 'DB_USER', 'value': 'user'}, {'name': 'DB_PASSWORD', 'value': 'password'}, {'name': 'JAVA_OPTS', 'value': '-server -Xms64m -Xmx896m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true  --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED --add-exports=jdk.unsupported/sun.reflect=ALL-UNNAMED'}], 'name': 'accounts', 'readinessProbe': {'path': '/realms/master'}, 'resources': [{'dst': '/tmp/realm.json', 'name': 'realm-config', 'src': 'realm.json'}], 'secrets': {}, 'secured': False, 'service': {'auto': True, 'name': 'accounts', 'port': 8080}, 'subdomain': 'accounts', 'uri_role_mapping': [{'roles': ['administrator'], 'uri': '/*'}], 'use_services': []}
@@ -161,4 +182,4 @@ def internal_data(self, value):
 
 
 if __name__ == "__main__":
-    test_property_access()
+    test_property_access()

libraries/models/docs/DatabaseConfig.md

@@ -2,6 +2,8 @@
 from ch_cli_tools.configurationgenerator import *
 from ch_cli_tools.preprocessing import preprocess_build_overrides, generate_hash_based_image_tags
 import pytest
+import shutil
+import subprocess
 
 HERE = os.path.dirname(os.path.realpath(__file__))
 RESOURCES = os.path.join(HERE, 'resources')
@@ -12,6 +14,24 @@ def exists(path):
     return path.exists()
 
 
+def render_helm_chart(chart_path):
+    completed = subprocess.run(
+        ["helm", "template", str(chart_path)],
+        check=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    return [manifest for manifest in yaml.safe_load_all(completed.stdout) if manifest]
+
+
+def find_manifest(manifests, kind, name):
+    for manifest in manifests:
+        if manifest.get("kind") == kind and manifest.get("metadata", {}).get("name") == name:
+            return manifest
+    raise AssertionError(f"Could not find {kind}/{name}")
+
+
 def test_collect_helm_values(tmp_path):
     out_folder = tmp_path / 'test_collect_helm_values'
     values = create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_folder, include=['samples', 'myapp'],
@@ -294,6 +314,57 @@ def test_clear_unused_dbconfig(tmp_path):
     assert db_config['postgres'] is None
 
 
+def test_cnpg_postgres_parameters_render_only_when_set(tmp_path):
+    out_folder = tmp_path / 'test_cnpg_postgres_parameters_render_only_when_set'
+    create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_folder, domain="my.local",
+                      env='withpostgres', local=False, include=["myapp"], exclude=["legacy"])
+
+    helm_path = out_folder / HELM_CHART_PATH
+    shutil.rmtree(helm_path / 'charts')
+    values_path = helm_path / 'values.yaml'
+    with open(values_path, 'r') as values_file:
+        values = yaml.safe_load(values_file)
+    postgres = values['apps']['myapp']['harness']['database']['postgres']
+    postgres['operator'] = True
+    postgres['parameters'] = {
+        # Simulate generated YAML values where on/off can be parsed as booleans before Helm renders the chart.
+        'autovacuum': True,
+        'max_connections': '200',
+        'shared_buffers': '1GB',
+        'synchronous_commit': True,
+        'track_io_timing': False,
+    }
+    with open(values_path, 'w') as values_file:
+        yaml.safe_dump(values, values_file)
+
+    manifests = render_helm_chart(helm_path)
+    db_name = values['apps']['myapp']['harness']['database']['name']
+    cluster = find_manifest(manifests, 'Cluster', db_name)
+    assert cluster['spec']['postgresql']['parameters'] == {
+        'autovacuum': 'true',
+        'max_connections': '200',
+        'shared_buffers': '1GB',
+        'synchronous_commit': 'true',
+        'track_io_timing': 'false',
+    }
+
+    postgres['parameters'] = {}
+    with open(values_path, 'w') as values_file:
+        yaml.safe_dump(values, values_file)
+
+    manifests = render_helm_chart(helm_path)
+    cluster = find_manifest(manifests, 'Cluster', db_name)
+    assert 'postgresql' not in cluster['spec']
+
+    postgres.pop('parameters')
+    with open(values_path, 'w') as values_file:
+        yaml.safe_dump(values, values_file)
+
+    manifests = render_helm_chart(helm_path)
+    cluster = find_manifest(manifests, 'Cluster', db_name)
+    assert 'postgresql' not in cluster['spec']
+
+
 def test_clear_all_dbconfig_if_nodb(tmp_path):
     out_folder = tmp_path / 'test_clear_all_dbconfig_if_nodb'