feat: Add generic database connection mechanism (#1163)

* WIP

* postgres -> postgresql

* Store username and password envs separately

* Better name modules

* Remove leftover code

* Switch to config-utils templating

* Add TODO marker

* Add MySQL

* Improve Derby JDBC driver

* empty commit

* Auto-create Derby database

* Update module structure

* clippy

* cargo fmt

* Implement CeleryConnection for Postgresql

* Support specifying the templating mechanism

* fix: Use $VAR instead of ${VAR} (for Airflow)

* Revert "fix: Use $VAR instead of ${VAR} (for Airflow)"

This reverts commit 8e05224.

It was actually mot needed :)

* Add lots of docs

* Improve docs

* Apply suggestions from code review

Co-authored-by: Malte Sander <malte.sander.it@gmail.com>

* refactor!: Take IntoIterator<Item = EnvVar> instead

* Remove too specific JDBC

* JDBC -> Jdbc and SQLAlchemy -> SqlAlchemy

* Update crates/stackable-operator/src/databases/databases/derby.rs

Co-authored-by: Techassi <git@techassi.dev>

* fix compilation

* refactor: Let connection_parameters_as_url_query_parameters return Option<String>

* Put "org.postgresql.Driver" behind POSTGRES_JDBC_DRIVER_CLASS constant

* refactor: Switch derby location from String to PathBuf

* Clarify what add_to_container does

* Remove bullet points

* Add docs on TemplatingMechanism

* Rename "databases" module to "database_connections"

* Explain context(false) and improve error messages

* Add TODO on connection param escaping

* Add comment on airflow-op switching to config files

* Address field name feedback; URI -> URL

* Leftovers I forgot to commit

* Rename env_var_from_secret to env_var_with_value_from_secret

* Add comment on snafu error

* Move Derby JDBC driver into consant

* Update CHANGELOG.md

---------

Co-authored-by: Malte Sander <malte.sander.it@gmail.com>
Co-authored-by: Techassi <git@techassi.dev>

Author: 3 people

crates/stackable-operator/CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+
+- Add generic database connection mechanism ([#1163]).
+
+### Changed
+
+- BREAKING: Change signature of `ContainerBuilder::add_env_vars` from `Vec<EnvVar>` to `IntoIterator<Item = EnvVar>` ([#1163]).
+
+[#1163]: https://github.com/stackabletech/operator-rs/pull/1163
+
 ## [0.109.0] - 2026-04-07
 
 ### Added

crates/stackable-operator/crds/DummyCluster.yaml

@@ -82,6 +82,208 @@ spec:
                       and `stopped` will take no effect until `reconciliationPaused` is set to false or removed.
                     type: boolean
                 type: object
+              databaseConnection:
+                oneOf:
+                - required:
+                  - postgresql
+                - required:
+                  - mysql
+                - required:
+                  - derby
+                - required:
+                  - redis
+                - required:
+                  - genericJdbc
+                - required:
+                  - genericSqlAlchemy
+                - required:
+                  - genericCelery
+                properties:
+                  derby:
+                    description: |-
+                      Connection settings for an embedded [Apache Derby](https://db.apache.org/derby/) database.
+
+                      Derby is an embedded, file-based Java database engine that requires no separate server process.
+                      It is typically used for development, testing, or as a lightweight metastore backend (e.g. for
+                      Apache Hive).
+                    properties:
+                      location:
+                        description: |-
+                          Path on the filesystem where Derby stores its database files.
+
+                          If not specified, defaults to `/tmp/derby/{unique_database_name}/derby.db`.
+                          The `{unique_database_name}` part is automatically handled by the operator and is added to
+                          prevent clashing database files. The `create=true` flag is always appended to the JDBC URL,
+                          so the database is created automatically if it does not yet exist at this location.
+                        nullable: true
+                        type: string
+                    type: object
+                  genericCelery:
+                    description: |-
+                      A generic Celery database connection for broker or result backend types not covered by a
+                      dedicated variant.
+
+                      Use this when you need a Celery-compatible connection that does not have a first-class
+                      connection type. The complete connection URL is read from a Secret, giving the user full
+                      control over the connection string.
+                    properties:
+                      connectionUrlSecretName:
+                        description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL.
+                        type: string
+                    required:
+                    - connectionUrlSecretName
+                    type: object
+                  genericJdbc:
+                    description: |-
+                      A generic JDBC database connection for database types not covered by a dedicated variant.
+
+                      Use this when you need to connect to a JDBC-compatible database that does not have a
+                      first-class connection type. You are responsible for providing the correct driver class name
+                      and a fully-formed JDBC URL as well as providing the needed classes on the Java classpath.
+                    properties:
+                      credentialsSecretName:
+                        description: |-
+                          Name of a Secret containing the `username` and `password` keys used to authenticate
+                          against the database.
+                        type: string
+                      driver:
+                        description: |-
+                          Fully-qualified Java class name of the JDBC driver, e.g. `org.postgresql.Driver` or
+                          `com.mysql.jdbc.Driver`. The driver JAR must be provided by you on the classpath.
+                        type: string
+                      url:
+                        description: |-
+                          The JDBC connection URL, e.g. `jdbc:postgresql://my-host:5432/mydb`. Credentials must
+                          not be embedded in this URL; they are instead injected via environment variables sourced
+                          from `credentials_secret`.
+                        format: uri
+                        type: string
+                    required:
+                    - credentialsSecretName
+                    - driver
+                    - url
+                    type: object
+                  genericSqlAlchemy:
+                    description: |-
+                      A generic SQLAlchemy database connection for database types not covered by a dedicated variant.
+
+                      Use this when you need to connect to a SQLAlchemy-compatible database that does not have a
+                      first-class connection type. The complete connection URL is read from a Secret, giving the user
+                      full control over the connection string including any driver-specific options.
+                    properties:
+                      connectionUrlSecretName:
+                        description: The name of the Secret that contains an `connectionUrl` key with the complete SQLAlchemy URL.
+                        type: string
+                    required:
+                    - connectionUrlSecretName
+                    type: object
+                  mysql:
+                    description: Connection settings for a [MySQL](https://www.mysql.com/) database.
+                    properties:
+                      credentialsSecretName:
+                        description: |-
+                          Name of a Secret containing the `username` and `password` keys used to authenticate
+                          against the MySQL server.
+                        type: string
+                      database:
+                        description: Name of the database (schema) to connect to.
+                        type: string
+                      host:
+                        description: Hostname or IP address of the MySQL server.
+                        type: string
+                      parameters:
+                        additionalProperties:
+                          type: string
+                        default: {}
+                        description: |-
+                          Additional map of connection parameters to append to the connection URL. The given
+                          `HashMap<String, String>` will be converted to query parameters in the form of
+                          `?param1=value1&param2=value2`.
+                        type: object
+                      port:
+                        default: 3306
+                        description: Port the MySQL server is listening on. Defaults to `3306`.
+                        format: uint16
+                        maximum: 65535.0
+                        minimum: 0.0
+                        type: integer
+                    required:
+                    - credentialsSecretName
+                    - database
+                    - host
+                    type: object
+                  postgresql:
+                    description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database.
+                    properties:
+                      credentialsSecretName:
+                        description: |-
+                          Name of a Secret containing the `username` and `password` keys used to authenticate
+                          against the PostgreSQL server.
+                        type: string
+                      database:
+                        description: Name of the database (schema) to connect to.
+                        type: string
+                      host:
+                        description: Hostname or IP address of the PostgreSQL server.
+                        type: string
+                      parameters:
+                        additionalProperties:
+                          type: string
+                        default: {}
+                        description: |-
+                          Additional map of JDBC connection parameters to append to the connection URL. The given
+                          `HashMap<String, String>` will be converted to query parameters in the form of
+                          `?param1=value1&param2=value2`.
+                        type: object
+                      port:
+                        default: 5432
+                        description: Port the PostgreSQL server is listening on. Defaults to `5432`.
+                        format: uint16
+                        maximum: 65535.0
+                        minimum: 0.0
+                        type: integer
+                    required:
+                    - credentialsSecretName
+                    - database
+                    - host
+                    type: object
+                  redis:
+                    description: |-
+                      Connection settings for a [Redis](https://redis.io/) instance.
+
+                      Redis is commonly used as a Celery message broker or result backend (e.g. for Apache Airflow).
+                    properties:
+                      credentialsSecretName:
+                        description: |-
+                          Name of a Secret containing the `username` and `password` keys used to authenticate
+                          against the Redis server.
+                        type: string
+                      databaseId:
+                        default: 0
+                        description: |-
+                          Numeric index of the Redis logical database to use. Defaults to `0`.
+
+                          Redis supports multiple logical databases within a single instance, identified by an
+                          integer index. Database `0` is the default.
+                        format: uint16
+                        maximum: 65535.0
+                        minimum: 0.0
+                        type: integer
+                      host:
+                        description: Hostname or IP address of the Redis server.
+                        type: string
+                      port:
+                        default: 6379
+                        description: Port the Redis server is listening on. Defaults to `6379`.
+                        format: uint16
+                        maximum: 65535.0
+                        minimum: 0.0
+                        type: integer
+                    required:
+                    - credentialsSecretName
+                    - host
+                    type: object
+                type: object
               domainName:
                 description: A validated domain name type conforming to RFC 1123, so e.g. not an IP address
                 type: string
@@ -2188,6 +2390,7 @@ spec:
             required:
             - clientAuthenticationDetails
             - clusterOperation
+            - databaseConnection
             - domainName
             - gitSync
             - hostName

crates/stackable-operator/src/builder/pod/container.rs

@@ -175,7 +175,10 @@ impl ContainerBuilder {
         self
     }
 
-    pub fn add_env_vars(&mut self, env_vars: Vec<EnvVar>) -> &mut Self {
+    pub fn add_env_vars<I>(&mut self, env_vars: I) -> &mut Self
+    where
+        I: IntoIterator<Item = EnvVar>,
+    {
         self.env.get_or_insert_with(Vec::new).extend(env_vars);
         self
     }

crates/stackable-operator/src/builder/pod/env.rs

@@ -0,0 +1,20 @@
+use k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector};
+
+pub fn env_var_with_value_from_secret(
+    env_var_name: impl Into<String>,
+    secret_name: impl Into<String>,
+    secret_key: impl Into<String>,
+) -> EnvVar {
+    EnvVar {
+        name: env_var_name.into(),
+        value_from: Some(EnvVarSource {
+            secret_key_ref: Some(SecretKeySelector {
+                name: secret_name.into(),
+                key: secret_key.into(),
+                ..Default::default()
+            }),
+            ..Default::default()
+        }),
+        ..Default::default()
+    }
+}

crates/stackable-operator/src/builder/pod/mod.rs

@@ -29,6 +29,7 @@ use crate::{
 };
 
 pub mod container;
+pub mod env;
 pub mod probe;
 pub mod resources;
 pub mod security;

crates/stackable-operator/src/crd/git_sync/v1alpha1_impl.rs

@@ -197,7 +197,7 @@ impl GitSyncResources {
                 one_time,
                 container_log_config,
             )])
-            .add_env_vars(env_vars.into())
+            .add_env_vars(env_vars.iter().cloned())
             .add_volume_mounts(volume_mounts.to_vec())
             .context(AddVolumeMountSnafu)?
             .resources(

crates/stackable-operator/src/crd/git_sync/v1alpha2_impl.rs

@@ -314,7 +314,7 @@ impl GitSyncResources {
                 container_log_config,
                 ca_cert_path,
             )])
-            .add_env_vars(env_vars.into())
+            .add_env_vars(env_vars.iter().cloned())
             .add_volume_mounts(volume_mounts.to_vec())
             .context(AddVolumeMountSnafu)?
             .resources(

crates/stackable-operator/src/database_connections/databases/derby.rs

@@ -0,0 +1,68 @@
+use std::path::PathBuf;
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use snafu::{OptionExt, ResultExt, Snafu};
+
+use crate::{
+    database_connections::{
+        TemplatingMechanism,
+        drivers::jdbc::{JdbcDatabaseConnection, JdbcDatabaseConnectionDetails},
+    },
+    utils::OptionExt as _,
+};
+
+/// Sadly the Derby driver class name is a bit complicated, e.g. for HMS up to 4.1.x we used
+/// `org.apache.derby.jdbc.EmbeddedDriver`, for HMS 4.2.x we used
+/// `org.apache.derby.iapi.jdbc.AutoloadedDriver`.
+pub const DERBY_JDBC_DRIVER_CLASS: &str = "org.apache.derby.jdbc.EmbeddedDriver";
+
+#[derive(Debug, Snafu)]
+pub enum Error {
+    #[snafu(display("failed to parse connection URL"))]
+    ParseConnectionUrl { source: url::ParseError },
+
+    #[snafu(display("invalid derby database location, likely as it contains non-utf8 characters"))]
+    NonUtf8Location { location: PathBuf },
+}
+
+/// Connection settings for an embedded [Apache Derby](https://db.apache.org/derby/) database.
+///
+/// Derby is an embedded, file-based Java database engine that requires no separate server process.
+/// It is typically used for development, testing, or as a lightweight metastore backend (e.g. for
+/// Apache Hive).
+#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DerbyConnection {
+    /// Path on the filesystem where Derby stores its database files.
+    ///
+    /// If not specified, defaults to `/tmp/derby/{unique_database_name}/derby.db`.
+    /// The `{unique_database_name}` part is automatically handled by the operator and is added to
+    /// prevent clashing database files. The `create=true` flag is always appended to the JDBC URL,
+    /// so the database is created automatically if it does not yet exist at this location.
+    pub location: Option<PathBuf>,
+}
+
+impl JdbcDatabaseConnection for DerbyConnection {
+    fn jdbc_connection_details_with_templating(
+        &self,
+        unique_database_name: &str,
+        _templating_mechanism: &TemplatingMechanism,
+    ) -> Result<JdbcDatabaseConnectionDetails, crate::database_connections::Error> {
+        let location = self.location.as_ref_or_else(|| {
+            PathBuf::from(format!("/tmp/derby/{unique_database_name}/derby.db"))
+        });
+        let location = location.to_str().with_context(|| NonUtf8LocationSnafu {
+            location: location.to_path_buf(),
+        })?;
+        let connection_url = format!("jdbc:derby:{location};create=true",);
+        let connection_url = connection_url.parse().context(ParseConnectionUrlSnafu)?;
+
+        Ok(JdbcDatabaseConnectionDetails {
+            driver: DERBY_JDBC_DRIVER_CLASS.to_owned(),
+            connection_url,
+            username_env: None,
+            password_env: None,
+        })
+    }
+}

crates/stackable-operator/src/database_connections/databases/mod.rs

@@ -0,0 +1,4 @@
+pub mod derby;
+pub mod mysql;
+pub mod postgresql;
+pub mod redis;