chore: smoke tests passing

Author: maltesander

extra/crds.yaml

@@ -1453,44 +1453,110 @@ spec:
                             type: string
                         type: object
                     type: object
-                  metadataStorageDatabase:
+                  metadataDatabase:
                     description: Druid requires an SQL database to store metadata into. Specify connection information here.
+                    oneOf:
+                    - required:
+                      - postgresql
+                    - required:
+                      - mysql
+                    - required:
+                      - derby
                     properties:
-                      connString:
+                      derby:
                         description: |-
-                          The connect string for the database, for Postgres this could look like:
-                          `jdbc:postgresql://postgresql-druid/druid`
-                        type: string
-                      credentialsSecret:
-                        description: |-
-                          A reference to a Secret containing the database credentials.
-                          The Secret needs to contain the keys `username` and `password`.
-                        nullable: true
-                        type: string
-                      dbType:
+                          Connection settings for an embedded [Apache Derby](https://db.apache.org/derby/) database.
+
+                          Derby is an embedded, file-based Java database engine that requires no separate server process.
+                          It is typically used for development, testing, or as a lightweight metastore backend (e.g. for
+                          Apache Hive).
+                        properties:
+                          location:
+                            description: |-
+                              Path on the filesystem where Derby stores its database files.
+
+                              If not specified, defaults to `/tmp/derby/{unique_database_name}/derby.db`.
+                              The `{unique_database_name}` part is automatically handled by the operator and is added to
+                              prevent clashing database files. The `create=true` flag is always appended to the JDBC URL,
+                              so the database is created automatically if it does not yet exist at this location.
+                            nullable: true
+                            type: string
+                        type: object
+                      mysql:
                         description: |-
-                          The database type. Supported values are: `derby`, `mysql` and `postgres`.
-                          Note that a Derby database created locally in the container is not persisted!
-                          Derby is not suitable for production use.
-                        enum:
-                        - derby
-                        - mysql
-                        - postgresql
-                        type: string
-                      host:
-                        description: The host, i.e. `postgresql-druid`.
-                        type: string
-                      port:
-                        description: The port, i.e. 5432
-                        format: uint16
-                        maximum: 65535.0
-                        minimum: 0.0
-                        type: integer
-                    required:
-                    - connString
-                    - dbType
-                    - host
-                    - port
+                          Connection settings for a [MySQL](https://www.mysql.com/) database.
+
+                          Please note that - due to license issues - we don't ship the mysql driver, you need to add
+                          it it yourself.
+                          See <https://docs.stackable.tech/home/stable/hive/usage-guide/database-driver/> for details.
+                        properties:
+                          credentialsSecretName:
+                            description: |-
+                              Name of a Secret containing the `username` and `password` keys used to authenticate
+                              against the MySQL server.
+                            type: string
+                          database:
+                            description: Name of the database (schema) to connect to.
+                            type: string
+                          host:
+                            description: Hostname or IP address of the MySQL server.
+                            type: string
+                          parameters:
+                            additionalProperties:
+                              type: string
+                            default: {}
+                            description: |-
+                              Additional map of connection parameters to append to the connection URL. The given
+                              `HashMap<String, String>` will be converted to query parameters in the form of
+                              `?param1=value1&param2=value2`.
+                            type: object
+                          port:
+                            default: 3306
+                            description: Port the MySQL server is listening on. Defaults to `3306`.
+                            format: uint16
+                            maximum: 65535.0
+                            minimum: 0.0
+                            type: integer
+                        required:
+                        - credentialsSecretName
+                        - database
+                        - host
+                        type: object
+                      postgresql:
+                        description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database.
+                        properties:
+                          credentialsSecretName:
+                            description: |-
+                              Name of a Secret containing the `username` and `password` keys used to authenticate
+                              against the PostgreSQL server.
+                            type: string
+                          database:
+                            description: Name of the database (schema) to connect to.
+                            type: string
+                          host:
+                            description: Hostname or IP address of the PostgreSQL server.
+                            type: string
+                          parameters:
+                            additionalProperties:
+                              type: string
+                            default: {}
+                            description: |-
+                              Additional map of JDBC connection parameters to append to the connection URL. The given
+                              `HashMap<String, String>` will be converted to query parameters in the form of
+                              `?param1=value1&param2=value2`.
+                            type: object
+                          port:
+                            default: 5432
+                            description: Port the PostgreSQL server is listening on. Defaults to `5432`.
+                            format: uint16
+                            maximum: 65535.0
+                            minimum: 0.0
+                            type: integer
+                        required:
+                        - credentialsSecretName
+                        - database
+                        - host
+                        type: object
                     type: object
                   tls:
                     default:
@@ -1529,7 +1595,7 @@ spec:
                     type: string
                 required:
                 - deepStorage
-                - metadataStorageDatabase
+                - metadataDatabase
                 - zookeeperConfigMapName
                 type: object
               clusterOperation:

rust/operator-binary/src/crd/database.rs

@@ -0,0 +1,63 @@
+use std::ops::Deref;
+
+use serde::{Deserialize, Serialize};
+use stackable_operator::{
+    database_connections::{
+        databases::{
+            derby::DerbyConnection, mysql::MysqlConnection, postgresql::PostgresqlConnection,
+        },
+        drivers::jdbc::JdbcDatabaseConnection,
+    },
+    schemars::{self, JsonSchema},
+};
+
+// metadata storage config properties
+pub const METADATA_STORAGE_TYPE: &str = "druid.metadata.storage.type";
+pub const METADATA_STORAGE_CONNECTOR_CONNECT_URI: &str =
+    "druid.metadata.storage.connector.connectURI";
+pub const METADATA_STORAGE_USER: &str = "druid.metadata.storage.connector.user";
+pub const METADATA_STORAGE_PASSWORD: &str = "druid.metadata.storage.connector.password";
+
+#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub enum MetadataDatabaseConnection {
+    // Docs are on the struct
+    Postgresql(PostgresqlConnection),
+
+    /// Connection settings for a [MySQL](https://www.mysql.com/) database.
+    ///
+    /// Please note that - due to license issues - we don't ship the mysql driver, you need to add
+    /// it it yourself.
+    /// See <https://docs.stackable.tech/home/stable/hive/usage-guide/database-driver/> for details.
+    Mysql(MysqlConnection),
+
+    // Docs are on the struct
+    Derby(DerbyConnection),
+    // We don't support generic (yet?), as we need to tell the metastore the `--dbtype` on startup,
+    // which is not known for generic connection. We could e.g. create a new struct with
+    // #[serde(flatten)] of the GenericJdbcDatabaseConnection and an additional field
+    // `metastoreDbType` (or similar).
+}
+
+impl MetadataDatabaseConnection {
+    /// Name of the database as it should be passed using the `--db-type` CLI argument to Hive
+    pub fn as_db_type(&self) -> &str {
+        match self {
+            MetadataDatabaseConnection::Postgresql(_) => "postgresql",
+            MetadataDatabaseConnection::Mysql(_) => "mysql",
+            MetadataDatabaseConnection::Derby(_) => "derby",
+        }
+    }
+}
+
+impl Deref for MetadataDatabaseConnection {
+    type Target = dyn JdbcDatabaseConnection;
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            Self::Postgresql(p) => p,
+            Self::Mysql(m) => m,
+            Self::Derby(d) => d,
+        }
+    }
+}

rust/operator-binary/src/crd/mod.rs

@@ -46,13 +46,15 @@ use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr};
 use crate::crd::{
     affinity::get_affinity,
     authorization::DruidAuthorization,
+    database::MetadataDatabaseConnection,
     resource::RoleResource,
     tls::{DruidTls, default_druid_tls},
 };
 
 pub mod affinity;
 pub mod authentication;
 pub mod authorization;
+pub mod database;
 pub mod memory;
 pub mod resource;
 pub mod security;
@@ -109,21 +111,12 @@ pub const AUTH_AUTHORIZERS_VALUE: &str = "[\"OpaAuthorizer\"]";
 pub const AUTH_AUTHORIZER_OPA_TYPE: &str = "druid.auth.authorizer.OpaAuthorizer.type";
 pub const AUTH_AUTHORIZER_OPA_TYPE_VALUE: &str = "opa";
 pub const AUTH_AUTHORIZER_OPA_URI: &str = "druid.auth.authorizer.OpaAuthorizer.opaUri";
-// metadata storage config properties
-const METADATA_STORAGE_TYPE: &str = "druid.metadata.storage.type";
-const METADATA_STORAGE_URI: &str = "druid.metadata.storage.connector.connectURI";
-const METADATA_STORAGE_HOST: &str = "druid.metadata.storage.connector.host";
-const METADATA_STORAGE_PORT: &str = "druid.metadata.storage.connector.port";
-const METADATA_STORAGE_USER: &str = "druid.metadata.storage.connector.user";
-const METADATA_STORAGE_PASSWORD: &str = "druid.metadata.storage.connector.password";
 // indexer properties
 pub const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray";
 // historical settings
 pub const PROCESSING_BUFFER_SIZE_BYTES: &str = "druid.processing.buffer.sizeBytes";
 pub const PROCESSING_NUM_MERGE_BUFFERS: &str = "druid.processing.numMergeBuffers";
 pub const PROCESSING_NUM_THREADS: &str = "druid.processing.numThreads";
-// extra
-pub const CREDENTIALS_SECRET_PROPERTY: &str = "credentialsSecret";
 // logs
 pub const MAX_DRUID_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity {
     value: 10.0,
@@ -136,10 +129,6 @@ pub const METRICS_PORT: u16 = 9090;
 
 pub const COOKIE_PASSPHRASE_ENV: &str = "OIDC_COOKIE_PASSPHRASE";
 
-// DB credentials - both of these are read from an env var by Druid with the ${env:...} syntax
-pub const DB_USERNAME_ENV: &str = "DB_USERNAME_ENV";
-pub const DB_PASSWORD_ENV: &str = "DB_PASSWORD_ENV";
-
 // Graceful shutdown timeouts
 const DEFAULT_BROKER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5);
 const DEFAULT_COORDINATOR_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5);
@@ -361,7 +350,7 @@ pub mod versioned {
         pub ingestion: Option<IngestionSpec>,
 
         /// Druid requires an SQL database to store metadata into. Specify connection information here.
-        pub metadata_storage_database: DatabaseConnectionSpec,
+        pub metadata_database: MetadataDatabaseConnection,
 
         /// TLS encryption settings for Druid, more information in the
         /// [security documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/security).
@@ -420,34 +409,6 @@ impl v1alpha1::DruidCluster {
         match file {
             JVM_CONFIG => {}
             RUNTIME_PROPS => {
-                let mds = &self.spec.cluster_config.metadata_storage_database;
-                result.insert(
-                    METADATA_STORAGE_TYPE.to_string(),
-                    Some(mds.db_type.to_string()),
-                );
-                result.insert(
-                    METADATA_STORAGE_URI.to_string(),
-                    Some(mds.conn_string.to_string()),
-                );
-                result.insert(
-                    METADATA_STORAGE_HOST.to_string(),
-                    Some(mds.host.to_string()),
-                );
-                result.insert(
-                    METADATA_STORAGE_PORT.to_string(),
-                    Some(mds.port.to_string()),
-                );
-                if mds.credentials_secret.is_some() {
-                    result.insert(
-                        METADATA_STORAGE_USER.to_string(),
-                        Some(format!("${{env:{DB_USERNAME_ENV}}}")),
-                    );
-                    result.insert(
-                        METADATA_STORAGE_PASSWORD.to_string(),
-                        Some(format!("${{env:{DB_PASSWORD_ENV}}}")),
-                    );
-                }
-
                 // OPA
                 if let Some(DruidAuthorization { opa: _ }) = &self.spec.cluster_config.authorization
                 {
@@ -1155,43 +1116,6 @@ impl DruidRole {
     }
 }
 
-#[derive(Clone, Debug, Default, Deserialize, JsonSchema, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct DatabaseConnectionSpec {
-    /// The database type. Supported values are: `derby`, `mysql` and `postgres`.
-    /// Note that a Derby database created locally in the container is not persisted!
-    /// Derby is not suitable for production use.
-    pub db_type: DbType,
-    /// The connect string for the database, for Postgres this could look like:
-    /// `jdbc:postgresql://postgresql-druid/druid`
-    pub conn_string: String,
-    /// The host, i.e. `postgresql-druid`.
-    pub host: String,
-    /// The port, i.e. 5432
-    pub port: u16,
-    /// A reference to a Secret containing the database credentials.
-    /// The Secret needs to contain the keys `username` and `password`.
-    pub credentials_secret: Option<String>,
-}
-
-#[derive(
-    Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize, Display, EnumString,
-)]
-pub enum DbType {
-    #[serde(rename = "derby")]
-    #[strum(serialize = "derby")]
-    #[default]
-    Derby,
-
-    #[serde(rename = "mysql")]
-    #[strum(serialize = "mysql")]
-    Mysql,
-
-    #[serde(rename = "postgresql")]
-    #[strum(serialize = "postgresql")]
-    Postgresql,
-}
-
 #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize, Display)]
 #[serde(rename_all = "camelCase")]
 pub enum DeepStorageSpec {