feat(hive): add MySQL support for Hive Metastore with configurable port

- Introduced `databaseType` option to switch between PostgreSQL (default) and MySQL for Hive Metastore.
- Added `databaseHostPort` to enable stable local ports for MySQL and PostgreSQL support containers.
- Updated Gradle plugin and test framework to integrate MySQL configuration.
- Enhanced documentation with examples for MySQL usage and configuration options.

Author: CoderYellow

CONTRIBUTING.md

@@ -96,7 +96,7 @@ Host-side Kafka Kerberos clients need a stable advertised host port. Existing Ke
 
 The project supports two Hive Metastore implementations:
 
-- `hiveMetastore = true`: open-source Hive HMS image plus external PostgreSQL.
+- `hiveMetastore = true`: open-source Hive HMS image plus external PostgreSQL by default, or MySQL with `databaseType = "mysql"`.
 - `clouderaHms = true`: Cloudera HMS image with embedded PostgreSQL.
 
 Keep their endpoint contract the same through `BigDataService.HIVE_METASTORE`. A test must enable only one HMS implementation.

README.md

@@ -6,6 +6,10 @@ Composable Testcontainers fixtures for local big-data integration tests.
 connection properties through a small Kotlin/JUnit API. Heavier setup such as
 S3 JCEKS generation, bucket creation, and Kafka Avro seeding lives in the
 optional `extensions` module so `core` and `junit5` stay lightweight.
+Open-source Hive Metastore uses PostgreSQL by default and can be switched to
+MySQL with TOML `databaseType = "mysql"` under `[hiveMetastore]`. Its support
+database uses a random host port by default; set `databaseHostPort` when you
+need a stable local port.
 
 ## Modules
 

autoconfigure/src/main/kotlin/org/openprojectx/bigdata/test/autoconfigure/BigDataTestAutoConfiguration.kt

@@ -5,6 +5,7 @@ import org.openprojectx.bigdata.test.core.BigDataService
 import org.openprojectx.bigdata.test.core.ContainerLogOptions
 import org.openprojectx.bigdata.test.core.HdfsOptions
 import org.openprojectx.bigdata.test.core.HttpTlsOptions
+import org.openprojectx.bigdata.test.core.HiveMetastoreDatabaseType
 import org.openprojectx.bigdata.test.core.HiveMetastoreDistribution
 import org.openprojectx.bigdata.test.core.HiveMetastoreOptions
 import org.openprojectx.bigdata.test.core.KafkaOptions
@@ -100,10 +101,12 @@ class BigDataTestAutoConfiguration {
                     enabled = true,
                     distribution = HiveMetastoreDistribution.OPEN_SOURCE,
                     image = properties.hiveMetastore.image,
-                    databaseImage = properties.hiveMetastore.databaseImage,
+                    databaseType = properties.hiveMetastore.databaseType,
+                    databaseImage = properties.hiveMetastore.databaseImageForType(),
                     databaseName = properties.hiveMetastore.databaseName,
                     databaseUser = properties.hiveMetastore.databaseUser,
                     databasePassword = properties.hiveMetastore.databasePassword,
+                    databaseHostPort = properties.hiveMetastore.databaseHostPort,
                     warehouseDir = properties.hiveMetastore.warehouseDir,
                     extraConfiguration = properties.hiveMetastore.extraConfiguration,
                     kerberos = KerberosAuthOptions(
@@ -199,4 +202,15 @@ class BigDataTestAutoConfiguration {
                 service.name.replace("_", "-").equals(this, ignoreCase = true) ||
                 service.name.replace("_", "").equals(this, ignoreCase = true)
         } ?: error("Unknown bigdata.test.container-log-levels service '$this'")
+
+    private fun BigDataTestProperties.HiveMetastore.databaseImageForType(): String =
+        when (databaseType) {
+            HiveMetastoreDatabaseType.POSTGRESQL -> databaseImage
+            HiveMetastoreDatabaseType.MYSQL ->
+                if (databaseImage == HiveMetastoreOptions.DEFAULT_POSTGRES_IMAGE) {
+                    HiveMetastoreOptions.DEFAULT_MYSQL_IMAGE
+                } else {
+                    databaseImage
+                }
+        }
 }

autoconfigure/src/main/kotlin/org/openprojectx/bigdata/test/autoconfigure/BigDataTestProperties.kt

@@ -1,5 +1,8 @@
 package org.openprojectx.bigdata.test.autoconfigure
 
+import org.openprojectx.bigdata.test.core.HiveMetastoreDatabaseType
+import org.openprojectx.bigdata.test.core.HiveMetastoreOptions
+
 import org.openprojectx.bigdata.test.core.ContainerLogMode
 import org.springframework.boot.context.properties.ConfigurationProperties
 
@@ -64,11 +67,13 @@ data class BigDataTestProperties(
 
     data class HiveMetastore(
         var enabled: Boolean = false,
-        var image: String = "ghcr.io/openprojectx/hive:3.1.3-hadoop-3.4.2-gcs-4.0.4-jdk17-0.1.4",
-        var databaseImage: String = "postgres:16-alpine",
+        var image: String = HiveMetastoreOptions.DEFAULT_IMAGE,
+        var databaseType: HiveMetastoreDatabaseType = HiveMetastoreDatabaseType.POSTGRESQL,
+        var databaseImage: String = HiveMetastoreOptions.DEFAULT_POSTGRES_IMAGE,
         var databaseName: String = "metastore",
         var databaseUser: String = "hive",
         var databasePassword: String = "hive",
+        var databaseHostPort: Int = 0,
         var warehouseDir: String = "/user/hive/warehouse",
         var extraConfiguration: Map<String, String> = emptyMap(),
         var kerberosEnabled: Boolean = false,

core/build.gradle.kts

@@ -8,8 +8,10 @@ dependencies {
     implementation(libs.bouncycastleProvider)
     api(libs.testcontainers)
     api(libs.testcontainersPostgresql)
+    api(libs.testcontainersMysql)
     api(libs.testcontainersKafka)
     api(libs.hiveDockerTestcontainers)
+    runtimeOnly(libs.mysqlConnector)
     testImplementation(libs.junitJupiterApi)
     testRuntimeOnly(libs.junitJupiterEngine)
     testRuntimeOnly(libs.junitPlatformLauncher)

core/src/main/kotlin/org/openprojectx/bigdata/test/core/BigDataTestKitOptions.kt

@@ -82,8 +82,10 @@ data class HdfsOptions(
 data class HiveMetastoreOptions(
     val enabled: Boolean = false,
     val distribution: HiveMetastoreDistribution = HiveMetastoreDistribution.OPEN_SOURCE,
-    val image: String = "ghcr.io/openprojectx/hive:3.1.3-hadoop-3.4.2-gcs-4.0.4-jdk17-0.1.4",
-    val databaseImage: String = "postgres:16-alpine",
+    val image: String = DEFAULT_IMAGE,
+    val databaseType: HiveMetastoreDatabaseType = HiveMetastoreDatabaseType.POSTGRESQL,
+    val databaseImage: String = DEFAULT_POSTGRES_IMAGE,
+    val databaseHostPort: Int = 0,
     val databaseName: String = "metastore",
     val databaseUser: String = "hive",
     val databasePassword: String = "hive",
@@ -96,13 +98,24 @@ data class HiveMetastoreOptions(
         servicePrincipal = "hive/hive-metastore.example.com@EXAMPLE.COM",
         keytabPath = "/kerby/keytabs/hive-metastore.keytab",
     ),
-)
+) {
+    companion object {
+        const val DEFAULT_IMAGE = "ghcr.io/openprojectx/hive:3.1.3-hadoop-3.4.2-gcs-4.0.4-jdk17-0.1.5"
+        const val DEFAULT_POSTGRES_IMAGE = "postgres:16-alpine"
+        const val DEFAULT_MYSQL_IMAGE = "mysql:8.0.44-bookworm"
+    }
+}
 
 enum class HiveMetastoreDistribution {
     OPEN_SOURCE,
     CLOUDERA,
 }
 
+enum class HiveMetastoreDatabaseType {
+    POSTGRESQL,
+    MYSQL,
+}
+
 data class KafkaOptions(
     val enabled: Boolean = false,
     val image: String = "apache/kafka:4.1.2",

core/src/main/kotlin/org/openprojectx/bigdata/test/core/config/BigDataTestConfig.kt

@@ -10,6 +10,7 @@ import org.openprojectx.bigdata.test.core.ContainerFileTransferOptions
 import org.openprojectx.bigdata.test.core.ContainerLogMode
 import org.openprojectx.bigdata.test.core.ContainerMountOptions
 import org.openprojectx.bigdata.test.core.ContainerPortOptions
+import org.openprojectx.bigdata.test.core.HiveMetastoreDatabaseType
 
 data class BigDataTestConfig(
     val images: BigDataTestImageConfig = BigDataTestImageConfig(),
@@ -80,6 +81,8 @@ data class BigDataTestHdfsConfig(
 }
 
 data class BigDataTestHiveMetastoreConfig(
+    val databaseType: HiveMetastoreDatabaseType? = null,
+    val databaseHostPort: Int? = null,
     val databaseName: String? = null,
     val databaseUser: String? = null,
     val databasePassword: String? = null,
@@ -89,6 +92,8 @@ data class BigDataTestHiveMetastoreConfig(
 ) {
     fun merge(override: BigDataTestHiveMetastoreConfig): BigDataTestHiveMetastoreConfig =
         BigDataTestHiveMetastoreConfig(
+            databaseType = override.databaseType ?: databaseType,
+            databaseHostPort = override.databaseHostPort ?: databaseHostPort,
             databaseName = override.databaseName ?: databaseName,
             databaseUser = override.databaseUser ?: databaseUser,
             databasePassword = override.databasePassword ?: databasePassword,
@@ -186,6 +191,7 @@ data class BigDataTestImageConfig(
     val hiveMetastore: String? = null,
     val clouderaHms: String? = null,
     val hiveMetastorePostgres: String? = null,
+    val hiveMetastoreMysql: String? = null,
     val kafka: String? = null,
     val schemaRegistry: String? = null,
     val kafkaUi: String? = null,
@@ -199,6 +205,7 @@ data class BigDataTestImageConfig(
             hiveMetastore = override.hiveMetastore ?: hiveMetastore,
             clouderaHms = override.clouderaHms ?: clouderaHms,
             hiveMetastorePostgres = override.hiveMetastorePostgres ?: hiveMetastorePostgres,
+            hiveMetastoreMysql = override.hiveMetastoreMysql ?: hiveMetastoreMysql,
             kafka = override.kafka ?: kafka,
             schemaRegistry = override.schemaRegistry ?: schemaRegistry,
             kafkaUi = override.kafkaUi ?: kafkaUi,
@@ -326,6 +333,7 @@ class BigDataTestConfigLoader(
                 hiveMetastore = images.string("hiveMetastore"),
                 clouderaHms = images.string("clouderaHms"),
                 hiveMetastorePostgres = images.string("hiveMetastorePostgres"),
+                hiveMetastoreMysql = images.string("hiveMetastoreMysql"),
                 kafka = images.string("kafka"),
                 schemaRegistry = images.string("schemaRegistry"),
                 kafkaUi = images.string("kafkaUi"),
@@ -374,6 +382,8 @@ class BigDataTestConfigLoader(
                 localHdfsSitePath = hdfs.string("localHdfsSitePath"),
             ),
             hiveMetastore = BigDataTestHiveMetastoreConfig(
+                databaseType = hiveMetastore.string("databaseType")?.let(::parseHiveMetastoreDatabaseType),
+                databaseHostPort = hiveMetastore.int("databaseHostPort"),
                 databaseName = hiveMetastore.string("databaseName"),
                 databaseUser = hiveMetastore.string("databaseUser"),
                 databasePassword = hiveMetastore.string("databasePassword"),
@@ -516,6 +526,13 @@ class BigDataTestConfigLoader(
             domain = values.string("domain"),
         )
 
+    private fun parseHiveMetastoreDatabaseType(value: String): HiveMetastoreDatabaseType =
+        when (value.trim().lowercase().replace("-", "").replace("_", "")) {
+            "postgres", "postgresql" -> HiveMetastoreDatabaseType.POSTGRESQL
+            "mysql" -> HiveMetastoreDatabaseType.MYSQL
+            else -> error("Hive Metastore databaseType must be one of postgresql, postgres, or mysql, got '$value'")
+        }
+
     private fun parseContainerCustomizations(
         tables: Map<String, Map<String, TomlValue>>,
     ): Map<BigDataService, ContainerCustomizationOptions> {