Skip to content

Commit 67679a8

Browse files
CoderYellowCoderYellow
committed
feat(hive): add MariaDB support for Cloudera HMS with customizable configuration
- Introduced `databaseType` option to switch between PostgreSQL (default) and MariaDB for Cloudera HMS. - Added support for embedding MariaDB with configurable `databaseName`, `databaseUser`, `databasePassword`, and `databaseHostPort`. - Updated `spring-bigdata-test.toml` with new Cloudera HMS configuration options. - Extended Gradle plugin and JUnit extension to support MariaDB configuration for Cloudera HMS. - Enhanced documentation with examples for configuring Cloudera HMS with MariaDB or PostgreSQL.
1 parent c291de9 commit 67679a8

14 files changed

Lines changed: 350 additions & 36 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ Open-source Hive Metastore uses PostgreSQL by default and can be switched to
1010
MySQL with TOML `databaseType = "mysql"` under `[hiveMetastore]`. Its support
1111
database uses a random host port by default; set `databaseHostPort` when you
1212
need a stable local port.
13+
Cloudera HMS defaults to `ghcr.io/openprojectx/cloudera-hms:0.1.74`; set
14+
`[clouderaHms] databaseType = "mariadb"` for the `0.1.74-mariadb` image.
15+
`[clouderaHms] databaseHostPort` can expose the embedded PostgreSQL/MariaDB
16+
port on a stable local port for troubleshooting.
1317

1418
## Modules
1519

‎autoconfigure/src/main/kotlin/org/openprojectx/bigdata/test/autoconfigure/BigDataTestAutoConfiguration.kt‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package org.openprojectx.bigdata.test.autoconfigure
22

33
import org.openprojectx.bigdata.test.core.BigDataTestKit
44
import org.openprojectx.bigdata.test.core.BigDataService
5+
import org.openprojectx.bigdata.test.core.ClouderaHmsDatabaseType
56
import org.openprojectx.bigdata.test.core.ContainerLogOptions
67
import org.openprojectx.bigdata.test.core.HdfsOptions
78
import org.openprojectx.bigdata.test.core.HttpTlsOptions
@@ -123,7 +124,12 @@ class BigDataTestAutoConfiguration {
123124
HiveMetastoreOptions(
124125
enabled = true,
125126
distribution = HiveMetastoreDistribution.CLOUDERA,
126-
image = properties.clouderaHms.image,
127+
image = properties.clouderaHms.imageForType(),
128+
clouderaDatabaseType = properties.clouderaHms.databaseType,
129+
databaseName = properties.clouderaHms.databaseName,
130+
databaseUser = properties.clouderaHms.databaseUser,
131+
databasePassword = properties.clouderaHms.databasePassword,
132+
databaseHostPort = properties.clouderaHms.databaseHostPort,
127133
warehouseDir = properties.clouderaHms.warehouseDir,
128134
extraConfiguration = properties.clouderaHms.extraConfiguration,
129135
kerberos = KerberosAuthOptions(
@@ -213,4 +219,15 @@ class BigDataTestAutoConfiguration {
213219
databaseImage
214220
}
215221
}
222+
223+
private fun BigDataTestProperties.ClouderaHms.imageForType(): String =
224+
when (databaseType) {
225+
ClouderaHmsDatabaseType.POSTGRESQL -> image
226+
ClouderaHmsDatabaseType.MARIADB ->
227+
if (image == HiveMetastoreOptions.DEFAULT_CLOUDERA_IMAGE) {
228+
HiveMetastoreOptions.DEFAULT_CLOUDERA_MARIADB_IMAGE
229+
} else {
230+
image
231+
}
232+
}
216233
}

‎autoconfigure/src/main/kotlin/org/openprojectx/bigdata/test/autoconfigure/BigDataTestProperties.kt‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package org.openprojectx.bigdata.test.autoconfigure
22

33
import org.openprojectx.bigdata.test.core.HiveMetastoreDatabaseType
44
import org.openprojectx.bigdata.test.core.HiveMetastoreOptions
5+
import org.openprojectx.bigdata.test.core.ClouderaHmsDatabaseType
56

67
import org.openprojectx.bigdata.test.core.ContainerLogMode
78
import org.springframework.boot.context.properties.ConfigurationProperties
@@ -74,14 +75,19 @@ data class BigDataTestProperties(
7475
var databaseUser: String = "hive",
7576
var databasePassword: String = "hive",
7677
var databaseHostPort: Int = 0,
77-
var warehouseDir: String = "/user/hive/warehouse",
78+
var warehouseDir: String = HiveMetastoreOptions.DEFAULT_CLOUDERA_WAREHOUSE_DIR,
7879
var extraConfiguration: Map<String, String> = emptyMap(),
7980
var kerberosEnabled: Boolean = false,
8081
)
8182

8283
data class ClouderaHms(
8384
var enabled: Boolean = false,
84-
var image: String = "ghcr.io/openprojectx/cloudera-hms:0.1.16",
85+
var image: String = HiveMetastoreOptions.DEFAULT_CLOUDERA_IMAGE,
86+
var databaseType: ClouderaHmsDatabaseType = ClouderaHmsDatabaseType.POSTGRESQL,
87+
var databaseName: String = "metastore",
88+
var databaseUser: String = "hive",
89+
var databasePassword: String = "hive",
90+
var databaseHostPort: Int = 0,
8591
var warehouseDir: String = "/user/hive/warehouse",
8692
var extraConfiguration: Map<String, String> = emptyMap(),
8793
var kerberosEnabled: Boolean = false,

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/BigDataTestKit.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,8 @@ class BigDataTestKit private constructor(
8080
options: HiveMetastoreOptions = HiveMetastoreOptions(
8181
enabled = true,
8282
distribution = HiveMetastoreDistribution.CLOUDERA,
83-
image = "ghcr.io/openprojectx/cloudera-hms:0.1.16",
83+
image = HiveMetastoreOptions.DEFAULT_CLOUDERA_IMAGE,
84+
warehouseDir = HiveMetastoreOptions.DEFAULT_CLOUDERA_WAREHOUSE_DIR,
8485
),
8586
): Builder =
8687
apply { hiveMetastore = options.copy(enabled = true, distribution = HiveMetastoreDistribution.CLOUDERA) }

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/BigDataTestKitOptions.kt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ data class HiveMetastoreOptions(
8484
val distribution: HiveMetastoreDistribution = HiveMetastoreDistribution.OPEN_SOURCE,
8585
val image: String = DEFAULT_IMAGE,
8686
val databaseType: HiveMetastoreDatabaseType = HiveMetastoreDatabaseType.POSTGRESQL,
87+
val clouderaDatabaseType: ClouderaHmsDatabaseType = ClouderaHmsDatabaseType.POSTGRESQL,
8788
val databaseImage: String = DEFAULT_POSTGRES_IMAGE,
8889
val databaseHostPort: Int = 0,
8990
val databaseName: String = "metastore",
@@ -101,6 +102,9 @@ data class HiveMetastoreOptions(
101102
) {
102103
companion object {
103104
const val DEFAULT_IMAGE = "ghcr.io/openprojectx/hive:3.1.3-hadoop-3.4.2-gcs-4.0.4-jdk17-0.1.5"
105+
const val DEFAULT_CLOUDERA_IMAGE = "ghcr.io/openprojectx/cloudera-hms:0.1.74"
106+
const val DEFAULT_CLOUDERA_MARIADB_IMAGE = "ghcr.io/openprojectx/cloudera-hms:0.1.74-mariadb"
107+
const val DEFAULT_CLOUDERA_WAREHOUSE_DIR = "/tmp/cloudera-hms/warehouse"
104108
const val DEFAULT_POSTGRES_IMAGE = "postgres:16-alpine"
105109
const val DEFAULT_MYSQL_IMAGE = "mysql:8.0.44-bookworm"
106110
}
@@ -116,6 +120,11 @@ enum class HiveMetastoreDatabaseType {
116120
MYSQL,
117121
}
118122

123+
enum class ClouderaHmsDatabaseType {
124+
POSTGRESQL,
125+
MARIADB,
126+
}
127+
119128
data class KafkaOptions(
120129
val enabled: Boolean = false,
121130
val image: String = "apache/kafka:4.1.2",

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/config/BigDataTestConfig.kt‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import org.openprojectx.bigdata.test.core.ContainerFileTransferOptions
1010
import org.openprojectx.bigdata.test.core.ContainerLogMode
1111
import org.openprojectx.bigdata.test.core.ContainerMountOptions
1212
import org.openprojectx.bigdata.test.core.ContainerPortOptions
13+
import org.openprojectx.bigdata.test.core.ClouderaHmsDatabaseType
1314
import org.openprojectx.bigdata.test.core.HiveMetastoreDatabaseType
1415

1516
data class BigDataTestConfig(
@@ -104,10 +105,22 @@ data class BigDataTestHiveMetastoreConfig(
104105
}
105106

106107
data class BigDataTestClouderaHmsConfig(
108+
val databaseType: ClouderaHmsDatabaseType? = null,
109+
val databaseHostPort: Int? = null,
110+
val databaseName: String? = null,
111+
val databaseUser: String? = null,
112+
val databasePassword: String? = null,
107113
val warehouseDir: String? = null,
108114
) {
109115
fun merge(override: BigDataTestClouderaHmsConfig): BigDataTestClouderaHmsConfig =
110-
BigDataTestClouderaHmsConfig(warehouseDir = override.warehouseDir ?: warehouseDir)
116+
BigDataTestClouderaHmsConfig(
117+
databaseType = override.databaseType ?: databaseType,
118+
databaseHostPort = override.databaseHostPort ?: databaseHostPort,
119+
databaseName = override.databaseName ?: databaseName,
120+
databaseUser = override.databaseUser ?: databaseUser,
121+
databasePassword = override.databasePassword ?: databasePassword,
122+
warehouseDir = override.warehouseDir ?: warehouseDir,
123+
)
111124
}
112125

113126
data class BigDataTestKafkaConfig(
@@ -190,6 +203,7 @@ data class BigDataTestImageConfig(
190203
val hdfs: String? = null,
191204
val hiveMetastore: String? = null,
192205
val clouderaHms: String? = null,
206+
val clouderaHmsMariadb: String? = null,
193207
val hiveMetastorePostgres: String? = null,
194208
val hiveMetastoreMysql: String? = null,
195209
val kafka: String? = null,
@@ -204,6 +218,7 @@ data class BigDataTestImageConfig(
204218
hdfs = override.hdfs ?: hdfs,
205219
hiveMetastore = override.hiveMetastore ?: hiveMetastore,
206220
clouderaHms = override.clouderaHms ?: clouderaHms,
221+
clouderaHmsMariadb = override.clouderaHmsMariadb ?: clouderaHmsMariadb,
207222
hiveMetastorePostgres = override.hiveMetastorePostgres ?: hiveMetastorePostgres,
208223
hiveMetastoreMysql = override.hiveMetastoreMysql ?: hiveMetastoreMysql,
209224
kafka = override.kafka ?: kafka,
@@ -332,6 +347,7 @@ class BigDataTestConfigLoader(
332347
hdfs = images.string("hdfs"),
333348
hiveMetastore = images.string("hiveMetastore"),
334349
clouderaHms = images.string("clouderaHms"),
350+
clouderaHmsMariadb = images.string("clouderaHmsMariadb"),
335351
hiveMetastorePostgres = images.string("hiveMetastorePostgres"),
336352
hiveMetastoreMysql = images.string("hiveMetastoreMysql"),
337353
kafka = images.string("kafka"),
@@ -391,7 +407,14 @@ class BigDataTestConfigLoader(
391407
localHiveSitePath = hiveMetastore.string("localHiveSitePath"),
392408
localMetastoreSitePath = hiveMetastore.string("localMetastoreSitePath"),
393409
),
394-
clouderaHms = BigDataTestClouderaHmsConfig(warehouseDir = clouderaHms.string("warehouseDir")),
410+
clouderaHms = BigDataTestClouderaHmsConfig(
411+
databaseType = clouderaHms.string("databaseType")?.let(::parseClouderaHmsDatabaseType),
412+
databaseHostPort = clouderaHms.int("databaseHostPort"),
413+
databaseName = clouderaHms.string("databaseName"),
414+
databaseUser = clouderaHms.string("databaseUser"),
415+
databasePassword = clouderaHms.string("databasePassword"),
416+
warehouseDir = clouderaHms.string("warehouseDir"),
417+
),
395418
kafka = BigDataTestKafkaConfig(
396419
schemaRegistryImage = kafka.string("schemaRegistryImage"),
397420
kafkaUiImage = kafka.string("kafkaUiImage"),
@@ -533,6 +556,13 @@ class BigDataTestConfigLoader(
533556
else -> error("Hive Metastore databaseType must be one of postgresql, postgres, or mysql, got '$value'")
534557
}
535558

559+
private fun parseClouderaHmsDatabaseType(value: String): ClouderaHmsDatabaseType =
560+
when (value.trim().lowercase().replace("-", "").replace("_", "")) {
561+
"postgres", "postgresql" -> ClouderaHmsDatabaseType.POSTGRESQL
562+
"mariadb", "maria" -> ClouderaHmsDatabaseType.MARIADB
563+
else -> error("Cloudera HMS databaseType must be one of postgresql, postgres, or mariadb, got '$value'")
564+
}
565+
536566
private fun parseContainerCustomizations(
537567
tables: Map<String, Map<String, TomlValue>>,
538568
): Map<BigDataService, ContainerCustomizationOptions> {

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/container/BigDataContainerFactory.kt‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import org.openprojectx.bigdata.test.core.BigDataEndpoint
44
import org.openprojectx.bigdata.test.core.BigDataHealthCheckMode
55
import org.openprojectx.bigdata.test.core.BigDataService
66
import org.openprojectx.bigdata.test.core.BigDataTestKitOptions
7+
import org.openprojectx.bigdata.test.core.ClouderaHmsDatabaseType
78
import org.openprojectx.bigdata.test.core.ContainerFileTransferOptions
89
import org.openprojectx.bigdata.test.core.ContainerLogMode
910
import org.openprojectx.bigdata.test.core.ContainerPortOptions
@@ -394,11 +395,14 @@ internal class BigDataContainerFactory(
394395

395396
private fun clouderaHms(): BigDataServiceContainer {
396397
val hive = options.hiveMetastore
398+
val databasePort = hive.clouderaDatabaseType.containerPort()
397399
val container = GenericBigDataContainer(hive.image)
398400
.withNetwork(network)
399401
.withNetworkAliases("hive-metastore", "hive-metastore.example.com")
400402
.withServicePort(9083, options.portBindings.hostPort(9083, options.portBindings.hiveMetastore))
403+
.withServicePort(databasePort, options.portBindings.hostPort(databasePort, hive.databaseHostPort))
401404
.withEnv("HMS_WAREHOUSE_DIR", hive.warehouseDir)
405+
.withClouderaHmsDatabase(hive)
402406
.waitingFor(Wait.forListeningPort().withStartupTimeout(Duration.ofMinutes(3)))
403407
if (hive.kerberos.enabled) {
404408
mountKerberos(container)
@@ -435,6 +439,10 @@ internal class BigDataContainerFactory(
435439
"spring.bigdata.test.hive-metastore.thrift-uri" to thriftUri,
436440
"bigdata.test.hive-metastore.hive-site" to clientXmlPaths.hiveSite,
437441
"bigdata.test.hive-metastore.metastore-site" to clientXmlPaths.metastoreSite,
442+
"bigdata.test.hive-metastore.database.type" to hive.clouderaDatabaseType.propertyValue(),
443+
"bigdata.test.hive-metastore.database.host" to container.host,
444+
"bigdata.test.hive-metastore.database.port" to container.getMappedPort(databasePort).toString(),
445+
"bigdata.test.hive-metastore.database.jdbc-url" to container.clouderaHmsJdbcUrl(hive, databasePort),
438446
) + tlsProperties +
439447
clientProperties +
440448
kerberosProperties("hive.metastore", hive.kerberos),
@@ -509,6 +517,37 @@ internal class BigDataContainerFactory(
509517
}
510518
}
511519

520+
private fun ClouderaHmsDatabaseType.propertyValue(): String =
521+
name.lowercase(Locale.ROOT)
522+
523+
private fun ClouderaHmsDatabaseType.containerPort(): Int =
524+
when (this) {
525+
ClouderaHmsDatabaseType.POSTGRESQL -> 5432
526+
ClouderaHmsDatabaseType.MARIADB -> 3306
527+
}
528+
529+
private fun GenericBigDataContainer.withClouderaHmsDatabase(hive: HiveMetastoreOptions): GenericBigDataContainer =
530+
apply {
531+
withEnv("HMS_DATABASE_TYPE", hive.clouderaDatabaseType.propertyValue())
532+
withEnv("POSTGRES_DB", hive.databaseName)
533+
withEnv("POSTGRES_USER", hive.databaseUser)
534+
withEnv("POSTGRES_PASSWORD", hive.databasePassword)
535+
withEnv("MARIADB_DATABASE", hive.databaseName)
536+
withEnv("MARIADB_USER", hive.databaseUser)
537+
withEnv("MARIADB_PASSWORD", hive.databasePassword)
538+
withEnv("MARIADB_RANDOM_ROOT_PASSWORD", "yes")
539+
withEnv("HMS_JDBC_USER", hive.databaseUser)
540+
withEnv("HMS_JDBC_PASSWORD", hive.databasePassword)
541+
}
542+
543+
private fun GenericBigDataContainer.clouderaHmsJdbcUrl(hive: HiveMetastoreOptions, databasePort: Int): String =
544+
when (hive.clouderaDatabaseType) {
545+
ClouderaHmsDatabaseType.POSTGRESQL ->
546+
"jdbc:postgresql://$host:${getMappedPort(databasePort)}/${hive.databaseName}"
547+
ClouderaHmsDatabaseType.MARIADB ->
548+
"jdbc:mariadb://$host:${getMappedPort(databasePort)}/${hive.databaseName}?useMysqlMetadata=true"
549+
}
550+
512551
private fun tlsKafka(kafka: KafkaOptions): BigDataServiceContainer {
513552
val kafkaHostPort = options.portBindings.hostPort(9092, options.portBindings.kafka)
514553
val container = if (kafkaHostPort == 0) {

0 commit comments

Comments
 (0)