Skip to content

Commit 7df6c4a

Browse files
CoderYellowCoderYellow
committed
feat: add native TLS support for Hive Metastore
- Introduced TLS configuration for Hive Metastore with support for Apache and Cloudera distributions. - Updated `BigDataContainerFactory` to configure and initialize Hive Metastore TLS, including truststore management. - Extended annotations, configuration, and examples to enable TLS for Hive Metastore. - Enhanced Spark and JUnit integration with TLS-aware Hive Metastore client setup. - Updated user guide with configuration examples and usage details for Hive Metastore TLS.
1 parent fb0c03f commit 7df6c4a

13 files changed

Lines changed: 150 additions & 15 deletions

File tree

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/BigDataService.kt‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,13 @@ enum class BigDataService(
2222
),
2323
HIVE_METASTORE(
2424
defaultPorts = mapOf("thrift" to 9083),
25-
endpointProperties = setOf("hive.metastore.uris", "spring.bigdata.test.hive-metastore.thrift-uri"),
25+
endpointProperties = setOf(
26+
"hive.metastore.uris",
27+
"spring.bigdata.test.hive-metastore.thrift-uri",
28+
"hive.metastore.use.SSL",
29+
"hive.metastore.truststore.path",
30+
"hive.metastore.truststore.password",
31+
),
2632
),
2733
KAFKA(
2834
defaultPorts = mapOf("bootstrap" to 9092),

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/BigDataTestKitOptions.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ data class HiveMetastoreOptions(
7474
val databasePassword: String = "hive",
7575
val warehouseDir: String = "/user/hive/warehouse",
7676
val extraConfiguration: Map<String, String> = emptyMap(),
77+
val tls: HttpTlsOptions = HttpTlsOptions(),
7778
val kerberos: KerberosAuthOptions = KerberosAuthOptions(
7879
servicePrincipal = "hive/hive-metastore.example.com@EXAMPLE.COM",
7980
keytabPath = "/kerby/keytabs/hive-metastore.keytab",

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/container/BigDataContainerFactory.kt‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -189,14 +189,16 @@ internal class BigDataContainerFactory(
189189
.withPostgres("hive-metastore-postgres", 5432, hive.databaseName, hive.databaseUser, hive.databasePassword)
190190
.withWarehousePath(hive.warehouseDir)
191191
container.withServicePort(9083, options.portBindings.hostPort(9083, options.portBindings.hiveMetastore))
192+
container.waitingFor(Wait.forListeningPort().withStartupTimeout(Duration.ofMinutes(5)))
192193
if (hive.kerberos.enabled) {
193194
mountKerberos(container)
194195
container
195196
.withEnv("KRB5_CONFIG", "/kerby/client/krb5.conf")
196197
.withEnv("SERVICE_OPTS", "-Djava.security.krb5.conf=/kerby/client/krb5.conf")
197198
}
199+
val tlsProperties = if (hive.tls.enabled) configureHiveMetastoreTls(container, hive.tls) else emptyMap()
198200
configureHiveDockerObjectStores(container)
199-
if (hive.extraConfiguration.isNotEmpty() || hive.kerberos.enabled) {
201+
if (hive.extraConfiguration.isNotEmpty() || hive.kerberos.enabled || hive.tls.enabled) {
200202
container
201203
.withEnv("HIVE_CUSTOM_CONF_DIR", "/bigdata-test/hive-conf")
202204
.withFileSystemBind(openSourceHiveConfigurationDirectory().toString(), "/bigdata-test/hive-conf", BindMode.READ_ONLY)
@@ -211,7 +213,7 @@ internal class BigDataContainerFactory(
211213
properties = mapOf(
212214
"hive.metastore.uris" to thriftUri,
213215
"spring.bigdata.test.hive-metastore.thrift-uri" to thriftUri,
214-
) + kerberosProperties("hive.metastore", hive.kerberos),
216+
) + tlsProperties + kerberosProperties("hive.metastore", hive.kerberos),
215217
)
216218
}
217219
}
@@ -233,6 +235,7 @@ internal class BigDataContainerFactory(
233235
.withEnv("HMS_CONF_HIVE_METASTORE_KERBEROS_KEYTAB_FILE", hive.kerberos.keytabPath)
234236
.withEnv("HMS_CONF_HADOOP_SECURITY_AUTHENTICATION", "kerberos")
235237
}
238+
val tlsProperties = if (hive.tls.enabled) configureHiveMetastoreTls(container, hive.tls) else emptyMap()
236239

237240
hiveMetastoreObjectStoreConfiguration().forEach { (key, value) ->
238241
container.withEnv("HMS_CONF_${encodeConfigKey(key)}", value)
@@ -250,7 +253,7 @@ internal class BigDataContainerFactory(
250253
properties = mapOf(
251254
"hive.metastore.uris" to thriftUri,
252255
"spring.bigdata.test.hive-metastore.thrift-uri" to thriftUri,
253-
) + kerberosProperties("hive.metastore", hive.kerberos),
256+
) + tlsProperties + kerberosProperties("hive.metastore", hive.kerberos),
254257
)
255258
}
256259
}
@@ -606,6 +609,9 @@ internal class BigDataContainerFactory(
606609
"hadoop.security.authentication" to "kerberos",
607610
)
608611
}
612+
if (hive.tls.enabled) {
613+
metastoreProperties += hiveMetastoreServerTlsProperties("/bigdata-test/tls/hive-metastore.p12")
614+
}
609615
metastoreProperties += hive.extraConfiguration
610616
val hadoopProperties = hiveMetastoreObjectStoreConfiguration()
611617

@@ -728,6 +734,39 @@ internal class BigDataContainerFactory(
728734
) + tlsMaterial.properties()
729735
}
730736

737+
private fun configureHiveMetastoreTls(
738+
container: GenericContainer<*>,
739+
tls: HttpTlsOptions,
740+
): Map<String, String> {
741+
val keyStore = tlsMaterial.keyStore(
742+
name = "hive-metastore",
743+
domain = tls.domain,
744+
sanDomains = listOf("hive-metastore", "hive-metastore.example.com"),
745+
)
746+
val keyStorePath = "/bigdata-test/tls/hive-metastore.p12"
747+
container.withFileSystemBind(keyStore.path.toString(), keyStorePath, BindMode.READ_ONLY)
748+
if (options.hiveMetastore.distribution == HiveMetastoreDistribution.CLOUDERA) {
749+
container.withEnv(
750+
"HMS_EXTRA_CONF",
751+
hiveMetastoreServerTlsProperties(keyStorePath)
752+
.map { (key, value) -> "$key=$value" }
753+
.joinToString("\n"),
754+
)
755+
}
756+
return mapOf(
757+
"hive.metastore.use.SSL" to "true",
758+
"hive.metastore.truststore.path" to tlsMaterial.trustStorePath.toString(),
759+
"hive.metastore.truststore.password" to tlsMaterial.trustStorePassword,
760+
) + tlsMaterial.properties()
761+
}
762+
763+
private fun hiveMetastoreServerTlsProperties(keyStorePath: String): Map<String, String> =
764+
mapOf(
765+
"hive.metastore.use.SSL" to "true",
766+
"hive.metastore.keystore.path" to keyStorePath,
767+
"hive.metastore.keystore.password" to tlsMaterial.trustStorePassword,
768+
)
769+
731770
private data class HttpTlsEndpoint(
732771
val host: String? = null,
733772
val port: Int? = null,

‎doc/user-guide.adoc‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,10 @@ domain = "kafka-ui.local"
286286
enabled = true
287287
domain = "localhost"
288288
289+
[hiveMetastoreTls]
290+
enabled = true
291+
domain = "localhost"
292+
289293
[hdfsWebTls]
290294
enabled = true
291295
domain = "hdfs.local"
@@ -313,7 +317,7 @@ schemaRegistryTls = 8443
313317

314318
Available TLS port fields are `hdfsWebTls`, `schemaRegistryTls`, `kafkaUiTls`, `localStackS3Tls`, and `fakeGcsTls`.
315319

316-
Non-HTTP services use their native TLS implementation rather than HAProxy to avoid protocol compatibility issues. Kafka TLS is native broker TLS because Kafka clients depend on broker metadata and advertised listeners.
320+
Non-HTTP services use their native TLS implementation rather than HAProxy to avoid protocol compatibility issues. Kafka TLS is native broker TLS because Kafka clients depend on broker metadata and advertised listeners. Hive Metastore TLS is native Thrift TLS on the normal metastore port.
317321

318322
Kafka TLS can be enabled from the annotation or TOML:
319323

@@ -329,6 +333,31 @@ enabled = true
329333
domain = "localhost"
330334
----
331335

336+
Hive Metastore TLS can be enabled for either `hiveMetastore` or `clouderaHms`:
337+
338+
[source,kotlin]
339+
----
340+
@BigDataTest(hiveMetastore = true, hiveMetastoreTls = true)
341+
----
342+
343+
[source,toml]
344+
----
345+
[services]
346+
hiveMetastore = true
347+
348+
[hiveMetastoreTls]
349+
enabled = true
350+
domain = "localhost"
351+
----
352+
353+
The HMS endpoint keeps the normal `thrift://host:port` URI. Hive clients switch to TLS through endpoint properties:
354+
355+
* `hive.metastore.use.SSL=true`
356+
* `hive.metastore.truststore.path`
357+
* `hive.metastore.truststore.password`
358+
359+
The JUnit extension also installs the generated truststore as JVM TLS properties for clients that use JSSE defaults.
360+
332361
The Kafka endpoint then exposes `security.protocol=SSL` for plaintext Kafka or `security.protocol=SASL_SSL` when Kafka Kerberos is also enabled. It also exposes `ssl.truststore.location`, `ssl.truststore.password`, and `ssl.truststore.type` for Kafka clients.
333362

334363
=== Dynamic and Fixed Ports
@@ -421,7 +450,7 @@ Each started service exposes a `BigDataEndpoint` with named ports and connection
421450
|`BigDataService.HIVE_METASTORE`
422451
|`thrift`
423452
|`9083`
424-
|`hive.metastore.uris`, `spring.bigdata.test.hive-metastore.thrift-uri`
453+
|`hive.metastore.uris`, `spring.bigdata.test.hive-metastore.thrift-uri`, `hive.metastore.use.SSL`, `hive.metastore.truststore.path`, `hive.metastore.truststore.password`
425454

426455
|`BigDataService.KAFKA`
427456
|`bootstrap`

‎example/spark/build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import org.gradle.api.attributes.Usage
66

77
plugins {
88
id("buildsrc.convention.kotlin-jvm")
9-
id("org.openprojectx.spark.platform") version "0.1.39"
9+
id("org.openprojectx.spark.platform") version "0.1.40"
1010
id("org.openprojectx.hadoop-native-loader") version "0.1.1"
1111

1212
}

‎example/spark/src/test/kotlin/org/openprojectx/bigdata/test/example/spark/SparkBigDataScenario.kt‎

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ abstract class SparkBigDataScenario {
8484
)
8585
assertHiveMetastoreTable(
8686
hiveMetastoreUri = context.environment.hiveMetastoreUri,
87+
hiveMetastoreTlsProperties = context.environment.hiveMetastoreTlsProperties,
8788
database = "hms_demo_$runId",
8889
table = "events_hms",
8990
)
@@ -92,6 +93,7 @@ abstract class SparkBigDataScenario {
9293
assertHiveExternalParquetTable(
9394
spark = context.spark,
9495
hiveMetastoreUri = context.environment.hiveMetastoreUri,
96+
hiveMetastoreTlsProperties = context.environment.hiveMetastoreTlsProperties,
9597
database = "hive_s3_demo_$runId",
9698
table = "events_parquet_s3",
9799
location = "s3a://${context.environment.s3Bucket}/hive-parquet/events_parquet_s3",
@@ -102,6 +104,7 @@ abstract class SparkBigDataScenario {
102104
assertHiveExternalParquetTable(
103105
spark = context.spark,
104106
hiveMetastoreUri = context.environment.hiveMetastoreUri,
107+
hiveMetastoreTlsProperties = context.environment.hiveMetastoreTlsProperties,
105108
database = "hive_gcs_demo_$runId",
106109
table = "events_parquet_gcs",
107110
location = context.environment.gcsIcebergDataPath,
@@ -147,6 +150,11 @@ abstract class SparkBigDataScenario {
147150
?: kafka.properties["sasl.kerberos.service.name"],
148151
kafkaJaasConfig = kafka.properties["sasl.jaas.config"],
149152
kafkaSslProperties = kafka.properties.filterKeys { it.startsWith("ssl.") },
153+
hiveMetastoreTlsProperties = hiveMetastore.properties.filterKeys {
154+
it == "hive.metastore.use.SSL" ||
155+
it == "hive.metastore.truststore.path" ||
156+
it == "hive.metastore.truststore.password"
157+
},
150158
)
151159
}
152160

@@ -185,6 +193,7 @@ abstract class SparkBigDataScenario {
185193
.config("spark.sql.warehouse.dir", "file:${Files.createTempDirectory("bigdata-test-spark-warehouse-")}")
186194
.config("spark.sql.statistics.size.autoUpdate.enabled", "false")
187195
.config("hive.metastore.uris", environment.hiveMetastoreUri)
196+
.configureHiveMetastoreTls(environment)
188197
.config("spark.hadoop.fs.defaultFS", environment.hdfsUri)
189198
.config("spark.hadoop.hadoop.security.credential.provider.path", environment.s3CredentialProviderPath)
190199
.config("spark.hadoop.fs.s3a.endpoint", environment.s3Endpoint)
@@ -231,6 +240,16 @@ abstract class SparkBigDataScenario {
231240
return this
232241
}
233242

243+
private fun SparkSession.Builder.configureHiveMetastoreTls(
244+
environment: SparkScenarioEnvironment,
245+
): SparkSession.Builder {
246+
environment.hiveMetastoreTlsProperties.forEach { (key, value) ->
247+
config(key, value)
248+
config("spark.hadoop.$key", value)
249+
}
250+
return this
251+
}
252+
234253
protected fun assertHdfsConfigStore(spark: SparkSession, hdfsUri: String, hdfsPath: String) {
235254
val exists =
236255
org.apache.hadoop.fs.FileSystem.get(URI.create(hdfsUri), spark.sparkContext().hadoopConfiguration())
@@ -298,9 +317,13 @@ abstract class SparkBigDataScenario {
298317
check(count == 2L) { "Expected two Iceberg rows in $identifier" }
299318
}
300319

301-
protected fun assertHiveMetastoreTable(hiveMetastoreUri: String, database: String, table: String) {
302-
val conf = HiveConf()
303-
conf.setVar(HiveConf.ConfVars.METASTOREURIS, hiveMetastoreUri)
320+
protected fun assertHiveMetastoreTable(
321+
hiveMetastoreUri: String,
322+
hiveMetastoreTlsProperties: Map<String, String>,
323+
database: String,
324+
table: String,
325+
) {
326+
val conf = hiveMetastoreConf(hiveMetastoreUri, hiveMetastoreTlsProperties)
304327
val client = hiveMetastoreClient(conf)
305328
try {
306329
check(client.getAllDatabases().contains(database)) { "Expected HMS database $database" }
@@ -321,6 +344,7 @@ abstract class SparkBigDataScenario {
321344
protected fun assertHiveExternalParquetTable(
322345
spark: SparkSession,
323346
hiveMetastoreUri: String,
347+
hiveMetastoreTlsProperties: Map<String, String>,
324348
database: String,
325349
table: String,
326350
location: String,
@@ -351,7 +375,7 @@ abstract class SparkBigDataScenario {
351375
val count = spark.table(identifier).where("storage = '$storageName'").count()
352376
check(count == 2L) { "Expected two Hive external Parquet rows in $identifier" }
353377
assertParquetFiles(spark, location)
354-
assertHiveMetastoreExternalParquetTable(hiveMetastoreUri, database, table, location)
378+
assertHiveMetastoreExternalParquetTable(hiveMetastoreUri, hiveMetastoreTlsProperties, database, table, location)
355379
}
356380

357381
private fun assertParquetFiles(spark: SparkSession, location: String) {
@@ -363,12 +387,12 @@ abstract class SparkBigDataScenario {
363387

364388
private fun assertHiveMetastoreExternalParquetTable(
365389
hiveMetastoreUri: String,
390+
hiveMetastoreTlsProperties: Map<String, String>,
366391
database: String,
367392
table: String,
368393
location: String
369394
) {
370-
val conf = HiveConf()
371-
conf.setVar(HiveConf.ConfVars.METASTOREURIS, hiveMetastoreUri)
395+
val conf = hiveMetastoreConf(hiveMetastoreUri, hiveMetastoreTlsProperties)
372396
val client = hiveMetastoreClient(conf)
373397
try {
374398
val hmsTable = client.getTable(database, table)
@@ -396,6 +420,16 @@ abstract class SparkBigDataScenario {
396420
return constructor.newInstance(conf) as IMetaStoreClient
397421
}
398422

423+
private fun hiveMetastoreConf(
424+
hiveMetastoreUri: String,
425+
hiveMetastoreTlsProperties: Map<String, String>,
426+
): HiveConf {
427+
val conf = HiveConf()
428+
conf.setVar(HiveConf.ConfVars.METASTOREURIS, hiveMetastoreUri)
429+
hiveMetastoreTlsProperties.forEach { (key, value) -> conf.set(key, value) }
430+
return conf
431+
}
432+
399433
private fun icebergTableProperties(vararg properties: Pair<String, String?>): String {
400434
val entries = properties.mapNotNull { (key, value) ->
401435
value?.let { "'$key'='$it'" }
@@ -437,4 +471,5 @@ data class SparkScenarioEnvironment(
437471
val kafkaKerberosServiceName: String?,
438472
val kafkaJaasConfig: String?,
439473
val kafkaSslProperties: Map<String, String>,
474+
val hiveMetastoreTlsProperties: Map<String, String>,
440475
)

‎example/spark/src/test/resources/spark-bigdata-test-apache-hms-kerberos.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,7 @@ kafkaKerberos = true
55

66
[ports]
77
kafka = 19092
8+
9+
[hiveMetastoreTls]
10+
enabled = true
11+
domain = "localhost"

‎example/spark/src/test/resources/spark-bigdata-test-apache-hms.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,6 @@
11
[services]
22
hiveMetastore = true
3+
4+
[hiveMetastoreTls]
5+
enabled = true
6+
domain = "localhost"

‎example/spark/src/test/resources/spark-bigdata-test-cloudera-hms-kerberos.toml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,8 @@ kafka = 19092
88

99
[kafkaTls]
1010
enabled = true
11-
domain = "localhost"
11+
domain = "localhost"
12+
13+
[hiveMetastoreTls]
14+
enabled = true
15+
domain = "localhost"

‎example/spark/src/test/resources/spark-bigdata-test-cloudera-hms.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,6 @@
11
[services]
22
clouderaHms = true
3+
4+
[hiveMetastoreTls]
5+
enabled = true
6+
domain = "localhost"

0 commit comments

Comments
 (0)