Skip to content

Commit d49a262

Browse files
CoderYellowCoderYellow
committed
feat(hive): add support for local Hive Metastore client XML provisioning
- Introduced `localHiveSitePath` and `localMetastoreSitePath` options for writing host-side Hive Metastore client XML files. - Updated `BigDataContainerFactory` to dynamically generate and manage `hive-site.xml` and `metastore-site.xml`. - Enhanced Gradle plugin to propagate and configure these options. - Updated user documentation with usage details and configuration examples.
1 parent 76cf86d commit d49a262

9 files changed

Lines changed: 100 additions & 14 deletions

File tree

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/BigDataTestKitOptions.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ data class HiveMetastoreOptions(
8888
val databaseUser: String = "hive",
8989
val databasePassword: String = "hive",
9090
val warehouseDir: String = "/user/hive/warehouse",
91+
val localHiveSitePath: String? = null,
92+
val localMetastoreSitePath: String? = null,
9193
val extraConfiguration: Map<String, String> = emptyMap(),
9294
val tls: HttpTlsOptions = HttpTlsOptions(),
9395
val kerberos: KerberosAuthOptions = KerberosAuthOptions(

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/config/BigDataTestConfig.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,17 @@ data class BigDataTestHiveMetastoreConfig(
8484
val databaseUser: String? = null,
8585
val databasePassword: String? = null,
8686
val warehouseDir: String? = null,
87+
val localHiveSitePath: String? = null,
88+
val localMetastoreSitePath: String? = null,
8789
) {
8890
fun merge(override: BigDataTestHiveMetastoreConfig): BigDataTestHiveMetastoreConfig =
8991
BigDataTestHiveMetastoreConfig(
9092
databaseName = override.databaseName ?: databaseName,
9193
databaseUser = override.databaseUser ?: databaseUser,
9294
databasePassword = override.databasePassword ?: databasePassword,
9395
warehouseDir = override.warehouseDir ?: warehouseDir,
96+
localHiveSitePath = override.localHiveSitePath ?: localHiveSitePath,
97+
localMetastoreSitePath = override.localMetastoreSitePath ?: localMetastoreSitePath,
9498
)
9599
}
96100

@@ -374,6 +378,8 @@ class BigDataTestConfigLoader(
374378
databaseUser = hiveMetastore.string("databaseUser"),
375379
databasePassword = hiveMetastore.string("databasePassword"),
376380
warehouseDir = hiveMetastore.string("warehouseDir"),
381+
localHiveSitePath = hiveMetastore.string("localHiveSitePath"),
382+
localMetastoreSitePath = hiveMetastore.string("localMetastoreSitePath"),
377383
),
378384
clouderaHms = BigDataTestClouderaHmsConfig(warehouseDir = clouderaHms.string("warehouseDir")),
379385
kafka = BigDataTestKafkaConfig(

‎core/src/main/kotlin/org/openprojectx/bigdata/test/core/container/BigDataContainerFactory.kt‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import org.openprojectx.bigdata.test.core.ContainerPortOptions
1010
import org.openprojectx.bigdata.test.core.HdfsOptions
1111
import org.openprojectx.bigdata.test.core.HttpTlsOptions
1212
import org.openprojectx.bigdata.test.core.HiveMetastoreDistribution
13+
import org.openprojectx.bigdata.test.core.HiveMetastoreOptions
1314
import org.openprojectx.bigdata.test.core.KerberosAuthOptions
1415
import org.openprojectx.bigdata.test.core.KafkaOptions
1516
import org.openprojectx.bigdata.test.core.KerberosOptions
@@ -56,6 +57,13 @@ internal class BigDataContainerFactory(
5657
?: kerberosDir
5758
?: Files.createTempDirectory("bigdata-test-hdfs-client-")
5859
}
60+
private val hiveMetastoreClientDir: Path by lazy {
61+
listOf(options.hiveMetastore.localHiveSitePath, options.hiveMetastore.localMetastoreSitePath)
62+
.firstNotNullOfOrNull { it?.let { path -> Path.of(path).parent } }
63+
?.let { Files.createDirectories(it) }
64+
?: kerberosDir
65+
?: Files.createTempDirectory("bigdata-test-hms-client-")
66+
}
5967
private val tlsMaterial: TlsMaterial by lazy { TlsMaterial(options.tls.copy(enabled = true)) }
6068

6169
fun healthCheck(service: BigDataService, container: GenericContainer<*>, endpoint: BigDataEndpoint) {
@@ -286,15 +294,19 @@ internal class BigDataContainerFactory(
286294
attachLogs("hive-metastore", applyContainerCustomizations(BigDataService.HIVE_METASTORE, container)),
287295
) {
288296
val thriftUri = container.thriftUri
297+
val clientProperties = hiveMetastoreClientProperties(thriftUri, tlsProperties)
298+
val clientXmlPaths = writeLocalHiveMetastoreClientXml(hive, clientProperties)
289299
BigDataEndpoint(
290300
service = BigDataService.HIVE_METASTORE,
291301
host = container.host,
292302
ports = mapOf("thrift" to container.getMappedPort(9083)),
293303
properties = mapOf(
294304
"hive.metastore.uris" to thriftUri,
295305
"spring.bigdata.test.hive-metastore.thrift-uri" to thriftUri,
306+
"bigdata.test.hive-metastore.hive-site" to clientXmlPaths.hiveSite,
307+
"bigdata.test.hive-metastore.metastore-site" to clientXmlPaths.metastoreSite,
296308
) + tlsProperties +
297-
hiveMetastoreClientKerberosProperties(hive.kerberos) +
309+
clientProperties +
298310
kerberosProperties("hive.metastore", hive.kerberos),
299311
)
300312
}
@@ -335,15 +347,19 @@ internal class BigDataContainerFactory(
335347
attachLogs("hive-metastore", applyContainerCustomizations(BigDataService.HIVE_METASTORE, container)),
336348
) {
337349
val thriftUri = "thrift://${container.host}:${container.getMappedPort(9083)}"
350+
val clientProperties = hiveMetastoreClientProperties(thriftUri, tlsProperties)
351+
val clientXmlPaths = writeLocalHiveMetastoreClientXml(hive, clientProperties)
338352
BigDataEndpoint(
339353
service = BigDataService.HIVE_METASTORE,
340354
host = container.host,
341355
ports = mapOf("thrift" to container.getMappedPort(9083)),
342356
properties = mapOf(
343357
"hive.metastore.uris" to thriftUri,
344358
"spring.bigdata.test.hive-metastore.thrift-uri" to thriftUri,
359+
"bigdata.test.hive-metastore.hive-site" to clientXmlPaths.hiveSite,
360+
"bigdata.test.hive-metastore.metastore-site" to clientXmlPaths.metastoreSite,
345361
) + tlsProperties +
346-
hiveMetastoreClientKerberosProperties(hive.kerberos) +
362+
clientProperties +
347363
kerberosProperties("hive.metastore", hive.kerberos),
348364
)
349365
}
@@ -1222,6 +1238,39 @@ internal class BigDataContainerFactory(
12221238
return path.toString()
12231239
}
12241240

1241+
private fun hiveMetastoreClientProperties(
1242+
thriftUri: String,
1243+
tlsProperties: Map<String, String>,
1244+
): Map<String, String> =
1245+
mapOf(
1246+
"hive.metastore.uris" to thriftUri,
1247+
) + hiveMetastoreHadoopConfiguration() +
1248+
hiveMetastoreClientKerberosProperties(options.hiveMetastore.kerberos) +
1249+
tlsProperties
1250+
1251+
private fun writeLocalHiveMetastoreClientXml(
1252+
hive: HiveMetastoreOptions,
1253+
properties: Map<String, String>,
1254+
): HiveMetastoreClientXmlPaths {
1255+
val hiveSitePath = hive.localHiveSitePath
1256+
?.let { Path.of(it) }
1257+
?: hiveMetastoreClientDir.resolve("hive-site.xml")
1258+
val metastoreSitePath = hive.localMetastoreSitePath
1259+
?.let { Path.of(it) }
1260+
?: hiveMetastoreClientDir.resolve("metastore-site.xml")
1261+
writeConfigurationXml(hiveSitePath, properties)
1262+
writeConfigurationXml(metastoreSitePath, properties)
1263+
return HiveMetastoreClientXmlPaths(
1264+
hiveSite = hiveSitePath.toString(),
1265+
metastoreSite = metastoreSitePath.toString(),
1266+
)
1267+
}
1268+
1269+
private data class HiveMetastoreClientXmlPaths(
1270+
val hiveSite: String,
1271+
val metastoreSite: String,
1272+
)
1273+
12251274
private fun hdfsSiteSecurity(kerberos: KerberosAuthOptions): String =
12261275
if (kerberos.enabled) {
12271276
"""

‎doc/user-guide.adoc‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,17 @@ hdfsWeb = 9870
300300

301301
`localHdfsSitePath` is optional. When set, the framework writes a host-side Hadoop XML file with the resolved HDFS client properties, including the mapped NameNode URI, DataNode hostname mode, and Kerberos client settings when HDFS Kerberos is enabled. When omitted, the file is generated in framework-managed temporary material.
302302

303+
HMS client XML files can be provisioned the same way. These files are host-side client resources and intentionally contain only client-safe settings such as `hive.metastore.uris`, Kerberos client flags, TLS truststore settings, and object-store/Hadoop properties. They do not include HMS server database credentials.
304+
305+
[source,toml]
306+
----
307+
[hiveMetastore]
308+
localHiveSitePath = "build/bigdata-test/hive/hive-site.xml"
309+
localMetastoreSitePath = "build/bigdata-test/hive/metastore-site.xml"
310+
----
311+
312+
When omitted, both files are generated in framework-managed temporary material. The same options apply to Apache HMS and Cloudera HMS.
313+
303314
=== Container Customization Escape Hatch
304315

305316
Typed service options should cover the normal path, but integration-test containers sometimes need image-specific workarounds before the framework has first-class support. Use container customizations as an escape hatch for troubleshooting or temporary overrides.
@@ -1620,12 +1631,17 @@ The plugin injects raw endpoint properties, such as `fs.defaultFS`, `hive.metast
16201631
bigdata.test.endpoint.hdfs.properties.fs.defaultFS
16211632
bigdata.test.endpoint.hdfs.properties.bigdata.test.hdfs.hdfs-site
16221633
bigdata.test.endpoint.hdfs.ports.namenode
1634+
bigdata.test.endpoint.hive-metastore.properties.bigdata.test.hive-metastore.hive-site
1635+
bigdata.test.endpoint.hive-metastore.properties.bigdata.test.hive-metastore.metastore-site
1636+
bigdata.test.endpoint.hive-metastore.properties.hive.metastore.uris
16231637
bigdata.test.endpoint.localstack-s3.properties.aws.endpoint-url.s3
16241638
bigdata.test.extensions.s3-jceks.credential-provider.path
16251639
----
16261640

16271641
For HDFS clients that prefer Hadoop XML resources, read `bigdata.test.endpoint.hdfs.properties.bigdata.test.hdfs.hdfs-site` and add that file to the Hadoop `Configuration`. The Gradle plugin also injects the same value as an environment variable for launched tasks.
16281642

1643+
For HMS clients that prefer XML resources, read `bigdata.test.endpoint.hive-metastore.properties.bigdata.test.hive-metastore.hive-site` or `bigdata.test.endpoint.hive-metastore.properties.bigdata.test.hive-metastore.metastore-site` and add the file to the Hive/Hadoop client configuration. The Gradle plugin injects both values as system properties and environment variables for launched tasks.
1644+
16291645
Environment variable names are generated by uppercasing property keys and replacing non-alphanumeric characters with `_`:
16301646

16311647
[source,text]

‎example/spark/src/test/kotlin/org/openprojectx/bigdata/test/example/spark/SparkBigDataScenario.kt‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -240,18 +240,18 @@ abstract class SparkBigDataScenario {
240240

241241
private fun SparkSession.Builder.configureKerberos(environment: SparkScenarioEnvironment): SparkSession.Builder {
242242
environment.krb5Conf?.let { config("spark.hadoop.java.security.krb5.conf", it) }
243-
environment.kerberosClientPrincipal?.let {
244-
config("spark.hadoop.bigdata.test.kerberos.client.principal", it)
245-
}
246-
environment.kerberosClientKeytab?.let {
247-
config("spark.hadoop.bigdata.test.kerberos.client.keytab", it)
248-
}
249-
environment.kafkaKerberosServiceName?.let {
250-
config("spark.hadoop.bigdata.test.kafka.service.name", it)
251-
}
252-
environment.kafkaJaasConfig?.let {
253-
config("spark.hadoop.bigdata.test.kafka.jaas.config", it)
254-
}
243+
// environment.kerberosClientPrincipal?.let {
244+
// config("spark.hadoop.bigdata.test.kerberos.client.principal", it)
245+
// }
246+
// environment.kerberosClientKeytab?.let {
247+
// config("spark.hadoop.bigdata.test.kerberos.client.keytab", it)
248+
// }
249+
// environment.kafkaKerberosServiceName?.let {
250+
// config("spark.hadoop.bigdata.test.kafka.service.name", it)
251+
// }
252+
// environment.kafkaJaasConfig?.let {
253+
// config("spark.hadoop.bigdata.test.kafka.jaas.config", it)
254+
// }
255255
return this
256256
}
257257

‎example/spark/src/test/resources/spark-bigdata-test-common.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,4 @@ hdfsNameNode = 8020
2323

2424
[containerLogs]
2525
mode = "FILE"
26+
append = false

‎gradle-plugin/src/main/kotlin/org/openprojectx/bigdata/test/gradle/BigDataTestGradleExtension.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,8 @@ abstract class BigDataTestGradleHiveMetastore @Inject constructor(objects: Objec
168168
val databaseUser: Property<String> = objects.property(String::class.java).convention("hive")
169169
val databasePassword: Property<String> = objects.property(String::class.java).convention("hive")
170170
val warehouseDir: Property<String> = objects.property(String::class.java).convention("/user/hive/warehouse")
171+
val localHiveSitePath: Property<String> = objects.property(String::class.java).convention("")
172+
val localMetastoreSitePath: Property<String> = objects.property(String::class.java).convention("")
171173
val kerberosEnabled: Property<Boolean> = objects.property(Boolean::class.java).convention(false)
172174
}
173175

‎gradle-plugin/src/main/kotlin/org/openprojectx/bigdata/test/gradle/BigDataTestGradlePlugin.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ class BigDataTestGradlePlugin : Plugin<Project> {
101101
spec.parameters.hiveMetastoreDatabaseUser.set(extension.hiveMetastore.databaseUser)
102102
spec.parameters.hiveMetastoreDatabasePassword.set(extension.hiveMetastore.databasePassword)
103103
spec.parameters.hiveMetastoreWarehouseDir.set(extension.hiveMetastore.warehouseDir)
104+
spec.parameters.hiveMetastoreLocalHiveSitePath.set(extension.hiveMetastore.localHiveSitePath)
105+
spec.parameters.hiveMetastoreLocalMetastoreSitePath.set(extension.hiveMetastore.localMetastoreSitePath)
104106
spec.parameters.hiveMetastoreKerberosEnabled.set(extension.hiveMetastore.kerberosEnabled)
105107

106108
spec.parameters.clouderaHmsImage.set(extension.clouderaHms.image)
@@ -313,6 +315,8 @@ class BigDataTestGradlePlugin : Plugin<Project> {
313315
extension.hiveMetastore.databaseUser.tomlConvention(config.hiveMetastore.databaseUser)
314316
extension.hiveMetastore.databasePassword.tomlConvention(config.hiveMetastore.databasePassword)
315317
extension.hiveMetastore.warehouseDir.tomlConvention(config.hiveMetastore.warehouseDir)
318+
extension.hiveMetastore.localHiveSitePath.tomlConvention(config.hiveMetastore.localHiveSitePath)
319+
extension.hiveMetastore.localMetastoreSitePath.tomlConvention(config.hiveMetastore.localMetastoreSitePath)
316320
extension.clouderaHms.warehouseDir.tomlConvention(config.clouderaHms.warehouseDir)
317321

318322
extension.ports.sameHostPorts.tomlConvention(config.ports.sameHostPorts)

‎gradle-plugin/src/main/kotlin/org/openprojectx/bigdata/test/gradle/BigDataTestGradleService.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ abstract class BigDataTestGradleService : BuildService<BigDataTestGradleService.
9393
val hiveMetastoreDatabaseUser: Property<String>
9494
val hiveMetastoreDatabasePassword: Property<String>
9595
val hiveMetastoreWarehouseDir: Property<String>
96+
val hiveMetastoreLocalHiveSitePath: Property<String>
97+
val hiveMetastoreLocalMetastoreSitePath: Property<String>
9698
val hiveMetastoreKerberosEnabled: Property<Boolean>
9799

98100
val clouderaHmsImage: Property<String>
@@ -226,6 +228,8 @@ abstract class BigDataTestGradleService : BuildService<BigDataTestGradleService.
226228
databaseUser = parameters.hiveMetastoreDatabaseUser.get(),
227229
databasePassword = parameters.hiveMetastoreDatabasePassword.get(),
228230
warehouseDir = parameters.hiveMetastoreWarehouseDir.get(),
231+
localHiveSitePath = projectPath(parameters.hiveMetastoreLocalHiveSitePath.get()),
232+
localMetastoreSitePath = projectPath(parameters.hiveMetastoreLocalMetastoreSitePath.get()),
229233
kerberos = KerberosAuthOptions(
230234
enabled = parameters.hiveMetastoreKerberosEnabled.get(),
231235
servicePrincipal = "hive/hive-metastore.${parameters.kerberosDomain.get()}@${parameters.kerberosRealm.get()}",
@@ -241,6 +245,8 @@ abstract class BigDataTestGradleService : BuildService<BigDataTestGradleService.
241245
distribution = HiveMetastoreDistribution.CLOUDERA,
242246
image = parameters.clouderaHmsImage.get(),
243247
warehouseDir = parameters.clouderaHmsWarehouseDir.get(),
248+
localHiveSitePath = projectPath(parameters.hiveMetastoreLocalHiveSitePath.get()),
249+
localMetastoreSitePath = projectPath(parameters.hiveMetastoreLocalMetastoreSitePath.get()),
244250
kerberos = KerberosAuthOptions(
245251
enabled = parameters.clouderaHmsKerberosEnabled.get(),
246252
servicePrincipal = "hive/hive-metastore.${parameters.kerberosDomain.get()}@${parameters.kerberosRealm.get()}",

0 commit comments

Comments
 (0)