Skip to content

Commit 7e5f8cd

Browse files
committed
feat: refine Hive configuration loading and improve container file handling
- Replaced temporary directory-based Hive configuration with in-memory file transfer for better efficiency. - Enhanced container setup by switching to `withCopyToContainer` for custom Hive configuration and TLS files. - Simplified XML configuration generation with reusable `configurationXml` utility.
1 parent d8391d9 commit 7e5f8cd

1 file changed

Lines changed: 24 additions & 24 deletions

File tree

core/src/main/kotlin/org/openprojectx/bigdata/test/core/container/BigDataContainerFactory.kt

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -199,9 +199,10 @@ internal class BigDataContainerFactory(
199199
val tlsProperties = if (hive.tls.enabled) configureHiveMetastoreTls(container, hive.tls) else emptyMap()
200200
configureHiveDockerObjectStores(container)
201201
if (hive.extraConfiguration.isNotEmpty() || hive.kerberos.enabled || hive.tls.enabled) {
202-
container
203-
.withEnv("HIVE_CUSTOM_CONF_DIR", "/bigdata-test/hive-conf")
204-
.withFileSystemBind(openSourceHiveConfigurationDirectory().toString(), "/bigdata-test/hive-conf", BindMode.READ_ONLY)
202+
container.withEnv("HIVE_CUSTOM_CONF_DIR", "/bigdata-test/hive-conf")
203+
openSourceHiveConfigurationFiles().forEach { (fileName, content) ->
204+
container.withCopyToContainer(Transferable.of(content), "/bigdata-test/hive-conf/$fileName")
205+
}
205206
}
206207

207208
return BigDataServiceContainer(BigDataService.HIVE_METASTORE, attachLogs("hive-metastore", container)) {
@@ -591,9 +592,8 @@ internal class BigDataContainerFactory(
591592
}
592593
}
593594

594-
private fun openSourceHiveConfigurationDirectory(): Path {
595+
private fun openSourceHiveConfigurationFiles(): Map<String, String> {
595596
val hive = options.hiveMetastore
596-
val dir = Files.createTempDirectory("bigdata-test-hive-conf-")
597597
val metastoreProperties = linkedMapOf(
598598
"hive.metastore.warehouse.dir" to hive.warehouseDir,
599599
"javax.jdo.option.ConnectionURL" to "jdbc:postgresql://hive-metastore-postgres:5432/${hive.databaseName}",
@@ -615,29 +615,29 @@ internal class BigDataContainerFactory(
615615
metastoreProperties += hive.extraConfiguration
616616
val hadoopProperties = hiveMetastoreObjectStoreConfiguration()
617617

618-
writeConfigurationXml(dir.resolve("hive-site.xml"), metastoreProperties + hadoopProperties)
619-
writeConfigurationXml(dir.resolve("metastore-site.xml"), metastoreProperties + hadoopProperties)
620-
writeConfigurationXml(dir.resolve("core-site.xml"), hadoopProperties)
621-
return dir
618+
return mapOf(
619+
"hive-site.xml" to configurationXml(metastoreProperties + hadoopProperties),
620+
"metastore-site.xml" to configurationXml(metastoreProperties + hadoopProperties),
621+
"core-site.xml" to configurationXml(hadoopProperties),
622+
)
622623
}
623624

624625
private fun writeConfigurationXml(path: Path, properties: Map<String, String>) {
625-
Files.writeString(
626-
path,
627-
buildString {
628-
appendLine("<configuration>")
629-
properties.forEach { (key, value) ->
630-
appendLine(" <property>")
631-
appendLine(" <name>${xmlEscape(key)}</name>")
632-
appendLine(" <value>${xmlEscape(value)}</value>")
633-
appendLine(" </property>")
634-
}
635-
appendLine("</configuration>")
636-
},
637-
StandardCharsets.UTF_8,
638-
)
626+
Files.writeString(path, configurationXml(properties), StandardCharsets.UTF_8)
639627
}
640628

629+
private fun configurationXml(properties: Map<String, String>): String =
630+
buildString {
631+
appendLine("<configuration>")
632+
properties.forEach { (key, value) ->
633+
appendLine(" <property>")
634+
appendLine(" <name>${xmlEscape(key)}</name>")
635+
appendLine(" <value>${xmlEscape(value)}</value>")
636+
appendLine(" </property>")
637+
}
638+
appendLine("</configuration>")
639+
}
640+
641641
private fun httpTlsEndpoint(
642642
name: String,
643643
tls: HttpTlsOptions,
@@ -744,7 +744,7 @@ internal class BigDataContainerFactory(
744744
sanDomains = listOf("hive-metastore", "hive-metastore.example.com"),
745745
)
746746
val keyStorePath = "/bigdata-test/tls/hive-metastore.p12"
747-
container.withFileSystemBind(keyStore.path.toString(), keyStorePath, BindMode.READ_ONLY)
747+
container.withCopyToContainer(Transferable.of(Files.readAllBytes(keyStore.path)), keyStorePath)
748748
if (options.hiveMetastore.distribution == HiveMetastoreDistribution.CLOUDERA) {
749749
container.withEnv(
750750
"HMS_EXTRA_CONF",

0 commit comments

Comments
 (0)