feat: add TLS support for BigDataTest services and examples

- Introduced TLS configuration for HDFS, Kafka Schema Registry, Kafka UI, LocalStack S3, and Fake GCS in `BigDataTestKit` and `AutoConfiguration`.
- Implemented HTTP-to-HTTPS mapping and JVM TLS property management with configurable truststores.
- Updated examples, JUnit extensions, and container factories to include TLS options and integration with HAProxy gateways.
- Added BouncyCastle dependencies for certificate generation and management.
- Enhanced README with updated TLS-enabled configuration examples.
- Refactored core utilities to support dynamic TLS material generation, PEM file handling, and secure container setup.

Author: CoderYellow

README.md

@@ -65,6 +65,19 @@ records = [
 ]
 ```
 
+HTTP services can be exposed through an HAProxy TLS gateway from TOML:
+
+```toml
+[services]
+localStackS3 = true
+
+[localStackS3Tls]
+enabled = true
+domain = "localhost"
+```
+
+The endpoint properties then return HTTPS URLs and JVM truststore settings such as `javax.net.ssl.trustStore`.
+
 ## Run Examples
 
 Use the shared Gradle home when running this repository locally:

autoconfigure/src/main/kotlin/org/openprojectx/bigdata/test/autoconfigure/BigDataTestAutoConfiguration.kt

@@ -2,6 +2,7 @@ package org.openprojectx.bigdata.test.autoconfigure
 
 import org.openprojectx.bigdata.test.core.BigDataTestKit
 import org.openprojectx.bigdata.test.core.HdfsOptions
+import org.openprojectx.bigdata.test.core.HttpTlsOptions
 import org.openprojectx.bigdata.test.core.HiveMetastoreDistribution
 import org.openprojectx.bigdata.test.core.HiveMetastoreOptions
 import org.openprojectx.bigdata.test.core.KafkaOptions
@@ -37,13 +38,17 @@ class BigDataTestAutoConfiguration {
             )
         }
 
-        if (properties.tls.enabled) {
+        if (properties.tls.enabled || properties.anyHttpTlsEnabled()) {
             builder.withTls(
                 TlsOptions(
-                    enabled = true,
+                    enabled = properties.tls.enabled || properties.anyHttpTlsEnabled(),
+                    caCertPath = properties.tls.caCertPath,
+                    caKeyPath = properties.tls.caKeyPath,
+                    trustStorePath = properties.tls.trustStorePath,
+                    trustStorePassword = properties.tls.trustStorePassword,
+                    haproxyImage = properties.tls.haproxyImage,
                     certPath = properties.tls.certPath,
                     keyPath = properties.tls.keyPath,
-                    caCertPath = properties.tls.caCertPath,
                 ),
             )
         }
@@ -53,6 +58,7 @@ class BigDataTestAutoConfiguration {
                 HdfsOptions(
                     enabled = true,
                     image = properties.hdfs.image,
+                    webTls = properties.hdfs.webTls.toCore(),
                     kerberos = KerberosAuthOptions(
                         enabled = properties.hdfs.kerberosEnabled,
                         servicePrincipal = "nn/hdfs.example.com@${properties.kerberos.realm}",
@@ -110,8 +116,10 @@ class BigDataTestAutoConfiguration {
                     image = properties.kafka.image,
                     schemaRegistryEnabled = properties.kafka.schemaRegistryEnabled,
                     schemaRegistryImage = properties.kafka.schemaRegistryImage,
+                    schemaRegistryTls = properties.kafka.schemaRegistryTls.toCore(),
                     kafkaUiEnabled = properties.kafka.kafkaUiEnabled,
                     kafkaUiImage = properties.kafka.kafkaUiImage,
+                    kafkaUiTls = properties.kafka.kafkaUiTls.toCore(),
                     kerberos = KerberosAuthOptions(
                         enabled = properties.kafka.kerberosEnabled,
                         servicePrincipal = "kafka/broker1.example.com@${properties.kerberos.realm}",
@@ -127,13 +135,38 @@ class BigDataTestAutoConfiguration {
         }
 
         if (properties.localstackS3.enabled) {
-            builder.withLocalStackS3(ObjectStoreOptions(enabled = true, image = properties.localstackS3.image))
+            builder.withLocalStackS3(
+                ObjectStoreOptions(
+                    enabled = true,
+                    image = properties.localstackS3.image,
+                    tls = properties.localstackS3.tls.toCore(),
+                ),
+            )
         }
 
         if (properties.fakeGcs.enabled) {
-            builder.withFakeGcs(ObjectStoreOptions(enabled = true, image = properties.fakeGcs.image))
+            builder.withFakeGcs(
+                ObjectStoreOptions(
+                    enabled = true,
+                    image = properties.fakeGcs.image,
+                    tls = properties.fakeGcs.tls.toCore(defaultDomain = "storage.googleapis.com"),
+                ),
+            )
         }
 
         return builder.build().also { it.start() }
     }
+
+    private fun BigDataTestProperties.HttpTls.toCore(defaultDomain: String = "localhost"): HttpTlsOptions =
+        HttpTlsOptions(
+            enabled = enabled,
+            domain = domain.ifBlank { defaultDomain },
+        )
+
+    private fun BigDataTestProperties.anyHttpTlsEnabled(): Boolean =
+        hdfs.webTls.enabled ||
+            kafka.schemaRegistryTls.enabled ||
+            kafka.kafkaUiTls.enabled ||
+            localstackS3.tls.enabled ||
+            fakeGcs.tls.enabled
 }

autoconfigure/src/main/kotlin/org/openprojectx/bigdata/test/autoconfigure/BigDataTestProperties.kt

@@ -25,15 +25,27 @@ data class BigDataTestProperties(
 
     data class Tls(
         var enabled: Boolean = false,
+        var caCertPath: String? = null,
+        var caKeyPath: String? = null,
+        var trustStorePath: String? = null,
+        var trustStorePassword: String = "changeit",
+        var haproxyImage: String = "haproxy:3.0-alpine",
+        @Deprecated("Use caCertPath")
         var certPath: String? = null,
+        @Deprecated("Use caKeyPath")
         var keyPath: String? = null,
-        var caCertPath: String? = null,
+    )
+
+    data class HttpTls(
+        var enabled: Boolean = false,
+        var domain: String = "localhost",
     )
 
     data class Hdfs(
         var enabled: Boolean = false,
         var image: String = "apache/hadoop:3.5.0",
         var kerberosEnabled: Boolean = false,
+        var webTls: HttpTls = HttpTls(),
     )
 
     data class HiveMetastore(
@@ -61,14 +73,17 @@ data class BigDataTestProperties(
         var image: String = "apache/kafka:4.1.2",
         var schemaRegistryEnabled: Boolean = false,
         var schemaRegistryImage: String = "confluentinc/cp-schema-registry:7.8.0",
+        var schemaRegistryTls: HttpTls = HttpTls(),
         var kafkaUiEnabled: Boolean = false,
         var kafkaUiImage: String = "ghcr.io/kafbat/kafka-ui:latest",
+        var kafkaUiTls: HttpTls = HttpTls(),
         var kerberosEnabled: Boolean = false,
         var kafkaUiKerberosEnabled: Boolean = false,
     )
 
     data class ObjectStore(
         var enabled: Boolean = false,
         var image: String,
+        var tls: HttpTls = HttpTls(),
     )
 }

core/build.gradle.kts

@@ -4,6 +4,8 @@ plugins {
 
 
 dependencies {
+    implementation(libs.bouncycastlePkix)
+    implementation(libs.bouncycastleProvider)
     api(libs.testcontainers)
     api(libs.testcontainersPostgresql)
     api(libs.testcontainersKafka)

core/src/main/kotlin/org/openprojectx/bigdata/test/core/BigDataService.kt

@@ -17,8 +17,8 @@ enum class BigDataService(
         ),
     ),
     HDFS(
-        defaultPorts = mapOf("namenode" to 8020, "web" to 9870),
-        endpointProperties = setOf("fs.defaultFS", "spring.hadoop.fs-uri"),
+        defaultPorts = mapOf("namenode" to 8020, "web" to 9870, "web-tls" to 443),
+        endpointProperties = setOf("fs.defaultFS", "spring.hadoop.fs-uri", "dfs.namenode.https-address"),
     ),
     HIVE_METASTORE(
         defaultPorts = mapOf("thrift" to 9083),
@@ -29,15 +29,15 @@ enum class BigDataService(
         endpointProperties = setOf("bootstrap.servers", "spring.kafka.bootstrap-servers", "bootstrap.servers.internal"),
     ),
     SCHEMA_REGISTRY(
-        defaultPorts = mapOf("http" to 8085),
+        defaultPorts = mapOf("http" to 8085, "https" to 443),
         endpointProperties = setOf("schema.registry.url"),
     ),
     KAFKA_UI(
-        defaultPorts = mapOf("http" to 8080),
+        defaultPorts = mapOf("http" to 8080, "https" to 443),
         endpointProperties = setOf("bigdata.test.kafka-ui.url"),
     ),
     LOCALSTACK_S3(
-        defaultPorts = mapOf("edge" to 4566),
+        defaultPorts = mapOf("edge" to 4566, "https" to 443),
         endpointProperties = setOf(
             "spring.cloud.aws.s3.endpoint",
             "aws.endpoint-url.s3",
@@ -47,7 +47,7 @@ enum class BigDataService(
         ),
     ),
     FAKE_GCS(
-        defaultPorts = mapOf("http" to 4443),
+        defaultPorts = mapOf("http" to 4443, "https" to 443),
         endpointProperties = setOf("bigdata.test.gcs.endpoint", "google.cloud.storage.host"),
     ),
 }

core/src/main/kotlin/org/openprojectx/bigdata/test/core/BigDataTestKitOptions.kt

@@ -36,16 +36,28 @@ data class KerberosAuthOptions(
 
 data class TlsOptions(
     val enabled: Boolean = false,
+    val caCertPath: String? = null,
+    val caKeyPath: String? = null,
+    val trustStorePath: String? = null,
+    val trustStorePassword: String = "changeit",
+    val haproxyImage: String = "haproxy:3.0-alpine",
+    @Deprecated("Use caCertPath for the root CA certificate")
     val certPath: String? = null,
+    @Deprecated("Use caKeyPath for the root CA private key")
     val keyPath: String? = null,
-    val caCertPath: String? = null,
+)
+
+data class HttpTlsOptions(
+    val enabled: Boolean = false,
+    val domain: String = "localhost",
 )
 
 data class HdfsOptions(
     val enabled: Boolean = false,
     val image: String = "apache/hadoop:3.5.0",
     val nameNodePort: Int = 8020,
     val webPort: Int = 9870,
+    val webTls: HttpTlsOptions = HttpTlsOptions(),
     val kerberos: KerberosAuthOptions = KerberosAuthOptions(
         servicePrincipal = "nn/hdfs.example.com@EXAMPLE.COM",
         keytabPath = "/kerby/keytabs/hdfs-namenode.keytab",
@@ -78,8 +90,10 @@ data class KafkaOptions(
     val image: String = "apache/kafka:4.1.2",
     val schemaRegistryEnabled: Boolean = false,
     val schemaRegistryImage: String = "confluentinc/cp-schema-registry:7.8.0",
+    val schemaRegistryTls: HttpTlsOptions = HttpTlsOptions(),
     val kafkaUiEnabled: Boolean = false,
     val kafkaUiImage: String = "ghcr.io/kafbat/kafka-ui:latest",
+    val kafkaUiTls: HttpTlsOptions = HttpTlsOptions(),
     val clusterId: String = "MkU3OEVBNTcwNTJENDM2Qk",
     val kerberos: KerberosAuthOptions = KerberosAuthOptions(
         servicePrincipal = "kafka/localhost@EXAMPLE.COM",
@@ -94,6 +108,7 @@ data class KafkaOptions(
 data class ObjectStoreOptions(
     val enabled: Boolean = false,
     val image: String = "localstack/localstack:4.14.0",
+    val tls: HttpTlsOptions = HttpTlsOptions(),
 )
 
 
@@ -103,12 +118,17 @@ data class PortBindingOptions(
     val kerberosKdc: Int = 0,
     val hdfsNameNode: Int = 0,
     val hdfsWeb: Int = 0,
+    val hdfsWebTls: Int = 0,
     val hiveMetastore: Int = 0,
     val kafka: Int = 0,
     val schemaRegistry: Int = 0,
+    val schemaRegistryTls: Int = 0,
     val kafkaUi: Int = 0,
+    val kafkaUiTls: Int = 0,
     val localStackS3: Int = 0,
+    val localStackS3Tls: Int = 0,
     val fakeGcs: Int = 0,
+    val fakeGcsTls: Int = 0,
 ) {
     fun hostPort(containerPort: Int, configuredHostPort: Int): Int {
         require(configuredHostPort >= 0) { "Host port must be 0 for random binding or a positive fixed port" }