feat: add configuration options for Kerberos material and container log append mode

- Introduced Kerberos configuration options: `materialDirectory`, `localKrb5ConfPath`, and `localClientKeytabPath` for better control over generated Kerberos files.
- Added `append` flag for container log options to toggle between appending or overwriting logs. Updated default behavior to append for backward compatibility.
- Updated documentation examples and test configurations to reflect new features.
- Minor updates to Gradle Plugin, JUnit5, and Core modules to integrate changes.

Author: CoderYellow

core/src/main/kotlin/org/openprojectx/bigdata/test/core/BigDataTestKitOptions.kt

@@ -23,6 +23,9 @@ data class KerberosOptions(
     val domain: String = "example.com",
     val clientPrincipal: String = "app_user@EXAMPLE.COM",
     val clientPassword: String = "app-user-secret",
+    val materialDirectory: String? = null,
+    val localKrb5ConfPath: String? = null,
+    val localClientKeytabPath: String? = null,
     val users: List<KerberosUserOptions> = emptyList(),
     val startupTimeoutSeconds: Int = 120,
     val materialTimeoutSeconds: Int = 30,
@@ -163,6 +166,7 @@ enum class ContainerLogMode {
 data class ContainerLogOptions(
     val mode: ContainerLogMode = ContainerLogMode.NONE,
     val directory: String = "build/bigdata-test-container-logs",
+    val append: Boolean = true,
 )
 
 enum class BigDataHealthCheckMode {

core/src/main/kotlin/org/openprojectx/bigdata/test/core/container/BigDataContainerFactory.kt

@@ -41,7 +41,14 @@ internal class BigDataContainerFactory(
     private val network = Network.newNetwork()
     private val supportContainers = mutableListOf<Startable>()
     private val logConsumers = mutableListOf<Closeable>()
-    private val kerberosDir: Path? = if (kerberosRequired()) Files.createTempDirectory("bigdata-test-kerberos-") else null
+    private val kerberosDir: Path? =
+        if (kerberosRequired()) {
+            options.kerberos.materialDirectory
+                ?.let { Files.createDirectories(Path.of(it)) }
+                ?: Files.createTempDirectory("bigdata-test-kerberos-")
+        } else {
+            null
+        }
     private val tlsMaterial: TlsMaterial by lazy { TlsMaterial(options.tls.copy(enabled = true)) }
 
     fun healthCheck(service: BigDataService, container: GenericContainer<*>, endpoint: BigDataEndpoint) {
@@ -1045,7 +1052,11 @@ internal class BigDataContainerFactory(
                     Files.newOutputStream(
                         logDir.resolve("${sanitizeLogName(name)}.log"),
                         StandardOpenOption.CREATE,
-                        StandardOpenOption.APPEND,
+                        if (options.containerLogs.append) {
+                            StandardOpenOption.APPEND
+                        } else {
+                            StandardOpenOption.TRUNCATE_EXISTING
+                        },
                     ),
                     StandardCharsets.UTF_8,
                 )
@@ -1356,9 +1367,14 @@ internal class BigDataContainerFactory(
 
     private fun localKerberosPath(containerPath: String): String =
         when {
-            containerPath == "/kerby/client/krb5.conf" -> Path.of(kerberosDirectory())
-                .resolve("krb5-local.conf")
-                .toString()
+            containerPath == "/kerby/client/krb5.conf" ->
+                options.kerberos.localKrb5ConfPath ?: Path.of(kerberosDirectory())
+                    .resolve("krb5-local.conf")
+                    .toString()
+            containerPath == "/kerby/keytabs/client.keytab" ->
+                options.kerberos.localClientKeytabPath ?: Path.of(kerberosDirectory())
+                    .resolve(containerPath.removePrefix("/kerby/"))
+                    .toString()
             containerPath.startsWith("/kerby/") -> Path.of(kerberosDirectory())
                 .resolve(containerPath.removePrefix("/kerby/"))
                 .toString()

doc/user-guide.adoc

@@ -887,7 +887,13 @@ Write one file per service:
 ----
 val kit = BigDataTestKit.builder()
     .withHiveMetastore()
-    .withContainerLogsToDirectory("build/container-logs")
+    .withContainerLogs(
+        ContainerLogOptions(
+            mode = ContainerLogMode.FILE,
+            directory = "build/container-logs",
+            append = false,
+        ),
+    )
     .build()
 ----
 
@@ -911,6 +917,7 @@ val kit = BigDataTestKit.builder()
     schemaRegistry = true,
     containerLogMode = ContainerLogMode.FILE,
     containerLogDirectory = "build/container-logs",
+    containerLogAppend = false,
 )
 class MyTroubleshootingTest
 ----
@@ -933,6 +940,18 @@ localStackS3 = "TRACE"
 * `hive-metastore.log`
 * `hive-metastore-postgres.log`
 
+File logs append by default for backward compatibility. Set `append = false` in TOML, `containerLogAppend = false` on `@BigDataTest`, or `ContainerLogOptions(append = false)` programmatically to overwrite the files at the start of each kit run.
+
+TOML example:
+
+[source,toml]
+----
+[containerLogs]
+mode = "FILE"
+directory = "build/container-logs"
+append = false
+----
+
 For HDFS troubleshooting, use `DEBUG` or `TRACE`. The framework maps that to Hadoop's container-side log settings and tails the NameNode/DataNode log files so startup failures show the Hadoop process output, not only the Testcontainers wait-strategy error.
 
 [source,kotlin]
@@ -994,14 +1013,17 @@ Kerberos bootstrap timing is configurable from TOML. The defaults are intentiona
 [kerberos]
 clientPrincipal = "app_user@EXAMPLE.COM"
 clientPassword = "app-user-secret"
+materialDirectory = "build/bigdata-test/kerberos"
+localKrb5ConfPath = "build/bigdata-test/kerberos/krb5.conf"
+localClientKeytabPath = "build/bigdata-test/kerberos/app_user.keytab"
 startupTimeoutSeconds = 180
 materialTimeoutSeconds = 120
 adminAttempts = 120
 adminRetryDelaySeconds = 1
 debug = true
 ----
 
-`clientPrincipal` and `clientPassword` configure the generated app/client user and keytab. `startupTimeoutSeconds` controls how long Testcontainers waits for the KDC image to print its ready log. `materialTimeoutSeconds` controls the post-start wait for generated keytabs and client config to be readable. `adminAttempts` and `adminRetryDelaySeconds` are passed to the Kerby image as `KERBY_KADMIN_ATTEMPTS` and `KERBY_KADMIN_RETRY_DELAY_SECONDS`.
+`clientPrincipal` and `clientPassword` configure the generated app/client user and keytab. `materialDirectory` makes generated Kerberos material land in a stable local directory instead of a temporary directory. `localKrb5ConfPath` and `localClientKeytabPath` optionally pin the exact local files exposed through `bigdata.test.kerberos.krb5-conf` and `bigdata.test.kerberos.client-keytab`. `startupTimeoutSeconds` controls how long Testcontainers waits for the KDC image to print its ready log. `materialTimeoutSeconds` controls the post-start wait for generated keytabs and client config to be readable. `adminAttempts` and `adminRetryDelaySeconds` are passed to the Kerby image as `KERBY_KADMIN_ATTEMPTS` and `KERBY_KADMIN_RETRY_DELAY_SECONDS`.
 
 Set `debug = true` when troubleshooting Kerberos startup in CI. It passes `KERBY_DEBUG=true` to the Kerby image, which enables JDK Kerberos debug output, Kerby DEBUG logs on container stdout, and dumps the generated `kdc.conf`, server `krb5.conf`, client `krb5.conf`, backend config, and admin server config into the container log.
 
@@ -1013,6 +1035,11 @@ BigDataTestKit.builder()
     .withKerberos(
         KerberosOptions(
             enabled = true,
+            clientPrincipal = "app_user@EXAMPLE.COM",
+            clientPassword = "app-user-secret",
+            materialDirectory = "build/bigdata-test/kerberos",
+            localKrb5ConfPath = "build/bigdata-test/kerberos/krb5.conf",
+            localClientKeytabPath = "build/bigdata-test/kerberos/app_user.keytab",
             startupTimeoutSeconds = 180,
             materialTimeoutSeconds = 120,
             adminAttempts = 120,
@@ -1538,6 +1565,9 @@ bigDataTest {
     kerberos {
         clientPrincipal.set(providers.gradleProperty("bigdata.test.kerberos.clientPrincipal"))
         clientPassword.set(providers.gradleProperty("bigdata.test.kerberos.clientPassword"))
+        materialDirectory.set(layout.buildDirectory.dir("bigdata-test/kerberos").map { it.asFile.absolutePath })
+        localKrb5ConfPath.set(layout.buildDirectory.file("bigdata-test/kerberos/krb5.conf").map { it.asFile.absolutePath })
+        localClientKeytabPath.set(layout.buildDirectory.file("bigdata-test/kerberos/app_user.keytab").map { it.asFile.absolutePath })
     }
 }
 ----

example/spark/src/test/kotlin/org/openprojectx/bigdata/test/example/spark/SparkBigDataTestExample.kt

@@ -9,7 +9,7 @@ import org.openprojectx.bigdata.test.junit5.BigDataTest
 @BigDataTest(
     config = [
         "classpath:spark-bigdata-test-common.toml",
-        "classpath:spark-bigdata-test-cloudera-hms-kerberos.toml",
+        "classpath:spark-bigdata-test-apache-hms-kerberos.toml",
     ],
 )
 class SparkBigDataTestExample : SparkBigDataScenario() {

example/spark/src/test/resources/spark-bigdata-test-apache-hms-kerberos.toml

@@ -11,6 +11,9 @@ materialTimeoutSeconds = 120
 adminAttempts = 120
 adminRetryDelaySeconds = 1
 debug = true
+materialDirectory = "build/bigdata-test/kerberos"
+localKrb5ConfPath = "build/bigdata-test/kerberos/krb5.conf"
+localClientKeytabPath = "build/bigdata-test/kerberos/app_user.keytab"
 
 [ports]
 kafka = 19092

example/spring-gradle-plugin/src/main/resources/spring-bigdata-test.toml

@@ -6,6 +6,9 @@ localStackS3 = true
 [kerberos]
 realm = "EXAMPLE.COM"
 domain = "example.com"
+materialDirectory = "build/bigdata-test/kerberos"
+localKrb5ConfPath = "build/bigdata-test/kerberos/krb5.conf"
+localClientKeytabPath = "build/bigdata-test/kerberos/app_user.keytab"
 debug = true
 
 [hdfs]

gradle-plugin/src/main/kotlin/org/openprojectx/bigdata/test/gradle/BigDataTestGradleConfig.kt

@@ -51,6 +51,9 @@ internal data class BigDataTestGradleConfig(
         val domain: String? = null,
         val clientPrincipal: String? = null,
         val clientPassword: String? = null,
+        val materialDirectory: String? = null,
+        val localKrb5ConfPath: String? = null,
+        val localClientKeytabPath: String? = null,
         val startupTimeoutSeconds: Int? = null,
         val materialTimeoutSeconds: Int? = null,
         val adminAttempts: Int? = null,
@@ -104,6 +107,7 @@ internal data class BigDataTestGradleConfig(
     data class ContainerLogs(
         val mode: ContainerLogMode? = null,
         val directory: String? = null,
+        val append: Boolean? = null,
     )
 
     fun merge(override: BigDataTestGradleConfig): BigDataTestGradleConfig =
@@ -140,6 +144,9 @@ internal data class BigDataTestGradleConfig(
                 domain = override.kerberos.domain ?: kerberos.domain,
                 clientPrincipal = override.kerberos.clientPrincipal ?: kerberos.clientPrincipal,
                 clientPassword = override.kerberos.clientPassword ?: kerberos.clientPassword,
+                materialDirectory = override.kerberos.materialDirectory ?: kerberos.materialDirectory,
+                localKrb5ConfPath = override.kerberos.localKrb5ConfPath ?: kerberos.localKrb5ConfPath,
+                localClientKeytabPath = override.kerberos.localClientKeytabPath ?: kerberos.localClientKeytabPath,
                 startupTimeoutSeconds = override.kerberos.startupTimeoutSeconds ?: kerberos.startupTimeoutSeconds,
                 materialTimeoutSeconds = override.kerberos.materialTimeoutSeconds ?: kerberos.materialTimeoutSeconds,
                 adminAttempts = override.kerberos.adminAttempts ?: kerberos.adminAttempts,
@@ -182,6 +189,7 @@ internal data class BigDataTestGradleConfig(
             containerLogs = ContainerLogs(
                 mode = override.containerLogs.mode ?: containerLogs.mode,
                 directory = override.containerLogs.directory ?: containerLogs.directory,
+                append = override.containerLogs.append ?: containerLogs.append,
             ),
             containerLogLevels = containerLogLevels + override.containerLogLevels,
         )
@@ -241,6 +249,9 @@ internal class BigDataTestGradleConfigLoader(
                 domain = kerberos.string("domain"),
                 clientPrincipal = kerberos.string("clientPrincipal"),
                 clientPassword = kerberos.string("clientPassword"),
+                materialDirectory = kerberos.string("materialDirectory"),
+                localKrb5ConfPath = kerberos.string("localKrb5ConfPath"),
+                localClientKeytabPath = kerberos.string("localClientKeytabPath"),
                 startupTimeoutSeconds = kerberos.int("startupTimeoutSeconds"),
                 materialTimeoutSeconds = kerberos.int("materialTimeoutSeconds"),
                 adminAttempts = kerberos.int("adminAttempts"),
@@ -285,6 +296,7 @@ internal class BigDataTestGradleConfigLoader(
             containerLogs = BigDataTestGradleConfig.ContainerLogs(
                 mode = containerLogs.string("mode")?.let { ContainerLogMode.valueOf(it.uppercase()) },
                 directory = containerLogs.string("directory"),
+                append = containerLogs.boolean("append"),
             ),
             containerLogLevels = containerLogLevels.mapValues { (key, value) ->
                 value.asString("containerLogLevels.$key")

gradle-plugin/src/main/kotlin/org/openprojectx/bigdata/test/gradle/BigDataTestGradleExtension.kt

@@ -132,6 +132,9 @@ abstract class BigDataTestGradleKerberos @Inject constructor(objects: ObjectFact
     val domain: Property<String> = objects.property(String::class.java).convention("example.com")
     val clientPrincipal: Property<String> = objects.property(String::class.java).convention("app_user@EXAMPLE.COM")
     val clientPassword: Property<String> = objects.property(String::class.java).convention("app-user-secret")
+    val materialDirectory: Property<String> = objects.property(String::class.java).convention("")
+    val localKrb5ConfPath: Property<String> = objects.property(String::class.java).convention("")
+    val localClientKeytabPath: Property<String> = objects.property(String::class.java).convention("")
     val startupTimeoutSeconds: Property<Int> = objects.property(Int::class.java).convention(120)
     val materialTimeoutSeconds: Property<Int> = objects.property(Int::class.java).convention(30)
     val adminAttempts: Property<Int> = objects.property(Int::class.java).convention(30)
@@ -187,4 +190,5 @@ abstract class BigDataTestGradleObjectStore @Inject constructor(objects: ObjectF
 abstract class BigDataTestGradleContainerLogs @Inject constructor(objects: ObjectFactory) {
     val mode: Property<ContainerLogMode> = objects.property(ContainerLogMode::class.java).convention(ContainerLogMode.NONE)
     val directory: Property<String> = objects.property(String::class.java).convention("build/bigdata-test-container-logs")
+    val append: Property<Boolean> = objects.property(Boolean::class.java).convention(true)
 }

gradle-plugin/src/main/kotlin/org/openprojectx/bigdata/test/gradle/BigDataTestGradlePlugin.kt

@@ -71,6 +71,9 @@ class BigDataTestGradlePlugin : Plugin<Project> {
             spec.parameters.kerberosDomain.set(extension.kerberos.domain)
             spec.parameters.kerberosClientPrincipal.set(extension.kerberos.clientPrincipal)
             spec.parameters.kerberosClientPassword.set(extension.kerberos.clientPassword)
+            spec.parameters.kerberosMaterialDirectory.set(extension.kerberos.materialDirectory)
+            spec.parameters.kerberosLocalKrb5ConfPath.set(extension.kerberos.localKrb5ConfPath)
+            spec.parameters.kerberosLocalClientKeytabPath.set(extension.kerberos.localClientKeytabPath)
             spec.parameters.kerberosStartupTimeoutSeconds.set(extension.kerberos.startupTimeoutSeconds)
             spec.parameters.kerberosMaterialTimeoutSeconds.set(extension.kerberos.materialTimeoutSeconds)
             spec.parameters.kerberosAdminAttempts.set(extension.kerberos.adminAttempts)
@@ -111,6 +114,7 @@ class BigDataTestGradlePlugin : Plugin<Project> {
 
             spec.parameters.containerLogMode.set(extension.containerLogs.mode)
             spec.parameters.containerLogDirectory.set(extension.containerLogs.directory)
+            spec.parameters.containerLogAppend.set(extension.containerLogs.append)
         }
 
         val startTask = project.tasks.register("bigDataTestStart", BigDataTestStartTask::class.java) { task ->
@@ -283,6 +287,9 @@ class BigDataTestGradlePlugin : Plugin<Project> {
         extension.kerberos.domain.tomlConvention(config.kerberos.domain)
         extension.kerberos.clientPrincipal.tomlConvention(config.kerberos.clientPrincipal)
         extension.kerberos.clientPassword.tomlConvention(config.kerberos.clientPassword)
+        extension.kerberos.materialDirectory.tomlConvention(config.kerberos.materialDirectory)
+        extension.kerberos.localKrb5ConfPath.tomlConvention(config.kerberos.localKrb5ConfPath)
+        extension.kerberos.localClientKeytabPath.tomlConvention(config.kerberos.localClientKeytabPath)
         extension.kerberos.startupTimeoutSeconds.tomlConvention(config.kerberos.startupTimeoutSeconds)
         extension.kerberos.materialTimeoutSeconds.tomlConvention(config.kerberos.materialTimeoutSeconds)
         extension.kerberos.adminAttempts.tomlConvention(config.kerberos.adminAttempts)
@@ -317,6 +324,7 @@ class BigDataTestGradlePlugin : Plugin<Project> {
 
         extension.containerLogs.mode.tomlConvention(config.containerLogs.mode)
         extension.containerLogs.directory.tomlConvention(config.containerLogs.directory)
+        extension.containerLogs.append.tomlConvention(config.containerLogs.append)
         extension.containerLogLevels.tomlConvention(config.containerLogLevels.takeIf { it.isNotEmpty() })
     }
 

gradle-plugin/src/main/kotlin/org/openprojectx/bigdata/test/gradle/BigDataTestGradleService.kt

@@ -60,6 +60,9 @@ abstract class BigDataTestGradleService : BuildService<BigDataTestGradleService.
         val kerberosDomain: Property<String>
         val kerberosClientPrincipal: Property<String>
         val kerberosClientPassword: Property<String>
+        val kerberosMaterialDirectory: Property<String>
+        val kerberosLocalKrb5ConfPath: Property<String>
+        val kerberosLocalClientKeytabPath: Property<String>
         val kerberosStartupTimeoutSeconds: Property<Int>
         val kerberosMaterialTimeoutSeconds: Property<Int>
         val kerberosAdminAttempts: Property<Int>
@@ -100,6 +103,7 @@ abstract class BigDataTestGradleService : BuildService<BigDataTestGradleService.
 
         val containerLogMode: Property<ContainerLogMode>
         val containerLogDirectory: Property<String>
+        val containerLogAppend: Property<Boolean>
     }
 
     private var kit: BigDataTestKit? = null
@@ -170,6 +174,7 @@ abstract class BigDataTestGradleService : BuildService<BigDataTestGradleService.
                 ContainerLogOptions(
                     mode = parameters.containerLogMode.get(),
                     directory = parameters.containerLogDirectory.get(),
+                    append = parameters.containerLogAppend.get(),
                 ),
             )
 
@@ -278,6 +283,9 @@ abstract class BigDataTestGradleService : BuildService<BigDataTestGradleService.
             domain = parameters.kerberosDomain.get(),
             clientPrincipal = parameters.kerberosClientPrincipal.get(),
             clientPassword = parameters.kerberosClientPassword.get(),
+            materialDirectory = parameters.kerberosMaterialDirectory.get().ifBlank { null },
+            localKrb5ConfPath = parameters.kerberosLocalKrb5ConfPath.get().ifBlank { null },
+            localClientKeytabPath = parameters.kerberosLocalClientKeytabPath.get().ifBlank { null },
             startupTimeoutSeconds = parameters.kerberosStartupTimeoutSeconds.get(),
             materialTimeoutSeconds = parameters.kerberosMaterialTimeoutSeconds.get(),
             adminAttempts = parameters.kerberosAdminAttempts.get(),