feat(hdfs): validate HDFS container environment variables and update configuration options

- Added `validateContainerEnvironment` to prevent conflicts with reserved Apache Hadoop environment variable formats.
- Updated `spring-bigdata-test.toml` to adjust container log options and enable HDFS.
- Introduced validation logic to handle unsupported configuration formats in the Hadoop image.
- Enhanced `BigDataContainerFactory` with environment variable conflict detection.
- Updated Gradle plugin to support project-relative container log directories.

Author: CoderYellow

README.md

@@ -91,6 +91,8 @@ HADOOP_OPTS = "-Dsun.security.krb5.debug=true"
 
 Programmatic tests can also call `customizeContainer(...)` for last-resort Testcontainers access.
 
+For HDFS, avoid arbitrary env names whose second token is an Apache Hadoop image config format such as `ENV`, `CONF`, `XML`, or `SH`. For example, `TEST_ENV=TEST` is parsed by `apache/hadoop:3.5.0` as an env-to-config instruction and fails before HDFS starts. Use a non-reserved name such as `TEST_VALUE`, or use the image convention intentionally, for example `CORE_XML_fs_defaultFS=...`.
+
 Optional CLI health checks can run after container startup:
 
 ```toml

core/src/main/kotlin/org/openprojectx/bigdata/test/core/container/BigDataContainerFactory.kt

@@ -982,7 +982,10 @@ internal class BigDataContainerFactory(
         val customization = options.containerCustomizations[service] ?: return container
         customization.networkMode?.let { container.withNetworkMode(it) }
         customization.ports.forEach { container.addPort(it) }
-        customization.environment.forEach { (name, value) -> container.withEnv(name, value) }
+        customization.environment.forEach { (name, value) ->
+            validateContainerEnvironment(service, name)
+            container.withEnv(name, value)
+        }
         customization.files.forEach { container.addFile(it) }
         customization.mounts.forEach { mount ->
             container.withFileSystemBind(
@@ -995,6 +998,27 @@ internal class BigDataContainerFactory(
         return container
     }
 
+    private fun validateContainerEnvironment(service: BigDataService, name: String) {
+        if (service != BigDataService.HDFS) return
+        val conflict = hadoopImageEnvToConfConflict(name) ?: return
+        error(
+            "HDFS container environment variable '$name' conflicts with the apache/hadoop image env-to-conf " +
+                "parser. '$conflict' is treated as a config file format by the image entrypoint. " +
+                "Use a non-reserved name, for example '${name}_VALUE'. For image-native config generation, " +
+                "use a supported format with a property key suffix, for example 'CORE_XML_fs_defaultFS'. " +
+                "Avoid the image's env/sh/cfg/conf formats; they are broken in apache/hadoop:3.5.0.",
+        )
+    }
+
+    private fun hadoopImageEnvToConfConflict(name: String): String? {
+        val parts = name.split(Regex("[_.]"))
+        if (parts.size < 2) return null
+        val format = parts[1].lowercase(Locale.ROOT)
+        if (format in HADOOP_IMAGE_ENV_TO_CONF_BROKEN_FORMATS) return format
+        if (parts.size == 2 && format in HADOOP_IMAGE_ENV_TO_CONF_SUPPORTED_FORMATS) return format
+        return null
+    }
+
     private fun runCliHealthCheck(service: BigDataService, container: GenericContainer<*>, timeoutSeconds: Long) {
         val command = cliHealthCheckCommand(service) ?: return
         val result = container.execInContainer("sh", "-lc", withShellTimeout(timeoutSeconds, command))
@@ -1519,4 +1543,9 @@ internal class BigDataContainerFactory(
           .${options.domain} = ${options.realm}
           ${options.domain} = ${options.realm}
         """.trimIndent()
+
+    private companion object {
+        val HADOOP_IMAGE_ENV_TO_CONF_SUPPORTED_FORMATS = setOf("xml", "properties", "yaml", "yml")
+        val HADOOP_IMAGE_ENV_TO_CONF_BROKEN_FORMATS = setOf("env", "sh", "cfg", "conf")
+    }
 }

doc/user-guide.adoc

@@ -350,6 +350,48 @@ Mounts are useful for local debugging, but copied files are more reliable in CI
 
 Available service names in container TOML are `kerberos`, `hdfs`, `hiveMetastore`, `hms`, `kafka`, `schemaRegistry`, `kafkaUi`, `localStackS3`, and `fakeGcs`.
 
+==== HDFS Environment Variable Names
+
+The HDFS service currently defaults to the `apache/hadoop:3.5.0` image. Its entrypoint runs the image's own `envtoconf.py` bootstrap before the `bigdata-test` HDFS startup command. This is an Apache Hadoop Docker image convention, not a general Hadoop runtime specification.
+
+That image parser treats environment variable names whose second token is a known config format as instructions to generate files under `HADOOP_CONF_DIR`. Tokens are split on `_` and `.`. For example:
+
+[source,text]
+----
+CORE_XML_fs_defaultFS=hdfs://hdfs:8020
+HDFS_XML_dfs_replication=1
+----
+
+Those variables ask the image to generate `core.xml` and `hdfs.xml` content. This can coexist with `bigdata-test` when you intentionally use the image-native convention.
+
+Do not use arbitrary HDFS custom environment variable names whose second token is one of the image's reserved config format names:
+
+[source,text]
+----
+xml, properties, yaml, yml, env, sh, cfg, conf
+----
+
+For example, this looks harmless but conflicts with the image parser:
+
+[source,toml]
+----
+[containers.hdfs.env]
+TEST_ENV = "TEST"
+----
+
+`TEST_ENV` is interpreted as "create `test.env`" with no config key. In `apache/hadoop:3.5.0`, the `env`, `sh`, `cfg`, and `conf` transforms are also buggy because they iterate parsed keys incorrectly. Names such as `TEST_ENV`, `TEST_ENV_FOO`, `APP_CONF`, and `FOO_SH_BAR` can fail before HDFS starts.
+
+Use a non-reserved name instead:
+
+[source,toml]
+----
+[containers.hdfs.env]
+TEST_VALUE = "TEST"
+MYAPP_ENV_VALUE = "TEST"
+----
+
+For Hadoop XML settings, prefer the typed `bigdata-test` options or copied XML files. If you intentionally use the image convention, use one of the working formats with a property suffix, such as `CORE_XML_fs_defaultFS`. Avoid the image's `env`, `sh`, `cfg`, and `conf` formats with `apache/hadoop:3.5.0`.
+
 Programmatic tests can use the same portable operations:
 
 [source,kotlin]

example/spring-gradle-plugin/src/main/resources/spring-bigdata-extensions.toml

@@ -1,5 +1,5 @@
 [kerberosMaterial]
-enabled = true
+enabled = false
 
 [s3Jceks]
 enabled = true

example/spring-gradle-plugin/src/main/resources/spring-bigdata-test.toml

@@ -1,7 +1,11 @@
 [services]
-kerberos = true
-hdfsKerberos = true
+kerberos = false
+hdfsKerberos = false
+hdfs = true
+
 localStackS3 = true
+#schemaRegistry = true
+#kafka = true
 
 [kerberos]
 realm = "EXAMPLE.COM"
@@ -22,11 +26,16 @@ hdfsWeb = 9870
 localStackS3 = 4566
 
 [containerLogs]
-mode = "STDOUT"
+mode = "FILE"
+directory = "build/container-logs"
+append = false
 
 [containerLogLevels]
 kerberos = "DEBUG"
 hdfs = "DEBUG"
 
 [containers.localstackS3.env]
 DISABLE_CORS_CHECKS = "1"
+
+#[containers.hdfs.env]
+#TEST_ENV = "TEST"

gradle-plugin/src/main/kotlin/org/openprojectx/bigdata/test/gradle/BigDataTestGradleService.kt

@@ -182,7 +182,8 @@ abstract class BigDataTestGradleService : BuildService<BigDataTestGradleService.
             .withContainerLogs(
                 ContainerLogOptions(
                     mode = parameters.containerLogMode.get(),
-                    directory = parameters.containerLogDirectory.get(),
+                    directory = projectPath(parameters.containerLogDirectory.get())
+                        ?: parameters.containerLogDirectory.get(),
                     append = parameters.containerLogAppend.get(),
                 ),
             )