feat(config): simplify TOML parsing with tomlj and enhance upload source support

- Replaced custom TOML parser with `tomlj` library for improved maintainability and compliance.
- Added support for inline `sources` arrays in `s3Uploads` and `gcsUploads` configurations.
- Updated tests to validate new configuration styles.
- Enhanced user documentation with guidance on using inline and table-array forms.

Author: CoderYellow

doc/user-guide.adoc

@@ -1351,6 +1351,30 @@ Outputs:
 The S3 and GCS upload extensions create the target bucket by default and upload local files or folders after the object-store services start.
 They use the AWS S3 SDK and Google Cloud Storage SDK against the endpoints exposed by `BigDataTestKit`.
 
+[source,toml]
+----
+[[s3Uploads]]
+id = "seed-s3"
+bucket = "demo-s3"
+prefix = "input"
+sources = [
+  { source = "file:src/test/resources/data/users.json", key = "input/users.json", contentType = "application/json" },
+  { source = "src/test/resources/data/events", prefix = "events" },
+]
+
+[[gcsUploads]]
+id = "seed-gcs"
+bucket = "demo-gcs"
+project = "bigdata-test"
+prefix = "input"
+createBucket = true
+sources = [
+  { source = "classpath:data/products.json", key = "input/products.json", contentType = "application/json" },
+]
+----
+
+The equivalent table-array form is also supported and is easier to edit when each source has several fields:
+
 [source,toml]
 ----
 [[s3Uploads]]
@@ -1380,8 +1404,8 @@ key = "input/products.json"
 contentType = "application/json"
 ----
 
-Use `[[s3Uploads.sources]]` and `[[gcsUploads.sources]]` for upload sources.
-Do not use an inline `sources = [ ... ]` array in project examples; the documented and tested TOML form is a table array, which is easier to extend and avoids parser differences around inline tables.
+Both `sources = [ { ... } ]` and `[[s3Uploads.sources]]` / `[[gcsUploads.sources]]` are valid TOML and supported by the extension config loader.
+Use inline arrays for short declarations, and table arrays when the source list gets longer.
 
 Programmatic config:
 

extensions/build.gradle.kts

@@ -46,6 +46,7 @@ dependencies {
     compileOnly(libs.sparkHive)
     compileOnly(libs.servletApi)
     implementation(libs.kotlinxSerialization)
+    implementation(libs.tomlj)
 
     shadedRuntime(libs.hadoopClientApi)
     shadedRuntime(libs.hadoopClientRuntime)
@@ -71,6 +72,7 @@ dependencies {
     shadedRuntime(libs.icebergAwsBundle)
     shadedRuntime(libs.servletApi)
     shadedRuntime(libs.kotlinxSerialization)
+    shadedRuntime(libs.tomlj)
 
     testImplementation(libs.junitJupiterApi)
     testImplementation(libs.hadoopClientApi)

extensions/src/main/kotlin/org/openprojectx/bigdata/test/extensions/config/BigDataExtensionsConfigLoader.kt

@@ -25,6 +25,14 @@ import org.openprojectx.bigdata.test.extensions.objectstore.ObjectStoreUploadSou
 import org.openprojectx.bigdata.test.extensions.objectstore.S3BucketExtension
 import org.openprojectx.bigdata.test.extensions.objectstore.S3UploadExtension
 import org.openprojectx.bigdata.test.extensions.spark.SparkSqlPreparationExtensionProvider
+import org.tomlj.Toml
+import org.tomlj.TomlArray
+import org.tomlj.TomlTable
+import java.math.BigDecimal
+import java.time.LocalDate
+import java.time.LocalDateTime
+import java.time.LocalTime
+import java.time.OffsetDateTime
 import java.util.ServiceLoader
 
 class BigDataExtensionsConfigLoader(
@@ -180,242 +188,34 @@ class BigDataExtensionsConfigLoader(
 
 private object TomlConfigParser {
     fun parse(text: String): JsonObject {
-        val root: MutableMap<String, Any> = linkedMapOf()
-        var current: MutableMap<String, Any> = root
-        logicalLines(text).forEach { logicalLine ->
-            val index = logicalLine.line
-            val line = logicalLine.text
-            when {
-                line.isEmpty() -> Unit
-                line.startsWith("[[") && line.endsWith("]]") ->
-                    current = resolveArrayTable(root, line.substring(2, line.length - 2).trim(), index)
-                line.startsWith("[") && line.endsWith("]") ->
-                    current = resolveTable(root, line.substring(1, line.length - 1).trim(), index)
-                else -> {
-                    val (key, value) = splitKeyValue(line, index)
-                    current[key] = parseValue(value, index)
-                }
-            }
-        }
-        return toJson(root).jsonObject
-    }
-
-    private data class TomlLine(val text: String, val line: Int)
-
-    private fun logicalLines(text: String): List<TomlLine> {
-        val lines = mutableListOf<TomlLine>()
-        val buffer = StringBuilder()
-        var startLine = 0
-        var depth = 0
-        text.lineSequence().forEachIndexed { index, raw ->
-            val line = stripComment(raw).trim()
-            if (line.isEmpty() && buffer.isEmpty()) return@forEachIndexed
-            if (buffer.isEmpty()) startLine = index else buffer.append(' ')
-            buffer.append(line)
-            depth += nestingDelta(line)
-            if (depth == 0) {
-                lines += TomlLine(buffer.toString().trim(), startLine)
-                buffer.clear()
-            }
-        }
-        require(buffer.isEmpty()) { "TOML line ${startLine + 1}: unclosed array or inline table" }
-        return lines
-    }
-
-    private fun resolveTable(root: MutableMap<String, Any>, path: String, line: Int): MutableMap<String, Any> {
-        var current = root
-        parsePath(path, line).forEach { segment ->
-            current = when (val existing = current[segment]) {
-                null -> linkedMapOf<String, Any>().also { current[segment] = it }
-                is MutableMap<*, *> -> existing.asStringMap(line)
-                is MutableList<*> -> existing.lastOrNull().asStringMap(line)
-                else -> error("TOML line ${line + 1}: '$segment' is not a table")
-            }
-        }
-        return current
-    }
-
-    private fun resolveArrayTable(root: MutableMap<String, Any>, path: String, line: Int): MutableMap<String, Any> {
-        val segments = parsePath(path, line)
-        var current = root
-        segments.dropLast(1).forEach { segment ->
-            current = when (val existing = current[segment]) {
-                null -> linkedMapOf<String, Any>().also { current[segment] = it }
-                is MutableMap<*, *> -> existing.asStringMap(line)
-                is MutableList<*> -> existing.lastOrNull().asStringMap(line)
-                else -> error("TOML line ${line + 1}: '$segment' is not a table")
-            }
+        val result = Toml.parse(text)
+        require(!result.hasErrors()) {
+            result.errors().joinToString(separator = System.lineSeparator()) { it.toString() }
+        }
+        return result.toJsonObject()
+    }
+
+    private fun TomlTable.toJsonObject(): JsonObject =
+        JsonObject(keySet().associateWith { key -> get(key).toJsonElement() })
+
+    private fun TomlArray.toJsonArray(): JsonArray =
+        JsonArray((0 until size()).map { index -> get(index).toJsonElement() })
+
+    private fun Any?.toJsonElement(): JsonElement =
+        when (this) {
+            null -> JsonPrimitive("null")
+            is String -> JsonPrimitive(this)
+            is Boolean -> JsonPrimitive(this)
+            is Long -> JsonPrimitive(this)
+            is Double -> JsonPrimitive(this)
+            is BigDecimal -> JsonPrimitive(this)
+            is TomlTable -> toJsonObject()
+            is TomlArray -> toJsonArray()
+            is OffsetDateTime,
+            is LocalDateTime,
+            is LocalDate,
+            is LocalTime,
+            -> JsonPrimitive(toString())
+            else -> error("Unsupported TOML value type: ${this::class}")
         }
-        val name = segments.last()
-        val array = when (val existing = current[name]) {
-            null -> mutableListOf<MutableMap<String, Any>>().also { current[name] = it }
-            is MutableList<*> -> existing.asMutableList(line)
-            else -> error("TOML line ${line + 1}: '$name' is not an array of tables")
-        }
-        return linkedMapOf<String, Any>().also { array += it }
-    }
-
-    private fun parseValue(value: String, line: Int): Any {
-        val trimmed = value.trim()
-        return when {
-            trimmed.startsWith('"') && trimmed.endsWith('"') -> parseString(trimmed, line)
-            trimmed.startsWith("{") && trimmed.endsWith("}") -> parseInlineTable(trimmed, line)
-            trimmed.startsWith("[") && trimmed.endsWith("]") -> parseArray(trimmed, line)
-            trimmed == "true" -> true
-            trimmed == "false" -> false
-            trimmed.toIntOrNull() != null -> trimmed.toInt()
-            trimmed.toLongOrNull() != null -> trimmed.toLong()
-            trimmed.toDoubleOrNull() != null -> trimmed.toDouble()
-            else -> error("TOML line ${line + 1}: unsupported value '$trimmed'")
-        }
-    }
-
-    private fun parseInlineTable(value: String, line: Int): Map<String, Any> {
-        val body = value.substring(1, value.length - 1).trim()
-        if (body.isEmpty()) return emptyMap()
-        return splitTopLevel(body, ',').associate { item ->
-            val (key, itemValue) = splitKeyValue(item, line)
-            key to parseValue(itemValue, line)
-        }
-    }
-
-    private fun parseArray(value: String, line: Int): List<Any> {
-        val body = value.substring(1, value.length - 1).trim()
-        if (body.isEmpty()) return emptyList()
-        return splitTopLevel(body, ',').map { parseValue(it, line) }
-    }
-
-    private fun splitKeyValue(lineText: String, line: Int): Pair<String, String> {
-        val index = lineText.indexOfTopLevel('=')
-        require(index > 0) { "TOML line ${line + 1}: expected key = value" }
-        return parseKey(lineText.substring(0, index).trim(), line) to lineText.substring(index + 1).trim()
-    }
-
-    private fun parseKey(key: String, line: Int): String {
-        require(key.isNotEmpty()) { "TOML line ${line + 1}: empty key" }
-        return if (key.startsWith('"') && key.endsWith('"')) parseString(key, line) else key
-    }
-
-    private fun parsePath(path: String, line: Int): List<String> =
-        path.split('.').map { parseKey(it.trim(), line) }.also {
-            require(it.isNotEmpty() && it.none(String::isEmpty)) { "TOML line ${line + 1}: invalid table path '$path'" }
-        }
-
-    private fun parseString(value: String, line: Int): String {
-        val body = value.substring(1, value.length - 1)
-        val result = StringBuilder()
-        var escaped = false
-        body.forEach { char ->
-            if (escaped) {
-                result.append(
-                    when (char) {
-                        'b' -> '\b'
-                        't' -> '\t'
-                        'n' -> '\n'
-                        'f' -> '\u000C'
-                        'r' -> '\r'
-                        '"' -> '"'
-                        '\\' -> '\\'
-                        else -> error("TOML line ${line + 1}: unsupported escape '\\$char'")
-                    },
-                )
-                escaped = false
-            } else if (char == '\\') {
-                escaped = true
-            } else {
-                result.append(char)
-            }
-        }
-        require(!escaped) { "TOML line ${line + 1}: dangling string escape" }
-        return result.toString()
-    }
-
-    private fun stripComment(raw: String): String {
-        var inString = false
-        var escaped = false
-        raw.forEachIndexed { index, char ->
-            when {
-                escaped -> escaped = false
-                char == '\\' && inString -> escaped = true
-                char == '"' -> inString = !inString
-                char == '#' && !inString -> return raw.substring(0, index)
-            }
-        }
-        return raw
-    }
-
-    private fun String.indexOfTopLevel(target: Char): Int {
-        var inString = false
-        var escaped = false
-        var depth = 0
-        forEachIndexed { index, char ->
-            when {
-                escaped -> escaped = false
-                char == '\\' && inString -> escaped = true
-                char == '"' -> inString = !inString
-                !inString && (char == '{' || char == '[') -> depth++
-                !inString && (char == '}' || char == ']') -> depth--
-                !inString && depth == 0 && char == target -> return index
-            }
-        }
-        return -1
-    }
-
-    private fun splitTopLevel(value: String, delimiter: Char): List<String> {
-        val items = mutableListOf<String>()
-        var start = 0
-        var inString = false
-        var escaped = false
-        var depth = 0
-        value.forEachIndexed { index, char ->
-            when {
-                escaped -> escaped = false
-                char == '\\' && inString -> escaped = true
-                char == '"' -> inString = !inString
-                !inString && (char == '{' || char == '[') -> depth++
-                !inString && (char == '}' || char == ']') -> depth--
-                !inString && depth == 0 && char == delimiter -> {
-                    items += value.substring(start, index).trim()
-                    start = index + 1
-                }
-            }
-        }
-        items += value.substring(start).trim()
-        return items.filter { it.isNotEmpty() }
-    }
-
-    private fun nestingDelta(value: String): Int {
-        var inString = false
-        var escaped = false
-        var delta = 0
-        value.forEach { char ->
-            when {
-                escaped -> escaped = false
-                char == '\\' && inString -> escaped = true
-                char == '"' -> inString = !inString
-                !inString && (char == '{' || char == '[') -> delta++
-                !inString && (char == '}' || char == ']') -> delta--
-            }
-        }
-        return delta
-    }
-
-    private fun toJson(value: Any): JsonElement = when (value) {
-        is String -> JsonPrimitive(value)
-        is Boolean -> JsonPrimitive(value)
-        is Int -> JsonPrimitive(value)
-        is Long -> JsonPrimitive(value)
-        is Double -> JsonPrimitive(value)
-        is Map<*, *> -> JsonObject(value.mapKeys { it.key as String }.mapValues { toJson(it.value ?: "null") })
-        is List<*> -> JsonArray(value.map { toJson(it ?: "null") })
-        else -> error("Unsupported TOML value type: ${value::class}")
-    }
-
-    @Suppress("UNCHECKED_CAST")
-    private fun Any?.asStringMap(line: Int): MutableMap<String, Any> =
-        this as? MutableMap<String, Any> ?: error("TOML line ${line + 1}: expected table")
-
-    @Suppress("UNCHECKED_CAST")
-    private fun Any?.asMutableList(line: Int): MutableList<MutableMap<String, Any>> =
-        this as? MutableList<MutableMap<String, Any>> ?: error("TOML line ${line + 1}: expected array of tables")
 }

extensions/src/test/kotlin/org/openprojectx/bigdata/test/extensions/objectstore/ObjectStoreUploadExtensionsTest.kt

@@ -28,7 +28,7 @@ class ObjectStoreUploadExtensionsTest {
     }
 
     @Test
-    fun `loads upload extensions from toml`() {
+    fun `loads upload extensions with inline sources from toml`() {
         val extensions = BigDataExtensionsConfigLoader().load(
             """
             [[s3Uploads]]
@@ -58,6 +58,31 @@ class ObjectStoreUploadExtensionsTest {
         assertEquals("seed-gcs", extensions[1].id)
     }
 
+    @Test
+    fun `loads upload extensions with source table arrays from toml`() {
+        val extensions = BigDataExtensionsConfigLoader().load(
+            """
+            [[s3Uploads]]
+            id = "seed-s3"
+            bucket = "demo"
+            prefix = "input"
+
+            [[s3Uploads.sources]]
+            source = "classpath:data/a.txt"
+            key = "input/a.txt"
+            contentType = "text/plain"
+
+            [[s3Uploads.sources]]
+            source = "file:build/seed"
+            prefix = "nested"
+            """.trimIndent(),
+        )
+
+        assertEquals(1, extensions.size)
+        assertTrue(extensions.single() is S3UploadExtension)
+        assertEquals("seed-s3", extensions.single().id)
+    }
+
     @Test
     fun `uploads files and directories to s3`(@TempDir tempDir: Path) {
         Files.writeString(tempDir.resolve("single.txt"), "single")