no consistent versions and two sparks

Signed-off-by: Robert Kruszewski <github@robertk.io>

Author: robert3005

java/build.gradle.kts

@@ -5,7 +5,6 @@ import net.ltgt.gradle.errorprone.errorprone
 
 plugins {
     id("com.diffplug.spotless") version "8.4.0"
-    id("com.palantir.consistent-versions") version "3.15.0"
     id("com.palantir.git-version") version "5.0.0"
     id("net.ltgt.errorprone") version "5.1.0" apply false
     id("com.google.protobuf") version "0.9.6" apply false
@@ -33,9 +32,9 @@ allprojects {
         apply(plugin = "net.ltgt.errorprone")
 
         dependencies {
-            "errorprone"("com.google.errorprone:error_prone_core")
-            "errorprone"("com.jakewharton.nopen:nopen-checker")
-            "compileOnly"("com.jakewharton.nopen:nopen-annotations")
+            "errorprone"("com.google.errorprone:error_prone_core:2.36.0")
+            "errorprone"("com.jakewharton.nopen:nopen-checker:1.0.1")
+            "compileOnly"("com.jakewharton.nopen:nopen-annotations:1.0.1")
         }
 
         spotless {

java/gradle/libs.versions.toml

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+[versions]
+arrow = "18.3.0"
+errorprone = "2.36.0"
+guava = "33.5.0-jre"
+immutables = "2.12.1"
+junit-jupiter = "6.0.3"
+logback = "1.5.32"
+netty = "4.2.7.Final"
+nopen = "1.0.1"
+protobuf = "4.33.5"
+slf4j = "2.0.17"
+spark3 = "3.5.8"
+spark4 = "4.1.1"
+
+[libraries]
+arrow-c-data = { module = "org.apache.arrow:arrow-c-data", version.ref = "arrow" }
+arrow-memory-core = { module = "org.apache.arrow:arrow-memory-core", version.ref = "arrow" }
+arrow-memory-netty = { module = "org.apache.arrow:arrow-memory-netty", version.ref = "arrow" }
+errorprone-annotations = { module = "com.google.errorprone:error_prone_annotations", version.ref = "errorprone" }
+errorprone-core = { module = "com.google.errorprone:error_prone_core", version.ref = "errorprone" }
+guava = { module = "com.google.guava:guava", version.ref = "guava" }
+immutables-value = { module = "org.immutables:value", version.ref = "immutables" }
+junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit-jupiter" }
+junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit-jupiter" }
+logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" }
+netty-bom = { module = "io.netty:netty-bom", version.ref = "netty" }
+nopen-annotations = { module = "com.jakewharton.nopen:nopen-annotations", version.ref = "nopen" }
+nopen-checker = { module = "com.jakewharton.nopen:nopen-checker", version.ref = "nopen" }
+protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" }
+slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
+slf4j-simple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4j" }

java/settings.gradle.kts

@@ -19,4 +19,8 @@ rootProject.name = "vortex-root"
 
 // API bindings
 include("vortex-jni")
-include("vortex-spark")
+include("vortex-spark_2.12")
+project(":vortex-spark_2.12").projectDir = file("vortex-spark")
+
+include("vortex-spark_2.13")
+project(":vortex-spark_2.13").projectDir = file("vortex-spark")

java/testfiles/Cargo.lock

@@ -12,32 +12,35 @@ plugins {
 }
 
 dependencies {
-    implementation("org.apache.arrow:arrow-c-data")
-    implementation("org.apache.arrow:arrow-memory-core")
-    implementation("org.apache.arrow:arrow-memory-netty")
+    // Align Netty versions across Arrow and Spark
+    implementation(platform(libs.netty.bom))
 
-    compileOnly("org.immutables:value")
-    annotationProcessor("org.immutables:value")
+    implementation(libs.arrow.c.data)
+    implementation(libs.arrow.memory.core)
+    implementation(libs.arrow.memory.netty)
 
-    errorprone("com.google.errorprone:error_prone_core")
-    errorprone("com.jakewharton.nopen:nopen-checker")
+    compileOnly(libs.immutables.value)
+    annotationProcessor(libs.immutables.value)
 
-    implementation("com.google.guava:guava")
-    implementation("com.google.protobuf:protobuf-java")
-    compileOnly("com.google.errorprone:error_prone_annotations")
-    compileOnly("com.jakewharton.nopen:nopen-annotations")
+    errorprone(libs.errorprone.core)
+    errorprone(libs.nopen.checker)
+
+    implementation(libs.guava)
+    implementation(libs.protobuf.java)
+    compileOnly(libs.errorprone.annotations)
+    compileOnly(libs.nopen.annotations)
 
     // Logging
-    implementation("org.slf4j:slf4j-api:2.0.17")
-    testRuntimeOnly("ch.qos.logback:logback-classic:1.5.32")
+    implementation(libs.slf4j.api)
+    testRuntimeOnly(libs.logback.classic)
 }
 
 testing {
     suites {
         val test by getting(JvmTestSuite::class) {
             useJUnitJupiter()
             dependencies {
-                implementation("org.junit.jupiter:junit-jupiter-params")
+                implementation(libs.junit.jupiter.params)
             }
         }
     }
@@ -47,7 +50,7 @@ mavenPublishing {
     coordinates(groupId = "dev.vortex", artifactId = "vortex-jni", version = "${rootProject.version}")
     publishToMavenCentral()
 
-    signAllPublications()
+//    signAllPublications()
 
     pom {
         name = "vortex-jni"
@@ -90,7 +93,7 @@ tasks.withType<Test>().all {
 
 protobuf {
     protoc {
-        artifact = "com.google.protobuf:protoc:4.33.5"
+        artifact = "com.google.protobuf:protoc:${libs.versions.protobuf.get()}"
     }
 }
 

java/vortex-jni/build.gradle.kts

@@ -3,24 +3,45 @@
 
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 
-apply(plugin = "com.vanniktech.maven.publish")
-
 plugins {
     `java-library`
     `jvm-test-suite`
     id("com.gradleup.shadow") version "9.4.0"
 }
 
+// Derive Scala and Spark versions from the Gradle project name (vortex-spark_2.12 or vortex-spark_2.13)
+val scalaVersion: String = project.name.substringAfterLast("_")
+val sparkVersion: String =
+    when (scalaVersion) {
+        "2.12" -> {
+            libs.versions.spark3.get()
+        }
+
+        "2.13" -> {
+            libs.versions.spark4.get()
+        }
+
+        else -> {
+            throw GradleException(
+                "Unsupported Scala version: $scalaVersion (project name must end with _2.12 or _2.13)",
+            )
+        }
+    }
+
+// Both vortex-spark_2.12 and vortex-spark_2.13 share this projectDir.
+// Give each its own build directory to avoid output conflicts.
+layout.buildDirectory = layout.projectDirectory.dir("build/${project.name}")
+
 dependencies {
-    compileOnly("org.apache.spark:spark-catalyst_2.13")
-    compileOnly("org.apache.spark:spark-sql_2.13")
+    compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion")
+    compileOnly("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion")
     api(project(":vortex-jni", configuration = "shadow"))
 
-    compileOnly("org.immutables:value")
-    annotationProcessor("org.immutables:value")
+    compileOnly(libs.immutables.value)
+    annotationProcessor(libs.immutables.value)
 
-    implementation("com.google.guava:guava")
-    implementation("org.slf4j:slf4j-api:2.0.17")
+    implementation(libs.guava)
+    implementation(libs.slf4j.api)
 }
 
 testing {
@@ -29,27 +50,30 @@ testing {
             useJUnitJupiter()
 
             dependencies {
-                implementation("org.junit.jupiter:junit-jupiter:6.0.3")
-                implementation("org.apache.spark:spark-core_2.13")
-                implementation("org.apache.spark:spark-sql_2.13")
-                runtimeOnly("org.slf4j:slf4j-simple:2.0.17")
-                // S3Mock Testcontainers for testing S3 integration (avoids classpath conflicts)
+                implementation(libs.junit.jupiter)
+                implementation("org.apache.spark:spark-core_$scalaVersion:$sparkVersion")
+                implementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion")
                 implementation("com.adobe.testing:s3mock-testcontainers")
                 implementation("org.testcontainers:junit-jupiter")
+                runtimeOnly(libs.slf4j.simple)
+                if (scalaVersion == "2.12") {
+                    // Spark 3.5 marks javax.servlet-api as provided; needed at test runtime for MetricsServlet
+                    runtimeOnly("javax.servlet:javax.servlet-api:4.0.1")
+                }
             }
         }
     }
 }
 
 mavenPublishing {
-    coordinates(groupId = "dev.vortex", artifactId = "vortex-spark", version = "${rootProject.version}")
+    coordinates(groupId = "dev.vortex", artifactId = "vortex-spark_$scalaVersion", version = "${rootProject.version}")
 
     publishToMavenCentral()
 
-    signAllPublications()
+//    signAllPublications()
 
     pom {
-        name = "vortex-spark"
+        name = "vortex-spark_$scalaVersion"
         description = project.description
         url = "https://vortex.dev"
         inceptionYear = "2025"