Spark: Initial support for 4.2.0-preview2

Author: manuzhang

.github/workflows/spark-ci.yml

@@ -73,7 +73,7 @@ jobs:
     strategy:
       matrix:
         jvm: [17, 21]
-        spark: ['3.4', '3.5', '4.0', '4.1']
+        spark: ['3.4', '3.5', '4.0', '4.1', '4.2']
         scala: ['2.12', '2.13']
         exclude:
           # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369)
@@ -84,6 +84,8 @@ jobs:
             scala: '2.12'
           - spark: '4.1'
             scala: '2.12'
+          - spark: '4.2'
+            scala: '2.12'
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.gitignore

@@ -40,6 +40,8 @@ spark/v4.0/spark/benchmark/*
 spark/v4.0/spark-extensions/benchmark/*
 spark/v4.1/spark/benchmark/*
 spark/v4.1/spark-extensions/benchmark/*
+spark/v4.2/spark/benchmark/*
+spark/v4.2/spark-extensions/benchmark/*
 */benchmark/*
 
 __pycache__/

build.gradle

@@ -118,6 +118,9 @@ allprojects {
     mavenCentral()
     mavenLocal()
     maven { url 'https://jitpack.io/' }
+    maven {
+      url 'https://repository.apache.org/content/repositories/orgapachespark-1515/'
+    }
   }
 }
 

dev/stage-binaries.sh

@@ -20,7 +20,7 @@
 
 SCALA_VERSION=2.12
 FLINK_VERSIONS=1.20,2.0,2.1
-SPARK_VERSIONS=3.4,3.5,4.0,4.1
+SPARK_VERSIONS=3.4,3.5,4.0,4.1,4.2
 KAFKA_VERSIONS=3
 
 ./gradlew -Prelease -DscalaVersion=$SCALA_VERSION -DflinkVersions=$FLINK_VERSIONS -DsparkVersions=$SPARK_VERSIONS -DkafkaVersions=$KAFKA_VERSIONS publishApachePublicationToMavenRepository --no-parallel --no-configuration-cache

gradle.properties

@@ -18,8 +18,8 @@ jmhJsonOutputPath=build/reports/jmh/results.json
 jmhIncludeRegex=.*
 systemProp.defaultFlinkVersions=2.1
 systemProp.knownFlinkVersions=1.20,2.0,2.1
-systemProp.defaultSparkVersions=4.1
-systemProp.knownSparkVersions=3.4,3.5,4.0,4.1
+systemProp.defaultSparkVersions=4.2
+systemProp.knownSparkVersions=3.4,3.5,4.0,4.1,4.2
 systemProp.defaultKafkaVersions=3
 systemProp.knownKafkaVersions=3
 systemProp.defaultScalaVersion=2.12

gradle/libs.versions.toml

@@ -91,6 +91,7 @@ spark34 = "3.4.4"
 spark35 = "3.5.8"
 spark40 = "4.0.2"
 spark41 = "4.1.1"
+spark42 = "4.2.0-preview2"
 sqlite-jdbc = "3.51.1.0"
 testcontainers = "2.0.3"
 tez08 = { strictly = "0.8.4"}  # see rich version usage explanation above

jmh.gradle

@@ -58,6 +58,11 @@ if (sparkVersions.contains("4.1")) {
     jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.1_2.13"))
 }
 
+if (sparkVersions.contains("4.2")) {
+    jmhProjects.add(project(":iceberg-spark:iceberg-spark-4.2_2.13"))
+    jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.2_2.13"))
+}
+
 configure(jmhProjects) {
   apply plugin: 'me.champeau.jmh'
   apply plugin: 'io.morethan.jmhreport'

settings.gradle

@@ -187,6 +187,18 @@ if (sparkVersions.contains("4.1")) {
     project(":iceberg-spark:spark-runtime-4.1_2.13").name = "iceberg-spark-runtime-4.1_2.13"
 }
 
+if (sparkVersions.contains("4.2")) {
+    include ":iceberg-spark:spark-4.2_2.13"
+    include ":iceberg-spark:spark-extensions-4.2_2.13"
+    include ":iceberg-spark:spark-runtime-4.2_2.13"
+    project(":iceberg-spark:spark-4.2_2.13").projectDir = file('spark/v4.2/spark')
+    project(":iceberg-spark:spark-4.2_2.13").name = "iceberg-spark-4.2_2.13"
+    project(":iceberg-spark:spark-extensions-4.2_2.13").projectDir = file('spark/v4.2/spark-extensions')
+    project(":iceberg-spark:spark-extensions-4.2_2.13").name = "iceberg-spark-extensions-4.2_2.13"
+    project(":iceberg-spark:spark-runtime-4.2_2.13").projectDir = file('spark/v4.2/spark-runtime')
+    project(":iceberg-spark:spark-runtime-4.2_2.13").name = "iceberg-spark-runtime-4.2_2.13"
+}
+
 if (kafkaVersions.contains("3")) {
   include 'kafka-connect'
   project(':kafka-connect').name = 'iceberg-kafka-connect'

spark/build.gradle

@@ -35,3 +35,7 @@ if (sparkVersions.contains("4.0")) {
 if (sparkVersions.contains("4.1")) {
     apply from: file("$projectDir/v4.1/build.gradle")
 }
+
+if (sparkVersions.contains("4.2")) {
+    apply from: file("$projectDir/v4.2/build.gradle")
+}

spark/v4.2/build.gradle

@@ -17,13 +17,13 @@
  * under the License.
  */
 
-String sparkMajorVersion = '4.1'
+String sparkMajorVersion = '4.2'
 String scalaVersion = '2.13'
 
 JavaVersion javaVersion = JavaVersion.current()
 Boolean javaVersionSupported = javaVersion == JavaVersion.VERSION_17 || javaVersion == JavaVersion.VERSION_21
 if (!javaVersionSupported) {
-  logger.warn("Skip Spark 4.1 build which requires JDK 17 or 21 but was executed with JDK " + javaVersion)
+  logger.warn("Skip Spark 4.2 build which requires JDK 17 or 21 but was executed with JDK " + javaVersion)
 }
 
 def sparkProjects = [
@@ -51,14 +51,6 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
   apply plugin: 'scala'
   apply plugin: 'com.github.alisiikh.scalastyle'
 
-  // Set target to JDK17 for Spark 4.1 to fix following error
-  // "spark/v4.1/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala:52:12: Class java.lang.Record not found"
-  tasks.withType(ScalaCompile.class) {
-    sourceCompatibility = "17"
-    targetCompatibility = "17"
-    scalaCompileOptions.additionalParameters.add("-release:17")
-  }
-
   sourceSets {
     main {
       scala.srcDirs = ['src/main/scala', 'src/main/java']
@@ -82,17 +74,17 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
 
     compileOnly libs.errorprone.annotations
     compileOnly libs.avro.avro
-    compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
+    compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark42.get()}") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.apache.arrow'
       exclude group: 'org.apache.parquet'
       // to make sure netty libs only come from project(':iceberg-arrow')
       exclude group: 'io.netty', module: 'netty-buffer'
       exclude group: 'io.netty', module: 'netty-common'
-      exclude group: 'org.roaringbitmap'
+      exclude group: 'com.github.RoaringBitmap.RoaringBitmap'
     }
 
-    // TODO: datafusion-comet Spark 4.1 support
+    // TODO: datafusion-comet Spark 4.2 support
     compileOnly "org.apache.datafusion:comet-spark-spark4.0_2.13:${libs.versions.comet.get()}"
 
     implementation libs.parquet.column
@@ -174,14 +166,14 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
     compileOnly project(':iceberg-core')
     compileOnly project(':iceberg-common')
     compileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
-    compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
+    compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark42.get()}") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.apache.arrow'
       exclude group: 'org.apache.parquet'
       // to make sure netty libs only come from project(':iceberg-arrow')
       exclude group: 'io.netty', module: 'netty-buffer'
       exclude group: 'io.netty', module: 'netty-common'
-      exclude group: 'org.roaringbitmap'
+      exclude group: 'com.github.RoaringBitmap.RoaringBitmap'
     }
     compileOnly libs.errorprone.annotations
 
@@ -203,7 +195,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
     testImplementation libs.avro.avro
     testImplementation libs.parquet.hadoop
     testImplementation libs.awaitility
-    // TODO: datafusion-comet Spark 4.1 support
+    // TODO: datafusion-comet Spark 4.2 support
     testImplementation "org.apache.datafusion:comet-spark-spark4.0_2.13:${libs.versions.comet.get()}"
     testImplementation(testFixtures(project(':iceberg-parquet')))
 
@@ -277,7 +269,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
     }
 
     integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
-    integrationImplementation("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
+    integrationImplementation("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark42.get()}") {
       exclude group: 'org.roaringbitmap'
     }
     integrationImplementation libs.junit.jupiter