Spark: Initial support for 4.2.0

Add Spark 4.2 module support and include follow-up compatibility fixes for Spark 4.2 view command handling and test expectations.

Co-authored-by: Codex <codex@openai.com>

Author: manuzhang

.github/workflows/spark-ci.yml

@@ -82,7 +82,7 @@ jobs:
       max-parallel: 20
       matrix:
         jvm: [17, 21]
-        spark: ['3.5', '4.0', '4.1']
+        spark: ['3.5', '4.0', '4.1', '4.2']
         scala: ['2.12', '2.13']
         # Split iceberg-spark (core) from iceberg-spark-extensions/-runtime
         # so they run as concurrent jobs rather than serially in one job.
@@ -94,6 +94,8 @@ jobs:
             scala: '2.12'
           - spark: '4.1'
             scala: '2.12'
+          - spark: '4.2'
+            scala: '2.12'
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.gitignore

@@ -37,6 +37,8 @@ spark/v4.0/spark/benchmark/*
 spark/v4.0/spark-extensions/benchmark/*
 spark/v4.1/spark/benchmark/*
 spark/v4.1/spark-extensions/benchmark/*
+spark/v4.2/spark/benchmark/*
+spark/v4.2/spark-extensions/benchmark/*
 */benchmark/*
 
 __pycache__/

build.gradle

@@ -117,6 +117,9 @@ allprojects {
   repositories {
     mavenCentral()
     mavenLocal()
+    maven {
+      url "https://repository.apache.org/content/repositories/orgapachespark-1523/"
+    }
   }
 }
 

dev/stage-binaries.sh

@@ -20,13 +20,13 @@
 
 SCALA_VERSION=2.12
 FLINK_VERSIONS=1.20,2.0,2.1
-SPARK_VERSIONS=3.5,4.0,4.1
+SPARK_VERSIONS=3.5,4.0,4.1,4.2
 KAFKA_VERSIONS=3
 
 ./gradlew -Prelease -DscalaVersion=$SCALA_VERSION -DflinkVersions=$FLINK_VERSIONS -DsparkVersions=$SPARK_VERSIONS -DkafkaVersions=$KAFKA_VERSIONS publishApachePublicationToMavenRepository --no-parallel --no-configuration-cache
 
 # Also publish Scala 2.13 Artifacts for versions that support it.
 # Flink does not yet support 2.13 (and is largely dropping a user-facing dependency on Scala). Hive doesn't need a Scala specification.
 ./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.5 :iceberg-spark:iceberg-spark-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.5_2.13:publishApachePublicationToMavenRepository --no-parallel --no-configuration-cache
-# Spark 4.0 only supports Scala 2.13. no need to specify scalaVersion
+# Spark 4.0+ only supports Scala 2.13. no need to specify scalaVersion
 

gradle.properties

@@ -18,8 +18,8 @@ jmhJsonOutputPath=build/reports/jmh/results.json
 jmhIncludeRegex=.*
 systemProp.defaultFlinkVersions=2.1
 systemProp.knownFlinkVersions=1.20,2.0,2.1
-systemProp.defaultSparkVersions=4.1
-systemProp.knownSparkVersions=3.5,4.0,4.1
+systemProp.defaultSparkVersions=4.2
+systemProp.knownSparkVersions=3.5,4.0,4.1,4.2
 systemProp.defaultKafkaVersions=3
 systemProp.knownKafkaVersions=3
 systemProp.defaultScalaVersion=2.12

gradle/libs.versions.toml

@@ -89,6 +89,7 @@ snowflake-jdbc = "3.28.0"
 spark35 = "3.5.8"
 spark40 = "4.0.3"
 spark41 = "4.1.2"
+spark42 = "4.2.0"
 sqlite-jdbc = "3.53.2.0"
 testcontainers = "2.0.5"
 tez08 = { strictly = "0.8.4"}  # see rich version usage explanation above

jmh.gradle

@@ -53,6 +53,11 @@ if (sparkVersions.contains("4.1")) {
     jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.1_2.13"))
 }
 
+if (sparkVersions.contains("4.2")) {
+    jmhProjects.add(project(":iceberg-spark:iceberg-spark-4.2_2.13"))
+    jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.2_2.13"))
+}
+
 configure(jmhProjects) {
   apply plugin: 'me.champeau.jmh'
   apply plugin: 'io.morethan.jmhreport'

settings.gradle

@@ -175,6 +175,18 @@ if (sparkVersions.contains("4.1")) {
     project(":iceberg-spark:spark-runtime-4.1_2.13").name = "iceberg-spark-runtime-4.1_2.13"
 }
 
+if (sparkVersions.contains("4.2")) {
+    include ":iceberg-spark:spark-4.2_2.13"
+    include ":iceberg-spark:spark-extensions-4.2_2.13"
+    include ":iceberg-spark:spark-runtime-4.2_2.13"
+    project(":iceberg-spark:spark-4.2_2.13").projectDir = file('spark/v4.2/spark')
+    project(":iceberg-spark:spark-4.2_2.13").name = "iceberg-spark-4.2_2.13"
+    project(":iceberg-spark:spark-extensions-4.2_2.13").projectDir = file('spark/v4.2/spark-extensions')
+    project(":iceberg-spark:spark-extensions-4.2_2.13").name = "iceberg-spark-extensions-4.2_2.13"
+    project(":iceberg-spark:spark-runtime-4.2_2.13").projectDir = file('spark/v4.2/spark-runtime')
+    project(":iceberg-spark:spark-runtime-4.2_2.13").name = "iceberg-spark-runtime-4.2_2.13"
+}
+
 if (kafkaVersions.contains("3")) {
   include 'kafka-connect'
   project(':kafka-connect').name = 'iceberg-kafka-connect'

spark/build.gradle

@@ -31,3 +31,7 @@ if (sparkVersions.contains("4.0")) {
 if (sparkVersions.contains("4.1")) {
     apply from: file("$projectDir/v4.1/build.gradle")
 }
+
+if (sparkVersions.contains("4.2")) {
+    apply from: file("$projectDir/v4.2/build.gradle")
+}

spark/v4.1/spark-runtime/runtime-deps.txt

@@ -1,7 +1,7 @@
-com.fasterxml.jackson.core:jackson-annotations:2.21
+com.fasterxml.jackson.core:jackson-annotations:2.22
 com.fasterxml.jackson.core:jackson-core:2.15
 com.fasterxml.jackson.core:jackson-databind:2.15
-com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21
+com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22
 com.github.ben-manes.caffeine:caffeine:2.9
 com.google.errorprone:error_prone_annotations:2.10
 com.google.flatbuffers:flatbuffers-java:23.5