Skip to content

Commit 0d5d05d

Browse files
manuzhangmanuzhang
committed
Spark: Initial support for 4.2.0-preview1
1 parent 7b6a5cf commit 0d5d05d

42 files changed

Lines changed: 110 additions & 77 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/spark-ci.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ jobs:
7373
strategy:
7474
matrix:
7575
jvm: [11, 17, 21]
76-
spark: ['3.4', '3.5', '4.0', '4.1']
76+
spark: ['3.4', '3.5', '4.0', '4.1', '4.2']
7777
scala: ['2.12', '2.13']
7878
exclude:
7979
# Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369)
@@ -82,12 +82,16 @@ jobs:
8282
spark: '4.0'
8383
- jvm: 11
8484
spark: '4.1'
85+
- jvm: 11
86+
spark: '4.2'
8587
- jvm: 21
8688
spark: '3.4'
8789
- spark: '4.0'
8890
scala: '2.12'
8991
- spark: '4.1'
9092
scala: '2.12'
93+
- spark: '4.2'
94+
scala: '2.12'
9195
env:
9296
SPARK_LOCAL_IP: localhost
9397
steps:

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ spark/v4.0/spark/benchmark/*
4040
spark/v4.0/spark-extensions/benchmark/*
4141
spark/v4.1/spark/benchmark/*
4242
spark/v4.1/spark-extensions/benchmark/*
43+
spark/v4.2/spark/benchmark/*
44+
spark/v4.2/spark-extensions/benchmark/*
4345
*/benchmark/*
4446

4547
__pycache__/

‎build.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,9 @@ allprojects {
120120
repositories {
121121
mavenCentral()
122122
mavenLocal()
123+
maven {
124+
url 'https://repository.apache.org/content/repositories/orgapachespark-1512/'
125+
}
123126
}
124127
}
125128

‎dev/stage-binaries.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
SCALA_VERSION=2.12
2222
FLINK_VERSIONS=1.20,2.0,2.1
23-
SPARK_VERSIONS=3.4,3.5,4.0,4.1
23+
SPARK_VERSIONS=3.4,3.5,4.0,4.1,4.2
2424
KAFKA_VERSIONS=3
2525

2626
./gradlew -Prelease -DscalaVersion=$SCALA_VERSION -DflinkVersions=$FLINK_VERSIONS -DsparkVersions=$SPARK_VERSIONS -DkafkaVersions=$KAFKA_VERSIONS publishApachePublicationToMavenRepository --no-parallel --no-configuration-cache

‎gradle.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ jmhJsonOutputPath=build/reports/jmh/results.json
1818
jmhIncludeRegex=.*
1919
systemProp.defaultFlinkVersions=2.1
2020
systemProp.knownFlinkVersions=1.20,2.0,2.1
21-
systemProp.defaultSparkVersions=4.1
22-
systemProp.knownSparkVersions=3.4,3.5,4.0,4.1
21+
systemProp.defaultSparkVersions=4.2
22+
systemProp.knownSparkVersions=3.4,3.5,4.0,4.1,4.2
2323
systemProp.defaultKafkaVersions=3
2424
systemProp.knownKafkaVersions=3
2525
systemProp.defaultScalaVersion=2.12

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ spark34 = "3.4.4"
8888
spark35 = "3.5.7"
8989
spark40 = "4.0.1"
9090
spark41 = "4.1.0"
91+
spark42 = "4.2.0-preview1"
9192
sqlite-jdbc = "3.51.1.0"
9293
testcontainers = "2.0.3"
9394
tez08 = { strictly = "0.8.4"} # see rich version usage explanation above

‎jmh.gradle‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ if (sparkVersions.contains("4.1")) {
5858
jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.1_2.13"))
5959
}
6060

61+
if (sparkVersions.contains("4.2")) {
62+
jmhProjects.add(project(":iceberg-spark:iceberg-spark-4.2_2.13"))
63+
jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.2_2.13"))
64+
}
65+
6166
configure(jmhProjects) {
6267
apply plugin: 'me.champeau.jmh'
6368
apply plugin: 'io.morethan.jmhreport'

‎settings.gradle‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,18 @@ if (sparkVersions.contains("4.1")) {
187187
project(":iceberg-spark:spark-runtime-4.1_2.13").name = "iceberg-spark-runtime-4.1_2.13"
188188
}
189189

190+
if (sparkVersions.contains("4.2")) {
191+
include ":iceberg-spark:spark-4.2_2.13"
192+
include ":iceberg-spark:spark-extensions-4.2_2.13"
193+
include ":iceberg-spark:spark-runtime-4.2_2.13"
194+
project(":iceberg-spark:spark-4.2_2.13").projectDir = file('spark/v4.2/spark')
195+
project(":iceberg-spark:spark-4.2_2.13").name = "iceberg-spark-4.2_2.13"
196+
project(":iceberg-spark:spark-extensions-4.2_2.13").projectDir = file('spark/v4.2/spark-extensions')
197+
project(":iceberg-spark:spark-extensions-4.2_2.13").name = "iceberg-spark-extensions-4.2_2.13"
198+
project(":iceberg-spark:spark-runtime-4.2_2.13").projectDir = file('spark/v4.2/spark-runtime')
199+
project(":iceberg-spark:spark-runtime-4.2_2.13").name = "iceberg-spark-runtime-4.2_2.13"
200+
}
201+
190202
if (kafkaVersions.contains("3")) {
191203
include 'kafka-connect'
192204
project(':kafka-connect').name = 'iceberg-kafka-connect'

‎spark/build.gradle‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,7 @@ if (sparkVersions.contains("4.0")) {
3535
if (sparkVersions.contains("4.1")) {
3636
apply from: file("$projectDir/v4.1/build.gradle")
3737
}
38+
39+
if (sparkVersions.contains("4.2")) {
40+
apply from: file("$projectDir/v4.2/build.gradle")
41+
}

‎spark/v4.2/build.gradle‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@
1717
* under the License.
1818
*/
1919

20-
String sparkMajorVersion = '4.1'
20+
String sparkMajorVersion = '4.2'
2121
String scalaVersion = '2.13'
2222

2323
JavaVersion javaVersion = JavaVersion.current()
2424
Boolean javaVersionSupported = javaVersion == JavaVersion.VERSION_17 || javaVersion == JavaVersion.VERSION_21
2525
if (!javaVersionSupported) {
26-
logger.warn("Skip Spark 4.1 build which requires JDK 17 or 21 but was executed with JDK " + javaVersion)
26+
logger.warn("Skip Spark 4.2 build which requires JDK 17 or 21 but was executed with JDK " + javaVersion)
2727
}
2828

2929
def sparkProjects = [
@@ -51,8 +51,8 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
5151
apply plugin: 'scala'
5252
apply plugin: 'com.github.alisiikh.scalastyle'
5353

54-
// Set target to JDK17 for Spark 4.1 to fix following error
55-
// "spark/v4.1/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala:52:12: Class java.lang.Record not found"
54+
// Set target to JDK17 for Spark 4.2 to fix following error
55+
// "spark/v4.2/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala:52:12: Class java.lang.Record not found"
5656
tasks.withType(ScalaCompile.class) {
5757
sourceCompatibility = "17"
5858
targetCompatibility = "17"
@@ -82,17 +82,17 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
8282

8383
compileOnly libs.errorprone.annotations
8484
compileOnly libs.avro.avro
85-
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
85+
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark42.get()}") {
8686
exclude group: 'org.apache.avro', module: 'avro'
8787
exclude group: 'org.apache.arrow'
8888
exclude group: 'org.apache.parquet'
8989
// to make sure netty libs only come from project(':iceberg-arrow')
9090
exclude group: 'io.netty', module: 'netty-buffer'
9191
exclude group: 'io.netty', module: 'netty-common'
92-
exclude group: 'org.roaringbitmap'
92+
exclude group: 'com.github.RoaringBitmap.RoaringBitmap'
9393
}
9494

95-
// TODO: datafusion-comet Spark 4.1 support
95+
// TODO: datafusion-comet Spark 4.2 support
9696
compileOnly "org.apache.datafusion:comet-spark-spark4.0_2.13:${libs.versions.comet.get()}"
9797

9898
implementation libs.parquet.column
@@ -174,14 +174,14 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
174174
compileOnly project(':iceberg-core')
175175
compileOnly project(':iceberg-common')
176176
compileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
177-
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
177+
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark42.get()}") {
178178
exclude group: 'org.apache.avro', module: 'avro'
179179
exclude group: 'org.apache.arrow'
180180
exclude group: 'org.apache.parquet'
181181
// to make sure netty libs only come from project(':iceberg-arrow')
182182
exclude group: 'io.netty', module: 'netty-buffer'
183183
exclude group: 'io.netty', module: 'netty-common'
184-
exclude group: 'org.roaringbitmap'
184+
exclude group: 'com.github.RoaringBitmap.RoaringBitmap'
185185
}
186186
compileOnly libs.errorprone.annotations
187187

@@ -203,7 +203,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
203203
testImplementation libs.avro.avro
204204
testImplementation libs.parquet.hadoop
205205
testImplementation libs.awaitility
206-
// TODO: datafusion-comet Spark 4.1 support
206+
// TODO: datafusion-comet Spark 4.2 support
207207
testImplementation "org.apache.datafusion:comet-spark-spark4.0_2.13:${libs.versions.comet.get()}"
208208
testImplementation(testFixtures(project(':iceberg-parquet')))
209209

@@ -277,7 +277,9 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
277277
}
278278

279279
integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
280-
integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}"
280+
integrationImplementation("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark42.get()}") {
281+
exclude group: 'com.github.RoaringBitmap.RoaringBitmap'
282+
}
281283
integrationImplementation libs.junit.jupiter
282284
integrationImplementation libs.junit.platform.launcher
283285
integrationImplementation libs.slf4j.simple

0 commit comments

Comments
 (0)