Skip to content

Commit a15674c

Browse files
manuzhangmanuzhang
committed
Spark: Initial support for 4.2.0-preview3
1 parent 28de29f commit a15674c

44 files changed

Lines changed: 106 additions & 108 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/spark-ci.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ jobs:
7777
max-parallel: 15
7878
matrix:
7979
jvm: [17, 21]
80-
spark: ['3.4', '3.5', '4.0', '4.1']
80+
spark: ['3.4', '3.5', '4.0', '4.1', '4.2']
8181
scala: ['2.12', '2.13']
8282
exclude:
8383
# Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369)
@@ -88,6 +88,8 @@ jobs:
8888
scala: '2.12'
8989
- spark: '4.1'
9090
scala: '2.12'
91+
- spark: '4.2'
92+
scala: '2.12'
9193
env:
9294
SPARK_LOCAL_IP: localhost
9395
steps:

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ spark/v4.0/spark/benchmark/*
4040
spark/v4.0/spark-extensions/benchmark/*
4141
spark/v4.1/spark/benchmark/*
4242
spark/v4.1/spark-extensions/benchmark/*
43+
spark/v4.2/spark/benchmark/*
44+
spark/v4.2/spark-extensions/benchmark/*
4345
*/benchmark/*
4446

4547
__pycache__/

‎build.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,9 @@ allprojects {
117117
repositories {
118118
mavenCentral()
119119
mavenLocal()
120+
maven {
121+
url "https://repository.apache.org/content/repositories/orgapachespark-1516/"
122+
}
120123
}
121124
}
122125

‎dev/stage-binaries.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
SCALA_VERSION=2.12
2222
FLINK_VERSIONS=1.20,2.0,2.1
23-
SPARK_VERSIONS=3.4,3.5,4.0,4.1
23+
SPARK_VERSIONS=3.4,3.5,4.0,4.1,4.2
2424
KAFKA_VERSIONS=3
2525

2626
./gradlew -Prelease -DscalaVersion=$SCALA_VERSION -DflinkVersions=$FLINK_VERSIONS -DsparkVersions=$SPARK_VERSIONS -DkafkaVersions=$KAFKA_VERSIONS publishApachePublicationToMavenRepository --no-parallel --no-configuration-cache

‎gradle.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ jmhJsonOutputPath=build/reports/jmh/results.json
1818
jmhIncludeRegex=.*
1919
systemProp.defaultFlinkVersions=2.1
2020
systemProp.knownFlinkVersions=1.20,2.0,2.1
21-
systemProp.defaultSparkVersions=4.1
22-
systemProp.knownSparkVersions=3.4,3.5,4.0,4.1
21+
systemProp.defaultSparkVersions=4.2
22+
systemProp.knownSparkVersions=3.4,3.5,4.0,4.1,4.2
2323
systemProp.defaultKafkaVersions=3
2424
systemProp.knownKafkaVersions=3
2525
systemProp.defaultScalaVersion=2.12

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ spark34 = "3.4.4"
8989
spark35 = "3.5.8"
9090
spark40 = "4.0.2"
9191
spark41 = "4.1.1"
92+
spark42 = "4.2.0-preview3"
9293
sqlite-jdbc = "3.51.2.0"
9394
testcontainers = "2.0.3"
9495
tez08 = { strictly = "0.8.4"} # see rich version usage explanation above

‎jmh.gradle‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ if (sparkVersions.contains("4.1")) {
5858
jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.1_2.13"))
5959
}
6060

61+
if (sparkVersions.contains("4.2")) {
62+
jmhProjects.add(project(":iceberg-spark:iceberg-spark-4.2_2.13"))
63+
jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.2_2.13"))
64+
}
65+
6166
configure(jmhProjects) {
6267
apply plugin: 'me.champeau.jmh'
6368
apply plugin: 'io.morethan.jmhreport'

‎settings.gradle‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,18 @@ if (sparkVersions.contains("4.1")) {
187187
project(":iceberg-spark:spark-runtime-4.1_2.13").name = "iceberg-spark-runtime-4.1_2.13"
188188
}
189189

190+
if (sparkVersions.contains("4.2")) {
191+
include ":iceberg-spark:spark-4.2_2.13"
192+
include ":iceberg-spark:spark-extensions-4.2_2.13"
193+
include ":iceberg-spark:spark-runtime-4.2_2.13"
194+
project(":iceberg-spark:spark-4.2_2.13").projectDir = file('spark/v4.2/spark')
195+
project(":iceberg-spark:spark-4.2_2.13").name = "iceberg-spark-4.2_2.13"
196+
project(":iceberg-spark:spark-extensions-4.2_2.13").projectDir = file('spark/v4.2/spark-extensions')
197+
project(":iceberg-spark:spark-extensions-4.2_2.13").name = "iceberg-spark-extensions-4.2_2.13"
198+
project(":iceberg-spark:spark-runtime-4.2_2.13").projectDir = file('spark/v4.2/spark-runtime')
199+
project(":iceberg-spark:spark-runtime-4.2_2.13").name = "iceberg-spark-runtime-4.2_2.13"
200+
}
201+
190202
if (kafkaVersions.contains("3")) {
191203
include 'kafka-connect'
192204
project(':kafka-connect').name = 'iceberg-kafka-connect'

‎spark/build.gradle‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,7 @@ if (sparkVersions.contains("4.0")) {
3535
if (sparkVersions.contains("4.1")) {
3636
apply from: file("$projectDir/v4.1/build.gradle")
3737
}
38+
39+
if (sparkVersions.contains("4.2")) {
40+
apply from: file("$projectDir/v4.2/build.gradle")
41+
}

‎spark/v4.2/build.gradle‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@
1717
* under the License.
1818
*/
1919

20-
String sparkMajorVersion = '4.1'
20+
String sparkMajorVersion = '4.2'
2121
String scalaVersion = '2.13'
2222

2323
JavaVersion javaVersion = JavaVersion.current()
2424
Boolean javaVersionSupported = javaVersion == JavaVersion.VERSION_17 || javaVersion == JavaVersion.VERSION_21
2525
if (!javaVersionSupported) {
26-
logger.warn("Skip Spark 4.1 build which requires JDK 17 or 21 but was executed with JDK " + javaVersion)
26+
logger.warn("Skip Spark 4.2 build which requires JDK 17 or 21 but was executed with JDK " + javaVersion)
2727
}
2828

2929
def sparkProjects = [
@@ -74,17 +74,17 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
7474

7575
compileOnly libs.errorprone.annotations
7676
compileOnly libs.avro.avro
77-
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
77+
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark42.get()}") {
7878
exclude group: 'org.apache.avro', module: 'avro'
7979
exclude group: 'org.apache.arrow'
8080
exclude group: 'org.apache.parquet'
8181
// to make sure netty libs only come from project(':iceberg-arrow')
8282
exclude group: 'io.netty', module: 'netty-buffer'
8383
exclude group: 'io.netty', module: 'netty-common'
84-
exclude group: 'org.roaringbitmap'
84+
exclude group: 'com.github.RoaringBitmap.RoaringBitmap'
8585
}
8686

87-
// TODO: datafusion-comet Spark 4.1 support
87+
// TODO: datafusion-comet Spark 4.2 support
8888
compileOnly "org.apache.datafusion:comet-spark-spark4.0_2.13:${libs.versions.comet.get()}"
8989

9090
implementation libs.parquet.column
@@ -166,14 +166,14 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
166166
compileOnly project(':iceberg-core')
167167
compileOnly project(':iceberg-common')
168168
compileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
169-
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
169+
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark42.get()}") {
170170
exclude group: 'org.apache.avro', module: 'avro'
171171
exclude group: 'org.apache.arrow'
172172
exclude group: 'org.apache.parquet'
173173
// to make sure netty libs only come from project(':iceberg-arrow')
174174
exclude group: 'io.netty', module: 'netty-buffer'
175175
exclude group: 'io.netty', module: 'netty-common'
176-
exclude group: 'org.roaringbitmap'
176+
exclude group: 'com.github.RoaringBitmap.RoaringBitmap'
177177
}
178178
compileOnly libs.errorprone.annotations
179179

@@ -195,7 +195,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
195195
testImplementation libs.avro.avro
196196
testImplementation libs.parquet.hadoop
197197
testImplementation libs.awaitility
198-
// TODO: datafusion-comet Spark 4.1 support
198+
// TODO: datafusion-comet Spark 4.2 support
199199
testImplementation "org.apache.datafusion:comet-spark-spark4.0_2.13:${libs.versions.comet.get()}"
200200
testImplementation(testFixtures(project(':iceberg-parquet')))
201201

@@ -269,7 +269,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
269269
}
270270

271271
integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
272-
integrationImplementation("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
272+
integrationImplementation("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark42.get()}") {
273273
exclude group: 'org.roaringbitmap'
274274
}
275275
integrationImplementation libs.junit.jupiter

0 commit comments

Comments
 (0)