Skip to content

Commit d413f68

Browse files
manuzhangmanuzhang
committed
Spark: Copy back 4.2 as 4.1
1 parent 3be38eb commit d413f68

616 files changed

Lines changed: 145529 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎spark/v4.1/build.gradle‎

Lines changed: 348 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,348 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
String sparkMajorVersion = '4.1'
21+
String scalaVersion = '2.13'
22+
23+
JavaVersion javaVersion = JavaVersion.current()
24+
Boolean javaVersionSupported = javaVersion == JavaVersion.VERSION_17 || javaVersion == JavaVersion.VERSION_21
25+
if (!javaVersionSupported) {
26+
logger.warn("Skip Spark 4.1 build which requires JDK 17 or 21 but was executed with JDK " + javaVersion)
27+
}
28+
29+
def sparkProjects = [
30+
project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}"),
31+
project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}"),
32+
project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}"),
33+
]
34+
35+
configure(sparkProjects) {
36+
tasks.configureEach {
37+
onlyIf { javaVersionSupported }
38+
}
39+
configurations {
40+
all {
41+
resolutionStrategy {
42+
force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson215.get()}"
43+
force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson215.get()}"
44+
force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson215.get()}"
45+
}
46+
}
47+
}
48+
}
49+
50+
project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
51+
apply plugin: 'scala'
52+
apply plugin: 'com.github.alisiikh.scalastyle'
53+
54+
sourceSets {
55+
main {
56+
scala.srcDirs = ['src/main/scala', 'src/main/java']
57+
java.srcDirs = []
58+
}
59+
}
60+
61+
dependencies {
62+
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
63+
api project(':iceberg-api')
64+
annotationProcessor libs.immutables.value
65+
compileOnly libs.immutables.value
66+
implementation project(':iceberg-common')
67+
implementation project(':iceberg-core')
68+
implementation project(':iceberg-data')
69+
implementation project(':iceberg-orc')
70+
implementation project(':iceberg-parquet')
71+
implementation project(':iceberg-arrow')
72+
implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
73+
implementation("org.apache.datasketches:datasketches-java:${libs.versions.datasketches.get()}")
74+
75+
compileOnly libs.errorprone.annotations
76+
compileOnly libs.avro.avro
77+
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
78+
exclude group: 'org.apache.avro', module: 'avro'
79+
exclude group: 'org.apache.arrow'
80+
exclude group: 'org.apache.parquet'
81+
// to make sure netty libs only come from project(':iceberg-arrow')
82+
exclude group: 'io.netty', module: 'netty-buffer'
83+
exclude group: 'io.netty', module: 'netty-common'
84+
exclude group: 'org.roaringbitmap'
85+
}
86+
87+
implementation libs.parquet.column
88+
implementation libs.parquet.hadoop
89+
90+
implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
91+
exclude group: 'org.apache.hadoop'
92+
exclude group: 'commons-lang'
93+
// These artifacts are shaded and included in the orc-core fat jar
94+
exclude group: 'com.google.protobuf', module: 'protobuf-java'
95+
exclude group: 'org.apache.hive', module: 'hive-storage-api'
96+
}
97+
98+
implementation(libs.arrow.vector) {
99+
exclude group: 'io.netty', module: 'netty-buffer'
100+
exclude group: 'io.netty', module: 'netty-common'
101+
exclude group: 'com.google.code.findbugs', module: 'jsr305'
102+
}
103+
104+
implementation libs.caffeine
105+
106+
testImplementation(libs.hadoop3.minicluster) {
107+
exclude group: 'org.apache.avro', module: 'avro'
108+
// to make sure netty libs only come from project(':iceberg-arrow')
109+
exclude group: 'io.netty', module: 'netty-buffer'
110+
exclude group: 'io.netty', module: 'netty-common'
111+
}
112+
testImplementation project(path: ':iceberg-hive-metastore')
113+
testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
114+
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
115+
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
116+
testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
117+
testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
118+
transitive = false
119+
}
120+
testImplementation libs.sqlite.jdbc
121+
testImplementation libs.awaitility
122+
testImplementation(testFixtures(project(':iceberg-parquet')))
123+
// runtime dependencies for running REST Catalog based integration test
124+
testRuntimeOnly libs.jetty.servlet
125+
}
126+
127+
test {
128+
useJUnitPlatform()
129+
}
130+
131+
tasks.withType(Test) {
132+
// Vectorized reads need more memory
133+
maxHeapSize '3160m'
134+
}
135+
}
136+
137+
project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") {
138+
apply plugin: 'java-library'
139+
apply plugin: 'scala'
140+
apply plugin: 'com.github.alisiikh.scalastyle'
141+
apply plugin: 'antlr'
142+
143+
configurations {
144+
/*
145+
The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This
146+
bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate
147+
runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with
148+
the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add
149+
the runtime dependency back to only the runtime configuration manually.
150+
*/
151+
implementation {
152+
extendsFrom = extendsFrom.findAll { it != configurations.antlr }
153+
}
154+
}
155+
156+
dependencies {
157+
implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
158+
implementation libs.roaringbitmap
159+
160+
compileOnly "org.scala-lang:scala-library"
161+
compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow')
162+
compileOnly project(':iceberg-api')
163+
compileOnly project(':iceberg-core')
164+
compileOnly project(':iceberg-common')
165+
compileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
166+
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
167+
exclude group: 'org.apache.avro', module: 'avro'
168+
exclude group: 'org.apache.arrow'
169+
exclude group: 'org.apache.parquet'
170+
// to make sure netty libs only come from project(':iceberg-arrow')
171+
exclude group: 'io.netty', module: 'netty-buffer'
172+
exclude group: 'io.netty', module: 'netty-common'
173+
exclude group: 'org.roaringbitmap'
174+
}
175+
compileOnly libs.errorprone.annotations
176+
177+
testImplementation project(path: ':iceberg-data')
178+
testImplementation project(path: ':iceberg-parquet')
179+
testImplementation project(path: ':iceberg-hive-metastore')
180+
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
181+
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
182+
testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
183+
testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
184+
testImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
185+
testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
186+
transitive = false
187+
}
188+
// runtime dependencies for running REST Catalog based integration test
189+
testRuntimeOnly libs.jetty.servlet
190+
testRuntimeOnly libs.sqlite.jdbc
191+
192+
testImplementation libs.avro.avro
193+
testImplementation libs.parquet.hadoop
194+
testImplementation libs.awaitility
195+
testImplementation(testFixtures(project(':iceberg-parquet')))
196+
197+
// Required because we remove antlr plugin dependencies from the compile configuration, see note above
198+
runtimeOnly libs.antlr.runtime413
199+
antlr libs.antlr.antlr413
200+
}
201+
202+
test {
203+
useJUnitPlatform()
204+
}
205+
206+
generateGrammarSource {
207+
maxHeapSize = "64m"
208+
arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions']
209+
}
210+
}
211+
212+
project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}") {
213+
apply plugin: 'com.gradleup.shadow'
214+
215+
tasks.jar.dependsOn tasks.shadowJar
216+
217+
sourceSets {
218+
integration {
219+
java.srcDir "$projectDir/src/integration/java"
220+
resources.srcDir "$projectDir/src/integration/resources"
221+
}
222+
}
223+
224+
configurations {
225+
implementation {
226+
exclude group: 'org.apache.spark'
227+
// included in Spark
228+
exclude group: 'org.slf4j'
229+
exclude group: 'org.apache.commons'
230+
exclude group: 'commons-pool'
231+
exclude group: 'commons-codec'
232+
exclude group: 'org.xerial.snappy'
233+
exclude group: 'javax.xml.bind'
234+
exclude group: 'javax.annotation'
235+
exclude group: 'com.github.luben'
236+
exclude group: 'com.ibm.icu'
237+
exclude group: 'org.glassfish'
238+
exclude group: 'org.abego.treelayout'
239+
exclude group: 'org.antlr'
240+
exclude group: 'org.scala-lang'
241+
exclude group: 'org.scala-lang.modules'
242+
}
243+
}
244+
245+
dependencies {
246+
api project(':iceberg-api')
247+
implementation project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
248+
implementation project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
249+
implementation project(':iceberg-aws')
250+
implementation project(':iceberg-azure')
251+
implementation(project(':iceberg-aliyun')) {
252+
exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
253+
exclude group: 'org.apache.httpcomponents', module: 'httpclient'
254+
exclude group: 'commons-logging', module: 'commons-logging'
255+
}
256+
implementation project(':iceberg-gcp')
257+
implementation project(':iceberg-bigquery')
258+
implementation project(':iceberg-hive-metastore')
259+
implementation(project(':iceberg-nessie')) {
260+
exclude group: 'com.google.code.findbugs', module: 'jsr305'
261+
}
262+
implementation (project(':iceberg-snowflake')) {
263+
exclude group: 'net.snowflake' , module: 'snowflake-jdbc'
264+
}
265+
266+
integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
267+
integrationImplementation("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
268+
exclude group: 'org.roaringbitmap'
269+
}
270+
integrationImplementation libs.junit.jupiter
271+
integrationImplementation libs.junit.platform.launcher
272+
integrationImplementation libs.slf4j.simple
273+
integrationImplementation libs.assertj.core
274+
integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
275+
integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
276+
integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
277+
integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
278+
279+
// runtime dependencies for running Hive Catalog based integration test
280+
integrationRuntimeOnly project(':iceberg-hive-metastore')
281+
// runtime dependencies for running REST Catalog based integration test
282+
integrationRuntimeOnly project(path: ':iceberg-core', configuration: 'testArtifacts')
283+
integrationRuntimeOnly (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
284+
transitive = false
285+
}
286+
integrationRuntimeOnly libs.jetty.servlet
287+
integrationRuntimeOnly libs.sqlite.jdbc
288+
289+
// Not allowed on our classpath, only the runtime jar is allowed
290+
integrationCompileOnly project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
291+
integrationCompileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
292+
integrationCompileOnly project(':iceberg-api')
293+
}
294+
295+
shadowJar {
296+
configurations = [project.configurations.runtimeClasspath]
297+
298+
zip64 true
299+
300+
// include the LICENSE and NOTICE files for the shaded Jar
301+
from(projectDir) {
302+
include 'LICENSE'
303+
include 'NOTICE'
304+
}
305+
306+
// Relocate dependencies to avoid conflicts
307+
relocate 'com.google.errorprone', 'org.apache.iceberg.shaded.com.google.errorprone'
308+
relocate 'com.google.flatbuffers', 'org.apache.iceberg.shaded.com.google.flatbuffers'
309+
relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
310+
relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
311+
relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
312+
relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
313+
relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded'
314+
relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer'
315+
relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
316+
relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
317+
relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
318+
relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
319+
relocate 'org.apache.hc.client5', 'org.apache.iceberg.shaded.org.apache.hc.client5'
320+
relocate 'org.apache.hc.core5', 'org.apache.iceberg.shaded.org.apache.hc.core5'
321+
// relocate Arrow and related deps to shade Iceberg specific version
322+
relocate 'io.netty', 'org.apache.iceberg.shaded.io.netty'
323+
relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow'
324+
relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch'
325+
relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
326+
relocate 'org.roaringbitmap', 'org.apache.iceberg.shaded.org.roaringbitmap'
327+
relocate 'org.apache.datasketches', 'org.apache.iceberg.shaded.org.apache.datasketches'
328+
329+
archiveClassifier.set(null)
330+
}
331+
332+
task integrationTest(type: Test) {
333+
useJUnitPlatform()
334+
description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}"
335+
group = "verification"
336+
jvmArgs += project.property('extraJvmArgs')
337+
testClassesDirs = sourceSets.integration.output.classesDirs
338+
classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path)
339+
inputs.file(shadowJar.archiveFile.get().asFile.path)
340+
}
341+
integrationTest.dependsOn shadowJar
342+
check.dependsOn integrationTest
343+
344+
jar {
345+
enabled = false
346+
}
347+
}
348+

0 commit comments

Comments
 (0)