Skip to content

Commit 04eee63

Browse files
manuzhangcodex
manuzhang
and
codex
committed
Spark: Copy back 4.2 as 4.1
Co-authored-by: Codex <codex@openai.com>
1 parent bc060a6 commit 04eee63

629 files changed

Lines changed: 150863 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎spark/v4.1/build.gradle‎

Lines changed: 327 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
String sparkMajorVersion = '4.1'
21+
String scalaVersion = '2.13'
22+
23+
def sparkProjects = [
24+
project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}"),
25+
project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}"),
26+
project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}"),
27+
]
28+
29+
configure(sparkProjects) {
30+
configurations {
31+
all {
32+
resolutionStrategy {
33+
force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson221.get()}"
34+
force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson221.get()}"
35+
force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson221.get()}"
36+
}
37+
}
38+
}
39+
}
40+
41+
project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
42+
apply plugin: 'scala'
43+
apply plugin: 'com.github.alisiikh.scalastyle'
44+
45+
sourceSets {
46+
main {
47+
scala.srcDirs = ['src/main/scala', 'src/main/java']
48+
java.srcDirs = []
49+
}
50+
}
51+
52+
dependencies {
53+
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
54+
api project(':iceberg-api')
55+
annotationProcessor libs.immutables.value
56+
compileOnly libs.immutables.value
57+
implementation project(':iceberg-common')
58+
implementation project(':iceberg-core')
59+
implementation project(':iceberg-data')
60+
implementation project(':iceberg-orc')
61+
implementation project(':iceberg-parquet')
62+
implementation project(':iceberg-arrow')
63+
implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
64+
implementation("org.apache.datasketches:datasketches-java:${libs.versions.datasketches.get()}")
65+
66+
compileOnly libs.errorprone.annotations
67+
compileOnly libs.avro.avro
68+
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
69+
exclude group: 'org.apache.avro', module: 'avro'
70+
exclude group: 'org.apache.arrow'
71+
exclude group: 'org.apache.parquet'
72+
// to make sure netty libs only come from project(':iceberg-arrow')
73+
exclude group: 'io.netty', module: 'netty-buffer'
74+
exclude group: 'io.netty', module: 'netty-common'
75+
exclude group: 'org.roaringbitmap'
76+
}
77+
78+
implementation libs.parquet.column
79+
implementation libs.parquet.hadoop
80+
81+
implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
82+
exclude group: 'org.apache.hadoop'
83+
exclude group: 'commons-lang'
84+
// These artifacts are shaded and included in the orc-core fat jar
85+
exclude group: 'com.google.protobuf', module: 'protobuf-java'
86+
exclude group: 'org.apache.hive', module: 'hive-storage-api'
87+
}
88+
89+
implementation(libs.arrow.vector) {
90+
exclude group: 'io.netty', module: 'netty-buffer'
91+
exclude group: 'io.netty', module: 'netty-common'
92+
exclude group: 'com.google.code.findbugs', module: 'jsr305'
93+
}
94+
95+
implementation libs.caffeine
96+
97+
testImplementation(libs.hadoop3.minicluster) {
98+
exclude group: 'org.apache.avro', module: 'avro'
99+
// to make sure netty libs only come from project(':iceberg-arrow')
100+
exclude group: 'io.netty', module: 'netty-buffer'
101+
exclude group: 'io.netty', module: 'netty-common'
102+
}
103+
testImplementation project(path: ':iceberg-hive-metastore')
104+
testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
105+
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
106+
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
107+
testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
108+
testImplementation project(path: ':iceberg-orc', configuration: 'testArtifacts')
109+
testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements'))
110+
testImplementation libs.awaitility
111+
testImplementation(testFixtures(project(':iceberg-parquet')))
112+
}
113+
114+
test {
115+
useJUnitPlatform()
116+
}
117+
118+
tasks.withType(Test) {
119+
// Vectorized reads need more memory
120+
maxHeapSize '3160m'
121+
}
122+
}
123+
124+
project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") {
125+
apply plugin: 'java-library'
126+
apply plugin: 'scala'
127+
apply plugin: 'com.github.alisiikh.scalastyle'
128+
apply plugin: 'antlr'
129+
130+
configurations {
131+
/*
132+
The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This
133+
bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate
134+
runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with
135+
the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add
136+
the runtime dependency back to only the runtime configuration manually.
137+
*/
138+
implementation {
139+
extendsFrom = extendsFrom.findAll { it != configurations.antlr }
140+
}
141+
}
142+
143+
dependencies {
144+
implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
145+
implementation libs.roaringbitmap
146+
147+
compileOnly "org.scala-lang:scala-library"
148+
compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow')
149+
compileOnly project(':iceberg-api')
150+
compileOnly project(':iceberg-core')
151+
compileOnly project(':iceberg-common')
152+
compileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
153+
compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
154+
exclude group: 'org.apache.avro', module: 'avro'
155+
exclude group: 'org.apache.arrow'
156+
exclude group: 'org.apache.parquet'
157+
// to make sure netty libs only come from project(':iceberg-arrow')
158+
exclude group: 'io.netty', module: 'netty-buffer'
159+
exclude group: 'io.netty', module: 'netty-common'
160+
exclude group: 'org.roaringbitmap'
161+
}
162+
compileOnly libs.errorprone.annotations
163+
164+
testImplementation project(path: ':iceberg-data')
165+
testImplementation project(path: ':iceberg-parquet')
166+
testImplementation project(path: ':iceberg-hive-metastore')
167+
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
168+
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
169+
testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
170+
testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
171+
testImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
172+
testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements'))
173+
testImplementation libs.avro.avro
174+
testImplementation libs.parquet.hadoop
175+
testImplementation libs.awaitility
176+
testImplementation(testFixtures(project(':iceberg-parquet')))
177+
178+
// Required because we remove antlr plugin dependencies from the compile configuration, see note above
179+
runtimeOnly libs.antlr.runtime413
180+
antlr libs.antlr.antlr413
181+
}
182+
183+
test {
184+
useJUnitPlatform()
185+
}
186+
187+
generateGrammarSource {
188+
maxHeapSize = "64m"
189+
arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions']
190+
}
191+
}
192+
193+
project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}") {
194+
apply plugin: 'com.gradleup.shadow'
195+
196+
tasks.jar.dependsOn tasks.shadowJar
197+
198+
sourceSets {
199+
integration {
200+
java.srcDir "$projectDir/src/integration/java"
201+
resources.srcDir "$projectDir/src/integration/resources"
202+
}
203+
}
204+
205+
configurations {
206+
implementation {
207+
exclude group: 'org.apache.spark'
208+
// included in Spark
209+
exclude group: 'org.slf4j'
210+
exclude group: 'org.apache.commons'
211+
exclude group: 'commons-pool'
212+
exclude group: 'commons-codec'
213+
exclude group: 'org.xerial.snappy'
214+
exclude group: 'javax.xml.bind'
215+
exclude group: 'javax.annotation'
216+
exclude group: 'com.github.luben'
217+
exclude group: 'com.ibm.icu'
218+
exclude group: 'org.glassfish'
219+
exclude group: 'org.abego.treelayout'
220+
exclude group: 'org.antlr'
221+
exclude group: 'org.scala-lang'
222+
exclude group: 'org.scala-lang.modules'
223+
}
224+
}
225+
226+
dependencies {
227+
api project(':iceberg-api')
228+
implementation project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
229+
implementation project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
230+
implementation project(':iceberg-aws')
231+
implementation project(':iceberg-azure')
232+
implementation(project(':iceberg-aliyun')) {
233+
exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
234+
exclude group: 'org.apache.httpcomponents', module: 'httpclient'
235+
exclude group: 'commons-logging', module: 'commons-logging'
236+
}
237+
implementation project(':iceberg-gcp')
238+
implementation project(':iceberg-bigquery')
239+
implementation project(':iceberg-hive-metastore')
240+
implementation(project(':iceberg-nessie')) {
241+
exclude group: 'com.google.code.findbugs', module: 'jsr305'
242+
}
243+
implementation (project(':iceberg-snowflake')) {
244+
exclude group: 'net.snowflake' , module: 'snowflake-jdbc'
245+
}
246+
247+
integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
248+
integrationImplementation("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark41.get()}") {
249+
exclude group: 'org.roaringbitmap'
250+
}
251+
integrationImplementation libs.junit.jupiter
252+
integrationImplementation libs.junit.platform.launcher
253+
integrationImplementation libs.slf4j.simple
254+
integrationImplementation libs.assertj.core
255+
integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
256+
integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
257+
integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
258+
integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
259+
260+
// runtime dependencies for running Hive Catalog based integration test
261+
integrationRuntimeOnly project(':iceberg-hive-metastore')
262+
// runtime dependencies for running REST Catalog based integration test
263+
integrationRuntimeOnly project(path: ':iceberg-core', configuration: 'testArtifacts')
264+
integrationRuntimeOnly (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements'))
265+
266+
// Not allowed on our classpath, only the runtime jar is allowed
267+
integrationCompileOnly project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
268+
integrationCompileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
269+
integrationCompileOnly project(':iceberg-api')
270+
}
271+
272+
shadowJar {
273+
configurations = [project.configurations.runtimeClasspath]
274+
275+
zip64 true
276+
277+
// include the LICENSE and NOTICE files for the shaded Jar
278+
from(projectDir) {
279+
include 'LICENSE'
280+
include 'NOTICE'
281+
}
282+
283+
// Relocate dependencies to avoid conflicts
284+
relocate 'com.google.errorprone', 'org.apache.iceberg.shaded.com.google.errorprone'
285+
relocate 'com.google.flatbuffers', 'org.apache.iceberg.shaded.com.google.flatbuffers'
286+
relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
287+
relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
288+
relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
289+
relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
290+
relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded'
291+
relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer'
292+
relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
293+
relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
294+
relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
295+
relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
296+
relocate 'org.apache.hc.client5', 'org.apache.iceberg.shaded.org.apache.hc.client5'
297+
relocate 'org.apache.hc.core5', 'org.apache.iceberg.shaded.org.apache.hc.core5'
298+
// relocate Arrow and related deps to shade Iceberg specific version
299+
relocate 'io.netty', 'org.apache.iceberg.shaded.io.netty'
300+
relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow'
301+
relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch'
302+
relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
303+
relocate 'org.roaringbitmap', 'org.apache.iceberg.shaded.org.roaringbitmap'
304+
relocate 'org.apache.datasketches', 'org.apache.iceberg.shaded.org.apache.datasketches'
305+
306+
archiveClassifier.set(null)
307+
}
308+
309+
task integrationTest(type: Test) {
310+
useJUnitPlatform()
311+
description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}"
312+
group = "verification"
313+
jvmArgs += project.property('extraJvmArgs')
314+
testClassesDirs = sourceSets.integration.output.classesDirs
315+
classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path)
316+
inputs.file(shadowJar.archiveFile.get().asFile.path)
317+
}
318+
integrationTest.dependsOn shadowJar
319+
check.dependsOn integrationTest
320+
321+
jar {
322+
enabled = false
323+
}
324+
325+
apply from: "${rootDir}/runtime-deps.gradle"
326+
}
327+

0 commit comments

Comments
 (0)