Skip to content

Commit 9a3bbfa

Browse files
committed
feat: add mavenLocal support and enhance Spark project configuration
- Added `mavenLocal()` repository to both plugin and dependency resolution management for improved local development flexibility. - Introduced `sparkPlatform` configuration in Spark example with support for line, variants, and addons. - Updated Spark example dependencies to direct Maven coordinates for clarity and version consistency.
1 parent 8b10553 commit 9a3bbfa

2 files changed

Lines changed: 26 additions & 21 deletions

File tree

example/spark/build.gradle.kts

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,36 +2,37 @@ import org.gradle.api.artifacts.component.ModuleComponentIdentifier
22

33
plugins {
44
id("buildsrc.convention.kotlin-jvm")
5+
id("org.openprojectx.spark.platform") version "0.1.38-SNAPSHOT"
56
}
67

78
description = "Spark JUnit 5 example for bigdata-test"
89

9-
configurations.all {
10-
resolutionStrategy.force("org.apache.kafka:kafka-clients:3.4.1")
11-
resolutionStrategy.capabilitiesResolution {
12-
withCapability("org.lz4:lz4-java") {
13-
select(candidates.first {
14-
val id = it.id
15-
id is ModuleComponentIdentifier && id.group == "at.yawk.lz4"
16-
})
17-
because("at.yawk.lz4:lz4-java 1.10.1 is the maintained fork with newer fixes")
18-
}
19-
}
10+
sparkPlatform {
11+
line.set("spark3")
12+
variants.set(listOf("iceberg"))
13+
addons.set(
14+
listOf(
15+
"hadoopAws",
16+
"hadoopGcs",
17+
"icebergAws",
18+
)
19+
)
20+
2021
}
2122

2223
dependencies {
2324
testImplementation(project(":junit5"))
2425
testImplementation(project(":extensions"))
25-
testImplementation(libs.sparkSql)
26-
testImplementation(libs.sparkHive)
27-
testImplementation(libs.sparkSqlKafka)
28-
testImplementation(libs.sparkAvro)
29-
testImplementation(libs.hadoopAws)
30-
testImplementation(libs.icebergSparkRuntime)
31-
testImplementation(libs.icebergAwsBundle)
32-
testImplementation(libs.gcsConnector)
33-
testImplementation(libs.gcsio)
34-
testImplementation(libs.gcsUtilHadoop)
26+
testImplementation("org.apache.spark:spark-sql_2.12")
27+
testImplementation("org.apache.spark:spark-hive_2.12")
28+
testImplementation("org.apache.spark:spark-sql-kafka-0-10_2.12")
29+
testImplementation("org.apache.spark:spark-avro_2.12")
30+
testImplementation("org.apache.hadoop:hadoop-aws")
31+
testImplementation("org.apache.iceberg:iceberg-spark-runtime-3.5_2.12")
32+
testImplementation("org.apache.iceberg:iceberg-aws-bundle")
33+
testImplementation("com.google.cloud.bigdataoss:gcs-connector")
34+
testImplementation("com.google.cloud.bigdataoss:gcsio")
35+
testImplementation("com.google.cloud.bigdataoss:util-hadoop")
3536
testImplementation(libs.junitJupiterApi)
3637
testRuntimeOnly(libs.junitJupiterEngine)
3738
testRuntimeOnly(libs.junitPlatformLauncher)

settings.gradle.kts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import java.io.File
22

33
pluginManagement {
44
repositories {
5+
mavenLocal()
6+
57
val isCi = System.getenv().containsKey("CI") ||
68
System.getenv().containsKey("GITHUB_ACTIONS") ||
79
System.getenv().containsKey("JENKINS_HOME")
@@ -17,6 +19,8 @@ pluginManagement {
1719

1820
dependencyResolutionManagement {
1921
repositories {
22+
mavenLocal()
23+
2024
val isCi = System.getenv().containsKey("CI") ||
2125
System.getenv().containsKey("GITHUB_ACTIONS") ||
2226
System.getenv().containsKey("JENKINS_HOME")

0 commit comments

Comments
 (0)