diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json b/.github/trigger_files/IO_Iceberg_Integration_Tests.json index b73af5e61a43..7ab7bcd9a9c6 100644 --- a/.github/trigger_files/IO_Iceberg_Integration_Tests.json +++ b/.github/trigger_files/IO_Iceberg_Integration_Tests.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 1 + "modification": 2 } diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json index 5abe02fc09c7..3a009261f4f9 100644 --- a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json +++ b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 1 + "modification": 2 } diff --git a/.github/trigger_files/beam_PostCommit_SQL.json b/.github/trigger_files/beam_PostCommit_SQL.json index 833fd9b0d174..5df3841d2363 100644 --- a/.github/trigger_files/beam_PostCommit_SQL.json +++ b/.github/trigger_files/beam_PostCommit_SQL.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run ", - "modification": 2 + "modification": 3 } diff --git a/sdks/java/extensions/sql/iceberg/build.gradle b/sdks/java/extensions/sql/iceberg/build.gradle index 1e319c97a8e2..893a485e7d86 100644 --- a/sdks/java/extensions/sql/iceberg/build.gradle +++ b/sdks/java/extensions/sql/iceberg/build.gradle @@ -48,6 +48,7 @@ dependencies { testImplementation library.java.google_api_services_bigquery testImplementation "org.apache.iceberg:iceberg-api:1.9.2" testImplementation "org.apache.iceberg:iceberg-core:1.9.2" + testRuntimeOnly library.java.bigdataoss_util_hadoop testImplementation project(":sdks:java:io:google-cloud-platform") testImplementation project(":sdks:java:extensions:google-cloud-platform-core") } diff --git a/sdks/java/io/expansion-service/build.gradle b/sdks/java/io/expansion-service/build.gradle index 4e4c15f367cb..4683b4fd6f50 100644 --- a/sdks/java/io/expansion-service/build.gradle +++ b/sdks/java/io/expansion-service/build.gradle @@ -57,6 +57,16 @@ configurations.runtimeClasspath { // Pin nimbus-jose-jwt to 9.37.4 to fix CVE-2025-53864 (transitive via hadoop-auth) resolutionStrategy.force 'com.nimbusds:nimbus-jose-jwt:9.37.4' + + // [iceberg] + // bigdataoss:gcs-connector and parquet:parquet-hadoop have conflicts with global hadoop-common:3.4.2 + // upgrading gcs-connector to 4.0.0 would be fine, because it uses hadoop-common 3.4.2 + // but parquet-hadoop is still at 3.3.0 + // so for now we need to pin hadoop to 3.3.6 until parquet-hadoop releases a version that uses hadoop 3.4.2+ + resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6' } shadowJar { @@ -83,12 +93,10 @@ dependencies { permitUnusedDeclared project(":sdks:java:extensions:kafka-factories") runtimeOnly project(":sdks:java:io:amazon-web-services2") // FileSystem may be used by Iceberg AddFiles - if (JavaVersion.current().compareTo(JavaVersion.VERSION_11) >= 0 && project.findProperty('testJavaVersion') != '8') { - // iceberg ended support for Java 8 in 1.7.0 - runtimeOnly project(":sdks:java:io:iceberg") - runtimeOnly project(":sdks:java:io:iceberg:hive") - runtimeOnly project(path: ":sdks:java:io:iceberg:bqms", configuration: "shadow") - } + runtimeOnly project(":sdks:java:io:iceberg") + runtimeOnly project(":sdks:java:io:iceberg:hive") + runtimeOnly project(path: ":sdks:java:io:iceberg:bqms", configuration: "shadow") + runtimeOnly library.java.bigdataoss_util_hadoop runtimeOnly library.java.kafka_clients runtimeOnly library.java.slf4j_jdk14 diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index bbd55fee2fc8..2dae0335f9c9 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -59,7 +59,7 @@ dependencies { implementation "org.apache.iceberg:iceberg-parquet:$iceberg_version" implementation "org.apache.iceberg:iceberg-orc:$iceberg_version" implementation "org.apache.iceberg:iceberg-data:$iceberg_version" - implementation library.java.hadoop_common + implementation "org.apache.hadoop:hadoop-common:3.3.6" // TODO(https://github.com/apache/beam/issues/21156): Determine how to build without this dependency provided "org.immutables:value:2.8.8" permitUnusedDeclared "org.immutables:value:2.8.8" @@ -70,6 +70,7 @@ dependencies { runtimeOnly "org.apache.iceberg:iceberg-azure:$iceberg_version" runtimeOnly "org.apache.iceberg:iceberg-azure-bundle:$iceberg_version" runtimeOnly library.java.bigdataoss_gcs_connector + runtimeOnly library.java.bigdataoss_util_hadoop runtimeOnly library.java.hadoop_client testImplementation project(":sdks:java:managed") @@ -117,6 +118,14 @@ dependencies { configurations.all { // iceberg-core needs avro:1.12.0 resolutionStrategy.force 'org.apache.avro:avro:1.12.0' + // bigdataoss:gcs-connector and parquet:parquet-hadoop have conflicts with global hadoop-common:3.4.2 + // upgrading gcs-connector to 4.0.0 would be fine, because it uses hadoop-common 3.4.2 + // but parquet-hadoop is still at 3.3.0 + // so for now we need to pin hadoop to 3.3.6 until parquet-hadoop releases a version that uses hadoop 3.4.2+ + resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6' } hadoopVersions.each {kv -> diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java index 56f21d2bc8b5..45ecc7cf71c3 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java @@ -19,7 +19,7 @@ import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.hadoop.util.Sets.newHashSet; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets.newHashSet; import com.google.auto.value.AutoValue; import java.io.Serializable;