Merge pull request #38503 from Abacn/cp

Cherry-pick #38488 and #38500 into release branch

Author: Amar3tto

.github/trigger_files/IO_Iceberg_Integration_Tests.json

@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run.",
-    "modification": 1
+    "modification": 2
 }

.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run.",
-  "modification": 1
+  "modification": 2
 }

.github/trigger_files/beam_PostCommit_SQL.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run ",
-  "modification": 2
+  "modification": 3
 }

sdks/java/extensions/sql/iceberg/build.gradle

@@ -48,6 +48,7 @@ dependencies {
   testImplementation library.java.google_api_services_bigquery
   testImplementation "org.apache.iceberg:iceberg-api:1.9.2"
   testImplementation "org.apache.iceberg:iceberg-core:1.9.2"
+  testRuntimeOnly library.java.bigdataoss_util_hadoop
   testImplementation project(":sdks:java:io:google-cloud-platform")
   testImplementation project(":sdks:java:extensions:google-cloud-platform-core")
 }

sdks/java/io/expansion-service/build.gradle

@@ -57,6 +57,16 @@ configurations.runtimeClasspath {
 
   // Pin nimbus-jose-jwt to 9.37.4 to fix CVE-2025-53864 (transitive via hadoop-auth)
   resolutionStrategy.force 'com.nimbusds:nimbus-jose-jwt:9.37.4'
+
+  // [iceberg]
+  // bigdataoss:gcs-connector and parquet:parquet-hadoop have conflicts with global hadoop-common:3.4.2
+  // upgrading gcs-connector to 4.0.0 would be fine, because it uses hadoop-common 3.4.2
+  // but parquet-hadoop is still at 3.3.0
+  // so for now we need to pin hadoop to 3.3.6 until parquet-hadoop releases a version that uses hadoop 3.4.2+
+  resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6'
+  resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6'
+  resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6'
+  resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6'
 }
 
 shadowJar {
@@ -83,12 +93,10 @@ dependencies {
   permitUnusedDeclared project(":sdks:java:extensions:kafka-factories")
   runtimeOnly project(":sdks:java:io:amazon-web-services2") // FileSystem may be used by Iceberg AddFiles
 
-  if (JavaVersion.current().compareTo(JavaVersion.VERSION_11) >= 0 && project.findProperty('testJavaVersion') != '8') {
-    // iceberg ended support for Java 8 in 1.7.0
-    runtimeOnly project(":sdks:java:io:iceberg")
-    runtimeOnly project(":sdks:java:io:iceberg:hive")
-    runtimeOnly project(path: ":sdks:java:io:iceberg:bqms", configuration: "shadow")
-  }
+  runtimeOnly project(":sdks:java:io:iceberg")
+  runtimeOnly project(":sdks:java:io:iceberg:hive")
+  runtimeOnly project(path: ":sdks:java:io:iceberg:bqms", configuration: "shadow")
+  runtimeOnly library.java.bigdataoss_util_hadoop
 
   runtimeOnly library.java.kafka_clients
   runtimeOnly library.java.slf4j_jdk14

sdks/java/io/iceberg/build.gradle

@@ -59,7 +59,7 @@ dependencies {
     implementation "org.apache.iceberg:iceberg-parquet:$iceberg_version"
     implementation "org.apache.iceberg:iceberg-orc:$iceberg_version"
     implementation "org.apache.iceberg:iceberg-data:$iceberg_version"
-    implementation library.java.hadoop_common
+    implementation "org.apache.hadoop:hadoop-common:3.3.6"
     // TODO(https://github.com/apache/beam/issues/21156): Determine how to build without this dependency
     provided "org.immutables:value:2.8.8"
     permitUnusedDeclared "org.immutables:value:2.8.8"
@@ -70,6 +70,7 @@ dependencies {
     runtimeOnly "org.apache.iceberg:iceberg-azure:$iceberg_version"
     runtimeOnly "org.apache.iceberg:iceberg-azure-bundle:$iceberg_version"
     runtimeOnly library.java.bigdataoss_gcs_connector
+    runtimeOnly library.java.bigdataoss_util_hadoop
     runtimeOnly library.java.hadoop_client
 
     testImplementation project(":sdks:java:managed")
@@ -117,6 +118,14 @@ dependencies {
 configurations.all {
   // iceberg-core needs avro:1.12.0
   resolutionStrategy.force 'org.apache.avro:avro:1.12.0'
+  // bigdataoss:gcs-connector and parquet:parquet-hadoop have conflicts with global hadoop-common:3.4.2
+  // upgrading gcs-connector to 4.0.0 would be fine, because it uses hadoop-common 3.4.2
+  // but parquet-hadoop is still at 3.3.0
+  // so for now we need to pin hadoop to 3.3.6 until parquet-hadoop releases a version that uses hadoop 3.4.2+
+  resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6'
+  resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6'
+  resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6'
+  resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6'
 }
 
 hadoopVersions.each {kv ->

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java

@@ -19,7 +19,7 @@
 
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument;
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.util.Sets.newHashSet;
+import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets.newHashSet;
 
 import com.google.auto.value.AutoValue;
 import java.io.Serializable;