Upgrade Spanner to the latest version (#37552)

* Upgrade Spanner to the latest version

* disable gRPC gcp extension

* Remove extra characters

* Update documentation

* Update the documentation

* disable gRPC gcp extension in tests

* Fix spotless

Author: sakthivelmanii

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -613,8 +613,6 @@ class BeamModulePlugin implements Plugin<Project> {
     def google_ads_version = "33.0.0"
     def google_clients_version = "2.0.0"
     def google_cloud_bigdataoss_version = "2.2.26"
-    // [bomupgrader] determined by: com.google.cloud:google-cloud-spanner, consistent with: google_cloud_platform_libraries_bom
-    def google_cloud_spanner_version = "6.104.0"
     def google_code_gson_version = "2.10.1"
     def google_oauth_clients_version = "1.34.1"
     // [bomupgrader] determined by: io.grpc:grpc-netty, consistent with: google_cloud_platform_libraries_bom
@@ -763,10 +761,7 @@ class BeamModulePlugin implements Plugin<Project> {
         // libraries-bom version on sdks/java/container/license_scripts/dep_urls_java.yaml
         google_cloud_platform_libraries_bom         : "com.google.cloud:libraries-bom:26.75.0",
         google_cloud_secret_manager                 : "com.google.cloud:google-cloud-secretmanager", // google_cloud_platform_libraries_bom sets version
-        // TODO(#35868) remove pinned google_cloud_spanner_bom after tests or upstream fixed
-        google_cloud_spanner_bom                    : "com.google.cloud:google-cloud-spanner-bom:$google_cloud_spanner_version",
         google_cloud_spanner                        : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version
-        google_cloud_spanner_test                   : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version:tests",
         google_cloud_storage                        : "com.google.cloud:google-cloud-storage", // google_cloud_platform_libraries_bom sets version
         google_cloud_tink                           : "com.google.crypto.tink:tink:1.19.0",
         google_cloud_vertexai                       : "com.google.cloud:google-cloud-vertexai", // google_cloud_platform_libraries_bom sets version

sdks/java/bom/gcp/build.gradle

@@ -20,17 +20,7 @@ apply from: '../common.gradle'
 
 dependencies {
   api platform(project(":sdks:java:bom"))
-  api platform(project.library.java.google_cloud_spanner_bom)
-  api platform(project.library.java.google_cloud_platform_libraries_bom) {
-    // TODO(https://github.com/apache/beam/issues/37328) remove exclude and google_cloud_spanner_bom after upstream and/or tests fixed
-    exclude group: "com.google.cloud", module: "google-cloud-spanner"
-    exclude group: "com.google.api.grpc", module: "proto-google-cloud-spanner-v1"
-    exclude group: "com.google.api.grpc", module: "proto-google-cloud-spanner-admin-instance-v1"
-    exclude group: "com.google.api.grpc", module: "proto-google-cloud-spanner-admin-database-v1"
-    exclude group: "com.google.api.grpc", module: "grpc-google-cloud-spanner-v1"
-    exclude group: "com.google.api.grpc", module: "grpc-google-cloud-spanner-admin-instance-v1"
-    exclude group: "com.google.api.grpc", module: "grpc-google-cloud-spanner-admin-database-v1"
-  }
+  api platform(project.library.java.google_cloud_platform_libraries_bom)
   constraints {
     api project.library.java.guava
   }
@@ -42,4 +32,4 @@ publishing {
       artifactId = 'beam-sdks-java-google-cloud-platform-bom'
     }
   }
-}
+}

sdks/java/io/google-cloud-platform/build.gradle

@@ -31,17 +31,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Google Cloud Platform"
 ext.summary = "IO library to read and write Google Cloud Platform systems from Beam."
 
 dependencies {
-  implementation(enforcedPlatform(library.java.google_cloud_platform_libraries_bom)) {
-    // TODO(https://github.com/apache/beam/issues/35868) remove exclude after upstream and/or tests fixed
-    exclude group: "com.google.cloud", module: "google-cloud-spanner"
-    exclude group: "com.google.api.grpc", module: "proto-google-cloud-spanner-v1"
-    exclude group: "com.google.api.grpc", module: "proto-google-cloud-spanner-admin-instance-v1"
-    exclude group: "com.google.api.grpc", module: "proto-google-cloud-spanner-admin-database-v1"
-    exclude group: "com.google.api.grpc", module: "grpc-google-cloud-spanner-v1"
-    exclude group: "com.google.api.grpc", module: "grpc-google-cloud-spanner-admin-instance-v1"
-    exclude group: "com.google.api.grpc", module: "grpc-google-cloud-spanner-admin-database-v1"
-  }
-  implementation(enforcedPlatform(library.java.google_cloud_spanner_bom))
+  implementation(enforcedPlatform(library.java.google_cloud_platform_libraries_bom))
   implementation project(path: ":model:pipeline", configuration: "shadow")
   implementation project(":runners:core-java")
   implementation project(path: ":sdks:java:core", configuration: "shadow")
@@ -164,7 +154,7 @@ dependencies {
   testImplementation library.java.mockito_core
   testRuntimeOnly library.java.mockito_inline
   testImplementation library.java.joda_time
-  testImplementation library.java.google_cloud_spanner_test
+  testImplementation "com.google.cloud:google-cloud-spanner::tests"
   testImplementation library.java.google_cloud_bigtable_emulator
   testRuntimeOnly library.java.slf4j_jdk14
   // everit_json is needed for Pubsub SchemaTransform that relies on JSON-schema translation.
@@ -358,4 +348,4 @@ task postCommit {
   description = "Integration tests of GCP connectors using the DirectRunner."
   dependsOn integrationTest
   dependsOn integrationTestKms
-}
+}

sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerAccessor.java

@@ -113,6 +113,11 @@ public static SpannerAccessor getOrCreate(SpannerConfig spannerConfig) {
   static SpannerOptions buildSpannerOptions(SpannerConfig spannerConfig) {
     SpannerOptions.Builder builder = SpannerOptions.newBuilder();
 
+    // TODO(https://github.com/apache/beam/issues/37451) Disable gRPC gcp extension which was
+    // causing the application thread to stall.
+    // Remove this once Spanner fixes the hanging issue
+    builder.disableGrpcGcpExtension();
+
     Set<Code> retryableCodes = new HashSet<>();
     if (spannerConfig.getRetryableCodes() != null) {
       retryableCodes.addAll(spannerConfig.getRetryableCodes());

sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadIT.java

@@ -114,7 +114,12 @@ public void setUp() throws Exception {
       project = options.as(GcpOptions.class).getProject();
     }
 
-    spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService();
+    spanner =
+        SpannerOptions.newBuilder()
+            .setProjectId(project)
+            .disableGrpcGcpExtension()
+            .build()
+            .getService();
 
     databaseName = generateDatabaseName();
     pgDatabaseName = "pg-" + databaseName;

sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java

@@ -121,7 +121,12 @@ public void setUp() throws Exception {
       project = options.as(GcpOptions.class).getProject();
     }
 
-    spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService();
+    spanner =
+        SpannerOptions.newBuilder()
+            .setProjectId(project)
+            .disableGrpcGcpExtension()
+            .build()
+            .getService();
 
     databaseName = generateDatabaseName();
     pgDatabaseName = "pg-" + databaseName;

sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/IntegrationTestEnv.java

@@ -77,7 +77,12 @@ protected void before() throws Throwable {
     instanceId = options.getInstanceId();
     generateDatabaseIds(options);
     spanner =
-        SpannerOptions.newBuilder().setProjectId(projectId).setHost(host).build().getService();
+        SpannerOptions.newBuilder()
+            .setProjectId(projectId)
+            .setHost(host)
+            .disableGrpcGcpExtension()
+            .build()
+            .getService();
     databaseAdminClient = spanner.getDatabaseAdminClient();
     metadataTableName = generateTableName(METADATA_TABLE_NAME_PREFIX);