Skip to content

Commit 78f779e

Browse files
AbacnAbacn
committed
Upgrade gcsio to 3.1.16
1 parent 2190c9f commit 78f779e

3 files changed

Lines changed: 31 additions & 56 deletions

File tree

‎buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -611,7 +611,7 @@ class BeamModulePlugin implements Plugin<Project> {
611611
def gax_version = "2.79.0"
612612
def google_ads_version = "33.0.0"
613613
def google_clients_version = "2.0.0"
614-
def google_cloud_bigdataoss_version = "2.2.26"
614+
def google_cloud_bigdataoss_version = "3.1.16"
615615
def google_code_gson_version = "2.10.1"
616616
def google_oauth_clients_version = "1.34.1"
617617
// [bomupgrader] determined by: io.grpc:grpc-netty, consistent with: google_cloud_platform_libraries_bom
@@ -702,9 +702,9 @@ class BeamModulePlugin implements Plugin<Project> {
702702
aws_java_sdk2_profiles : "software.amazon.awssdk:profiles:$aws_java_sdk2_version",
703703
azure_sdk_bom : "com.azure:azure-sdk-bom:1.2.14",
704704
bigdataoss_gcsio : "com.google.cloud.bigdataoss:gcsio:$google_cloud_bigdataoss_version",
705-
bigdataoss_gcs_connector : "com.google.cloud.bigdataoss:gcs-connector:hadoop2-$google_cloud_bigdataoss_version",
705+
bigdataoss_gcs_connector : "com.google.cloud.bigdataoss:gcs-connector:$google_cloud_bigdataoss_version",
706706
bigdataoss_util : "com.google.cloud.bigdataoss:util:$google_cloud_bigdataoss_version",
707-
bigdataoss_util_hadoop : "com.google.cloud.bigdataoss:util-hadoop:hadoop2-$google_cloud_bigdataoss_version",
707+
bigdataoss_util_hadoop : "com.google.cloud.bigdataoss:util-hadoop:$google_cloud_bigdataoss_version",
708708
byte_buddy : "net.bytebuddy:byte-buddy:1.17.7",
709709
cassandra_driver_core : "com.datastax.cassandra:cassandra-driver-core:$cassandra_driver_version",
710710
cassandra_driver_mapping : "com.datastax.cassandra:cassandra-driver-mapping:$cassandra_driver_version",

‎sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilV1.java‎

Lines changed: 16 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
import com.google.api.client.http.HttpHeaders;
3131
import com.google.api.client.http.HttpRequestInitializer;
3232
import com.google.api.client.http.HttpStatusCodes;
33-
import com.google.api.client.http.HttpTransport;
3433
import com.google.api.client.util.BackOff;
3534
import com.google.api.client.util.Sleeper;
3635
import com.google.api.services.storage.Storage;
@@ -53,7 +52,6 @@
5352
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
5453
import java.io.FileNotFoundException;
5554
import java.io.IOException;
56-
import java.lang.reflect.Method;
5755
import java.nio.channels.SeekableByteChannel;
5856
import java.nio.channels.WritableByteChannel;
5957
import java.nio.file.AccessDeniedException;
@@ -267,8 +265,12 @@ public boolean shouldRetry(IOException e) {
267265
.setReadChannelOptions(gcsReadOptions)
268266
.setGrpcEnabled(shouldUseGrpc)
269267
.build();
270-
googleCloudStorage =
271-
createGoogleCloudStorage(googleCloudStorageOptions, storageClient, credentials);
268+
try {
269+
googleCloudStorage =
270+
createGoogleCloudStorage(googleCloudStorageOptions, storageClient, credentials);
271+
} catch (IOException e) {
272+
throw new RuntimeException(e);
273+
}
272274
this.batchRequestSupplier =
273275
() -> {
274276
// Capture reference to this so that the most recent storageClient and initializer
@@ -725,48 +727,16 @@ public WritableByteChannel create(GcsPath path, CreateOptions options) throws IO
725727
}
726728

727729
GoogleCloudStorage createGoogleCloudStorage(
728-
GoogleCloudStorageOptions options, Storage storage, Credentials credentials) {
729-
try {
730-
return new GoogleCloudStorageImpl(options, storage, credentials);
731-
} catch (NoSuchMethodError e) {
732-
// gcs-connector 3.x drops the direct constructor and exclusively uses Builder
733-
// TODO eliminate reflection once Beam drops Java 8 support and upgrades to gcsio 3.x
734-
try {
735-
final Method builderMethod = GoogleCloudStorageImpl.class.getMethod("builder");
736-
Object builder = builderMethod.invoke(null);
737-
final Class<?> builderClass =
738-
Class.forName(
739-
"com.google.cloud.hadoop.gcsio.AutoBuilder_GoogleCloudStorageImpl_Builder");
740-
741-
final Method setOptionsMethod =
742-
builderClass.getMethod("setOptions", GoogleCloudStorageOptions.class);
743-
setOptionsMethod.setAccessible(true);
744-
builder = setOptionsMethod.invoke(builder, options);
745-
746-
final Method setHttpTransportMethod =
747-
builderClass.getMethod("setHttpTransport", HttpTransport.class);
748-
setHttpTransportMethod.setAccessible(true);
749-
builder =
750-
setHttpTransportMethod.invoke(builder, storage.getRequestFactory().getTransport());
751-
752-
final Method setCredentialsMethod =
753-
builderClass.getMethod("setCredentials", Credentials.class);
754-
setCredentialsMethod.setAccessible(true);
755-
builder = setCredentialsMethod.invoke(builder, credentials);
756-
757-
final Method setHttpRequestInitializerMethod =
758-
builderClass.getMethod("setHttpRequestInitializer", HttpRequestInitializer.class);
759-
setHttpRequestInitializerMethod.setAccessible(true);
760-
builder = setHttpRequestInitializerMethod.invoke(builder, httpRequestInitializer);
761-
762-
final Method buildMethod = builderClass.getMethod("build");
763-
buildMethod.setAccessible(true);
764-
return (GoogleCloudStorage) buildMethod.invoke(builder);
765-
} catch (Exception reflectionError) {
766-
throw new RuntimeException(
767-
"Failed to construct GoogleCloudStorageImpl from gcsio 3.x Builder", reflectionError);
768-
}
769-
}
730+
GoogleCloudStorageOptions options, Storage storage, Credentials credentials)
731+
throws IOException {
732+
return GoogleCloudStorageImpl.builder()
733+
.setOptions(options)
734+
.setHttpTransport(storage.getRequestFactory().getTransport())
735+
.setCredentials(credentials)
736+
// gcsio 3 expects httpRequestInitializer to be either absent or
737+
// com.google.cloud.hadoop.util.RetryHttpInitializer when credentials not provided
738+
.setHttpRequestInitializer(credentials != null ? httpRequestInitializer : null)
739+
.build();
770740
}
771741

772742
/**

‎sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilTest.java‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -184,16 +184,19 @@ public void testCreationWithExplicitGoogleCloudStorageReadOptions() throws Excep
184184
GoogleCloudStorageReadOptions readOptions =
185185
GoogleCloudStorageReadOptions.builder()
186186
.setFadvise(GoogleCloudStorageReadOptions.Fadvise.AUTO)
187-
.setSupportGzipEncoding(true)
188-
.setFastFailOnNotFound(false)
187+
.setGzipEncodingSupportEnabled(true)
188+
.setFastFailOnNotFoundEnabled(false)
189189
.build();
190190

191191
GcsOptions pipelineOptions = PipelineOptionsFactory.as(GcsOptions.class);
192192
pipelineOptions.setGoogleCloudStorageReadOptions(readOptions);
193193

194194
GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
195195
GoogleCloudStorage googleCloudStorageMock = Mockito.spy(GoogleCloudStorage.class);
196-
Mockito.when(googleCloudStorageMock.open(Mockito.any(), Mockito.any()))
196+
Mockito.when(
197+
googleCloudStorageMock.open(
198+
Mockito.any(StorageResourceId.class),
199+
Mockito.any(GoogleCloudStorageReadOptions.class)))
197200
.thenReturn(Mockito.mock(SeekableByteChannel.class));
198201
gcsUtil.delegate.setCloudStorageImpl(googleCloudStorageMock);
199202

@@ -1006,7 +1009,7 @@ public void testGCSChannelCloseIdempotent() throws IOException {
10061009
GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
10071010
GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
10081011
GoogleCloudStorageReadOptions readOptions =
1009-
GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(false).build();
1012+
GoogleCloudStorageReadOptions.builder().setFastFailOnNotFoundEnabled(false).build();
10101013

10111014
gcsUtil.delegate.setCloudStorageImpl(
10121015
GoogleCloudStorageOptions.builder()
@@ -1026,7 +1029,7 @@ public void testGCSReadMetricsIsSet() {
10261029
GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
10271030
GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
10281031
GoogleCloudStorageReadOptions readOptions =
1029-
GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(true).build();
1032+
GoogleCloudStorageReadOptions.builder().setFastFailOnNotFoundEnabled(true).build();
10301033
gcsUtil.delegate.setCloudStorageImpl(
10311034
GoogleCloudStorageOptions.builder()
10321035
.setAppName("Beam")
@@ -1673,8 +1676,10 @@ public static GcsUtilV1Mock createMockWithMockStorage(
16731676
.thenReturn(Channels.newChannel(new ByteArrayOutputStream()));
16741677
} else {
16751678
SeekableByteChannel seekableByteChannel = new SeekableInMemoryByteChannel(readPayload);
1676-
Mockito.when(googleCloudStorageMock.open(Mockito.any())).thenReturn(seekableByteChannel);
1677-
Mockito.when(googleCloudStorageMock.open(Mockito.any(), Mockito.any()))
1679+
Mockito.when(googleCloudStorageMock.open(Mockito.any(StorageResourceId.class)))
1680+
.thenReturn(seekableByteChannel);
1681+
Mockito.when(
1682+
googleCloudStorageMock.open(Mockito.any(StorageResourceId.class), Mockito.any()))
16781683
.thenReturn(seekableByteChannel);
16791684
}
16801685
return gcsUtilMock;

0 commit comments

Comments
 (0)