Skip to content

Commit f891ea2

Browse files
zoewanggzoewangg
authored
Merge branch 'master' into zoewang/fixEndpointAwareAuthSchemeDirectory
2 parents 4e1d023 + e12b2c9 commit f891ea2

9 files changed

Lines changed: 146 additions & 9 deletions

File tree

‎.changes/next-release/feature-s3-fe521b0.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "feature",
3+
"category": "s3",
4+
"contributor": "",
5+
"description": "Add CRT shouldStream config as CRT_MEMORY_BUFFER_DISABLED SDK advanced client option"
6+
}

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/client/config/SdkAdvancedAsyncClientOption.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,19 @@ public final class SdkAdvancedAsyncClientOption<T> extends ClientOption<T> {
5555
public static final SdkAdvancedAsyncClientOption<Executor> FUTURE_COMPLETION_EXECUTOR =
5656
new SdkAdvancedAsyncClientOption<>(Executor.class);
5757

58+
/**
59+
* Advanced configuration for the native S3CrtAsyncClient, which only applies for file-based multipart uploads.
60+
* By default, the S3CrtAsyncClient will skip buffering parts in memory for large objects.
61+
* <p>
62+
* When set to false, the client will not buffer parts for large object, but will buffer parts for smaller objects .
63+
* When set to true, the client will skip buffering parts even for small objects.
64+
* Parts for large objects are never buffered.
65+
* <p>
66+
* Default to {@code false}.
67+
*/
68+
public static final SdkAdvancedAsyncClientOption<Boolean> CRT_MEMORY_BUFFER_DISABLED =
69+
new SdkAdvancedAsyncClientOption<>(Boolean.class);
70+
5871
private SdkAdvancedAsyncClientOption(Class<T> valueClass) {
5972
super(valueClass);
6073
}

‎services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/S3CrtClientPutObjectIntegrationTest.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import software.amazon.awssdk.core.ResponseBytes;
3737
import software.amazon.awssdk.core.ResponseInputStream;
3838
import software.amazon.awssdk.core.async.AsyncRequestBody;
39+
import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
3940
import software.amazon.awssdk.core.exception.SdkClientException;
4041
import software.amazon.awssdk.core.sync.ResponseTransformer;
4142
import software.amazon.awssdk.crt.CrtResource;
@@ -79,6 +80,25 @@ public static void teardown() throws IOException {
7980

8081
@Test
8182
void putObject_fileRequestBody_objectSentCorrectly() throws Exception {
83+
S3AsyncClient crtClientWithMemoryBufferDisabled = S3CrtAsyncClient.builder()
84+
.credentialsProvider(AwsTestBase.CREDENTIALS_PROVIDER_CHAIN)
85+
.region(S3IntegrationTestBase.DEFAULT_REGION)
86+
.advancedOption(SdkAdvancedAsyncClientOption.CRT_MEMORY_BUFFER_DISABLED, true)
87+
.build();
88+
89+
AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath());
90+
crtClientWithMemoryBufferDisabled.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join();
91+
92+
ResponseInputStream<GetObjectResponse> objContent = S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY),
93+
ResponseTransformer.toInputStream());
94+
95+
byte[] expectedSum = ChecksumUtils.computeCheckSum(Files.newInputStream(testFile.toPath()));
96+
97+
Assertions.assertThat(ChecksumUtils.computeCheckSum(objContent)).isEqualTo(expectedSum);
98+
}
99+
100+
@Test
101+
void putObject_withMemoryBufferDisabled_fileRequestBody_objectSentCorrectly() throws Exception {
82102
AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath());
83103
s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join();
84104

@@ -90,6 +110,7 @@ void putObject_fileRequestBody_objectSentCorrectly() throws Exception {
90110
Assertions.assertThat(ChecksumUtils.computeCheckSum(objContent)).isEqualTo(expectedSum);
91111
}
92112

113+
93114
@Test
94115
void putObject_file_objectSentCorrectly() throws Exception {
95116
s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), testFile.toPath()).join();

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import java.net.URI;
1919
import java.nio.file.Path;
20+
import java.util.Map;
2021
import java.util.concurrent.CompletableFuture;
2122
import java.util.concurrent.Executor;
2223
import java.util.concurrent.ThreadPoolExecutor;
@@ -25,6 +26,7 @@
2526
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
2627
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
2728
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
29+
import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
2830
import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
2931
import software.amazon.awssdk.identity.spi.IdentityProvider;
3032
import software.amazon.awssdk.regions.Region;
@@ -361,6 +363,21 @@ default S3CrtAsyncClientBuilder retryConfiguration(Consumer<S3CrtRetryConfigurat
361363
*/
362364
S3CrtAsyncClientBuilder disableS3ExpressSessionAuth(Boolean disableS3ExpressSessionAuth);
363365

366+
/**
367+
* Configure an advanced async option. These values are used very rarely, and the majority of SDK customers can ignore
368+
* them.
369+
*
370+
* @param option The option to configure.
371+
* @param value The value of the option.
372+
* @param <T> The type of the option.
373+
*/
374+
<T> S3CrtAsyncClientBuilder advancedOption(SdkAdvancedAsyncClientOption<T> option, T value);
375+
376+
/**
377+
* Configure the map of advanced override options. This will override all values currently configured. The values in the
378+
* map must match the key type of the map, or a runtime exception will be raised.
379+
*/
380+
S3CrtAsyncClientBuilder advancedOptions(Map<SdkAdvancedAsyncClientOption<?>, ?> advancedOptions);
364381

365382
@Override
366383
S3AsyncClient build();

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.nio.file.Path;
3232
import java.util.ArrayList;
3333
import java.util.List;
34+
import java.util.Map;
3435
import java.util.concurrent.CompletableFuture;
3536
import java.util.concurrent.Executor;
3637
import software.amazon.awssdk.annotations.SdkInternalApi;
@@ -47,6 +48,7 @@
4748
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
4849
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
4950
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
51+
import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
5052
import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
5153
import software.amazon.awssdk.core.interceptor.Context;
5254
import software.amazon.awssdk.core.interceptor.ExecutionAttribute;
@@ -78,6 +80,7 @@
7880
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
7981
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
8082
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
83+
import software.amazon.awssdk.utils.AttributeMap;
8184
import software.amazon.awssdk.utils.CollectionUtils;
8285
import software.amazon.awssdk.utils.Validate;
8386

@@ -222,7 +225,8 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau
222225
.readBufferSizeInBytes(builder.readBufferSizeInBytes)
223226
.httpConfiguration(builder.httpConfiguration)
224227
.thresholdInBytes(builder.thresholdInBytes)
225-
.maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes);
228+
.maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes)
229+
.advancedOptions(builder.advancedOptions.build());
226230

227231
if (builder.retryConfiguration != null) {
228232
nativeClientBuilder.standardRetryOptions(
@@ -257,6 +261,7 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB
257261
private Executor futureCompletionExecutor;
258262
private Boolean disableS3ExpressSessionAuth;
259263

264+
private AttributeMap.Builder advancedOptions = AttributeMap.builder();
260265

261266
@Override
262267
public DefaultS3CrtClientBuilder credentialsProvider(AwsCredentialsProvider credentialsProvider) {
@@ -388,6 +393,18 @@ public DefaultS3CrtClientBuilder disableS3ExpressSessionAuth(Boolean disableS3Ex
388393
return this;
389394
}
390395

396+
@Override
397+
public <T> DefaultS3CrtClientBuilder advancedOption(SdkAdvancedAsyncClientOption<T> option, T value) {
398+
this.advancedOptions.put(option, value);
399+
return this;
400+
}
401+
402+
@Override
403+
public DefaultS3CrtClientBuilder advancedOptions(Map<SdkAdvancedAsyncClientOption<?>, ?> advancedOptions) {
404+
this.advancedOptions.putAll(advancedOptions);
405+
return this;
406+
}
407+
391408
@Override
392409
public S3CrtAsyncClient build() {
393410
return new DefaultS3CrtAsyncClient(this);

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import software.amazon.awssdk.crt.http.HttpProxyEnvironmentVariableSetting;
4949
import software.amazon.awssdk.crt.http.HttpRequest;
5050
import software.amazon.awssdk.crt.s3.ChecksumConfig;
51+
import software.amazon.awssdk.crt.s3.FileIoOptions;
5152
import software.amazon.awssdk.crt.s3.ResumeToken;
5253
import software.amazon.awssdk.crt.s3.S3Client;
5354
import software.amazon.awssdk.crt.s3.S3ClientOptions;
@@ -127,6 +128,8 @@ private S3ClientOptions createS3ClientOption() {
127128
.ifPresent(options::withConnectTimeoutMs);
128129
Optional.ofNullable(s3NativeClientConfiguration.httpMonitoringOptions())
129130
.ifPresent(options::withHttpMonitoringOptions);
131+
Optional.ofNullable(s3NativeClientConfiguration.memoryBufferDisabled())
132+
.ifPresent(memoryBufferDisabled -> options.withFileIoOptions(new FileIoOptions(memoryBufferDisabled, 0.0, false)));
130133
return options;
131134
}
132135

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
package software.amazon.awssdk.services.s3.internal.crt;
1717

18+
import static software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption.CRT_MEMORY_BUFFER_DISABLED;
1819
import static software.amazon.awssdk.crtcore.CrtConfigurationUtils.resolveHttpMonitoringOptions;
1920
import static software.amazon.awssdk.crtcore.CrtConfigurationUtils.resolveProxy;
2021

@@ -33,6 +34,7 @@
3334
import software.amazon.awssdk.identity.spi.IdentityProvider;
3435
import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
3536
import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration;
37+
import software.amazon.awssdk.utils.AttributeMap;
3638
import software.amazon.awssdk.utils.Logger;
3739
import software.amazon.awssdk.utils.SdkAutoCloseable;
3840
import software.amazon.awssdk.utils.Validate;
@@ -64,6 +66,7 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable {
6466
private final HttpMonitoringOptions httpMonitoringOptions;
6567
private final Boolean useEnvironmentVariableProxyOptionsValues;
6668
private final long maxNativeMemoryLimitInBytes;
69+
private final Boolean memoryBufferDisabled;
6770

6871
public S3NativeClientConfiguration(Builder builder) {
6972
this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() :
@@ -113,6 +116,8 @@ public S3NativeClientConfiguration(Builder builder) {
113116
}
114117
this.standardRetryOptions = builder.standardRetryOptions;
115118
this.useEnvironmentVariableProxyOptionsValues = resolveUseEnvironmentVariableValues(builder);
119+
this.memoryBufferDisabled =
120+
builder.advancedOptions == null ? null : builder.advancedOptions.get(CRT_MEMORY_BUFFER_DISABLED);
116121
}
117122

118123
private static Boolean resolveUseEnvironmentVariableValues(Builder builder) {
@@ -191,6 +196,10 @@ public Long readBufferSizeInBytes() {
191196
return readBufferSizeInBytes;
192197
}
193198

199+
public Boolean memoryBufferDisabled() {
200+
return memoryBufferDisabled;
201+
}
202+
194203
@Override
195204
public void close() {
196205
clientBootstrap.close();
@@ -213,6 +222,8 @@ public static final class Builder {
213222
private Long thresholdInBytes;
214223
private Long maxNativeMemoryLimitInBytes;
215224

225+
private AttributeMap advancedOptions;
226+
216227
private Builder() {
217228
}
218229

@@ -274,5 +285,10 @@ public Builder thresholdInBytes(Long thresholdInBytes) {
274285
this.thresholdInBytes = thresholdInBytes;
275286
return this;
276287
}
288+
289+
public Builder advancedOptions(AttributeMap advancedOptions) {
290+
this.advancedOptions = advancedOptions;
291+
return this;
292+
}
277293
}
278294
}

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClientTest.java‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
2727
import software.amazon.awssdk.core.async.AsyncRequestBody;
2828
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
29+
import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
2930
import software.amazon.awssdk.core.interceptor.Context;
3031
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
3132
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
@@ -37,6 +38,7 @@
3738
import software.amazon.awssdk.services.s3.endpoints.S3ClientContextParams;
3839
import software.amazon.awssdk.services.s3.internal.crossregion.S3CrossRegionAsyncClient;
3940
import software.amazon.awssdk.utils.AttributeMap;
41+
import software.amazon.awssdk.utils.MapUtils;
4042

4143
class DefaultS3CrtAsyncClientTest {
4244

@@ -102,7 +104,7 @@ void invalidConfig_shouldThrowException(long value) {
102104
void crtClient_with_crossRegionAccessEnabled_asTrue() {
103105
try (S3AsyncClient crossRegionCrtClient = S3AsyncClient.crtBuilder().crossRegionAccessEnabled(true).build()) {
104106
assertThat(crossRegionCrtClient).isInstanceOf(DefaultS3CrtAsyncClient.class);
105-
assertThat(((DelegatingS3AsyncClient)crossRegionCrtClient).delegate()).isInstanceOf(S3CrossRegionAsyncClient.class);
107+
assertThat(((DelegatingS3AsyncClient) crossRegionCrtClient).delegate()).isInstanceOf(S3CrossRegionAsyncClient.class);
106108
}
107109
}
108110

@@ -115,7 +117,7 @@ void crtClient_with_crossRegionAccessEnabled_asFalse() {
115117

116118
try (S3AsyncClient defaultCrtClient = S3AsyncClient.crtBuilder().build()) {
117119
assertThat(defaultCrtClient).isInstanceOf(DefaultS3CrtAsyncClient.class);
118-
assertThat(((DelegatingS3AsyncClient)defaultCrtClient).delegate()).isNotInstanceOf(S3CrossRegionAsyncClient.class);
120+
assertThat(((DelegatingS3AsyncClient) defaultCrtClient).delegate()).isNotInstanceOf(S3CrossRegionAsyncClient.class);
119121
}
120122
}
121123

@@ -140,4 +142,23 @@ void defaultClient_credentialsProvidersNotSingleton() {
140142
.isNotEqualTo(DefaultCredentialsProvider.create());
141143
}
142144
}
145+
146+
@Test
147+
void build_withAdvancedOptions() {
148+
try (DefaultS3CrtAsyncClient client = (DefaultS3CrtAsyncClient) S3AsyncClient
149+
.crtBuilder()
150+
.advancedOption(SdkAdvancedAsyncClientOption.CRT_MEMORY_BUFFER_DISABLED, true)
151+
.build()) {
152+
assertThat(client).isNotNull();
153+
assertThat(client).isInstanceOf(DefaultS3CrtAsyncClient.class);
154+
}
155+
156+
try (DefaultS3CrtAsyncClient client = (DefaultS3CrtAsyncClient) S3AsyncClient
157+
.crtBuilder()
158+
.advancedOptions(MapUtils.of(SdkAdvancedAsyncClientOption.CRT_MEMORY_BUFFER_DISABLED, true))
159+
.build()) {
160+
assertThat(client).isNotNull();
161+
assertThat(client).isInstanceOf(DefaultS3CrtAsyncClient.class);
162+
}
163+
}
143164
}

0 commit comments

Comments
 (0)