Skip to content

Commit be935eb

Browse files
Fred1155Fred1155
authored
Add support of Request-level credentials override in DefaultS3CrtAsyncClient (#6793)
* add request level creds override for crt * changelog added * change decription * change integration test to wiremock test * added test for not happy path
1 parent 3ac0f42 commit be935eb

File tree

6 files changed

+249
-25
lines changed

6 files changed

+249
-25
lines changed

‎.changes/next-release/feature-AmazonS3-bf3403d.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "feature",
3+
"category": "Amazon S3",
4+
"contributor": "",
5+
"description": "Added support of Request-level credentials override in DefaultS3CrtAsyncClient. See [#5354](https://github.com/aws/aws-sdk-java-v2/issues/5354)."
6+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java‎

Lines changed: 31 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -423,24 +423,37 @@ public void afterMarshalling(Context.AfterMarshalling context,
423423
existingHttpAttributes.toBuilder() :
424424
SdkHttpExecutionAttributes.builder();
425425

426-
SdkHttpExecutionAttributes attributes =
427-
builder.put(OPERATION_NAME,
428-
executionAttributes.getAttribute(SdkExecutionAttribute.OPERATION_NAME))
429-
.put(HTTP_CHECKSUM, executionAttributes.getAttribute(SdkInternalExecutionAttribute.HTTP_CHECKSUM))
430-
.put(SIGNING_REGION, executionAttributes.getAttribute(AwsSignerExecutionAttribute.SIGNING_REGION))
431-
.put(S3InternalSdkHttpExecutionAttribute.OBJECT_FILE_PATH,
432-
executionAttributes.getAttribute(OBJECT_FILE_PATH))
433-
.put(USE_S3_EXPRESS_AUTH, S3ExpressUtils.useS3ExpressAuthScheme(executionAttributes))
434-
.put(SIGNING_NAME, executionAttributes.getAttribute(SERVICE_SIGNING_NAME))
435-
.put(REQUEST_CHECKSUM_CALCULATION,
436-
executionAttributes.getAttribute(SdkInternalExecutionAttribute.REQUEST_CHECKSUM_CALCULATION))
437-
.put(RESPONSE_CHECKSUM_VALIDATION,
438-
executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESPONSE_CHECKSUM_VALIDATION))
439-
.put(S3InternalSdkHttpExecutionAttribute.RESPONSE_FILE_PATH,
440-
executionAttributes.getAttribute(RESPONSE_FILE_PATH))
441-
.put(S3InternalSdkHttpExecutionAttribute.RESPONSE_FILE_OPTION,
442-
executionAttributes.getAttribute(RESPONSE_FILE_OPTION))
443-
.build();
426+
builder.put(OPERATION_NAME,
427+
executionAttributes.getAttribute(SdkExecutionAttribute.OPERATION_NAME))
428+
.put(HTTP_CHECKSUM, executionAttributes.getAttribute(SdkInternalExecutionAttribute.HTTP_CHECKSUM))
429+
.put(SIGNING_REGION, executionAttributes.getAttribute(AwsSignerExecutionAttribute.SIGNING_REGION))
430+
.put(S3InternalSdkHttpExecutionAttribute.OBJECT_FILE_PATH,
431+
executionAttributes.getAttribute(OBJECT_FILE_PATH))
432+
.put(USE_S3_EXPRESS_AUTH, S3ExpressUtils.useS3ExpressAuthScheme(executionAttributes))
433+
.put(SIGNING_NAME, executionAttributes.getAttribute(SERVICE_SIGNING_NAME))
434+
.put(REQUEST_CHECKSUM_CALCULATION,
435+
executionAttributes.getAttribute(SdkInternalExecutionAttribute.REQUEST_CHECKSUM_CALCULATION))
436+
.put(RESPONSE_CHECKSUM_VALIDATION,
437+
executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESPONSE_CHECKSUM_VALIDATION))
438+
.put(S3InternalSdkHttpExecutionAttribute.RESPONSE_FILE_PATH,
439+
executionAttributes.getAttribute(RESPONSE_FILE_PATH))
440+
.put(S3InternalSdkHttpExecutionAttribute.RESPONSE_FILE_OPTION,
441+
executionAttributes.getAttribute(RESPONSE_FILE_OPTION));
442+
443+
SdkRequest request = context.request();
444+
if (request instanceof AwsRequest) {
445+
((AwsRequest) request).overrideConfiguration().ifPresent(config -> {
446+
AwsRequestOverrideConfiguration awsConfig = (AwsRequestOverrideConfiguration) config;
447+
awsConfig.credentialsIdentityProvider().ifPresent(credentialsProvider -> {
448+
CrtCredentialsProviderAdapter adapter =
449+
new CrtCredentialsProviderAdapter(credentialsProvider);
450+
builder.put(S3InternalSdkHttpExecutionAttribute.CRT_CREDENTIALS_PROVIDER_ADAPTER,
451+
adapter);
452+
});
453+
});
454+
}
455+
456+
SdkHttpExecutionAttributes attributes = builder.build();
444457

445458
// We rely on CRT to perform checksum validation, disable SDK flexible checksum implementation
446459
executionAttributes.putAttribute(SdkInternalExecutionAttribute.HTTP_CHECKSUM, null);
@@ -468,11 +481,6 @@ private static void validateOverrideConfiguration(SdkRequest request) {
468481
throw new UnsupportedOperationException("Request-level signer override is not supported");
469482
}
470483

471-
// TODO: support request-level credential override
472-
if (overrideConfiguration.credentialsIdentityProvider().isPresent()) {
473-
throw new UnsupportedOperationException("Request-level credentials override is not supported");
474-
}
475-
476484
if (!CollectionUtils.isNullOrEmpty(overrideConfiguration.metricPublishers())) {
477485
throw new UnsupportedOperationException("Request-level Metric Publishers override is not supported");
478486
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
4444
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
4545
import software.amazon.awssdk.core.interceptor.trait.HttpChecksum;
46+
import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
4647
import software.amazon.awssdk.crt.auth.signing.AwsSigningConfig;
4748
import software.amazon.awssdk.crt.http.HttpHeader;
4849
import software.amazon.awssdk.crt.http.HttpProxyEnvironmentVariableSetting;
@@ -186,6 +187,9 @@ public CompletableFuture<Void> execute(AsyncExecuteRequest asyncRequest) {
186187
requestOptions = requestOptions.withResponseFileOption(responseFileOption);
187188
}
188189

190+
CrtCredentialsProviderAdapter requestCredentialsAdapter =
191+
httpExecutionAttributes.getAttribute(S3InternalSdkHttpExecutionAttribute.CRT_CREDENTIALS_PROVIDER_ADAPTER);
192+
189193
try {
190194
S3MetaRequestWrapper requestWrapper = new S3MetaRequestWrapper(crtS3Client.makeMetaRequest(requestOptions));
191195
s3MetaRequestFuture.complete(requestWrapper);
@@ -196,16 +200,30 @@ public CompletableFuture<Void> execute(AsyncExecuteRequest asyncRequest) {
196200
if (observable != null) {
197201
observable.subscribe(requestWrapper);
198202
}
203+
} catch (Throwable t) {
204+
if (requestCredentialsAdapter != null) {
205+
requestCredentialsAdapter.close();
206+
}
207+
throw t;
199208
} finally {
200209
signingConfig.close();
201210
}
202211

212+
if (requestCredentialsAdapter != null) {
213+
executeFuture.whenComplete((result, error) -> requestCredentialsAdapter.close());
214+
}
215+
203216
return executeFuture;
204217
}
205218

206219
private AwsSigningConfig awsSigningConfig(Region signingRegion, SdkHttpExecutionAttributes httpExecutionAttributes) {
220+
CrtCredentialsProviderAdapter requestAdapter =
221+
httpExecutionAttributes.getAttribute(S3InternalSdkHttpExecutionAttribute.CRT_CREDENTIALS_PROVIDER_ADAPTER);
222+
CredentialsProvider effectiveCredentials =
223+
requestAdapter != null ? requestAdapter.crtCredentials() : s3ClientOptions.getCredentialsProvider();
224+
207225
AwsSigningConfig defaultS3SigningConfig =
208-
AwsSigningConfig.getDefaultS3SigningConfig(s3ClientOptions.getRegion(), s3ClientOptions.getCredentialsProvider());
226+
AwsSigningConfig.getDefaultS3SigningConfig(s3ClientOptions.getRegion(), effectiveCredentials);
209227

210228
// Override the region only if the signing region has changed from the previously configured region.
211229
if (signingRegion != null && !s3ClientOptions.getRegion().equals(signingRegion.id())) {

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3InternalSdkHttpExecutionAttribute.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@ public final class S3InternalSdkHttpExecutionAttribute<T> extends SdkHttpExecuti
6464
public static final S3InternalSdkHttpExecutionAttribute<S3MetaRequestOptions.ResponseFileOption> RESPONSE_FILE_OPTION =
6565
new S3InternalSdkHttpExecutionAttribute<>(S3MetaRequestOptions.ResponseFileOption.class);
6666

67+
public static final S3InternalSdkHttpExecutionAttribute<CrtCredentialsProviderAdapter> CRT_CREDENTIALS_PROVIDER_ADAPTER =
68+
new S3InternalSdkHttpExecutionAttribute<>(CrtCredentialsProviderAdapter.class);
69+
6770
private S3InternalSdkHttpExecutionAttribute(Class<T> valueClass) {
6871
super(valueClass);
6972
}

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/crt/S3CrtRequestLevelCredentialsWireMockTest.java‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.crt;
17+
18+
import static com.github.tomakehurst.wiremock.client.WireMock.containing;
19+
import static com.github.tomakehurst.wiremock.client.WireMock.get;
20+
import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor;
21+
import static com.github.tomakehurst.wiremock.client.WireMock.head;
22+
import static com.github.tomakehurst.wiremock.client.WireMock.put;
23+
import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor;
24+
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
25+
import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
26+
import static com.github.tomakehurst.wiremock.client.WireMock.verify;
27+
28+
import com.github.tomakehurst.wiremock.client.WireMock;
29+
import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
30+
import com.github.tomakehurst.wiremock.junit5.WireMockTest;
31+
import java.net.URI;
32+
import java.nio.charset.StandardCharsets;
33+
import org.junit.jupiter.api.AfterEach;
34+
import org.junit.jupiter.api.BeforeAll;
35+
import org.junit.jupiter.api.BeforeEach;
36+
import org.junit.jupiter.api.Test;
37+
import org.junit.jupiter.api.Timeout;
38+
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
39+
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
40+
import software.amazon.awssdk.core.async.AsyncRequestBody;
41+
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
42+
import software.amazon.awssdk.crt.Log;
43+
import software.amazon.awssdk.regions.Region;
44+
import software.amazon.awssdk.services.s3.S3AsyncClient;
45+
46+
/**
47+
* WireMock tests verifying that request-level credential overrides are used for signing
48+
* with the S3 CRT client. Verifies the Authorization header contains the expected access key.
49+
*/
50+
@WireMockTest
51+
@Timeout(10)
52+
public class S3CrtRequestLevelCredentialsWireMockTest {
53+
54+
private static final String BUCKET = "my-bucket";
55+
private static final String KEY = "my-key";
56+
private static final String PATH = String.format("/%s/%s", BUCKET, KEY);
57+
private static final byte[] CONTENT = "hello".getBytes(StandardCharsets.UTF_8);
58+
59+
private static final StaticCredentialsProvider CLIENT_CREDENTIALS =
60+
StaticCredentialsProvider.create(AwsBasicCredentials.create("clientAccessKey", "clientSecretKey"));
61+
62+
private static final StaticCredentialsProvider REQUEST_CREDENTIALS =
63+
StaticCredentialsProvider.create(AwsBasicCredentials.create("requestAccessKey", "requestSecretKey"));
64+
65+
private S3AsyncClient s3;
66+
67+
@BeforeAll
68+
public static void setUpBeforeAll() {
69+
System.setProperty("aws.crt.debugnative", "true");
70+
Log.initLoggingToStdout(Log.LogLevel.Warn);
71+
}
72+
73+
@BeforeEach
74+
public void setup(WireMockRuntimeInfo wiremock) {
75+
stubFor(head(urlPathEqualTo(PATH))
76+
.willReturn(WireMock.aResponse().withStatus(200)
77+
.withHeader("ETag", "etag")
78+
.withHeader("Content-Length",
79+
Integer.toString(CONTENT.length))));
80+
stubFor(get(urlPathEqualTo(PATH))
81+
.willReturn(WireMock.aResponse().withStatus(200)
82+
.withHeader("Content-Type", "text/plain")
83+
.withBody(CONTENT)));
84+
stubFor(put(urlPathEqualTo(PATH))
85+
.willReturn(WireMock.aResponse().withStatus(200)
86+
.withHeader("ETag", "etag")));
87+
88+
s3 = S3AsyncClient.crtBuilder()
89+
.endpointOverride(URI.create("http://localhost:" + wiremock.getHttpPort()))
90+
.credentialsProvider(CLIENT_CREDENTIALS)
91+
.forcePathStyle(true)
92+
.region(Region.US_EAST_1)
93+
.build();
94+
}
95+
96+
@AfterEach
97+
public void tearDown() {
98+
s3.close();
99+
}
100+
101+
@Test
102+
void getObject_withRequestLevelCredentials_shouldSignWithOverrideCredentials() {
103+
s3.getObject(
104+
b -> b.bucket(BUCKET).key(KEY)
105+
.overrideConfiguration(o -> o.credentialsProvider(REQUEST_CREDENTIALS)),
106+
AsyncResponseTransformer.toBytes()).join();
107+
108+
verify(getRequestedFor(urlPathEqualTo(PATH))
109+
.withHeader("Authorization", containing("Credential=requestAccessKey/")));
110+
}
111+
112+
@Test
113+
void getObject_withoutRequestLevelCredentials_shouldSignWithClientCredentials() {
114+
s3.getObject(
115+
b -> b.bucket(BUCKET).key(KEY),
116+
AsyncResponseTransformer.toBytes()).join();
117+
118+
verify(getRequestedFor(urlPathEqualTo(PATH))
119+
.withHeader("Authorization", containing("Credential=clientAccessKey/")));
120+
}
121+
122+
@Test
123+
void putObject_withRequestLevelCredentials_shouldSignWithOverrideCredentials() {
124+
s3.putObject(
125+
b -> b.bucket(BUCKET).key(KEY)
126+
.overrideConfiguration(o -> o.credentialsProvider(REQUEST_CREDENTIALS)),
127+
AsyncRequestBody.fromString("hello")).join();
128+
129+
verify(putRequestedFor(urlPathEqualTo(PATH))
130+
.withHeader("Authorization", containing("Credential=requestAccessKey/")));
131+
}
132+
133+
@Test
134+
void putObject_withoutRequestLevelCredentials_shouldSignWithClientCredentials() {
135+
s3.putObject(
136+
b -> b.bucket(BUCKET).key(KEY),
137+
AsyncRequestBody.fromString("hello")).join();
138+
139+
verify(putRequestedFor(urlPathEqualTo(PATH))
140+
.withHeader("Authorization", containing("Credential=clientAccessKey/")));
141+
}
142+
}

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package software.amazon.awssdk.services.s3.internal.crt;
1717

1818
import static org.assertj.core.api.Assertions.assertThat;
19+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
1920
import static org.mockito.ArgumentMatchers.any;
2021
import static org.mockito.Mockito.verify;
2122
import static org.mockito.Mockito.when;
@@ -27,6 +28,7 @@
2728
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.RESPONSE_CHECKSUM_VALIDATION;
2829
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.RESPONSE_FILE_OPTION;
2930
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.RESPONSE_FILE_PATH;
31+
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.CRT_CREDENTIALS_PROVIDER_ADAPTER;
3032
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.SIGNING_NAME;
3133
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.SIGNING_REGION;
3234
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.USE_S3_EXPRESS_AUTH;
@@ -50,6 +52,7 @@
5052
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
5153
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
5254
import software.amazon.awssdk.core.interceptor.trait.HttpChecksum;
55+
import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
5356
import software.amazon.awssdk.crt.auth.signing.AwsSigningConfig;
5457
import software.amazon.awssdk.crt.http.HttpProxyEnvironmentVariableSetting;
5558
import software.amazon.awssdk.crt.http.HttpRequest;
@@ -60,7 +63,6 @@
6063
import software.amazon.awssdk.crt.s3.S3ClientOptions;
6164
import software.amazon.awssdk.crt.s3.S3MetaRequest;
6265
import software.amazon.awssdk.crt.s3.S3MetaRequestOptions;
63-
import software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandler;
6466
import software.amazon.awssdk.http.SdkHttpMethod;
6567
import software.amazon.awssdk.http.SdkHttpRequest;
6668
import software.amazon.awssdk.http.async.AsyncExecuteRequest;
@@ -623,6 +625,51 @@ public void responseFilePathAndOption_shouldPassToCrt() {
623625
assertThat(actual.getResponseFileOption()).isEqualTo(S3MetaRequestOptions.ResponseFileOption.CREATE_OR_APPEND);
624626
}
625627

628+
@Test
629+
public void execute_withRequestLevelCredentials_shouldCloseAdapterOnCompletion() {
630+
CrtCredentialsProviderAdapter adapter = Mockito.mock(CrtCredentialsProviderAdapter.class);
631+
when(adapter.crtCredentials()).thenReturn(Mockito.mock(CredentialsProvider.class));
632+
S3MetaRequest metaRequest = Mockito.mock(S3MetaRequest.class);
633+
when(s3Client.makeMetaRequest(any(S3MetaRequestOptions.class))).thenReturn(metaRequest);
634+
635+
AsyncExecuteRequest asyncExecuteRequest =
636+
getExecuteRequestBuilder()
637+
.putHttpExecutionAttribute(OPERATION_NAME, "GetObject")
638+
.putHttpExecutionAttribute(SIGNING_REGION, Region.US_WEST_2)
639+
.putHttpExecutionAttribute(SIGNING_NAME, "s3")
640+
.putHttpExecutionAttribute(CRT_CREDENTIALS_PROVIDER_ADAPTER, adapter)
641+
.build();
642+
643+
CompletableFuture<Void> future = asyncHttpClient.execute(asyncExecuteRequest);
644+
645+
Mockito.verify(adapter, Mockito.never()).close();
646+
647+
future.complete(null);
648+
649+
Mockito.verify(adapter).close();
650+
}
651+
652+
@Test
653+
void execute_whenMakeMetaRequestThrows_shouldCloseAdapter() {
654+
CrtCredentialsProviderAdapter adapter = Mockito.mock(CrtCredentialsProviderAdapter.class);
655+
when(adapter.crtCredentials()).thenReturn(Mockito.mock(CredentialsProvider.class));
656+
when(s3Client.makeMetaRequest(any(S3MetaRequestOptions.class)))
657+
.thenThrow(new RuntimeException("boom"));
658+
659+
AsyncExecuteRequest asyncExecuteRequest =
660+
getExecuteRequestBuilder()
661+
.putHttpExecutionAttribute(OPERATION_NAME, "GetObject")
662+
.putHttpExecutionAttribute(SIGNING_REGION, Region.US_WEST_2)
663+
.putHttpExecutionAttribute(SIGNING_NAME, "s3")
664+
.putHttpExecutionAttribute(CRT_CREDENTIALS_PROVIDER_ADAPTER, adapter)
665+
.build();
666+
667+
assertThatThrownBy(() -> asyncHttpClient.execute(asyncExecuteRequest))
668+
.isInstanceOf(RuntimeException.class);
669+
670+
Mockito.verify(adapter).close();
671+
}
672+
626673
private AsyncExecuteRequest.Builder getExecuteRequestBuilder() {
627674
return getExecuteRequestBuilder(443);
628675
}

0 commit comments

Comments
 (0)