Skip to content

Commit 40fdcc5

Browse files
author
Dhananjay Badaya
committed
Add support for request-level configuration override in AsyncClient multipart upload
When using the S3 AsyncClient for multipart uploads and copies, request-level override configuration set on the original PutObjectRequest or CopyObjectRequest was not being propagated to the sub-requests (CreateMultipartUpload, UploadPart, CompleteMultipartUpload, AbortMultipartUpload, ListParts, HeadObject, UploadPartCopy). This change ensures overrideConfiguration is forwarded from the original request to all derived multipart sub-requests in SdkPojoConversionUtils and CopyObjectHelper.
1 parent 07af8f9 commit 40fdcc5

4 files changed

Lines changed: 171 additions & 0 deletions

File tree

‎.changes/next-release/bugfix-AWSS3-52d62ca.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "bugfix",
3+
"category": "AWS S3",
4+
"contributor": "dbadaya1",
5+
"description": "Fixed request-level override configuration not being propagated to sub-requests during multipart uploads and copies in the S3 AsyncClient."
6+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@ private CompletableFuture<CompleteMultipartUploadResponse> completeMultipartUplo
173173
.sseCustomerAlgorithm(copyObjectRequest.sseCustomerAlgorithm())
174174
.sseCustomerKey(copyObjectRequest.sseCustomerKey())
175175
.sseCustomerKeyMD5(copyObjectRequest.sseCustomerKeyMD5())
176+
.overrideConfiguration(copyObjectRequest.overrideConfiguration().orElse(null))
176177
.build();
177178
return s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest);
178179
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/SdkPojoConversionUtils.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ public static UploadPartRequest toUploadPartRequest(PutObjectRequest putObjectRe
164164
UploadPartRequest.Builder builder = UploadPartRequest.builder();
165165
validateRequestFields(putObjectRequest, builder.build(), PUT_OBJECT_TO_UPLOAD_PART_ALLOWED_FIELDS);
166166
setSdkFields(builder, putObjectRequest, PUT_OBJECT_REQUEST_TO_UPLOAD_PART_FIELDS_TO_IGNORE);
167+
builder.overrideConfiguration(putObjectRequest.overrideConfiguration().orElse(null));
167168
return builder.uploadId(uploadId).partNumber(partNumber).build();
168169
}
169170

@@ -175,6 +176,7 @@ public static CompleteMultipartUploadRequest toCompleteMultipartUploadRequest(Pu
175176
setSdkFields(builder, putObjectRequest);
176177

177178
builder.mpuObjectSize(contentLength);
179+
builder.overrideConfiguration(putObjectRequest.overrideConfiguration().orElse(null));
178180

179181
if (S3ChecksumUtils.checksumValueSpecified(putObjectRequest)) {
180182
builder.checksumType(ChecksumType.FULL_OBJECT);
@@ -188,6 +190,7 @@ public static CreateMultipartUploadRequest toCreateMultipartUploadRequest(PutObj
188190
CreateMultipartUploadRequest.Builder builder = CreateMultipartUploadRequest.builder();
189191
validateRequestFields(putObjectRequest, builder.build(), PUT_OBJECT_TO_UPLOAD_PART_ALLOWED_FIELDS);
190192
setSdkFields(builder, putObjectRequest);
193+
builder.overrideConfiguration(putObjectRequest.overrideConfiguration().orElse(null));
191194

192195
if (S3ChecksumUtils.checksumValueSpecified(putObjectRequest)) {
193196
builder.checksumType(ChecksumType.FULL_OBJECT);
@@ -212,6 +215,7 @@ public static HeadObjectRequest toHeadObjectRequest(CopyObjectRequest copyObject
212215
.sseCustomerAlgorithm(copyObjectRequest.copySourceSSECustomerAlgorithm())
213216
.sseCustomerKey(copyObjectRequest.copySourceSSECustomerKey())
214217
.sseCustomerKeyMD5(copyObjectRequest.copySourceSSECustomerKeyMD5())
218+
.overrideConfiguration(copyObjectRequest.overrideConfiguration().orElse(null))
215219
.build();
216220
}
217221

@@ -238,6 +242,7 @@ public static ListPartsRequest toListPartsRequest(String uploadId, PutObjectRequ
238242
ListPartsRequest.Builder builder = ListPartsRequest.builder();
239243
validateRequestFields(putObjectRequest, builder.build(), PUT_OBJECT_TO_UPLOAD_PART_ALLOWED_FIELDS);
240244
setSdkFields(builder, putObjectRequest);
245+
builder.overrideConfiguration(putObjectRequest.overrideConfiguration().orElse(null));
241246
return builder.uploadId(uploadId).build();
242247
}
243248

@@ -247,6 +252,7 @@ public static CreateMultipartUploadRequest toCreateMultipartUploadRequest(CopyOb
247252
setSdkFields(builder, copyObjectRequest);
248253
builder.bucket(copyObjectRequest.destinationBucket());
249254
builder.key(copyObjectRequest.destinationKey());
255+
builder.overrideConfiguration(copyObjectRequest.overrideConfiguration().orElse(null));
250256
return builder.build();
251257
}
252258

@@ -275,13 +281,15 @@ public static AbortMultipartUploadRequest.Builder toAbortMultipartUploadRequest(
275281
setSdkFields(builder, copyObjectRequest);
276282
builder.bucket(copyObjectRequest.destinationBucket());
277283
builder.key(copyObjectRequest.destinationKey());
284+
builder.overrideConfiguration(copyObjectRequest.overrideConfiguration().orElse(null));
278285
return builder;
279286
}
280287

281288
public static AbortMultipartUploadRequest.Builder toAbortMultipartUploadRequest(PutObjectRequest putObjectRequest) {
282289
AbortMultipartUploadRequest.Builder builder = AbortMultipartUploadRequest.builder();
283290
validateRequestFields(putObjectRequest, builder.build(), PUT_OBJECT_TO_UPLOAD_PART_ALLOWED_FIELDS);
284291
setSdkFields(builder, putObjectRequest);
292+
builder.overrideConfiguration(putObjectRequest.overrideConfiguration().orElse(null));
285293
return builder;
286294
}
287295

@@ -296,6 +304,7 @@ public static UploadPartCopyRequest toUploadPartCopyRequest(CopyObjectRequest co
296304
.uploadId(uploadId)
297305
.bucket(copyObjectRequest.destinationBucket())
298306
.key(copyObjectRequest.destinationKey())
307+
.overrideConfiguration(copyObjectRequest.overrideConfiguration().orElse(null))
299308
.build();
300309
}
301310

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/SdkPojoConversionUtilsTest.java‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.commons.lang3.RandomStringUtils;
3434
import org.junit.jupiter.api.Test;
3535
import software.amazon.awssdk.awscore.DefaultAwsResponseMetadata;
36+
import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
3637
import software.amazon.awssdk.core.SdkField;
3738
import software.amazon.awssdk.core.SdkPojo;
3839
import software.amazon.awssdk.http.SdkHttpFullResponse;
@@ -244,6 +245,160 @@ void toListPartsRequest_putObject_shouldCopyProperties() {
244245
assertThat(convertedObject.uploadId()).isEqualTo("uploadId");
245246
}
246247

248+
@Test
249+
void toCreateMultipartUploadRequest_putObjectWithOverrideConfig_shouldPropagateOverrideConfig() {
250+
AwsRequestOverrideConfiguration overrideConfig = AwsRequestOverrideConfiguration.builder()
251+
.putHeader("x-custom", "value")
252+
.build();
253+
PutObjectRequest request = PutObjectRequest.builder()
254+
.bucket("bucket")
255+
.key("key")
256+
.overrideConfiguration(overrideConfig)
257+
.build();
258+
CreateMultipartUploadRequest converted = SdkPojoConversionUtils.toCreateMultipartUploadRequest(request);
259+
assertThat(converted.overrideConfiguration()).isPresent();
260+
assertThat(converted.overrideConfiguration().get()).isEqualTo(overrideConfig);
261+
}
262+
263+
@Test
264+
void toUploadPartRequest_withOverrideConfig_shouldPropagateOverrideConfig() {
265+
AwsRequestOverrideConfiguration overrideConfig = AwsRequestOverrideConfiguration.builder()
266+
.putHeader("x-custom", "value")
267+
.build();
268+
PutObjectRequest request = PutObjectRequest.builder()
269+
.bucket("bucket")
270+
.key("key")
271+
.overrideConfiguration(overrideConfig)
272+
.build();
273+
UploadPartRequest converted = SdkPojoConversionUtils.toUploadPartRequest(request, 1, "uploadId");
274+
assertThat(converted.overrideConfiguration()).isPresent();
275+
assertThat(converted.overrideConfiguration().get()).isEqualTo(overrideConfig);
276+
}
277+
278+
@Test
279+
void toCompleteMultipartUploadRequest_withOverrideConfig_shouldPropagateOverrideConfig() {
280+
AwsRequestOverrideConfiguration overrideConfig = AwsRequestOverrideConfiguration.builder()
281+
.putHeader("x-custom", "value")
282+
.build();
283+
PutObjectRequest request = PutObjectRequest.builder()
284+
.bucket("bucket")
285+
.key("key")
286+
.overrideConfiguration(overrideConfig)
287+
.build();
288+
CompletedPart[] parts = { CompletedPart.builder().partNumber(1).build() };
289+
CompleteMultipartUploadRequest converted =
290+
SdkPojoConversionUtils.toCompleteMultipartUploadRequest(request, "uploadId", parts, 100L);
291+
assertThat(converted.overrideConfiguration()).isPresent();
292+
assertThat(converted.overrideConfiguration().get()).isEqualTo(overrideConfig);
293+
}
294+
295+
@Test
296+
void toAbortMultipartUploadRequest_putObjectWithOverrideConfig_shouldPropagateOverrideConfig() {
297+
AwsRequestOverrideConfiguration overrideConfig = AwsRequestOverrideConfiguration.builder()
298+
.putHeader("x-custom", "value")
299+
.build();
300+
PutObjectRequest request = PutObjectRequest.builder()
301+
.bucket("bucket")
302+
.key("key")
303+
.overrideConfiguration(overrideConfig)
304+
.build();
305+
AbortMultipartUploadRequest converted = SdkPojoConversionUtils.toAbortMultipartUploadRequest(request).build();
306+
assertThat(converted.overrideConfiguration()).isPresent();
307+
assertThat(converted.overrideConfiguration().get()).isEqualTo(overrideConfig);
308+
}
309+
310+
@Test
311+
void toListPartsRequest_withOverrideConfig_shouldPropagateOverrideConfig() {
312+
AwsRequestOverrideConfiguration overrideConfig = AwsRequestOverrideConfiguration.builder()
313+
.putHeader("x-custom", "value")
314+
.build();
315+
PutObjectRequest request = PutObjectRequest.builder()
316+
.bucket("bucket")
317+
.key("key")
318+
.overrideConfiguration(overrideConfig)
319+
.build();
320+
ListPartsRequest converted = SdkPojoConversionUtils.toListPartsRequest("uploadId", request);
321+
assertThat(converted.overrideConfiguration()).isPresent();
322+
assertThat(converted.overrideConfiguration().get()).isEqualTo(overrideConfig);
323+
}
324+
325+
@Test
326+
void toCreateMultipartUploadRequest_copyObjectWithOverrideConfig_shouldPropagateOverrideConfig() {
327+
AwsRequestOverrideConfiguration overrideConfig = AwsRequestOverrideConfiguration.builder()
328+
.putHeader("x-custom", "value")
329+
.build();
330+
CopyObjectRequest request = CopyObjectRequest.builder()
331+
.sourceBucket("src")
332+
.sourceKey("srcKey")
333+
.destinationBucket("dst")
334+
.destinationKey("dstKey")
335+
.overrideConfiguration(overrideConfig)
336+
.build();
337+
CreateMultipartUploadRequest converted = SdkPojoConversionUtils.toCreateMultipartUploadRequest(request);
338+
assertThat(converted.overrideConfiguration()).isPresent();
339+
assertThat(converted.overrideConfiguration().get()).isEqualTo(overrideConfig);
340+
}
341+
342+
@Test
343+
void toHeadObjectRequest_withOverrideConfig_shouldPropagateOverrideConfig() {
344+
AwsRequestOverrideConfiguration overrideConfig = AwsRequestOverrideConfiguration.builder()
345+
.putHeader("x-custom", "value")
346+
.build();
347+
CopyObjectRequest request = CopyObjectRequest.builder()
348+
.sourceBucket("src")
349+
.sourceKey("srcKey")
350+
.overrideConfiguration(overrideConfig)
351+
.build();
352+
HeadObjectRequest converted = SdkPojoConversionUtils.toHeadObjectRequest(request);
353+
assertThat(converted.overrideConfiguration()).isPresent();
354+
assertThat(converted.overrideConfiguration().get()).isEqualTo(overrideConfig);
355+
}
356+
357+
@Test
358+
void toAbortMultipartUploadRequest_copyObjectWithOverrideConfig_shouldPropagateOverrideConfig() {
359+
AwsRequestOverrideConfiguration overrideConfig = AwsRequestOverrideConfiguration.builder()
360+
.putHeader("x-custom", "value")
361+
.build();
362+
CopyObjectRequest request = CopyObjectRequest.builder()
363+
.sourceBucket("src")
364+
.sourceKey("srcKey")
365+
.destinationBucket("dst")
366+
.destinationKey("dstKey")
367+
.overrideConfiguration(overrideConfig)
368+
.build();
369+
AbortMultipartUploadRequest converted = SdkPojoConversionUtils.toAbortMultipartUploadRequest(request).build();
370+
assertThat(converted.overrideConfiguration()).isPresent();
371+
assertThat(converted.overrideConfiguration().get()).isEqualTo(overrideConfig);
372+
}
373+
374+
@Test
375+
void toUploadPartCopyRequest_withOverrideConfig_shouldPropagateOverrideConfig() {
376+
AwsRequestOverrideConfiguration overrideConfig = AwsRequestOverrideConfiguration.builder()
377+
.putHeader("x-custom", "value")
378+
.build();
379+
CopyObjectRequest request = CopyObjectRequest.builder()
380+
.sourceBucket("src")
381+
.sourceKey("srcKey")
382+
.destinationBucket("dst")
383+
.destinationKey("dstKey")
384+
.overrideConfiguration(overrideConfig)
385+
.build();
386+
UploadPartCopyRequest converted = SdkPojoConversionUtils.toUploadPartCopyRequest(request, 1, "uploadId",
387+
"bytes=0-1024");
388+
assertThat(converted.overrideConfiguration()).isPresent();
389+
assertThat(converted.overrideConfiguration().get()).isEqualTo(overrideConfig);
390+
}
391+
392+
@Test
393+
void toCreateMultipartUploadRequest_putObjectWithoutOverrideConfig_shouldNotHaveOverrideConfig() {
394+
PutObjectRequest request = PutObjectRequest.builder()
395+
.bucket("bucket")
396+
.key("key")
397+
.build();
398+
CreateMultipartUploadRequest converted = SdkPojoConversionUtils.toCreateMultipartUploadRequest(request);
399+
assertThat(converted.overrideConfiguration()).isNotPresent();
400+
}
401+
247402
private static void verifyFieldsAreCopied(SdkPojo requestConvertedFrom,
248403
SdkPojo requestConvertedTo,
249404
Set<String> fieldsToIgnore,

0 commit comments

Comments
 (0)