From 8e513289dc77a66958a8d7011c844f82afb2c2bc Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 25 Nov 2024 14:05:14 -0800 Subject: [PATCH 01/24] step 1 - refactoring --- .../ContentMetadataEncodingStrategy.java | 10 ++++---- .../s3/internal/InstructionFileConfig.java | 1 + .../MultipartUploadObjectPipeline.java | 7 +++--- .../ObjectMetadataEncodingStrategy.java | 23 +++++++++++++++---- .../internal/PutEncryptedObjectPipeline.java | 8 ++----- 5 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java index 3045ae0eb..3c0c83879 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java @@ -2,13 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.encryption.s3.internal; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.encryption.s3.materials.EncryptionMaterials; -import java.util.Map; - -@FunctionalInterface public interface ContentMetadataEncodingStrategy { - Map encodeMetadata(EncryptionMaterials materials, byte[] iv, - Map metadata); + PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, PutObjectRequest putObjectRequest); + CreateMultipartUploadRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, CreateMultipartUploadRequest createMultipartUploadRequest); + } diff --git a/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java b/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java index 4b70a15d1..b24da4104 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java +++ b/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java @@ -16,6 +16,7 @@ public class InstructionFileConfig { final private InstructionFileClientType _clientType; final private S3AsyncClient _s3AsyncClient; final private S3Client _s3Client; + final private boolean _enableInstructionFilePut; private InstructionFileConfig(final Builder builder) { _clientType = builder._clientType; diff --git a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java index b24d6d499..7e8dc7452 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java @@ -67,11 +67,10 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload MultipartEncryptedContent encryptedContent = _contentEncryptionStrategy.initMultipartEncryption(materials); - Map metadata = new HashMap<>(request.metadata()); - metadata = _contentMetadataEncodingStrategy.encodeMetadata(materials, encryptedContent.getIv(), metadata); - request = request.toBuilder() + CreateMultipartUploadRequest createMpuRequest = _contentMetadataEncodingStrategy.encodeMetadata(materials, encryptedContent.getIv(), request); + request = createMpuRequest.toBuilder() .overrideConfiguration(API_NAME_INTERCEPTOR) - .metadata(metadata).build(); + .build(); CreateMultipartUploadResponse response = _s3AsyncClient.createMultipartUpload(request).join(); diff --git a/src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java index c0f0d5a0c..b2b9ba4da 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java @@ -1,21 +1,37 @@ package software.amazon.encryption.s3.internal; import software.amazon.awssdk.protocols.jsoncore.JsonWriter; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.encryption.s3.S3EncryptionClientException; import software.amazon.encryption.s3.materials.EncryptedDataKey; import software.amazon.encryption.s3.materials.EncryptionMaterials; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.HashMap; import java.util.Map; public class ObjectMetadataEncodingStrategy implements ContentMetadataEncodingStrategy { private static final Base64.Encoder ENCODER = Base64.getEncoder(); - @Override - public Map encodeMetadata(EncryptionMaterials materials, byte[] iv, - Map metadata) { + public PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, PutObjectRequest putObjectRequest) { + Map newMetadata = addMetadataToMap(putObjectRequest.metadata(), materials, iv); + return putObjectRequest.toBuilder() + .metadata(newMetadata) + .build(); + } + + public CreateMultipartUploadRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, CreateMultipartUploadRequest createMultipartUploadRequest) { + Map newMetadata = addMetadataToMap(createMultipartUploadRequest.metadata(), materials, iv); + return createMultipartUploadRequest.toBuilder() + .metadata(newMetadata) + .build(); + } + + private Map addMetadataToMap(Map map, EncryptionMaterials materials, byte[] iv) { + Map metadata = new HashMap<>(map); EncryptedDataKey edk = materials.encryptedDataKeys().get(0); metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_V2, ENCODER.encodeToString(edk.encryptedDatakey())); metadata.put(MetadataKeyConstants.CONTENT_IV, ENCODER.encodeToString(iv)); @@ -37,5 +53,4 @@ public Map encodeMetadata(EncryptionMaterials materials, byte[] } return metadata; } - } diff --git a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java index ba7bbbb3b..f3ccb743b 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java @@ -14,8 +14,6 @@ import software.amazon.encryption.s3.materials.EncryptionMaterialsRequest; import java.security.SecureRandom; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.CompletableFuture; import static software.amazon.encryption.s3.internal.ApiNameVersion.API_NAME_INTERCEPTOR; @@ -70,12 +68,10 @@ public CompletableFuture putObject(PutObjectRequest request, EncryptedContent encryptedContent = _asyncContentEncryptionStrategy.encryptContent(materials, requestBody); - Map metadata = new HashMap<>(request.metadata()); - metadata = _contentMetadataEncodingStrategy.encodeMetadata(materials, encryptedContent.getIv(), metadata); - PutObjectRequest encryptedPutRequest = request.toBuilder() + PutObjectRequest modifiedRequest = _contentMetadataEncodingStrategy.encodeMetadata(materials, encryptedContent.getIv(), request); + PutObjectRequest encryptedPutRequest = modifiedRequest.toBuilder() .overrideConfiguration(API_NAME_INTERCEPTOR) .contentLength(encryptedContent.getCiphertextLength()) - .metadata(metadata) .build(); return _s3AsyncClient.putObject(encryptedPutRequest, encryptedContent.getAsyncCiphertext()); } From 898c55e682c72c8f4b6ee67ac14254e7a53cfbac Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 2 Dec 2024 09:40:10 -0800 Subject: [PATCH 02/24] wip - instruction file puts --- .../ContentMetadataEncodingStrategy.java | 70 +++++++++++++++++-- .../s3/internal/InstructionFileConfig.java | 39 +++++++++++ .../MultipartUploadObjectPipeline.java | 2 +- .../ObjectMetadataEncodingStrategy.java | 56 --------------- .../internal/PutEncryptedObjectPipeline.java | 10 ++- 5 files changed, 114 insertions(+), 63 deletions(-) delete mode 100644 src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java diff --git a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java index 3c0c83879..95d8132ce 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java @@ -1,14 +1,74 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 package software.amazon.encryption.s3.internal; +import software.amazon.awssdk.protocols.jsoncore.JsonWriter; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.encryption.s3.S3EncryptionClientException; +import software.amazon.encryption.s3.materials.EncryptedDataKey; import software.amazon.encryption.s3.materials.EncryptionMaterials; -public interface ContentMetadataEncodingStrategy { +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; - PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, PutObjectRequest putObjectRequest); - CreateMultipartUploadRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, CreateMultipartUploadRequest createMultipartUploadRequest); +public class ContentMetadataEncodingStrategy { + private static final Base64.Encoder ENCODER = Base64.getEncoder(); + private final InstructionFileConfig _instructionFileConfig; + + public ContentMetadataEncodingStrategy(InstructionFileConfig instructionFileConfig) { + _instructionFileConfig = instructionFileConfig; + } + + public PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, PutObjectRequest putObjectRequest) { + if (_instructionFileConfig.isInstructionFilePutEnabled()) { + // TODO: serialize inst file as string + final String metadataString = metadataToString(materials, iv); + _instructionFileConfig.putInstructionFile(putObjectRequest, ""); + // the original object is returned as-is + return putObjectRequest; + } else { + Map newMetadata = addMetadataToMap(putObjectRequest.metadata(), materials, iv); + return putObjectRequest.toBuilder() + .metadata(newMetadata) + .build(); + } + } + + public CreateMultipartUploadRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, CreateMultipartUploadRequest createMultipartUploadRequest) { + Map newMetadata = addMetadataToMap(createMultipartUploadRequest.metadata(), materials, iv); + return createMultipartUploadRequest.toBuilder() + .metadata(newMetadata) + .build(); + } + + private String metadataToString(EncryptionMaterials materials, byte[] iv) { + // this is just the metadata map serialized as JSON + return ""; + } + + private Map addMetadataToMap(Map map, EncryptionMaterials materials, byte[] iv) { + Map metadata = new HashMap<>(map); + EncryptedDataKey edk = materials.encryptedDataKeys().get(0); + metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_V2, ENCODER.encodeToString(edk.encryptedDatakey())); + metadata.put(MetadataKeyConstants.CONTENT_IV, ENCODER.encodeToString(iv)); + metadata.put(MetadataKeyConstants.CONTENT_CIPHER, materials.algorithmSuite().cipherName()); + metadata.put(MetadataKeyConstants.CONTENT_CIPHER_TAG_LENGTH, Integer.toString(materials.algorithmSuite().cipherTagLengthBits())); + metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_ALGORITHM, new String(edk.keyProviderInfo(), StandardCharsets.UTF_8)); + + try (JsonWriter jsonWriter = JsonWriter.create()) { + jsonWriter.writeStartObject(); + for (Map.Entry entry : materials.encryptionContext().entrySet()) { + jsonWriter.writeFieldName(entry.getKey()).writeValue(entry.getValue()); + } + jsonWriter.writeEndObject(); + + String jsonEncryptionContext = new String(jsonWriter.getBytes(), StandardCharsets.UTF_8); + metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_CONTEXT, jsonEncryptionContext); + } catch (JsonWriter.JsonGenerationException e) { + throw new S3EncryptionClientException("Cannot serialize encryption context to JSON.", e); + } + return metadata; + } } diff --git a/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java b/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java index b24da4104..ad067df5a 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java +++ b/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java @@ -1,11 +1,15 @@ package software.amazon.encryption.s3.internal; import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.encryption.s3.S3EncryptionClientException; /** @@ -22,6 +26,7 @@ private InstructionFileConfig(final Builder builder) { _clientType = builder._clientType; _s3Client = builder._s3Client; _s3AsyncClient = builder._s3AsyncClient; + _enableInstructionFilePut = builder._enableInstructionFilePut; } public static Builder builder() { @@ -34,6 +39,30 @@ public enum InstructionFileClientType { ASYNC } + boolean isInstructionFilePutEnabled() { + return _enableInstructionFilePut; + } + + PutObjectResponse putInstructionFile(PutObjectRequest request, String instructionFileContent) { + // This shouldn't happen in practice because the metadata strategy will evaluate + // if instruction file Puts are enabled before calling this method; check again anyway for robustness + if (!_enableInstructionFilePut) { + throw new S3EncryptionClientException("Enable Instruction File Put must be set to true in order to call PutObject with an instruction file!"); + } + switch (_clientType) { + case SYNCHRONOUS: + return _s3Client.putObject(request, RequestBody.fromString(instructionFileContent)); + case ASYNC: + return _s3AsyncClient.putObject(request, AsyncRequestBody.fromString(instructionFileContent)).join(); + case DISABLED: + // this should never happen because we check enablePut first + throw new S3EncryptionClientException("Instruction File has been disabled!"); + default: + // this should never happen + throw new S3EncryptionClientException("Unknown Instruction File Type"); + } + } + ResponseInputStream getInstructionFile(GetObjectRequest request) { switch (_clientType) { case SYNCHRONOUS: @@ -65,6 +94,7 @@ public static class Builder { private boolean _disableInstructionFile; private S3AsyncClient _s3AsyncClient; private S3Client _s3Client; + private boolean _enableInstructionFilePut; /** * When set to true, the S3 Encryption Client will not attempt to get instruction files. @@ -76,6 +106,11 @@ public Builder disableInstructionFile(boolean disableInstructionFile) { return this; } + public Builder enableInstructionFilePutObject(boolean enableInstructionFilePutObject) { + _enableInstructionFilePut = enableInstructionFilePutObject; + return this; + } + /** * Sets the S3 client to use to retrieve instruction files. * @param instructionFileClient @@ -95,6 +130,7 @@ public Builder instructionFileAsyncClient(S3AsyncClient instructionFileAsyncClie _s3AsyncClient = instructionFileAsyncClient; return this; } + public InstructionFileConfig build() { if ((_s3AsyncClient != null || _s3Client != null) && _disableInstructionFile) { throw new S3EncryptionClientException("Instruction Files have been disabled but a client has been passed!"); @@ -102,6 +138,9 @@ public InstructionFileConfig build() { if (_disableInstructionFile) { // We know both clients are null, so carry on. this._clientType = InstructionFileClientType.DISABLED; + if (_enableInstructionFilePut) { + throw new S3EncryptionClientException("Instruction Files must be enabled to enable Instruction Files for PutObject."); + } return new InstructionFileConfig(this); } if (_s3Client != null && _s3AsyncClient != null) { diff --git a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java index 7e8dc7452..2aad3d831 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java @@ -216,7 +216,7 @@ public void putLocalObject(RequestBody requestBody, String uploadId, OutputStrea public static class Builder { private final Map _multipartUploadMaterials = Collections.synchronizedMap(new HashMap<>()); - private final ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy = new ObjectMetadataEncodingStrategy(); + private final ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy = new ContentMetadataEncodingStrategy(); private S3AsyncClient _s3AsyncClient; private CryptographicMaterialsManager _cryptoMaterialsManager; private SecureRandom _secureRandom; diff --git a/src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java deleted file mode 100644 index b2b9ba4da..000000000 --- a/src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java +++ /dev/null @@ -1,56 +0,0 @@ -package software.amazon.encryption.s3.internal; - -import software.amazon.awssdk.protocols.jsoncore.JsonWriter; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.encryption.s3.S3EncryptionClientException; -import software.amazon.encryption.s3.materials.EncryptedDataKey; -import software.amazon.encryption.s3.materials.EncryptionMaterials; - -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; - -public class ObjectMetadataEncodingStrategy implements ContentMetadataEncodingStrategy { - - private static final Base64.Encoder ENCODER = Base64.getEncoder(); - - public PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, PutObjectRequest putObjectRequest) { - Map newMetadata = addMetadataToMap(putObjectRequest.metadata(), materials, iv); - return putObjectRequest.toBuilder() - .metadata(newMetadata) - .build(); - } - - public CreateMultipartUploadRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, CreateMultipartUploadRequest createMultipartUploadRequest) { - Map newMetadata = addMetadataToMap(createMultipartUploadRequest.metadata(), materials, iv); - return createMultipartUploadRequest.toBuilder() - .metadata(newMetadata) - .build(); - } - - private Map addMetadataToMap(Map map, EncryptionMaterials materials, byte[] iv) { - Map metadata = new HashMap<>(map); - EncryptedDataKey edk = materials.encryptedDataKeys().get(0); - metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_V2, ENCODER.encodeToString(edk.encryptedDatakey())); - metadata.put(MetadataKeyConstants.CONTENT_IV, ENCODER.encodeToString(iv)); - metadata.put(MetadataKeyConstants.CONTENT_CIPHER, materials.algorithmSuite().cipherName()); - metadata.put(MetadataKeyConstants.CONTENT_CIPHER_TAG_LENGTH, Integer.toString(materials.algorithmSuite().cipherTagLengthBits())); - metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_ALGORITHM, new String(edk.keyProviderInfo(), StandardCharsets.UTF_8)); - - try (JsonWriter jsonWriter = JsonWriter.create()) { - jsonWriter.writeStartObject(); - for (Map.Entry entry : materials.encryptionContext().entrySet()) { - jsonWriter.writeFieldName(entry.getKey()).writeValue(entry.getValue()); - } - jsonWriter.writeEndObject(); - - String jsonEncryptionContext = new String(jsonWriter.getBytes(), StandardCharsets.UTF_8); - metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_CONTEXT, jsonEncryptionContext); - } catch (JsonWriter.JsonGenerationException e) { - throw new S3EncryptionClientException("Cannot serialize encryption context to JSON.", e); - } - return metadata; - } -} diff --git a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java index f3ccb743b..2cac59153 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java @@ -81,7 +81,8 @@ public static class Builder { private CryptographicMaterialsManager _cryptoMaterialsManager; private SecureRandom _secureRandom; private AsyncContentEncryptionStrategy _asyncContentEncryptionStrategy; - private final ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy = new ObjectMetadataEncodingStrategy(); + private InstructionFileConfig _instructionFileConfig; + private ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy; private Builder() { } @@ -106,6 +107,11 @@ public Builder secureRandom(SecureRandom secureRandom) { return this; } + public Builder instructionFileConfig(InstructionFileConfig instructionFileConfig) { + this._instructionFileConfig = instructionFileConfig; + return this; + } + public PutEncryptedObjectPipeline build() { // Default to AesGcm since it is the only active (non-legacy) content encryption strategy if (_asyncContentEncryptionStrategy == null) { @@ -114,6 +120,8 @@ public PutEncryptedObjectPipeline build() { .secureRandom(_secureRandom) .build(); } + _contentMetadataEncodingStrategy = new ContentMetadataEncodingStrategy(_instructionFileConfig); + return new PutEncryptedObjectPipeline(this); } } From fd64f1c1de53eb03b9773dc1303a57eef6f3024d Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 10 Dec 2024 16:42:27 -0800 Subject: [PATCH 03/24] cleanup --- .../encryption/s3/S3EncryptionClient.java | 1 + .../ContentMetadataDecodingStrategy.java | 1 - .../ContentMetadataEncodingStrategy.java | 20 ++++- .../s3/internal/InstructionFileConfig.java | 22 +++++- .../s3/internal/MetadataKeyConstants.java | 2 + .../MultipartUploadObjectPipeline.java | 9 ++- .../S3EncryptionClientCompatibilityTest.java | 5 +- .../encryption/s3/S3EncryptionClientTest.java | 77 ++++++++++++++++++- 8 files changed, 127 insertions(+), 10 deletions(-) diff --git a/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java b/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java index 3af414462..ed7255632 100644 --- a/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java +++ b/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java @@ -205,6 +205,7 @@ public PutObjectResponse putObject(PutObjectRequest putObjectRequest, RequestBod .s3AsyncClient(_wrappedAsyncClient) .cryptoMaterialsManager(_cryptoMaterialsManager) .secureRandom(_secureRandom) + .instructionFileConfig(_instructionFileConfig) .build(); ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); diff --git a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataDecodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataDecodingStrategy.java index af3d09824..b79df6af6 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataDecodingStrategy.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataDecodingStrategy.java @@ -168,7 +168,6 @@ private ContentMetadata readFromMap(Map metadata, GetObjectRespo public ContentMetadata decode(GetObjectRequest request, GetObjectResponse response) { Map metadata = response.metadata(); - ContentMetadataDecodingStrategy strategy; if (metadata != null && metadata.containsKey(MetadataKeyConstants.CONTENT_IV) && (metadata.containsKey(MetadataKeyConstants.ENCRYPTED_DATA_KEY_V1) diff --git a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java index 95d8132ce..3a84cf2de 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java @@ -23,10 +23,9 @@ public ContentMetadataEncodingStrategy(InstructionFileConfig instructionFileConf public PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, PutObjectRequest putObjectRequest) { if (_instructionFileConfig.isInstructionFilePutEnabled()) { - // TODO: serialize inst file as string final String metadataString = metadataToString(materials, iv); - _instructionFileConfig.putInstructionFile(putObjectRequest, ""); - // the original object is returned as-is + _instructionFileConfig.putInstructionFile(putObjectRequest, metadataString); + // the original request object is returned as-is return putObjectRequest; } else { Map newMetadata = addMetadataToMap(putObjectRequest.metadata(), materials, iv); @@ -45,7 +44,20 @@ public CreateMultipartUploadRequest encodeMetadata(EncryptionMaterials materials private String metadataToString(EncryptionMaterials materials, byte[] iv) { // this is just the metadata map serialized as JSON - return ""; + // so first get the Map + final Map metadataMap = addMetadataToMap(new HashMap<>(), materials, iv); + // then serialize it + try (JsonWriter jsonWriter = JsonWriter.create()) { + jsonWriter.writeStartObject(); + for (Map.Entry entry : metadataMap.entrySet()) { + jsonWriter.writeFieldName(entry.getKey()).writeValue(entry.getValue()); + } + jsonWriter.writeEndObject(); + + return new String(jsonWriter.getBytes(), StandardCharsets.UTF_8); + } catch (JsonWriter.JsonGenerationException e) { + throw new S3EncryptionClientException("Cannot serialize materials to JSON.", e); + } } private Map addMetadataToMap(Map map, EncryptionMaterials materials, byte[] iv) { diff --git a/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java b/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java index ad067df5a..6d40d86cd 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java +++ b/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java @@ -12,6 +12,12 @@ import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.encryption.s3.S3EncryptionClientException; +import java.util.HashMap; +import java.util.Map; + +import static software.amazon.encryption.s3.S3EncryptionClientUtilities.INSTRUCTION_FILE_SUFFIX; +import static software.amazon.encryption.s3.internal.MetadataKeyConstants.INSTRUCTION_FILE; + /** * Provides configuration options for instruction file behaviors. */ @@ -49,11 +55,23 @@ PutObjectResponse putInstructionFile(PutObjectRequest request, String instructio if (!_enableInstructionFilePut) { throw new S3EncryptionClientException("Enable Instruction File Put must be set to true in order to call PutObject with an instruction file!"); } + + // Instruction file DOES NOT contain the same metadata as the actual object + Map instFileMetadata = new HashMap<>(1); + // It contains a key with no value identifying it as an instruction file + instFileMetadata.put(INSTRUCTION_FILE, ""); + + // In a future release, non-default suffixes will be supported. + // Use toBuilder to keep all other fields the same as the actual request + final PutObjectRequest instPutRequest = request.toBuilder() + .key(request.key() + INSTRUCTION_FILE_SUFFIX) + .metadata(instFileMetadata) + .build(); switch (_clientType) { case SYNCHRONOUS: - return _s3Client.putObject(request, RequestBody.fromString(instructionFileContent)); + return _s3Client.putObject(instPutRequest, RequestBody.fromString(instructionFileContent)); case ASYNC: - return _s3AsyncClient.putObject(request, AsyncRequestBody.fromString(instructionFileContent)).join(); + return _s3AsyncClient.putObject(instPutRequest, AsyncRequestBody.fromString(instructionFileContent)).join(); case DISABLED: // this should never happen because we check enablePut first throw new S3EncryptionClientException("Instruction File has been disabled!"); diff --git a/src/main/java/software/amazon/encryption/s3/internal/MetadataKeyConstants.java b/src/main/java/software/amazon/encryption/s3/internal/MetadataKeyConstants.java index 12c0c856f..722d3c48e 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/MetadataKeyConstants.java +++ b/src/main/java/software/amazon/encryption/s3/internal/MetadataKeyConstants.java @@ -13,4 +13,6 @@ public class MetadataKeyConstants { // This is usually an actual Java cipher e.g. AES/GCM/NoPadding public static final String CONTENT_CIPHER = "x-amz-cek-alg"; public static final String CONTENT_CIPHER_TAG_LENGTH = "x-amz-tag-len"; + // Only used in instruction files to identify them as such + public static final String INSTRUCTION_FILE = "x-amz-crypto-instr-file"; } diff --git a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java index 2aad3d831..c736f7130 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java @@ -216,12 +216,13 @@ public void putLocalObject(RequestBody requestBody, String uploadId, OutputStrea public static class Builder { private final Map _multipartUploadMaterials = Collections.synchronizedMap(new HashMap<>()); - private final ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy = new ContentMetadataEncodingStrategy(); + private ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy; private S3AsyncClient _s3AsyncClient; private CryptographicMaterialsManager _cryptoMaterialsManager; private SecureRandom _secureRandom; // To Create Cipher which is used in during uploadPart requests. private MultipartContentEncryptionStrategy _contentEncryptionStrategy; + private InstructionFileConfig _instructionFileConfig; private Builder() { } @@ -246,6 +247,11 @@ public Builder secureRandom(SecureRandom secureRandom) { return this; } + public Builder instructionFileConfig(InstructionFileConfig instructionFileConfig) { + this._instructionFileConfig = instructionFileConfig; + return this; + } + public MultipartUploadObjectPipeline build() { // Default to AesGcm since it is the only active (non-legacy) content encryption strategy if (_contentEncryptionStrategy == null) { @@ -254,6 +260,7 @@ public MultipartUploadObjectPipeline build() { .secureRandom(_secureRandom) .build(); } + _contentMetadataEncodingStrategy = new ContentMetadataEncodingStrategy(_instructionFileConfig); return new MultipartUploadObjectPipeline(this); } } diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java index 6a750f535..e2fc2758e 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java @@ -561,6 +561,7 @@ public void KmsV1toV3() { CryptoConfiguration v1Config = new CryptoConfiguration(CryptoMode.AuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile) .withAwsKmsRegion(KMS_REGION); AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() @@ -585,7 +586,7 @@ public void KmsV1toV3() { assertEquals(input, output); // Cleanup - deleteObject(BUCKET, objectKey, v3Client); +// deleteObject(BUCKET, objectKey, v3Client); v3Client.close(); } @@ -596,8 +597,10 @@ public void KmsContextV2toV3() { // V2 Client EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(KMS_KEY_ID); + CryptoConfigurationV2 config = new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption); AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() .withEncryptionMaterialsProvider(materialsProvider) + .withCryptoConfiguration(config) .build(); // V3 Client diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java index 182fc4841..4fadb4a83 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java @@ -1102,6 +1102,81 @@ public void testInstructionFileConfig() { s3Client.close(); } + @Test + public void testPutWithInstructionFile() { + final String objectKey = appendTestSuffix("instruction-file-put-object"); + final String objectKeyV2 = appendTestSuffix("instruction-file-put-object-v2"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + // Disabled client should fail + S3Client s3ClientDisabledInstructionFile = S3EncryptionClient.builder() + .wrappedClient(wrappedClient) + .instructionFileConfig(InstructionFileConfig.builder() + .disableInstructionFile(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + try { + s3ClientDisabledInstructionFile.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build()); + fail("expected exception"); + } catch (S3EncryptionClientException exception) { + assertTrue(exception.getMessage().contains("Exception encountered while fetching Instruction File.")); + } + + // Get the instruction file separately using a default client + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + + ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build()); + String output = objectResponse.asUtf8String(); + assertEquals(input, output); + + + // Temporary - Generate an instruction file in V2 to compare against V3 + // TODO: do this for other keyrings as well + EncryptionMaterialsProvider materialsProvider = + new StaticEncryptionMaterialsProvider(new KMSEncryptionMaterials(KMS_KEY_ID)); + CryptoConfigurationV2 cryptoConfig = + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); + + AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); + + v2Client.putObject(BUCKET, objectKeyV2, input); + + // Cleanup +// deleteObject(BUCKET, objectKey, s3Client); + s3ClientDisabledInstructionFile.close(); + s3Client.close(); + } + /** * A simple, reusable round-trip (encryption + decryption) using a given * S3Client. Useful for testing client configuration. @@ -1124,4 +1199,4 @@ private void simpleV3RoundTrip(final S3Client v3Client, final String objectKey) String output = objectResponse.asUtf8String(); assertEquals(input, output); } -} +} \ No newline at end of file From ed761d9ea4bd98642fba4981d9f48d6ad2d8ce54 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 29 Apr 2025 11:14:52 -0700 Subject: [PATCH 04/24] more tests in new class --- ...S3EncryptionClientInstructionFileTest.java | 221 ++++++++++++++++++ .../encryption/s3/S3EncryptionClientTest.java | 75 ------ 2 files changed, 221 insertions(+), 75 deletions(-) create mode 100644 src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java new file mode 100644 index 000000000..6c23ab7f2 --- /dev/null +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java @@ -0,0 +1,221 @@ +package software.amazon.encryption.s3; + +import com.amazonaws.services.s3.AmazonS3EncryptionClientV2; +import com.amazonaws.services.s3.AmazonS3EncryptionV2; +import com.amazonaws.services.s3.model.CryptoConfigurationV2; +import com.amazonaws.services.s3.model.CryptoMode; +import com.amazonaws.services.s3.model.CryptoStorageMode; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.KMSEncryptionMaterials; +import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.encryption.s3.internal.InstructionFileConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.BUCKET; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.KMS_KEY_ID; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.appendTestSuffix; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.deleteObject; + +public class S3EncryptionClientInstructionFileTest { + + @Test + public void testInstructionFileExists() { + final String objectKey = appendTestSuffix("instruction-file-put-object"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + // Get the instruction file separately using a default client + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + // Ensure its metadata identifies it as such + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + + // Ensure decryption succeeds + ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build()); + String output = objectResponse.asUtf8String(); + assertEquals(input, output); + + deleteObject(BUCKET, objectKey, s3Client); + s3Client.close(); + defaultClient.close(); + } + + @Test + public void testDisabledClientFails() { + final String objectKey = appendTestSuffix("instruction-file-put-object"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + // Put with Instruction File + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + // Disabled client should fail + S3Client s3ClientDisabledInstructionFile = S3EncryptionClient.builder() + .wrappedClient(wrappedClient) + .instructionFileConfig(InstructionFileConfig.builder() + .disableInstructionFile(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + try { + s3ClientDisabledInstructionFile.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build()); + fail("expected exception"); + } catch (S3EncryptionClientException exception) { + assertTrue(exception.getMessage().contains("Exception encountered while fetching Instruction File.")); + } + + deleteObject(BUCKET, objectKey, s3Client); + s3Client.close(); + s3ClientDisabledInstructionFile.close(); + } + + + /** + * This test is somewhat redundant given deletion itself is tested in + * e.g. deleteObjectWithInstructionFileSuccess, but is included anyway to be thorough + */ + @Test + public void testInstructionFileDelete() { + final String objectKey = appendTestSuffix("instruction-file-put-object"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + // Get the instruction file separately using a default client + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + // Ensure its metadata identifies it as such + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + + // Ensure decryption succeeds + ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build()); + String output = objectResponse.asUtf8String(); + assertEquals(input, output); + + deleteObject(BUCKET, objectKey, s3Client); + + try { + defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + fail("expected exception!"); + } catch (NoSuchKeyException e) { + // expected + } + + s3Client.close(); + defaultClient.close(); + } + @Test + public void testPutWithInstructionFile() { + final String objectKey = appendTestSuffix("instruction-file-put-object"); + final String objectKeyV2 = appendTestSuffix("instruction-file-put-object-v2"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + // Get the instruction file separately using a default client + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + + ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build()); + String output = objectResponse.asUtf8String(); + assertEquals(input, output); + + // Temporary - Generate an instruction file in V2 to compare against V3 + // TODO: do this for other keyrings as well + // TODO: Instead, make a V3ToV2 test + EncryptionMaterialsProvider materialsProvider = + new StaticEncryptionMaterialsProvider(new KMSEncryptionMaterials(KMS_KEY_ID)); + CryptoConfigurationV2 cryptoConfig = + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); + + AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); + + v2Client.putObject(BUCKET, objectKeyV2, input); + + // Cleanup +// deleteObject(BUCKET, objectKey, s3Client); + s3Client.close(); + } +} diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java index 4fadb4a83..d2520c6b2 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java @@ -1102,81 +1102,6 @@ public void testInstructionFileConfig() { s3Client.close(); } - @Test - public void testPutWithInstructionFile() { - final String objectKey = appendTestSuffix("instruction-file-put-object"); - final String objectKeyV2 = appendTestSuffix("instruction-file-put-object-v2"); - final String input = "SimpleTestOfV3EncryptionClient"; - S3Client wrappedClient = S3Client.create(); - S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); - - s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); - - // Disabled client should fail - S3Client s3ClientDisabledInstructionFile = S3EncryptionClient.builder() - .wrappedClient(wrappedClient) - .instructionFileConfig(InstructionFileConfig.builder() - .disableInstructionFile(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); - - try { - s3ClientDisabledInstructionFile.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); - fail("expected exception"); - } catch (S3EncryptionClientException exception) { - assertTrue(exception.getMessage().contains("Exception encountered while fetching Instruction File.")); - } - - // Get the instruction file separately using a default client - S3Client defaultClient = S3Client.create(); - ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey + ".instruction") - .build()); - assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); - - ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); - String output = objectResponse.asUtf8String(); - assertEquals(input, output); - - - // Temporary - Generate an instruction file in V2 to compare against V3 - // TODO: do this for other keyrings as well - EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new KMSEncryptionMaterials(KMS_KEY_ID)); - CryptoConfigurationV2 cryptoConfig = - new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) - .withStorageMode(CryptoStorageMode.InstructionFile); - - AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() - .withCryptoConfiguration(cryptoConfig) - .withEncryptionMaterialsProvider(materialsProvider) - .build(); - - v2Client.putObject(BUCKET, objectKeyV2, input); - - // Cleanup -// deleteObject(BUCKET, objectKey, s3Client); - s3ClientDisabledInstructionFile.close(); - s3Client.close(); - } - /** * A simple, reusable round-trip (encryption + decryption) using a given * S3Client. Useful for testing client configuration. From d6da2cf0d737bd30421384808c890b2bf86b2020 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 9 May 2025 13:38:29 -0700 Subject: [PATCH 05/24] v3 to v2 tests for default client --- ...S3EncryptionClientInstructionFileTest.java | 110 +++++++++++++++--- 1 file changed, 91 insertions(+), 19 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java index 6c23ab7f2..9d88d390f 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java @@ -5,6 +5,7 @@ import com.amazonaws.services.s3.model.CryptoConfigurationV2; import com.amazonaws.services.s3.model.CryptoMode; import com.amazonaws.services.s3.model.CryptoStorageMode; +import com.amazonaws.services.s3.model.EncryptionMaterials; import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; import com.amazonaws.services.s3.model.KMSEncryptionMaterials; import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; @@ -16,6 +17,13 @@ import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.encryption.s3.internal.InstructionFileConfig; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; + +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -164,10 +172,10 @@ public void testInstructionFileDelete() { s3Client.close(); defaultClient.close(); } + @Test - public void testPutWithInstructionFile() { - final String objectKey = appendTestSuffix("instruction-file-put-object"); - final String objectKeyV2 = appendTestSuffix("instruction-file-put-object-v2"); + public void testPutWithInstructionFileV3ToV2Kms() { + final String objectKey = appendTestSuffix("instruction-file-put-object-v3-to-v2-kms"); final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() @@ -183,26 +191,89 @@ public void testPutWithInstructionFile() { .key(objectKey) .build(), RequestBody.fromString(input)); - // Get the instruction file separately using a default client - S3Client defaultClient = S3Client.create(); - ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + EncryptionMaterialsProvider materialsProvider = + new StaticEncryptionMaterialsProvider(new KMSEncryptionMaterials(KMS_KEY_ID)); + CryptoConfigurationV2 cryptoConfig = + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); + + AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); + + String result = v2Client.getObjectAsString(BUCKET, objectKey); + assertEquals(input, result); + + // Cleanup + deleteObject(BUCKET, objectKey, s3Client); + s3Client.close(); + } + + @Test + public void testPutWithInstructionFileV3ToV2Aes() throws NoSuchAlgorithmException { + KeyGenerator keyGen = KeyGenerator.getInstance("AES"); + keyGen.init(256); + SecretKey aesKey = keyGen.generateKey(); + final String objectKey = appendTestSuffix("instruction-file-put-object-v3-to-v2-aes"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .aesKey(aesKey) + .build(); + + s3Client.putObject(builder -> builder .bucket(BUCKET) - .key(objectKey + ".instruction") - .build()); - assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + .key(objectKey) + .build(), RequestBody.fromString(input)); - ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder + EncryptionMaterialsProvider materialsProvider = + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(aesKey)); + CryptoConfigurationV2 cryptoConfig = + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); + + AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); + + String result = v2Client.getObjectAsString(BUCKET, objectKey); + assertEquals(input, result); + + // Cleanup + deleteObject(BUCKET, objectKey, s3Client); + s3Client.close(); + } + + @Test + public void testPutWithInstructionFileV3ToV2Rsa() throws NoSuchAlgorithmException { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyPair rsaKey = keyPairGen.generateKeyPair(); + + final String objectKey = appendTestSuffix("instruction-file-put-object-v3-to-v2-rsa"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .rsaKeyPair(rsaKey) + .build(); + + s3Client.putObject(builder -> builder .bucket(BUCKET) .key(objectKey) - .build()); - String output = objectResponse.asUtf8String(); - assertEquals(input, output); + .build(), RequestBody.fromString(input)); - // Temporary - Generate an instruction file in V2 to compare against V3 - // TODO: do this for other keyrings as well - // TODO: Instead, make a V3ToV2 test EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new KMSEncryptionMaterials(KMS_KEY_ID)); + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(rsaKey)); CryptoConfigurationV2 cryptoConfig = new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) .withStorageMode(CryptoStorageMode.InstructionFile); @@ -212,10 +283,11 @@ public void testPutWithInstructionFile() { .withEncryptionMaterialsProvider(materialsProvider) .build(); - v2Client.putObject(BUCKET, objectKeyV2, input); + String result = v2Client.getObjectAsString(BUCKET, objectKey); + assertEquals(input, result); // Cleanup -// deleteObject(BUCKET, objectKey, s3Client); + deleteObject(BUCKET, objectKey, s3Client); s3Client.close(); } } From 288ee6506d2443e91678398d1b3b0b3869ffd4b3 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Tue, 27 May 2025 13:02:10 -0700 Subject: [PATCH 06/24] Fixed nullptr exception in code --- .../encryption/s3/S3AsyncEncryptionClient.java | 13 +++++++++++-- .../s3/internal/PutEncryptedObjectPipeline.java | 4 +++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java b/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java index b748f50d7..ef95c9693 100644 --- a/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java +++ b/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java @@ -79,7 +79,7 @@ public class S3AsyncEncryptionClient extends DelegatingS3AsyncClient { private final boolean _enableDelayedAuthenticationMode; private final boolean _enableMultipartPutObject; private final long _bufferSize; - private InstructionFileConfig _instructionFileConfig; + private final InstructionFileConfig _instructionFileConfig; private S3AsyncEncryptionClient(Builder builder) { super(builder._wrappedClient); @@ -151,6 +151,8 @@ public CompletableFuture putObject(PutObjectRequest putObject .s3AsyncClient(_wrappedClient) .cryptoMaterialsManager(_cryptoMaterialsManager) .secureRandom(_secureRandom) + //Debugging: added this line + .instructionFileConfig(_instructionFileConfig) .build(); return pipeline.putObject(putObjectRequest, requestBody); @@ -162,13 +164,16 @@ private CompletableFuture multipartPutObject(PutObjectRequest // if the wrappedClient is a CRT, use it mpuClient = _wrappedClient; } else { - // else create a default CRT client + // else create a default CRT client (debugging: yes goes here) mpuClient = S3AsyncClient.crtCreate(); } + //The issue is here: after this step, the instruction file config is null + //Debugging: added this line to include the instruction file config PutEncryptedObjectPipeline pipeline = PutEncryptedObjectPipeline.builder() .s3AsyncClient(mpuClient) .cryptoMaterialsManager(_cryptoMaterialsManager) .secureRandom(_secureRandom) + .instructionFileConfig(_instructionFileConfig) .build(); // Ensures parts are not retried to avoid corrupting ciphertext AsyncRequestBody noRetryBody = new NoRetriesAsyncRequestBody(requestBody); @@ -289,6 +294,10 @@ public void close() { _instructionFileConfig.closeClient(); } + public InstructionFileConfig get_instructionFileConfig() { + return _instructionFileConfig; + } + // This is very similar to the S3EncryptionClient builder // Make sure to keep both clients in mind when adding new builder options public static class Builder implements S3AsyncClientBuilder { diff --git a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java index a4d5bbf78..35ae13b8e 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java @@ -24,7 +24,8 @@ public class PutEncryptedObjectPipeline { final private CryptographicMaterialsManager _cryptoMaterialsManager; final private AsyncContentEncryptionStrategy _asyncContentEncryptionStrategy; final private ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy; - + //Debugging: + final private InstructionFileConfig _instructionFileConfig; public static Builder builder() { return new Builder(); } @@ -34,6 +35,7 @@ private PutEncryptedObjectPipeline(Builder builder) { this._cryptoMaterialsManager = builder._cryptoMaterialsManager; this._asyncContentEncryptionStrategy = builder._asyncContentEncryptionStrategy; this._contentMetadataEncodingStrategy = builder._contentMetadataEncodingStrategy; + this._instructionFileConfig = builder._instructionFileConfig; } public CompletableFuture putObject(PutObjectRequest request, AsyncRequestBody requestBody) { From 9c1c50d72189d63d7ccf57feabb130179361fe5e Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 28 May 2025 09:09:31 -0700 Subject: [PATCH 07/24] testing whether v2 does or doesn't translate additional parameters for MPU + instructionFile --- .../S3EncryptionClientCompatibilityTest.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java index 790d34814..158d72fc4 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java @@ -15,9 +15,15 @@ import com.amazonaws.services.s3.model.EncryptedPutObjectRequest; import com.amazonaws.services.s3.model.EncryptionMaterials; import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.GetObjectMetadataRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; import com.amazonaws.services.s3.model.KMSEncryptionMaterials; import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.StorageClass; +import com.amazonaws.services.s3.model.UploadObjectRequest; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseBytes; @@ -28,20 +34,25 @@ import software.amazon.awssdk.services.s3.model.MetadataDirective; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.encryption.s3.internal.InstructionFileConfig; +import software.amazon.encryption.s3.utils.BoundedInputStream; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.security.KeyPair; import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static software.amazon.encryption.s3.S3EncryptionClient.builder; import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.BUCKET; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.KMS_KEY_ID; @@ -127,6 +138,7 @@ public void AesWrapV1toV3() { // Asserts final String input = "AesGcmV1toV3"; + System.out.println(System.getenv("AWS_S3EC_TEST_BUCKET")); v1Client.putObject(BUCKET, objectKey, input); ResponseBytes objectResponse = v3Client.getObjectAsBytes(builder -> builder @@ -210,6 +222,39 @@ public void AesGcmV2toV3WithInstructionFile() { deleteObject(BUCKET, objectKey, v3Client); v3Client.close(); } + @Test + public void multipartPutObjectWithOptionsAndInstructionFileV2() throws IOException, InterruptedException, ExecutionException { + final String objectKey = appendTestSuffix("multipart-put-object-with-options-and-instruction-file-v2"); + final long fileSizeLimit = 1024 * 1024 * 10; //sets file size limit to 10 MB + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + + //Now, we will create encryption client (v2) with instruction file config enabled and multipart upload enabled + EncryptionMaterialsProvider materialsProvider = + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(AES_KEY)); + CryptoConfigurationV2 cryptoConfig = + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); + AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); + UploadObjectRequest uploadObjectRequest = new UploadObjectRequest("s3ec-github-test-bucket-597133212884", objectKey, inputStream, new ObjectMetadata()) + .withPartSize(1024 * 1024 * 5) + .withStorageClass(StorageClass.Glacier); + v2Client.uploadObject(uploadObjectRequest); + + //Assert that the storage class on main object matches "GLACIER" + GetObjectMetadataRequest mainObjectRequest = new GetObjectMetadataRequest("s3ec-github-test-bucket-597133212884", objectKey); + ObjectMetadata mainObjectMetadata = v2Client.getObjectMetadata(mainObjectRequest); + assertEquals("GLACIER", mainObjectMetadata.getStorageClass()); + + //Assert that the instruction file does not contain storage class (V2) + GetObjectMetadataRequest instructionObjectRequest = new GetObjectMetadataRequest("s3ec-github-test-bucket-597133212884", objectKey + ".instruction"); + ObjectMetadata instructionFileMetadata = v2Client.getObjectMetadata(instructionObjectRequest); + + assertNotEquals("GLACIER", instructionFileMetadata.getStorageClass()); + + } @Test public void AesGcmV3toV1() { From ae6b6b40c7c77ebb4d7f78dc822cbef4a4f63346 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 28 May 2025 14:07:03 -0700 Subject: [PATCH 08/24] Fixed nullptr errors for MultipartUploadObjectPipeline regarding _instructionFileConfig --- .../encryption/s3/internal/MultipartUploadObjectPipeline.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java index c736f7130..ba0b5d6e9 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java @@ -42,6 +42,7 @@ public class MultipartUploadObjectPipeline { final private CryptographicMaterialsManager _cryptoMaterialsManager; final private MultipartContentEncryptionStrategy _contentEncryptionStrategy; final private ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy; + final private InstructionFileConfig _instructionFileConfig; /** * Map of data about in progress encrypted multipart uploads. */ @@ -53,6 +54,7 @@ private MultipartUploadObjectPipeline(Builder builder) { this._contentEncryptionStrategy = builder._contentEncryptionStrategy; this._contentMetadataEncodingStrategy = builder._contentMetadataEncodingStrategy; this._multipartUploadMaterials = builder._multipartUploadMaterials; + this._instructionFileConfig = builder._instructionFileConfig; } public static Builder builder() { From c9378cf384ecee9d073dae54d7562f2e0b6e0251 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 28 May 2025 14:49:56 -0700 Subject: [PATCH 09/24] Added another convertRequest method to convert CreateMultipartUploadRequest to PutObjectRequest --- .../s3/internal/ConvertSDKRequests.java | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java index 55d86ec3f..dacbf6682 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java @@ -12,6 +12,131 @@ public class ConvertSDKRequests { + public static PutObjectRequest convertRequest(CreateMultipartUploadRequest request) { + + final PutObjectRequest.Builder output = PutObjectRequest.builder(); + request + .toBuilder() + .sdkFields() + .forEach(f -> { + final Object value = f.getValueOrDefault(request); + if (value != null) { + switch (f.memberName()) { + case "ACL": + output.acl((String) value); + break; + case "Bucket": + output.bucket((String) value); + break; + case "BucketKeyEnabled": + output.bucketKeyEnabled((Boolean) value); + break; + case "CacheControl": + output.cacheControl((String) value); + break; + case "ChecksumAlgorithm": + output.checksumAlgorithm((String) value); + break; + case "ContentDisposition": + assert value instanceof String; + output.contentDisposition((String) value); + break; + case "ContentEncoding": + output.contentEncoding((String) value); + break; + case "ContentLanguage": + output.contentLanguage((String) value); + break; + case "ContentType": + output.contentType((String) value); + break; + case "ExpectedBucketOwner": + output.expectedBucketOwner((String) value); + break; + case "Expires": + output.expires((Instant) value); + break; + case "GrantFullControl": + output.grantFullControl((String) value); + break; + case "GrantRead": + output.grantRead((String) value); + break; + case "GrantReadACP": + output.grantReadACP((String) value); + break; + case "GrantWriteACP": + output.grantWriteACP((String) value); + break; + case "Key": + output.key((String) value); + break; + case "Metadata": + if (!isStringStringMap(value)) { + throw new IllegalArgumentException("Metadata must be a Map"); + } + @SuppressWarnings("unchecked") + Map metadata = (Map) value; + output.metadata(metadata); + break; + case "ObjectLockLegalHoldStatus": + output.objectLockLegalHoldStatus((String) value); + break; + case "ObjectLockMode": + output.objectLockMode((String) value); + break; + case "ObjectLockRetainUntilDate": + output.objectLockRetainUntilDate((Instant) value); + break; + case "RequestPayer": + output.requestPayer((String) value); + break; + case "ServerSideEncryption": + output.serverSideEncryption((String) value); + break; + case "SSECustomerAlgorithm": + output.sseCustomerAlgorithm((String) value); + break; + case "SSECustomerKey": + output.sseCustomerKey((String) value); + break; + case "SSEKMSEncryptionContext": + output.ssekmsEncryptionContext((String) value); + break; + case "SSEKMSKeyId": + output.ssekmsKeyId((String) value); + break; + case "StorageClass": + output.storageClass((String) value); + break; + case "Tagging": + output.tagging((String) value); + break; + case "WebsiteRedirectLocation": + output.websiteRedirectLocation((String) value); + break; + default: + // Rather than silently dropping the value, + // we loudly signal that we don't know how to handle this field. + throw new IllegalArgumentException( + f.locationName() + " is an unknown field. " + + "The S3 Encryption Client does not recognize this option and cannot set it on the PutObjectRequest." + + "This may be a new S3 feature." + + "Please report this to the Amazon S3 Encryption Client for Java: " + + "https://github.com/aws/amazon-s3-encryption-client-java/issues." + + "To work around this issue you can disable multi part upload," + + "use the Async client, or not set this value on PutObject." + + "You may be able to update this value after the PutObject request completes." + ); + } + } + }); + return output + // OverrideConfiguration is not as SDKField but still needs to be supported + .overrideConfiguration(request.overrideConfiguration().orElse(null)) + .build(); + } + public static CreateMultipartUploadRequest convertRequest(PutObjectRequest request) { final CreateMultipartUploadRequest.Builder output = CreateMultipartUploadRequest.builder(); From ef353d7e98aba0752b5a7285b827d99a2c66c897 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 28 May 2025 16:00:26 -0700 Subject: [PATCH 10/24] Added test cases for converting MultipartUploadRequests --- .../s3/internal/ConvertSDKRequestsTest.java | 271 +++++++++++++----- 1 file changed, 194 insertions(+), 77 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java b/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java index 2f7757d3c..6c2e6d1cc 100644 --- a/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java +++ b/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java @@ -3,6 +3,7 @@ import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; import software.amazon.awssdk.services.s3.model.*; +import java.awt.event.ComponentListener; import java.time.Duration; import java.time.Instant; import java.util.HashMap; @@ -15,8 +16,8 @@ class ConvertSDKRequestsTest { void testConvertPutObjectRequest_Bucket() { final String value = "test-bucket"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .bucket(value) - .build(); + .bucket(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.bucket()); @@ -26,8 +27,8 @@ void testConvertPutObjectRequest_Bucket() { void testConvertPutObjectRequest_ACL() { final ObjectCannedACL value = ObjectCannedACL.PRIVATE; PutObjectRequest originalRequest = PutObjectRequest.builder() - .acl(value) - .build(); + .acl(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.acl()); @@ -37,8 +38,8 @@ void testConvertPutObjectRequest_ACL() { void testConvertPutObjectRequest_ACL2() { final String value = ObjectCannedACL.PRIVATE.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .acl(value) - .build(); + .acl(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.acl().toString()); @@ -48,8 +49,8 @@ void testConvertPutObjectRequest_ACL2() { void testConvertPutObjectRequest_BucketKeyEnabled() { final Boolean value = true; PutObjectRequest originalRequest = PutObjectRequest.builder() - .bucketKeyEnabled(value) - .build(); + .bucketKeyEnabled(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.bucketKeyEnabled()); @@ -59,8 +60,8 @@ void testConvertPutObjectRequest_BucketKeyEnabled() { void testConvertPutObjectRequest_CacheControl() { final String value = "max-age=3600"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .cacheControl(value) - .build(); + .cacheControl(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.cacheControl()); @@ -70,8 +71,8 @@ void testConvertPutObjectRequest_CacheControl() { void testConvertPutObjectRequest_ChecksumAlgorithm() { final ChecksumAlgorithm value = ChecksumAlgorithm.SHA256; PutObjectRequest originalRequest = PutObjectRequest.builder() - .checksumAlgorithm(value) - .build(); + .checksumAlgorithm(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.checksumAlgorithm()); @@ -81,8 +82,8 @@ void testConvertPutObjectRequest_ChecksumAlgorithm() { void testConvertPutObjectRequest_ChecksumAlgorithm2() { final String value = ChecksumAlgorithm.SHA256.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .checksumAlgorithm(value) - .build(); + .checksumAlgorithm(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.checksumAlgorithm().toString()); @@ -92,8 +93,8 @@ void testConvertPutObjectRequest_ChecksumAlgorithm2() { void testConvertPutObjectRequest_ContentDisposition() { final String value = "attachment; filename=\"filename.jpg\""; PutObjectRequest originalRequest = PutObjectRequest.builder() - .contentDisposition(value) - .build(); + .contentDisposition(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.contentDisposition()); @@ -103,8 +104,8 @@ void testConvertPutObjectRequest_ContentDisposition() { void testConvertPutObjectRequest_ContentEncoding() { final String value = "gzip"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .contentEncoding(value) - .build(); + .contentEncoding(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.contentEncoding()); @@ -114,8 +115,8 @@ void testConvertPutObjectRequest_ContentEncoding() { void testConvertPutObjectRequest_ContentLanguage() { final String value = "en-US"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .contentLanguage(value) - .build(); + .contentLanguage(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.contentLanguage()); @@ -125,8 +126,8 @@ void testConvertPutObjectRequest_ContentLanguage() { void testConvertPutObjectRequest_ContentType() { final String value = "text/plain"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .contentType(value) - .build(); + .contentType(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.contentType()); @@ -136,8 +137,8 @@ void testConvertPutObjectRequest_ContentType() { void testConvertPutObjectRequest_ExpectedBucketOwner() { final String value = "owner123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .expectedBucketOwner(value) - .build(); + .expectedBucketOwner(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.expectedBucketOwner()); @@ -147,8 +148,8 @@ void testConvertPutObjectRequest_ExpectedBucketOwner() { void testConvertPutObjectRequest_Expires() { final Instant value = Instant.now(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .expires(value) - .build(); + .expires(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.expires()); @@ -158,8 +159,8 @@ void testConvertPutObjectRequest_Expires() { void testConvertPutObjectRequest_GrantFullControl() { final String value = "id=123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .grantFullControl(value) - .build(); + .grantFullControl(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.grantFullControl()); @@ -169,8 +170,8 @@ void testConvertPutObjectRequest_GrantFullControl() { void testConvertPutObjectRequest_GrantRead() { final String value = "id=123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .grantRead(value) - .build(); + .grantRead(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.grantRead()); @@ -180,8 +181,8 @@ void testConvertPutObjectRequest_GrantRead() { void testConvertPutObjectRequest_GrantReadACP() { final String value = "id=123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .grantReadACP(value) - .build(); + .grantReadACP(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.grantReadACP()); @@ -191,8 +192,8 @@ void testConvertPutObjectRequest_GrantReadACP() { void testConvertPutObjectRequest_GrantWriteACP() { final String value = "id=123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .grantWriteACP(value) - .build(); + .grantWriteACP(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.grantWriteACP()); @@ -202,8 +203,8 @@ void testConvertPutObjectRequest_GrantWriteACP() { void testConvertPutObjectRequest_Key() { final String value = "test-key"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .key(value) - .build(); + .key(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.key()); @@ -214,8 +215,8 @@ void testConvertPutObjectRequest_Metadata() { final Map value = new HashMap<>(); value.put("key1", "value1"); PutObjectRequest originalRequest = PutObjectRequest.builder() - .metadata(value) - .build(); + .metadata(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.metadata()); @@ -225,8 +226,8 @@ void testConvertPutObjectRequest_Metadata() { void testConvertPutObjectRequest_ObjectLockLegalHoldStatus() { final ObjectLockLegalHoldStatus value = ObjectLockLegalHoldStatus.ON; PutObjectRequest originalRequest = PutObjectRequest.builder() - .objectLockLegalHoldStatus(value) - .build(); + .objectLockLegalHoldStatus(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.objectLockLegalHoldStatus()); @@ -236,8 +237,8 @@ void testConvertPutObjectRequest_ObjectLockLegalHoldStatus() { void testConvertPutObjectRequest_ObjectLockLegalHoldStatus2() { final String value = ObjectLockLegalHoldStatus.ON.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .objectLockLegalHoldStatus(value) - .build(); + .objectLockLegalHoldStatus(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.objectLockLegalHoldStatus().toString()); @@ -247,8 +248,8 @@ void testConvertPutObjectRequest_ObjectLockLegalHoldStatus2() { void testConvertPutObjectRequest_ObjectLockMode() { final ObjectLockMode value = ObjectLockMode.GOVERNANCE; PutObjectRequest originalRequest = PutObjectRequest.builder() - .objectLockMode(value) - .build(); + .objectLockMode(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.objectLockMode()); @@ -258,8 +259,8 @@ void testConvertPutObjectRequest_ObjectLockMode() { void testConvertPutObjectRequest_ObjectLockMode2() { final String value = "GOVERNANCE"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .objectLockMode(value) - .build(); + .objectLockMode(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.objectLockMode().toString()); @@ -269,8 +270,8 @@ void testConvertPutObjectRequest_ObjectLockMode2() { void testConvertPutObjectRequest_ObjectLockRetainUntilDate() { final Instant value = Instant.now(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .objectLockRetainUntilDate(value) - .build(); + .objectLockRetainUntilDate(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.objectLockRetainUntilDate()); @@ -280,8 +281,8 @@ void testConvertPutObjectRequest_ObjectLockRetainUntilDate() { void testConvertPutObjectRequest_RequestPayer() { final RequestPayer value = RequestPayer.REQUESTER; PutObjectRequest originalRequest = PutObjectRequest.builder() - .requestPayer(value) - .build(); + .requestPayer(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.requestPayer()); @@ -291,8 +292,8 @@ void testConvertPutObjectRequest_RequestPayer() { void testConvertPutObjectRequest_RequestPayer2() { final String value = RequestPayer.REQUESTER.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .requestPayer(value) - .build(); + .requestPayer(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.requestPayer().toString()); @@ -302,8 +303,8 @@ void testConvertPutObjectRequest_RequestPayer2() { void testConvertPutObjectRequest_ServerSideEncryption() { final ServerSideEncryption value = ServerSideEncryption.AES256; PutObjectRequest originalRequest = PutObjectRequest.builder() - .serverSideEncryption(value) - .build(); + .serverSideEncryption(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.serverSideEncryption()); @@ -313,8 +314,8 @@ void testConvertPutObjectRequest_ServerSideEncryption() { void testConvertPutObjectRequest_ServerSideEncryption2() { final String value = ServerSideEncryption.AES256.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .serverSideEncryption(value) - .build(); + .serverSideEncryption(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.serverSideEncryption().toString()); @@ -324,8 +325,8 @@ void testConvertPutObjectRequest_ServerSideEncryption2() { void testConvertPutObjectRequest_SSECustomerAlgorithm() { final String value = "AES256"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .sseCustomerAlgorithm(value) - .build(); + .sseCustomerAlgorithm(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.sseCustomerAlgorithm()); @@ -335,8 +336,8 @@ void testConvertPutObjectRequest_SSECustomerAlgorithm() { void testConvertPutObjectRequest_SSECustomerKey() { final String value = "key123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .sseCustomerKey(value) - .build(); + .sseCustomerKey(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.sseCustomerKey()); @@ -346,8 +347,8 @@ void testConvertPutObjectRequest_SSECustomerKey() { void testConvertPutObjectRequest_SSEKMSKeyId() { final String value = "arn:aws:kms:region:123456789012:key/key-id"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .ssekmsKeyId(value) - .build(); + .ssekmsKeyId(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.ssekmsKeyId()); @@ -357,8 +358,8 @@ void testConvertPutObjectRequest_SSEKMSKeyId() { void testConvertPutObjectRequest_SSEKMSEncryptionContext() { final String value = "context123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .ssekmsEncryptionContext(value) - .build(); + .ssekmsEncryptionContext(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.ssekmsEncryptionContext()); @@ -368,8 +369,8 @@ void testConvertPutObjectRequest_SSEKMSEncryptionContext() { void testConvertPutObjectRequest_StorageClass() { final StorageClass value = StorageClass.STANDARD; PutObjectRequest originalRequest = PutObjectRequest.builder() - .storageClass(value) - .build(); + .storageClass(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.storageClass()); @@ -379,8 +380,8 @@ void testConvertPutObjectRequest_StorageClass() { void testConvertPutObjectRequest_StorageClass2() { final String value = StorageClass.STANDARD.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .storageClass(value) - .build(); + .storageClass(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.storageClass().toString()); @@ -390,8 +391,8 @@ void testConvertPutObjectRequest_StorageClass2() { void testConvertPutObjectRequest_Tagging() { final String value = "key1=value1&key2=value2"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .tagging(value) - .build(); + .tagging(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.tagging()); @@ -401,8 +402,8 @@ void testConvertPutObjectRequest_Tagging() { void testConvertPutObjectRequest_WebsiteRedirectLocation() { final String value = "/redirected"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .websiteRedirectLocation(value) - .build(); + .websiteRedirectLocation(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.websiteRedirectLocation()); @@ -411,12 +412,12 @@ void testConvertPutObjectRequest_WebsiteRedirectLocation() { @Test void testConvertPutObjectRequest_OverrideConfiguration() { final AwsRequestOverrideConfiguration value = AwsRequestOverrideConfiguration - .builder() - .apiCallAttemptTimeout(Duration.ofMillis(100)) - .build(); + .builder() + .apiCallAttemptTimeout(Duration.ofMillis(100)) + .build(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .overrideConfiguration(value) - .build(); + .overrideConfiguration(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertTrue(convertedRequest.overrideConfiguration().isPresent()); @@ -470,4 +471,120 @@ public void testConvertResponse() { assertNull(putResponse.ssekmsEncryptionContext()); assertNull(putResponse.size()); } + + @Test + public void testBasicConvertMultipartUploadRequest() { + // Create a MultipartUploadRequest with various fields set + CreateMultipartUploadRequest request = CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .build(); + PutObjectRequest result = ConvertSDKRequests.convertRequest(request); + assertEquals("test-bucket", result.bucket()); + assertEquals("test-key", result.key()); + assertNotNull(result); + } + + @Test + public void testConversionAllFields() { + Map metadata = new HashMap(); + metadata.put("test-key-1", "test-value-1"); + metadata.put("test-key-2", "test-value-2"); + metadata.put("test-key-3", "test-value-3"); + + Instant expires = Instant.now(); + Instant retainUntilDate = Instant.now(); + + CreateMultipartUploadRequest request = CreateMultipartUploadRequest.builder() + .acl("test-acl") + .bucket("test-bucket") + .bucketKeyEnabled(true) + .cacheControl("test-cache-control") + .checksumAlgorithm("test-checksum-algorithm") + .contentDisposition("test-content-disposition") + .contentEncoding("test-content-encoding") + .contentLanguage("test-content-language") + .contentType("test-content-type") + .expectedBucketOwner("test-bucket-owner") + .expires(expires) + .grantFullControl("test-grant-full-control") + .grantRead("test-grant-read") + .grantReadACP("test-grant-read-acp") + .grantWriteACP("test-grant-write-acp") + .key("test-key") + .metadata(metadata) + .objectLockLegalHoldStatus(ObjectLockLegalHoldStatus.OFF) + .objectLockMode(ObjectLockMode.COMPLIANCE) + .objectLockRetainUntilDate(retainUntilDate) + .requestPayer(RequestPayer.REQUESTER) + .serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE) + .sseCustomerAlgorithm("test-sse-customer-algorithm") + .sseCustomerKey("test-sse-customer-key") + .ssekmsEncryptionContext("test-ssekms-encryption-context") + .ssekmsKeyId("test-ssekms-key-id") + .storageClass(StorageClass.SNOW) + .tagging("test-tagging") + .websiteRedirectLocation("test-website-redirect-location") + .build(); + PutObjectRequest result = ConvertSDKRequests.convertRequest(request); + assertEquals("test-acl", result.aclAsString()); + assertEquals("test-bucket", result.bucket()); + assertEquals(true, result.bucketKeyEnabled()); + assertEquals("test-cache-control", result.cacheControl()); + assertEquals("test-checksum-algorithm", result.checksumAlgorithmAsString()); + assertEquals("test-content-disposition", result.contentDisposition()); + assertEquals("test-content-encoding", result.contentEncoding()); + assertEquals("test-content-language", result.contentLanguage()); + assertEquals("test-content-type", result.contentType()); + assertEquals("test-bucket-owner", result.expectedBucketOwner()); + assertEquals(expires, result.expires()); + assertEquals("test-grant-full-control", result.grantFullControl()); + assertEquals("test-grant-read", result.grantRead()); + assertEquals("test-grant-read-acp", result.grantReadACP()); + assertEquals("test-grant-write-acp", result.grantWriteACP()); + assertEquals("test-key", result.key()); + assertEquals(metadata, result.metadata()); + assertEquals(ObjectLockLegalHoldStatus.OFF.toString(), result.objectLockLegalHoldStatusAsString()); + assertEquals(ObjectLockMode.COMPLIANCE.toString(), result.objectLockModeAsString()); + assertEquals(retainUntilDate, result.objectLockRetainUntilDate()); + assertEquals(RequestPayer.REQUESTER.toString(), result.requestPayerAsString()); + assertEquals(ServerSideEncryption.AWS_KMS_DSSE.toString(), result.serverSideEncryptionAsString()); + assertEquals("test-sse-customer-algorithm", result.sseCustomerAlgorithm()); + assertEquals("test-sse-customer-key", result.sseCustomerKey()); + assertEquals("test-ssekms-encryption-context", result.ssekmsEncryptionContext()); + assertEquals("test-ssekms-key-id", result.ssekmsKeyId()); + assertEquals(StorageClass.SNOW.toString(), result.storageClassAsString()); + assertEquals("test-tagging", result.tagging()); + assertEquals("test-website-redirect-location", result.websiteRedirectLocation()); + } + + @Test + public void testConvertMultipartUploadRequestWithNullValues() { + CreateMultipartUploadRequest request = CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .tagging("test-tagging") + .objectLockMode(ObjectLockMode.COMPLIANCE) + .contentLanguage("test-content-language") + .grantReadACP("test-grant-read-acp") + .build(); + PutObjectRequest result = ConvertSDKRequests.convertRequest(request); + assertEquals("test-bucket", result.bucket()); + assertEquals("test-key", result.key()); + assertEquals("test-tagging", result.tagging()); + assertEquals(ObjectLockMode.COMPLIANCE.toString(), result.objectLockModeAsString()); + assertEquals("test-content-language", result.contentLanguage()); + assertEquals("test-grant-read-acp", result.grantReadACP()); + + assertNull(result.aclAsString()); + assertNull(result.grantFullControl()); + assertNull(result.grantRead()); + assertNull(result.storageClass()); + assertNull(result.websiteRedirectLocation()); + assertTrue(result.metadata().isEmpty()); + + } } + + + From fbdbe92fae6d1557bdc5bc31a0f72df6683710c0 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Thu, 29 May 2025 16:30:54 -0700 Subject: [PATCH 11/24] Added tests for S3EncryptonClient with Instruction File + Multipart enabled --- ...S3EncryptionClientInstructionFileTest.java | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java index 9d88d390f..a487007c9 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java @@ -9,26 +9,46 @@ import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; import com.amazonaws.services.s3.model.KMSEncryptionMaterials; import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; +import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.SdkPartType; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.encryption.s3.internal.InstructionFileConfig; +import software.amazon.encryption.s3.utils.BoundedInputStream; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.security.KeyPair; import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.BUCKET; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.KMS_KEY_ID; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.S3_REGION; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.appendTestSuffix; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.deleteObject; @@ -290,4 +310,149 @@ public void testPutWithInstructionFileV3ToV2Rsa() throws NoSuchAlgorithmExceptio deleteObject(BUCKET, objectKey, s3Client); s3Client.close(); } + + @Test + public void testMultipartPutWithInstructionFile() throws IOException, NoSuchAlgorithmException { + final String object_key = appendTestSuffix("test-multipart-put-instruction-file"); + + final long fileSizeLimit = 1024 * 1024 * 50; //50 MB + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + Map encryptionContext = new HashMap<>(); + encryptionContext.put("test-key", "test-value"); + + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(object_key), RequestBody.fromInputStream(inputStream, fileSizeLimit)); + + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(object_key + ".instruction") + .build()); + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + + ResponseInputStream getResponse = s3Client.getObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(object_key)); + + assertTrue(IOUtils.contentEquals(objectStreamForResult, getResponse)); + + deleteObject(BUCKET, object_key, s3Client); + s3Client.close(); + + } + + @Test + public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithmException, IOException { + final String object_key = appendTestSuffix("test-low-level-multipart-put-instruction-file"); + + final long fileSizeLimit = 1024 * 1024 * 50; + final int PART_SIZE = 10 * 1024 * 1024; + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyPair rsaKey = keyPairGen.generateKeyPair(); + + S3Client wrappedClient = S3Client.create(); + + S3Client v3Client = S3EncryptionClient.builder() + .rsaKeyPair(rsaKey) + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .enableDelayedAuthenticationMode(true) + .build(); + + + CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> + builder.bucket(BUCKET).key(object_key)); + + List partETags = new ArrayList<>(); + + int bytesRead, bytesSent = 0; + byte[] partData = new byte[PART_SIZE]; + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + int partsSent = 1; + while ((bytesRead = inputStream.read(partData, 0, partData.length)) != -1) { + outputStream.write(partData, 0, bytesRead); + if (bytesSent < PART_SIZE) { + bytesSent += bytesRead; + continue; + } + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .build(); + + final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); + + UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, + RequestBody.fromInputStream(partInputStream, partInputStream.available())); + + partETags.add(CompletedPart.builder() + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); + outputStream.reset(); + bytesSent = 0; + partsSent++; + } + inputStream.close(); + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .sdkPartType(SdkPartType.LAST) + .build(); + final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); + UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, + RequestBody.fromInputStream(partInputStream, partInputStream.available())); + partETags.add(CompletedPart.builder() + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); + v3Client.completeMultipartUpload(builder -> builder + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .multipartUpload(partBuilder -> partBuilder.parts(partETags))); + + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(object_key + ".instruction") + .build()); + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + + ResponseInputStream getResponse = v3Client.getObject(builder -> builder + .bucket(BUCKET) + .key(object_key)); + + assertTrue(IOUtils.contentEquals(objectStreamForResult, getResponse)); + + deleteObject(BUCKET, object_key, v3Client); + v3Client.close(); + } + } + From 9e25d8f8f999da89168b8a219d5096ba30142548 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 30 May 2025 08:17:43 -0700 Subject: [PATCH 12/24] added .instructionFileConfig for PutEncryptedObjectPipeline builder --- .../java/software/amazon/encryption/s3/S3EncryptionClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java b/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java index 752d6ab52..0c7f12a7b 100644 --- a/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java +++ b/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java @@ -1165,6 +1165,7 @@ public S3EncryptionClient build() { .s3AsyncClient(_wrappedAsyncClient) .cryptoMaterialsManager(_cryptoMaterialsManager) .secureRandom(_secureRandom) + .instructionFileConfig(_instructionFileConfig) .build(); return new S3EncryptionClient(this); From 7bf259cb2b260c43defb27f680311585c3955f9c Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 30 May 2025 08:21:15 -0700 Subject: [PATCH 13/24] cleaned up ContentMetadataEncodingStrategy class --- .../ContentMetadataEncodingStrategy.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java index 3a84cf2de..f805ae6ac 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java @@ -3,6 +3,7 @@ import software.amazon.awssdk.protocols.jsoncore.JsonWriter; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Request; import software.amazon.encryption.s3.S3EncryptionClientException; import software.amazon.encryption.s3.materials.EncryptedDataKey; import software.amazon.encryption.s3.materials.EncryptionMaterials; @@ -36,12 +37,19 @@ public PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, } public CreateMultipartUploadRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, CreateMultipartUploadRequest createMultipartUploadRequest) { - Map newMetadata = addMetadataToMap(createMultipartUploadRequest.metadata(), materials, iv); - return createMultipartUploadRequest.toBuilder() - .metadata(newMetadata) - .build(); + if(_instructionFileConfig.isInstructionFilePutEnabled()) { + final String metadataString = metadataToString(materials, iv); + PutObjectRequest putObjectRequest = ConvertSDKRequests.convertRequest(createMultipartUploadRequest); + _instructionFileConfig.putInstructionFile(putObjectRequest, metadataString); + // the original request object is returned as-is + return createMultipartUploadRequest; + } else { + Map newMetadata = addMetadataToMap(createMultipartUploadRequest.metadata(), materials, iv); + return createMultipartUploadRequest.toBuilder() + .metadata(newMetadata) + .build(); + } } - private String metadataToString(EncryptionMaterials materials, byte[] iv) { // this is just the metadata map serialized as JSON // so first get the Map From ff5e99cc117a6f5c7472af29fb879c9850ea08ce Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 30 May 2025 08:22:09 -0700 Subject: [PATCH 14/24] Added test cases to S3AsyncEncryptionClientTest regarging MPU + instructionFile --- .../s3/S3AsyncEncryptionClientTest.java | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java b/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java index 698b95b8d..5535969f4 100644 --- a/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java @@ -35,13 +35,18 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.StorageClass; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.encryption.s3.internal.ConvertSDKRequests; import software.amazon.encryption.s3.internal.InstructionFileConfig; import software.amazon.encryption.s3.materials.KmsKeyring; import software.amazon.encryption.s3.utils.BoundedInputStream; @@ -66,6 +71,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -829,6 +835,111 @@ public void testAsyncInstructionFileConfig() { s3ClientDisabledInstructionFile.close(); s3Client.close(); } + @Test + public void testAsyncInstructionFileConfigMultipart() { + final String objectKey = appendTestSuffix("test-multipart-async-instruction-file-config"); + final String input = "SimpleTestOfV3EncryptionClient"; + + AwsCredentialsProvider credentials = DefaultCredentialsProvider.create(); + S3Client wrappedClient = S3Client.create(); + S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .credentialsProvider(credentials) + .build(); + PutObjectRequest request = PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(); + CompletableFuture putObjectResponse = v3Client.putObject(request, AsyncRequestBody.fromString(input)); + putObjectResponse.join(); + + assertNotNull(putObjectResponse); + + ResponseBytes instructionFileResponse = wrappedClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + assertNotNull(instructionFileResponse); + assertTrue(instructionFileResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + + CompletableFuture> futureGetObj = v3Client.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), AsyncResponseTransformer.toBytes()); + ResponseBytes getResponse = futureGetObj.join(); + assertNotNull(getResponse); + assertEquals(input, getResponse.asUtf8String()); + + deleteObject(BUCKET, objectKey, v3Client); + + v3Client.close(); + } + @Test + public void testAsyncInstructionFileConfigMultipartWithOptions() { + final String objectKey = appendTestSuffix("test-multipart-async-instruction-file-config-options"); + final String input = "SimpleTestOfV3EncryptionClient"; + + AwsCredentialsProvider credentials = DefaultCredentialsProvider.create(); + S3Client wrappedClient = S3Client.create(); + S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .credentialsProvider(credentials) + .build(); + CreateMultipartUploadRequest multipartUploadRequest = CreateMultipartUploadRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .storageClass(StorageClass.STANDARD_IA) + .build(); + assertNotNull(multipartUploadRequest); + + PutObjectRequest putObjectRequest = ConvertSDKRequests.convertRequest(multipartUploadRequest); + + assertNotNull(putObjectRequest); + assertEquals(putObjectRequest.storageClassAsString(), multipartUploadRequest.storageClassAsString()); + + CompletableFuture putObjectResponse = v3Client.putObject(putObjectRequest, AsyncRequestBody.fromString(input)); + putObjectResponse.join(); + + assertNotNull(putObjectResponse); + + ResponseBytes instructionFileResponse = wrappedClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + assertNotNull(instructionFileResponse); + Map metadata = instructionFileResponse.response().metadata(); + assertTrue(metadata.containsKey("x-amz-crypto-instr-file")); + + HeadObjectResponse instructionHeadResponse = wrappedClient.headObject(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + assertNotNull(instructionHeadResponse.storageClass()); + assertEquals(instructionHeadResponse.storageClassAsString(), StorageClass.STANDARD_IA.toString()); + CompletableFuture> futureGetObj = v3Client.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), AsyncResponseTransformer.toBytes()); + ResponseBytes getResponse = futureGetObj.join(); + assertNotNull(getResponse); + assertEquals(input, getResponse.asUtf8String()); + + assertEquals(getResponse.response().storageClassAsString(), StorageClass.STANDARD_IA.toString()); + + deleteObject(BUCKET, objectKey, v3Client); + v3Client.close(); + + } @Test public void wrappedClientMultipartUploadThrowsException() throws IOException { From a396a7c9dcefa23a8db3ffd6477c3aaba9f1f1c9 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 30 May 2025 10:53:47 -0700 Subject: [PATCH 15/24] fixed import issues --- .../amazon/encryption/s3/internal/InstructionFileConfig.java | 1 - .../encryption/s3/S3EncryptionClientMultipartUploadTest.java | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java b/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java index 6d40d86cd..59cccc788 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java +++ b/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java @@ -34,7 +34,6 @@ private InstructionFileConfig(final Builder builder) { _s3AsyncClient = builder._s3AsyncClient; _enableInstructionFilePut = builder._enableInstructionFilePut; } - public static Builder builder() { return new Builder(); } diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java index 3f61fa2fb..d6535d525 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java @@ -67,6 +67,8 @@ public void multipartPutObjectAsync() throws IOException { final InputStream inputStream = new BoundedInputStream(fileSizeLimit); final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + + S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() .kmsKeyId(KMS_KEY_ID) .enableMultipartPutObject(true) @@ -78,11 +80,12 @@ public void multipartPutObjectAsync() throws IOException { encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); - + CompletableFuture futurePut = v3Client.putObject(builder -> builder .bucket(BUCKET) .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) .key(objectKey), AsyncRequestBody.fromInputStream(inputStream, fileSizeLimit, singleThreadExecutor)); + futurePut.join(); singleThreadExecutor.shutdown(); From b1c089809f24869ebb7aa92540d6963af9d50ea8 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 30 May 2025 11:19:32 -0700 Subject: [PATCH 16/24] removed debugging lines --- .../amazon/encryption/s3/S3AsyncEncryptionClient.java | 8 +------- .../s3/internal/PutEncryptedObjectPipeline.java | 1 - 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java b/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java index ef95c9693..9f7a13e1e 100644 --- a/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java +++ b/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java @@ -151,7 +151,6 @@ public CompletableFuture putObject(PutObjectRequest putObject .s3AsyncClient(_wrappedClient) .cryptoMaterialsManager(_cryptoMaterialsManager) .secureRandom(_secureRandom) - //Debugging: added this line .instructionFileConfig(_instructionFileConfig) .build(); @@ -164,11 +163,9 @@ private CompletableFuture multipartPutObject(PutObjectRequest // if the wrappedClient is a CRT, use it mpuClient = _wrappedClient; } else { - // else create a default CRT client (debugging: yes goes here) + // else create a default CRT client mpuClient = S3AsyncClient.crtCreate(); } - //The issue is here: after this step, the instruction file config is null - //Debugging: added this line to include the instruction file config PutEncryptedObjectPipeline pipeline = PutEncryptedObjectPipeline.builder() .s3AsyncClient(mpuClient) .cryptoMaterialsManager(_cryptoMaterialsManager) @@ -294,9 +291,6 @@ public void close() { _instructionFileConfig.closeClient(); } - public InstructionFileConfig get_instructionFileConfig() { - return _instructionFileConfig; - } // This is very similar to the S3EncryptionClient builder // Make sure to keep both clients in mind when adding new builder options diff --git a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java index 35ae13b8e..97070bf76 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java @@ -24,7 +24,6 @@ public class PutEncryptedObjectPipeline { final private CryptographicMaterialsManager _cryptoMaterialsManager; final private AsyncContentEncryptionStrategy _asyncContentEncryptionStrategy; final private ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy; - //Debugging: final private InstructionFileConfig _instructionFileConfig; public static Builder builder() { return new Builder(); From b4b2047af8961e0cc7ab64c8c94972141f617e29 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 30 May 2025 12:47:38 -0700 Subject: [PATCH 17/24] Replaced passing in ARN of my personal testing bucket with the BUCKET variable --- .../s3/S3EncryptionClientCompatibilityTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java index 158d72fc4..ea35fbf01 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java @@ -238,21 +238,21 @@ public void multipartPutObjectWithOptionsAndInstructionFileV2() throws IOExcepti .withCryptoConfiguration(cryptoConfig) .withEncryptionMaterialsProvider(materialsProvider) .build(); - UploadObjectRequest uploadObjectRequest = new UploadObjectRequest("s3ec-github-test-bucket-597133212884", objectKey, inputStream, new ObjectMetadata()) + UploadObjectRequest uploadObjectRequest = new UploadObjectRequest(BUCKET, objectKey, inputStream, new ObjectMetadata()) .withPartSize(1024 * 1024 * 5) - .withStorageClass(StorageClass.Glacier); + .withStorageClass(StorageClass.StandardInfrequentAccess); v2Client.uploadObject(uploadObjectRequest); //Assert that the storage class on main object matches "GLACIER" - GetObjectMetadataRequest mainObjectRequest = new GetObjectMetadataRequest("s3ec-github-test-bucket-597133212884", objectKey); + GetObjectMetadataRequest mainObjectRequest = new GetObjectMetadataRequest(BUCKET, objectKey); ObjectMetadata mainObjectMetadata = v2Client.getObjectMetadata(mainObjectRequest); - assertEquals("GLACIER", mainObjectMetadata.getStorageClass()); + assertEquals("STANDARD_IA", mainObjectMetadata.getStorageClass()); //Assert that the instruction file does not contain storage class (V2) - GetObjectMetadataRequest instructionObjectRequest = new GetObjectMetadataRequest("s3ec-github-test-bucket-597133212884", objectKey + ".instruction"); + GetObjectMetadataRequest instructionObjectRequest = new GetObjectMetadataRequest(BUCKET, objectKey + ".instruction"); ObjectMetadata instructionFileMetadata = v2Client.getObjectMetadata(instructionObjectRequest); - assertNotEquals("GLACIER", instructionFileMetadata.getStorageClass()); + assertNotEquals("STANDARD_IA", instructionFileMetadata.getStorageClass()); } From b9ae158b9f341178362e9fc1dbe341ac41e6131e Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 30 May 2025 13:46:50 -0700 Subject: [PATCH 18/24] removed unused import --- .../encryption/s3/internal/ContentMetadataEncodingStrategy.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java index f805ae6ac..ebe0143ee 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java @@ -3,7 +3,6 @@ import software.amazon.awssdk.protocols.jsoncore.JsonWriter; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.S3Request; import software.amazon.encryption.s3.S3EncryptionClientException; import software.amazon.encryption.s3.materials.EncryptedDataKey; import software.amazon.encryption.s3.materials.EncryptionMaterials; From 4f51c42d2006de68135ba942e16404ca935fd308 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Tue, 3 Jun 2025 16:25:31 -0700 Subject: [PATCH 19/24] fixed all changes requested from PR --- .../s3/examples/HelloWorldProgramExample.java | 43 +++++++++++++++++++ .../ContentMetadataEncodingStrategy.java | 2 + .../s3/internal/ConvertSDKRequests.java | 16 ++++--- .../MultipartUploadObjectPipeline.java | 3 ++ .../internal/PutEncryptedObjectPipeline.java | 3 ++ .../s3/S3AsyncEncryptionClientTest.java | 26 +++-------- .../S3EncryptionClientCompatibilityTest.java | 36 +--------------- ...S3EncryptionClientInstructionFileTest.java | 13 ++++-- .../s3/internal/ConvertSDKRequestsTest.java | 3 +- 9 files changed, 80 insertions(+), 65 deletions(-) create mode 100644 src/examples/java/software/amazon/encryption/s3/examples/HelloWorldProgramExample.java diff --git a/src/examples/java/software/amazon/encryption/s3/examples/HelloWorldProgramExample.java b/src/examples/java/software/amazon/encryption/s3/examples/HelloWorldProgramExample.java new file mode 100644 index 000000000..1192538f2 --- /dev/null +++ b/src/examples/java/software/amazon/encryption/s3/examples/HelloWorldProgramExample.java @@ -0,0 +1,43 @@ +package software.amazon.encryption.s3.examples; + +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.encryption.s3.S3EncryptionClient; +import software.amazon.awssdk.regions.Region; + +public class HelloWorldProgramExample { + public static void main(String[] args) { + //Create AWS KMS key (go to KMS to do this): + String kmsKeyId = "arn:aws:kms:us-east-2:597133212884:key/1483518b-144f-48d7-84ce-735ff8d6da98"; + //Think of object as the "plaintext message" + String object = "Hello World"; + //Created new bucket for this program...had to update permissions to fix bugs + String bucket = "testing-bucket-hello-world"; + //Object Key: Identifier of object in S3 + String object_key = "hello-world.txt"; + + try (S3Client v3Client = S3EncryptionClient.builder() + .kmsKeyId(kmsKeyId) + .enableLegacyUnauthenticatedModes(true) + .region(Region.US_EAST_2) + .build()) { + + v3Client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(object_key) + .build(), RequestBody.fromString(object)); + + String output = v3Client.getObjectAsBytes(builder -> builder + .bucket(bucket) + .key(object_key) + ).asUtf8String(); + + System.out.println("Object stored in S3 is: "+output); + + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} + diff --git a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java index ebe0143ee..cf4e86fa7 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java @@ -1,3 +1,5 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package software.amazon.encryption.s3.internal; import software.amazon.awssdk.protocols.jsoncore.JsonWriter; diff --git a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java index dacbf6682..e2a567fbd 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java @@ -13,7 +13,9 @@ public class ConvertSDKRequests { public static PutObjectRequest convertRequest(CreateMultipartUploadRequest request) { - + /*Converts a CreateMultipartUploadRequest into a PutObjectRequest by setting optional fields needed for + putInstructionFile operation. + */ final PutObjectRequest.Builder output = PutObjectRequest.builder(); request .toBuilder() @@ -119,13 +121,13 @@ public static PutObjectRequest convertRequest(CreateMultipartUploadRequest reque // Rather than silently dropping the value, // we loudly signal that we don't know how to handle this field. throw new IllegalArgumentException( - f.locationName() + " is an unknown field. " + + f.memberName() + " is an unknown field. " + "The S3 Encryption Client does not recognize this option and cannot set it on the PutObjectRequest." + "This may be a new S3 feature." + "Please report this to the Amazon S3 Encryption Client for Java: " + "https://github.com/aws/amazon-s3-encryption-client-java/issues." + - "To work around this issue you can disable multi part upload," + - "use the Async client, or not set this value on PutObject." + + "To work around this issue, you can disable Instruction File on PutObject or disable" + + "multi part upload, or use the Async client, or not set this value on PutObject." + "You may be able to update this value after the PutObject request completes." ); } @@ -138,7 +140,9 @@ public static PutObjectRequest convertRequest(CreateMultipartUploadRequest reque } public static CreateMultipartUploadRequest convertRequest(PutObjectRequest request) { - + /*Converts a PutObjectRequest into a CreateMultipartUploadRequest by setting optional fields needed for high-level + multipart upload operation. + */ final CreateMultipartUploadRequest.Builder output = CreateMultipartUploadRequest.builder(); request .toBuilder() @@ -251,7 +255,7 @@ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest reque // Rather than silently dropping the value, // we loudly signal that we don't know how to handle this field. throw new IllegalArgumentException( - f.locationName() + " is an unknown field. " + + f.memberName() + " is an unknown field. " + "The S3 Encryption Client does not recognize this option and cannot set it on the CreateMultipartUploadRequest." + "This may be a new S3 feature." + "Please report this to the Amazon S3 Encryption Client for Java: " + diff --git a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java index ba0b5d6e9..cb4dde03a 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java @@ -262,6 +262,9 @@ public MultipartUploadObjectPipeline build() { .secureRandom(_secureRandom) .build(); } + if(_instructionFileConfig == null) { + _instructionFileConfig = InstructionFileConfig.builder().build(); + } _contentMetadataEncodingStrategy = new ContentMetadataEncodingStrategy(_instructionFileConfig); return new MultipartUploadObjectPipeline(this); } diff --git a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java index 97070bf76..a311e210c 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java @@ -121,6 +121,9 @@ public PutEncryptedObjectPipeline build() { .secureRandom(_secureRandom) .build(); } + if(_instructionFileConfig == null) { + _instructionFileConfig = InstructionFileConfig.builder().build(); + } _contentMetadataEncodingStrategy = new ContentMetadataEncodingStrategy(_instructionFileConfig); return new PutEncryptedObjectPipeline(this); diff --git a/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java b/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java index 5535969f4..ebca61df8 100644 --- a/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java @@ -45,6 +45,7 @@ import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.awssdk.services.s3.model.StorageClassAnalysisDataExport; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; import software.amazon.encryption.s3.internal.ConvertSDKRequests; import software.amazon.encryption.s3.internal.InstructionFileConfig; @@ -883,8 +884,8 @@ public void testAsyncInstructionFileConfigMultipart() { public void testAsyncInstructionFileConfigMultipartWithOptions() { final String objectKey = appendTestSuffix("test-multipart-async-instruction-file-config-options"); final String input = "SimpleTestOfV3EncryptionClient"; + final StorageClass storageClass = StorageClass.STANDARD_IA; - AwsCredentialsProvider credentials = DefaultCredentialsProvider.create(); S3Client wrappedClient = S3Client.create(); S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() .instructionFileConfig(InstructionFileConfig.builder() @@ -893,25 +894,16 @@ public void testAsyncInstructionFileConfigMultipartWithOptions() { .build()) .kmsKeyId(KMS_KEY_ID) .enableMultipartPutObject(true) - .credentialsProvider(credentials) .build(); - CreateMultipartUploadRequest multipartUploadRequest = CreateMultipartUploadRequest.builder() + PutObjectRequest putObjectRequest = PutObjectRequest.builder() .bucket(BUCKET) .key(objectKey) - .storageClass(StorageClass.STANDARD_IA) + .storageClass(storageClass) .build(); - assertNotNull(multipartUploadRequest); - - PutObjectRequest putObjectRequest = ConvertSDKRequests.convertRequest(multipartUploadRequest); - - assertNotNull(putObjectRequest); - assertEquals(putObjectRequest.storageClassAsString(), multipartUploadRequest.storageClassAsString()); CompletableFuture putObjectResponse = v3Client.putObject(putObjectRequest, AsyncRequestBody.fromString(input)); putObjectResponse.join(); - assertNotNull(putObjectResponse); - ResponseBytes instructionFileResponse = wrappedClient.getObjectAsBytes(builder -> builder .bucket(BUCKET) .key(objectKey + ".instruction") @@ -920,12 +912,8 @@ public void testAsyncInstructionFileConfigMultipartWithOptions() { Map metadata = instructionFileResponse.response().metadata(); assertTrue(metadata.containsKey("x-amz-crypto-instr-file")); - HeadObjectResponse instructionHeadResponse = wrappedClient.headObject(builder -> builder - .bucket(BUCKET) - .key(objectKey + ".instruction") - .build()); - assertNotNull(instructionHeadResponse.storageClass()); - assertEquals(instructionHeadResponse.storageClassAsString(), StorageClass.STANDARD_IA.toString()); + assertEquals(storageClass.toString(), instructionFileResponse.response().storageClassAsString()); + CompletableFuture> futureGetObj = v3Client.getObject(builder -> builder .bucket(BUCKET) .key(objectKey) @@ -934,7 +922,7 @@ public void testAsyncInstructionFileConfigMultipartWithOptions() { assertNotNull(getResponse); assertEquals(input, getResponse.asUtf8String()); - assertEquals(getResponse.response().storageClassAsString(), StorageClass.STANDARD_IA.toString()); + assertEquals(getResponse.response().storageClassAsString(), storageClass.toString()); deleteObject(BUCKET, objectKey, v3Client); v3Client.close(); diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java index ea35fbf01..fb8dad2e9 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java @@ -138,7 +138,6 @@ public void AesWrapV1toV3() { // Asserts final String input = "AesGcmV1toV3"; - System.out.println(System.getenv("AWS_S3EC_TEST_BUCKET")); v1Client.putObject(BUCKET, objectKey, input); ResponseBytes objectResponse = v3Client.getObjectAsBytes(builder -> builder @@ -222,39 +221,6 @@ public void AesGcmV2toV3WithInstructionFile() { deleteObject(BUCKET, objectKey, v3Client); v3Client.close(); } - @Test - public void multipartPutObjectWithOptionsAndInstructionFileV2() throws IOException, InterruptedException, ExecutionException { - final String objectKey = appendTestSuffix("multipart-put-object-with-options-and-instruction-file-v2"); - final long fileSizeLimit = 1024 * 1024 * 10; //sets file size limit to 10 MB - final InputStream inputStream = new BoundedInputStream(fileSizeLimit); - - //Now, we will create encryption client (v2) with instruction file config enabled and multipart upload enabled - EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new EncryptionMaterials(AES_KEY)); - CryptoConfigurationV2 cryptoConfig = - new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) - .withStorageMode(CryptoStorageMode.InstructionFile); - AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() - .withCryptoConfiguration(cryptoConfig) - .withEncryptionMaterialsProvider(materialsProvider) - .build(); - UploadObjectRequest uploadObjectRequest = new UploadObjectRequest(BUCKET, objectKey, inputStream, new ObjectMetadata()) - .withPartSize(1024 * 1024 * 5) - .withStorageClass(StorageClass.StandardInfrequentAccess); - v2Client.uploadObject(uploadObjectRequest); - - //Assert that the storage class on main object matches "GLACIER" - GetObjectMetadataRequest mainObjectRequest = new GetObjectMetadataRequest(BUCKET, objectKey); - ObjectMetadata mainObjectMetadata = v2Client.getObjectMetadata(mainObjectRequest); - assertEquals("STANDARD_IA", mainObjectMetadata.getStorageClass()); - - //Assert that the instruction file does not contain storage class (V2) - GetObjectMetadataRequest instructionObjectRequest = new GetObjectMetadataRequest(BUCKET, objectKey + ".instruction"); - ObjectMetadata instructionFileMetadata = v2Client.getObjectMetadata(instructionObjectRequest); - - assertNotEquals("STANDARD_IA", instructionFileMetadata.getStorageClass()); - - } @Test public void AesGcmV3toV1() { @@ -633,7 +599,7 @@ public void KmsV1toV3() { assertEquals(input, output); // Cleanup -// deleteObject(BUCKET, objectKey, v3Client); + deleteObject(BUCKET, objectKey, v3Client); v3Client.close(); } diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java index a487007c9..3559986a0 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java @@ -21,6 +21,7 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.SdkPartType; +import software.amazon.awssdk.services.s3.model.StorageClass; import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.encryption.s3.internal.InstructionFileConfig; @@ -96,7 +97,7 @@ public void testInstructionFileExists() { @Test public void testDisabledClientFails() { - final String objectKey = appendTestSuffix("instruction-file-put-object"); + final String objectKey = appendTestSuffix("instruction-file-put-object-disabled-fails"); final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() @@ -144,7 +145,7 @@ public void testDisabledClientFails() { */ @Test public void testInstructionFileDelete() { - final String objectKey = appendTestSuffix("instruction-file-put-object"); + final String objectKey = appendTestSuffix("instruction-file-put-object-delete"); final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() @@ -318,6 +319,7 @@ public void testMultipartPutWithInstructionFile() throws IOException, NoSuchAlgo final long fileSizeLimit = 1024 * 1024 * 50; //50 MB final InputStream inputStream = new BoundedInputStream(fileSizeLimit); final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + final StorageClass storageClass = StorageClass.STANDARD_IA; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() @@ -326,6 +328,7 @@ public void testMultipartPutWithInstructionFile() throws IOException, NoSuchAlgo .enableInstructionFilePutObject(true) .build()) .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) .build(); Map encryptionContext = new HashMap<>(); @@ -334,6 +337,7 @@ public void testMultipartPutWithInstructionFile() throws IOException, NoSuchAlgo s3Client.putObject(builder -> builder .bucket(BUCKET) + .storageClass(storageClass) .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) .key(object_key), RequestBody.fromInputStream(inputStream, fileSizeLimit)); @@ -343,6 +347,7 @@ public void testMultipartPutWithInstructionFile() throws IOException, NoSuchAlgo .key(object_key + ".instruction") .build()); assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + assertEquals(storageClass.toString(), directInstGetResponse.response().storageClassAsString()); ResponseInputStream getResponse = s3Client.getObject(builder -> builder .bucket(BUCKET) @@ -364,6 +369,7 @@ public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithm final int PART_SIZE = 10 * 1024 * 1024; final InputStream inputStream = new BoundedInputStream(fileSizeLimit); final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + final StorageClass storageClass = StorageClass.STANDARD_IA; KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); keyPairGen.initialize(2048); @@ -382,7 +388,7 @@ public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithm CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> - builder.bucket(BUCKET).key(object_key)); + builder.bucket(BUCKET).key(object_key).storageClass(storageClass)); List partETags = new ArrayList<>(); @@ -443,6 +449,7 @@ public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithm .key(object_key + ".instruction") .build()); assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + assertEquals(storageClass.toString(), directInstGetResponse.response().storageClassAsString()); ResponseInputStream getResponse = v3Client.getObject(builder -> builder .bucket(BUCKET) diff --git a/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java b/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java index 6c2e6d1cc..cdfb7413d 100644 --- a/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java +++ b/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java @@ -3,7 +3,6 @@ import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; import software.amazon.awssdk.services.s3.model.*; -import java.awt.event.ComponentListener; import java.time.Duration; import java.time.Instant; import java.util.HashMap; @@ -486,7 +485,7 @@ public void testBasicConvertMultipartUploadRequest() { } @Test - public void testConversionAllFields() { + public void testConversionAllFieldsMultipartUploadRequestToPutObjectRequest() { Map metadata = new HashMap(); metadata.put("test-key-1", "test-value-1"); metadata.put("test-key-2", "test-value-2"); From bf2be6e0ce8321d43de3c5085788d3386312f967 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Tue, 3 Jun 2025 16:31:21 -0700 Subject: [PATCH 20/24] Removed HelloWorldProgramExample.java --- .../s3/examples/HelloWorldProgramExample.java | 43 ------------------- 1 file changed, 43 deletions(-) delete mode 100644 src/examples/java/software/amazon/encryption/s3/examples/HelloWorldProgramExample.java diff --git a/src/examples/java/software/amazon/encryption/s3/examples/HelloWorldProgramExample.java b/src/examples/java/software/amazon/encryption/s3/examples/HelloWorldProgramExample.java deleted file mode 100644 index 1192538f2..000000000 --- a/src/examples/java/software/amazon/encryption/s3/examples/HelloWorldProgramExample.java +++ /dev/null @@ -1,43 +0,0 @@ -package software.amazon.encryption.s3.examples; - -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.encryption.s3.S3EncryptionClient; -import software.amazon.awssdk.regions.Region; - -public class HelloWorldProgramExample { - public static void main(String[] args) { - //Create AWS KMS key (go to KMS to do this): - String kmsKeyId = "arn:aws:kms:us-east-2:597133212884:key/1483518b-144f-48d7-84ce-735ff8d6da98"; - //Think of object as the "plaintext message" - String object = "Hello World"; - //Created new bucket for this program...had to update permissions to fix bugs - String bucket = "testing-bucket-hello-world"; - //Object Key: Identifier of object in S3 - String object_key = "hello-world.txt"; - - try (S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(kmsKeyId) - .enableLegacyUnauthenticatedModes(true) - .region(Region.US_EAST_2) - .build()) { - - v3Client.putObject(PutObjectRequest.builder() - .bucket(bucket) - .key(object_key) - .build(), RequestBody.fromString(object)); - - String output = v3Client.getObjectAsBytes(builder -> builder - .bucket(bucket) - .key(object_key) - ).asUtf8String(); - - System.out.println("Object stored in S3 is: "+output); - - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} - From 02f6f48cfc434e687ea8c8bd42c80452b4330b0f Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 6 Jun 2025 09:21:07 -0700 Subject: [PATCH 21/24] Fixed the second PR round of reviews --- .../s3/internal/ConvertSDKRequests.java | 23 +++++++++++++------ .../s3/S3AsyncEncryptionClientTest.java | 4 ---- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java index e2a567fbd..024577103 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java @@ -12,10 +12,15 @@ public class ConvertSDKRequests { + /** + * Converts a CreateMultipartUploadRequest to a PutObjectRequest. This conversion is necessary when + * Instruction File PutObject is enabled and a multipart upload is performed.The method copies all the + * relevant fields from the CreateMultipartUploadRequest to the PutObjectRequest. + * @param request The CreateMultipartUploadRequest to convert + * @return The converted PutObjectRequest + * @throws IllegalArgumentException if the request contains an invalid field + */ public static PutObjectRequest convertRequest(CreateMultipartUploadRequest request) { - /*Converts a CreateMultipartUploadRequest into a PutObjectRequest by setting optional fields needed for - putInstructionFile operation. - */ final PutObjectRequest.Builder output = PutObjectRequest.builder(); request .toBuilder() @@ -138,11 +143,15 @@ public static PutObjectRequest convertRequest(CreateMultipartUploadRequest reque .overrideConfiguration(request.overrideConfiguration().orElse(null)) .build(); } - + /** + * Converts a PutObjectRequest to CreateMultipartUploadRequest.This conversion is necessary to convert an + * original PutObjectRequest into a CreateMultipartUploadRequest to initiate the + * multipart upload while maintaining the original request's configuration. + * @param request The PutObjectRequest to convert + * @return The converted CreateMultipartUploadRequest + * @throws IllegalArgumentException if the request contains an invalid field + */ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest request) { - /*Converts a PutObjectRequest into a CreateMultipartUploadRequest by setting optional fields needed for high-level - multipart upload operation. - */ final CreateMultipartUploadRequest.Builder output = CreateMultipartUploadRequest.builder(); request .toBuilder() diff --git a/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java b/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java index ebca61df8..2668918b0 100644 --- a/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java @@ -35,19 +35,15 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.StorageClass; -import software.amazon.awssdk.services.s3.model.StorageClassAnalysisDataExport; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; -import software.amazon.encryption.s3.internal.ConvertSDKRequests; import software.amazon.encryption.s3.internal.InstructionFileConfig; import software.amazon.encryption.s3.materials.KmsKeyring; import software.amazon.encryption.s3.utils.BoundedInputStream; From c319d4472ad85a472a85a35f8d907282d65170ef Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 6 Jun 2025 10:04:04 -0700 Subject: [PATCH 22/24] testMultipartPutWithInstructionFile() should be working now --- .../encryption/s3/S3EncryptionClientInstructionFileTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java index 3559986a0..9ca5b8352 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java @@ -14,7 +14,6 @@ import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CompletedPart; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; @@ -43,13 +42,11 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.BUCKET; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.KMS_KEY_ID; -import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.S3_REGION; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.appendTestSuffix; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.deleteObject; @@ -313,7 +310,7 @@ public void testPutWithInstructionFileV3ToV2Rsa() throws NoSuchAlgorithmExceptio } @Test - public void testMultipartPutWithInstructionFile() throws IOException, NoSuchAlgorithmException { + public void testMultipartPutWithInstructionFile() throws IOException { final String object_key = appendTestSuffix("test-multipart-put-instruction-file"); final long fileSizeLimit = 1024 * 1024 * 50; //50 MB From 0ed41a5c47d3eafcdefafa85c4fb4fd6179dced3 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Thu, 12 Jun 2025 15:04:47 -0700 Subject: [PATCH 23/24] fixed second round of PR --- .../s3/internal/ConvertSDKRequests.java | 8 +- .../s3/internal/ConvertSDKRequestsTest.java | 272 +++++++++--------- 2 files changed, 143 insertions(+), 137 deletions(-) diff --git a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java index 024577103..9337492ae 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java @@ -107,6 +107,9 @@ public static PutObjectRequest convertRequest(CreateMultipartUploadRequest reque case "SSECustomerKey": output.sseCustomerKey((String) value); break; + case "SSECustomerKeyMD5": + output.sseCustomerKeyMD5((String) value); + break; case "SSEKMSEncryptionContext": output.ssekmsEncryptionContext((String) value); break; @@ -175,8 +178,6 @@ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest reque case "ChecksumAlgorithm": output.checksumAlgorithm((String) value); break; - case "ChecksumType": - output.checksumType((ChecksumType) value); case "ContentDisposition": assert value instanceof String; output.contentDisposition((String) value); @@ -245,6 +246,9 @@ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest reque case "SSECustomerKey": output.sseCustomerKey((String) value); break; + case "SSECustomerKeyMD5": + output.sseCustomerKeyMD5((String) value); + break; case "SSEKMSEncryptionContext": output.ssekmsEncryptionContext((String) value); break; diff --git a/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java b/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java index cdfb7413d..fc167d502 100644 --- a/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java +++ b/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java @@ -1,4 +1,5 @@ package software.amazon.encryption.s3.internal; + import org.junit.jupiter.api.Test; import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; import software.amazon.awssdk.services.s3.model.*; @@ -7,6 +8,7 @@ import java.time.Instant; import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.*; class ConvertSDKRequestsTest { @@ -15,8 +17,8 @@ class ConvertSDKRequestsTest { void testConvertPutObjectRequest_Bucket() { final String value = "test-bucket"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .bucket(value) - .build(); + .bucket(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.bucket()); @@ -26,8 +28,8 @@ void testConvertPutObjectRequest_Bucket() { void testConvertPutObjectRequest_ACL() { final ObjectCannedACL value = ObjectCannedACL.PRIVATE; PutObjectRequest originalRequest = PutObjectRequest.builder() - .acl(value) - .build(); + .acl(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.acl()); @@ -37,8 +39,8 @@ void testConvertPutObjectRequest_ACL() { void testConvertPutObjectRequest_ACL2() { final String value = ObjectCannedACL.PRIVATE.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .acl(value) - .build(); + .acl(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.acl().toString()); @@ -48,8 +50,8 @@ void testConvertPutObjectRequest_ACL2() { void testConvertPutObjectRequest_BucketKeyEnabled() { final Boolean value = true; PutObjectRequest originalRequest = PutObjectRequest.builder() - .bucketKeyEnabled(value) - .build(); + .bucketKeyEnabled(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.bucketKeyEnabled()); @@ -59,8 +61,8 @@ void testConvertPutObjectRequest_BucketKeyEnabled() { void testConvertPutObjectRequest_CacheControl() { final String value = "max-age=3600"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .cacheControl(value) - .build(); + .cacheControl(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.cacheControl()); @@ -70,8 +72,8 @@ void testConvertPutObjectRequest_CacheControl() { void testConvertPutObjectRequest_ChecksumAlgorithm() { final ChecksumAlgorithm value = ChecksumAlgorithm.SHA256; PutObjectRequest originalRequest = PutObjectRequest.builder() - .checksumAlgorithm(value) - .build(); + .checksumAlgorithm(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.checksumAlgorithm()); @@ -81,8 +83,8 @@ void testConvertPutObjectRequest_ChecksumAlgorithm() { void testConvertPutObjectRequest_ChecksumAlgorithm2() { final String value = ChecksumAlgorithm.SHA256.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .checksumAlgorithm(value) - .build(); + .checksumAlgorithm(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.checksumAlgorithm().toString()); @@ -92,8 +94,8 @@ void testConvertPutObjectRequest_ChecksumAlgorithm2() { void testConvertPutObjectRequest_ContentDisposition() { final String value = "attachment; filename=\"filename.jpg\""; PutObjectRequest originalRequest = PutObjectRequest.builder() - .contentDisposition(value) - .build(); + .contentDisposition(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.contentDisposition()); @@ -103,8 +105,8 @@ void testConvertPutObjectRequest_ContentDisposition() { void testConvertPutObjectRequest_ContentEncoding() { final String value = "gzip"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .contentEncoding(value) - .build(); + .contentEncoding(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.contentEncoding()); @@ -114,8 +116,8 @@ void testConvertPutObjectRequest_ContentEncoding() { void testConvertPutObjectRequest_ContentLanguage() { final String value = "en-US"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .contentLanguage(value) - .build(); + .contentLanguage(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.contentLanguage()); @@ -125,8 +127,8 @@ void testConvertPutObjectRequest_ContentLanguage() { void testConvertPutObjectRequest_ContentType() { final String value = "text/plain"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .contentType(value) - .build(); + .contentType(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.contentType()); @@ -136,8 +138,8 @@ void testConvertPutObjectRequest_ContentType() { void testConvertPutObjectRequest_ExpectedBucketOwner() { final String value = "owner123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .expectedBucketOwner(value) - .build(); + .expectedBucketOwner(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.expectedBucketOwner()); @@ -147,8 +149,8 @@ void testConvertPutObjectRequest_ExpectedBucketOwner() { void testConvertPutObjectRequest_Expires() { final Instant value = Instant.now(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .expires(value) - .build(); + .expires(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.expires()); @@ -158,8 +160,8 @@ void testConvertPutObjectRequest_Expires() { void testConvertPutObjectRequest_GrantFullControl() { final String value = "id=123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .grantFullControl(value) - .build(); + .grantFullControl(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.grantFullControl()); @@ -169,8 +171,8 @@ void testConvertPutObjectRequest_GrantFullControl() { void testConvertPutObjectRequest_GrantRead() { final String value = "id=123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .grantRead(value) - .build(); + .grantRead(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.grantRead()); @@ -180,8 +182,8 @@ void testConvertPutObjectRequest_GrantRead() { void testConvertPutObjectRequest_GrantReadACP() { final String value = "id=123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .grantReadACP(value) - .build(); + .grantReadACP(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.grantReadACP()); @@ -191,8 +193,8 @@ void testConvertPutObjectRequest_GrantReadACP() { void testConvertPutObjectRequest_GrantWriteACP() { final String value = "id=123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .grantWriteACP(value) - .build(); + .grantWriteACP(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.grantWriteACP()); @@ -202,8 +204,8 @@ void testConvertPutObjectRequest_GrantWriteACP() { void testConvertPutObjectRequest_Key() { final String value = "test-key"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .key(value) - .build(); + .key(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.key()); @@ -214,8 +216,8 @@ void testConvertPutObjectRequest_Metadata() { final Map value = new HashMap<>(); value.put("key1", "value1"); PutObjectRequest originalRequest = PutObjectRequest.builder() - .metadata(value) - .build(); + .metadata(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.metadata()); @@ -225,8 +227,8 @@ void testConvertPutObjectRequest_Metadata() { void testConvertPutObjectRequest_ObjectLockLegalHoldStatus() { final ObjectLockLegalHoldStatus value = ObjectLockLegalHoldStatus.ON; PutObjectRequest originalRequest = PutObjectRequest.builder() - .objectLockLegalHoldStatus(value) - .build(); + .objectLockLegalHoldStatus(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.objectLockLegalHoldStatus()); @@ -236,8 +238,8 @@ void testConvertPutObjectRequest_ObjectLockLegalHoldStatus() { void testConvertPutObjectRequest_ObjectLockLegalHoldStatus2() { final String value = ObjectLockLegalHoldStatus.ON.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .objectLockLegalHoldStatus(value) - .build(); + .objectLockLegalHoldStatus(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.objectLockLegalHoldStatus().toString()); @@ -247,8 +249,8 @@ void testConvertPutObjectRequest_ObjectLockLegalHoldStatus2() { void testConvertPutObjectRequest_ObjectLockMode() { final ObjectLockMode value = ObjectLockMode.GOVERNANCE; PutObjectRequest originalRequest = PutObjectRequest.builder() - .objectLockMode(value) - .build(); + .objectLockMode(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.objectLockMode()); @@ -258,8 +260,8 @@ void testConvertPutObjectRequest_ObjectLockMode() { void testConvertPutObjectRequest_ObjectLockMode2() { final String value = "GOVERNANCE"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .objectLockMode(value) - .build(); + .objectLockMode(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.objectLockMode().toString()); @@ -269,8 +271,8 @@ void testConvertPutObjectRequest_ObjectLockMode2() { void testConvertPutObjectRequest_ObjectLockRetainUntilDate() { final Instant value = Instant.now(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .objectLockRetainUntilDate(value) - .build(); + .objectLockRetainUntilDate(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.objectLockRetainUntilDate()); @@ -280,8 +282,8 @@ void testConvertPutObjectRequest_ObjectLockRetainUntilDate() { void testConvertPutObjectRequest_RequestPayer() { final RequestPayer value = RequestPayer.REQUESTER; PutObjectRequest originalRequest = PutObjectRequest.builder() - .requestPayer(value) - .build(); + .requestPayer(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.requestPayer()); @@ -291,8 +293,8 @@ void testConvertPutObjectRequest_RequestPayer() { void testConvertPutObjectRequest_RequestPayer2() { final String value = RequestPayer.REQUESTER.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .requestPayer(value) - .build(); + .requestPayer(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.requestPayer().toString()); @@ -302,8 +304,8 @@ void testConvertPutObjectRequest_RequestPayer2() { void testConvertPutObjectRequest_ServerSideEncryption() { final ServerSideEncryption value = ServerSideEncryption.AES256; PutObjectRequest originalRequest = PutObjectRequest.builder() - .serverSideEncryption(value) - .build(); + .serverSideEncryption(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.serverSideEncryption()); @@ -313,8 +315,8 @@ void testConvertPutObjectRequest_ServerSideEncryption() { void testConvertPutObjectRequest_ServerSideEncryption2() { final String value = ServerSideEncryption.AES256.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .serverSideEncryption(value) - .build(); + .serverSideEncryption(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.serverSideEncryption().toString()); @@ -324,8 +326,8 @@ void testConvertPutObjectRequest_ServerSideEncryption2() { void testConvertPutObjectRequest_SSECustomerAlgorithm() { final String value = "AES256"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .sseCustomerAlgorithm(value) - .build(); + .sseCustomerAlgorithm(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.sseCustomerAlgorithm()); @@ -335,8 +337,8 @@ void testConvertPutObjectRequest_SSECustomerAlgorithm() { void testConvertPutObjectRequest_SSECustomerKey() { final String value = "key123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .sseCustomerKey(value) - .build(); + .sseCustomerKey(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.sseCustomerKey()); @@ -346,8 +348,8 @@ void testConvertPutObjectRequest_SSECustomerKey() { void testConvertPutObjectRequest_SSEKMSKeyId() { final String value = "arn:aws:kms:region:123456789012:key/key-id"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .ssekmsKeyId(value) - .build(); + .ssekmsKeyId(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.ssekmsKeyId()); @@ -357,8 +359,8 @@ void testConvertPutObjectRequest_SSEKMSKeyId() { void testConvertPutObjectRequest_SSEKMSEncryptionContext() { final String value = "context123"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .ssekmsEncryptionContext(value) - .build(); + .ssekmsEncryptionContext(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.ssekmsEncryptionContext()); @@ -368,8 +370,8 @@ void testConvertPutObjectRequest_SSEKMSEncryptionContext() { void testConvertPutObjectRequest_StorageClass() { final StorageClass value = StorageClass.STANDARD; PutObjectRequest originalRequest = PutObjectRequest.builder() - .storageClass(value) - .build(); + .storageClass(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.storageClass()); @@ -379,8 +381,8 @@ void testConvertPutObjectRequest_StorageClass() { void testConvertPutObjectRequest_StorageClass2() { final String value = StorageClass.STANDARD.toString(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .storageClass(value) - .build(); + .storageClass(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.storageClass().toString()); @@ -390,8 +392,8 @@ void testConvertPutObjectRequest_StorageClass2() { void testConvertPutObjectRequest_Tagging() { final String value = "key1=value1&key2=value2"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .tagging(value) - .build(); + .tagging(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.tagging()); @@ -401,8 +403,8 @@ void testConvertPutObjectRequest_Tagging() { void testConvertPutObjectRequest_WebsiteRedirectLocation() { final String value = "/redirected"; PutObjectRequest originalRequest = PutObjectRequest.builder() - .websiteRedirectLocation(value) - .build(); + .websiteRedirectLocation(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertEquals(value, convertedRequest.websiteRedirectLocation()); @@ -411,12 +413,12 @@ void testConvertPutObjectRequest_WebsiteRedirectLocation() { @Test void testConvertPutObjectRequest_OverrideConfiguration() { final AwsRequestOverrideConfiguration value = AwsRequestOverrideConfiguration - .builder() - .apiCallAttemptTimeout(Duration.ofMillis(100)) - .build(); + .builder() + .apiCallAttemptTimeout(Duration.ofMillis(100)) + .build(); PutObjectRequest originalRequest = PutObjectRequest.builder() - .overrideConfiguration(value) - .build(); + .overrideConfiguration(value) + .build(); final CreateMultipartUploadRequest convertedRequest = ConvertSDKRequests.convertRequest(originalRequest); assertNotNull(convertedRequest); assertTrue(convertedRequest.overrideConfiguration().isPresent()); @@ -427,24 +429,24 @@ void testConvertPutObjectRequest_OverrideConfiguration() { public void testConvertResponse() { // Create a CompleteMultipartUploadResponse with various fields set CompleteMultipartUploadResponse completeResponse = CompleteMultipartUploadResponse.builder() - .eTag("test-etag") - .expiration("test-expiration") - .checksumCRC32("test-crc32") - .checksumCRC32C("test-crc32c") - .checksumCRC64NVME("test-crc64") - .checksumSHA1("test-sha1") - .checksumSHA256("test-sha256") - .checksumType(ChecksumType.COMPOSITE) - .serverSideEncryption(ServerSideEncryption.AWS_KMS) - .versionId("test-version-id") - .ssekmsKeyId("test-kms-key-id") - .bucketKeyEnabled(true) - .requestCharged("requester") - // Fields that should be ignored - .location("test-location") - .bucket("test-bucket") - .key("test-key") - .build(); + .eTag("test-etag") + .expiration("test-expiration") + .checksumCRC32("test-crc32") + .checksumCRC32C("test-crc32c") + .checksumCRC64NVME("test-crc64") + .checksumSHA1("test-sha1") + .checksumSHA256("test-sha256") + .checksumType(ChecksumType.COMPOSITE) + .serverSideEncryption(ServerSideEncryption.AWS_KMS) + .versionId("test-version-id") + .ssekmsKeyId("test-kms-key-id") + .bucketKeyEnabled(true) + .requestCharged("requester") + // Fields that should be ignored + .location("test-location") + .bucket("test-bucket") + .key("test-key") + .build(); // Convert the response PutObjectResponse putResponse = ConvertSDKRequests.convertResponse(completeResponse); @@ -475,9 +477,9 @@ public void testConvertResponse() { public void testBasicConvertMultipartUploadRequest() { // Create a MultipartUploadRequest with various fields set CreateMultipartUploadRequest request = CreateMultipartUploadRequest.builder() - .bucket("test-bucket") - .key("test-key") - .build(); + .bucket("test-bucket") + .key("test-key") + .build(); PutObjectRequest result = ConvertSDKRequests.convertRequest(request); assertEquals("test-bucket", result.bucket()); assertEquals("test-key", result.key()); @@ -495,36 +497,36 @@ public void testConversionAllFieldsMultipartUploadRequestToPutObjectRequest() { Instant retainUntilDate = Instant.now(); CreateMultipartUploadRequest request = CreateMultipartUploadRequest.builder() - .acl("test-acl") - .bucket("test-bucket") - .bucketKeyEnabled(true) - .cacheControl("test-cache-control") - .checksumAlgorithm("test-checksum-algorithm") - .contentDisposition("test-content-disposition") - .contentEncoding("test-content-encoding") - .contentLanguage("test-content-language") - .contentType("test-content-type") - .expectedBucketOwner("test-bucket-owner") - .expires(expires) - .grantFullControl("test-grant-full-control") - .grantRead("test-grant-read") - .grantReadACP("test-grant-read-acp") - .grantWriteACP("test-grant-write-acp") - .key("test-key") - .metadata(metadata) - .objectLockLegalHoldStatus(ObjectLockLegalHoldStatus.OFF) - .objectLockMode(ObjectLockMode.COMPLIANCE) - .objectLockRetainUntilDate(retainUntilDate) - .requestPayer(RequestPayer.REQUESTER) - .serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE) - .sseCustomerAlgorithm("test-sse-customer-algorithm") - .sseCustomerKey("test-sse-customer-key") - .ssekmsEncryptionContext("test-ssekms-encryption-context") - .ssekmsKeyId("test-ssekms-key-id") - .storageClass(StorageClass.SNOW) - .tagging("test-tagging") - .websiteRedirectLocation("test-website-redirect-location") - .build(); + .acl("test-acl") + .bucket("test-bucket") + .bucketKeyEnabled(true) + .cacheControl("test-cache-control") + .checksumAlgorithm("test-checksum-algorithm") + .contentDisposition("test-content-disposition") + .contentEncoding("test-content-encoding") + .contentLanguage("test-content-language") + .contentType("test-content-type") + .expectedBucketOwner("test-bucket-owner") + .expires(expires) + .grantFullControl("test-grant-full-control") + .grantRead("test-grant-read") + .grantReadACP("test-grant-read-acp") + .grantWriteACP("test-grant-write-acp") + .key("test-key") + .metadata(metadata) + .objectLockLegalHoldStatus(ObjectLockLegalHoldStatus.OFF) + .objectLockMode(ObjectLockMode.COMPLIANCE) + .objectLockRetainUntilDate(retainUntilDate) + .requestPayer(RequestPayer.REQUESTER) + .serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE) + .sseCustomerAlgorithm("test-sse-customer-algorithm") + .sseCustomerKey("test-sse-customer-key") + .ssekmsEncryptionContext("test-ssekms-encryption-context") + .ssekmsKeyId("test-ssekms-key-id") + .storageClass(StorageClass.SNOW) + .tagging("test-tagging") + .websiteRedirectLocation("test-website-redirect-location") + .build(); PutObjectRequest result = ConvertSDKRequests.convertRequest(request); assertEquals("test-acl", result.aclAsString()); assertEquals("test-bucket", result.bucket()); @@ -560,13 +562,13 @@ public void testConversionAllFieldsMultipartUploadRequestToPutObjectRequest() { @Test public void testConvertMultipartUploadRequestWithNullValues() { CreateMultipartUploadRequest request = CreateMultipartUploadRequest.builder() - .bucket("test-bucket") - .key("test-key") - .tagging("test-tagging") - .objectLockMode(ObjectLockMode.COMPLIANCE) - .contentLanguage("test-content-language") - .grantReadACP("test-grant-read-acp") - .build(); + .bucket("test-bucket") + .key("test-key") + .tagging("test-tagging") + .objectLockMode(ObjectLockMode.COMPLIANCE) + .contentLanguage("test-content-language") + .grantReadACP("test-grant-read-acp") + .build(); PutObjectRequest result = ConvertSDKRequests.convertRequest(request); assertEquals("test-bucket", result.bucket()); assertEquals("test-key", result.key()); From 0349dec4072161fdd9073e7711009f514bf83974 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Thu, 12 Jun 2025 15:16:39 -0700 Subject: [PATCH 24/24] removed unused import --- .../amazon/encryption/s3/internal/ConvertSDKRequests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java index 9337492ae..e4c72a2a6 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java @@ -4,7 +4,6 @@ import java.util.Map; import org.apache.commons.logging.LogFactory; -import software.amazon.awssdk.services.s3.model.ChecksumType; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest;