diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java index a67bd28317a9..d2e0a362f1b6 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java @@ -19,11 +19,14 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.util.List; import javax.ws.rs.core.Response; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.http.HttpStatus; import org.apache.ratis.util.function.CheckedSupplier; @@ -104,6 +107,67 @@ public static Response deleteTagging( return subject.delete(bucket, key, null, ""); } + /** Initiate multipart upload. + * @return upload ID */ + public static String initiateMultipartUpload(ObjectEndpoint subject, String bucket, String key) + throws IOException, OS3Exception { + try (Response response = subject.initializeMultipartUpload(bucket, key)) { + assertEquals(HttpStatus.SC_OK, response.getStatus()); + MultipartUploadInitiateResponse entity = (MultipartUploadInitiateResponse) response.getEntity(); + String uploadID = entity.getUploadID(); + assertNotNull(uploadID, "uploadID == null"); + return uploadID; + } + } + + /** Upload part of multipart key. + * @return Part to be used for completion request */ + public static CompleteMultipartUploadRequest.Part uploadPart( + ObjectEndpoint subject, + String bucket, + String key, + int partNumber, + String uploadID, + String content + ) throws IOException, OS3Exception { + CompleteMultipartUploadRequest.Part part = new CompleteMultipartUploadRequest.Part(); + + try (Response response = put(subject, bucket, key, partNumber, uploadID, content)) { + assertEquals(HttpStatus.SC_OK, response.getStatus()); + String eTag = response.getHeaderString(OzoneConsts.ETAG); + assertNotNull(eTag); + part.setETag(eTag); + } + + part.setPartNumber(partNumber); + + return part; + } + + /** Complete multipart upload. */ + public static void completeMultipartUpload( + ObjectEndpoint subject, + String bucket, + String key, + String uploadID, + List parts + ) throws IOException, OS3Exception { + CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(); + completeMultipartUploadRequest.setPartList(parts); + + try (Response response = subject.completeMultipartUpload(bucket, key, uploadID, completeMultipartUploadRequest)) { + assertEquals(HttpStatus.SC_OK, response.getStatus()); + + CompleteMultipartUploadResponse completeMultipartUploadResponse = + (CompleteMultipartUploadResponse) response.getEntity(); + + assertEquals(bucket, completeMultipartUploadResponse.getBucket()); + assertEquals(key, completeMultipartUploadResponse.getKey()); + assertEquals(bucket, completeMultipartUploadResponse.getLocation()); + assertNotNull(completeMultipartUploadResponse.getETag()); + } + } + /** Verify response is success for {@code request}. */ public static void assertSucceeds(CheckedSupplier request) throws E { try (Response response = request.get()) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java index 9c46a718508f..518275c7026e 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java @@ -17,9 +17,9 @@ package org.apache.hadoop.ozone.s3.endpoint; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -55,17 +55,10 @@ public void testAbortMultipartUpload() throws Exception { .build(); - Response response = rest.initializeMultipartUpload(bucket, key); - - assertEquals(200, response.getStatus()); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); - + String uploadID = initiateMultipartUpload(rest, bucket, key); // Abort multipart upload - response = rest.delete(bucket, key, uploadID, null); + Response response = rest.delete(bucket, key, uploadID, null); assertEquals(204, response.getStatus()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java index 71113cec45ae..a425c166af4e 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java @@ -17,16 +17,14 @@ package org.apache.hadoop.ozone.s3.endpoint; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import jakarta.annotation.Nonnull; import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.Response; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.OzoneClient; @@ -52,21 +50,11 @@ public void testInitiateMultipartUpload() throws Exception { ObjectEndpoint rest = getObjectEndpoint(client, headers); - Response response = rest.initializeMultipartUpload(bucket, key); - - assertEquals(200, response.getStatus()); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); + String uploadID = initiateMultipartUpload(rest, bucket, key); // Calling again should return different uploadID. - response = rest.initializeMultipartUpload(bucket, key); - assertEquals(200, response.getStatus()); - multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - assertNotEquals(multipartUploadInitiateResponse.getUploadID(), uploadID); + String nextID = initiateMultipartUpload(rest, bucket, key); + assertNotEquals(uploadID, nextID); } @Test @@ -79,12 +67,7 @@ public void testInitiateMultipartUploadWithECKey() throws Exception { ObjectEndpoint rest = getObjectEndpoint(client, headers); client.getObjectStore().getS3Bucket(bucket) .setReplicationConfig(new ECReplicationConfig("rs-3-2-1024K")); - Response response = rest.initializeMultipartUpload(bucket, key); - - assertEquals(200, response.getStatus()); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); + initiateMultipartUpload(rest, bucket, key); } @Nonnull diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java index 30be715b5305..f42c5849aa2e 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java @@ -17,17 +17,16 @@ package org.apache.hadoop.ozone.s3.endpoint; -import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.uploadPart; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import org.apache.hadoop.ozone.OzoneConsts; @@ -63,32 +62,11 @@ public void setUp() throws Exception { .setClient(client) .build(); - Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET, - OzoneConsts.KEY); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - uploadID = multipartUploadInitiateResponse.getUploadID(); + uploadID = initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY); - assertEquals(200, response.getStatus()); - - String content = "Multipart Upload"; - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, - content.length(), 1, uploadID, null, null, body); - - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); - - response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, - content.length(), 2, uploadID, null, null, body); - - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); - - response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, - content.length(), 3, uploadID, null, null, body); - - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); + for (int i = 1; i <= 3; i++) { + uploadPart(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, i, uploadID, "Multipart Upload"); + } } @Test diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java index fde336f48079..53500bbd2e54 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java @@ -17,20 +17,19 @@ package org.apache.hadoop.ozone.s3.endpoint; -import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.singletonList; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.completeMultipartUpload; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.uploadPart; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -76,7 +75,7 @@ public void setUp() throws Exception { private String initiateMultipartUpload(String key) throws IOException, OS3Exception { - return initiateMultipartUpload(key, Collections.emptyMap()); + return EndpointTestUtils.initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET, key); } private String initiateMultipartUpload(String key, Map metadata) throws IOException, @@ -90,82 +89,19 @@ private String initiateMultipartUpload(String key, Map metadata) when(headers.getRequestHeaders()).thenReturn(metadataHeaders); - Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET, - key); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); - - assertEquals(200, response.getStatus()); - - return uploadID; - } - - private Part uploadPart(String key, String uploadID, int partNumber, String - content) throws IOException, OS3Exception { - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - Response response = rest.put(OzoneConsts.S3_BUCKET, key, content.length(), - partNumber, uploadID, null, null, body); - assertEquals(200, response.getStatus()); - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); - Part part = new Part(); - part.setETag(response.getHeaderString(OzoneConsts.ETAG)); - part.setPartNumber(partNumber); - - return part; - } - - private void completeMultipartUpload(String key, - CompleteMultipartUploadRequest completeMultipartUploadRequest, - String uploadID) throws IOException, OS3Exception { - Response response = rest.completeMultipartUpload(OzoneConsts.S3_BUCKET, key, - uploadID, completeMultipartUploadRequest); - - assertEquals(200, response.getStatus()); - - CompleteMultipartUploadResponse completeMultipartUploadResponse = - (CompleteMultipartUploadResponse) response.getEntity(); - - assertEquals(OzoneConsts.S3_BUCKET, - completeMultipartUploadResponse.getBucket()); - assertEquals(key, completeMultipartUploadResponse.getKey()); - assertEquals(OzoneConsts.S3_BUCKET, - completeMultipartUploadResponse.getLocation()); - assertNotNull(completeMultipartUploadResponse.getETag()); + return EndpointTestUtils.initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET, key); } @Test public void testMultipart() throws Exception { - - // Initiate multipart upload String uploadID = initiateMultipartUpload(OzoneConsts.KEY); - List partsList = new ArrayList<>(); - - // Upload parts - String content = "Multipart Upload 1"; - int partNumber = 1; - - Part part1 = uploadPart(OzoneConsts.KEY, uploadID, partNumber, content); - partsList.add(part1); - - content = "Multipart Upload 2"; - partNumber = 2; - Part part2 = uploadPart(OzoneConsts.KEY, uploadID, partNumber, content); - partsList.add(part2); - - // complete multipart upload - CompleteMultipartUploadRequest completeMultipartUploadRequest = new - CompleteMultipartUploadRequest(); - completeMultipartUploadRequest.setPartList(partsList); - - - completeMultipartUpload(OzoneConsts.KEY, completeMultipartUploadRequest, - uploadID); + List partsList = new ArrayList<>(); + partsList.add(uploadPart(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 1, uploadID, "Multipart Upload 1")); + partsList.add(uploadPart(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 2, uploadID, "Multipart Upload 2")); + completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, uploadID, partsList); } @Test @@ -177,21 +113,8 @@ public void testMultipartWithCustomMetadata() throws Exception { customMetadata.put("custom-key2", "custom-value2"); String uploadID = initiateMultipartUpload(key, customMetadata); - - List partsList = new ArrayList<>(); - - // Upload parts - String content = "Multipart Upload 1"; - int partNumber = 1; - - Part part1 = uploadPart(key, uploadID, partNumber, content); - partsList.add(part1); - - CompleteMultipartUploadRequest completeMultipartUploadRequest = new - CompleteMultipartUploadRequest(); - completeMultipartUploadRequest.setPartList(partsList); - - completeMultipartUpload(key, completeMultipartUploadRequest, uploadID); + Part part1 = uploadPart(rest, OzoneConsts.S3_BUCKET, key, 1, uploadID, "Multipart Upload 1"); + completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, key, uploadID, singletonList(part1)); Response headResponse = rest.head(OzoneConsts.S3_BUCKET, key); @@ -209,27 +132,17 @@ public void testMultipartInvalidPartOrderError() throws Exception { List partsList = new ArrayList<>(); // Upload parts - String content = "Multipart Upload 1"; - int partNumber = 1; - Part part1 = uploadPart(key, uploadID, partNumber, content); + Part part1 = uploadPart(rest, OzoneConsts.S3_BUCKET, key, 1, uploadID, "Multipart Upload 1"); // Change part number part1.setPartNumber(3); partsList.add(part1); - content = "Multipart Upload 2"; - partNumber = 2; - - Part part2 = uploadPart(key, uploadID, partNumber, content); - partsList.add(part2); + partsList.add(uploadPart(rest, OzoneConsts.S3_BUCKET, key, 2, uploadID, "Multipart Upload 2")); // complete multipart upload - CompleteMultipartUploadRequest completeMultipartUploadRequest = new - CompleteMultipartUploadRequest(); - completeMultipartUploadRequest.setPartList(partsList); - OS3Exception ex = - assertThrows(OS3Exception.class, - () -> completeMultipartUpload(key, completeMultipartUploadRequest, uploadID)); + OS3Exception ex = assertThrows(OS3Exception.class, + () -> completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, key, uploadID, partsList)); assertEquals(S3ErrorTable.INVALID_PART_ORDER.getCode(), ex.getCode()); } @@ -243,27 +156,16 @@ public void testMultipartInvalidPartError() throws Exception { List partsList = new ArrayList<>(); // Upload parts - String content = "Multipart Upload 1"; - int partNumber = 1; - - Part part1 = uploadPart(key, uploadID, partNumber, content); + Part part1 = uploadPart(rest, OzoneConsts.S3_BUCKET, key, 1, uploadID, "Multipart Upload 1"); // Change part name. part1.setETag("random"); partsList.add(part1); - content = "Multipart Upload 2"; - partNumber = 2; - - Part part2 = uploadPart(key, uploadID, partNumber, content); - partsList.add(part2); + partsList.add(uploadPart(rest, OzoneConsts.S3_BUCKET, key, 2, uploadID, "Multipart Upload 2")); // complete multipart upload - CompleteMultipartUploadRequest completeMultipartUploadRequest = new - CompleteMultipartUploadRequest(); - completeMultipartUploadRequest.setPartList(partsList); - OS3Exception ex = - assertThrows(OS3Exception.class, - () -> completeMultipartUpload(key, completeMultipartUploadRequest, uploadID)); + OS3Exception ex = assertThrows(OS3Exception.class, + () -> completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, key, uploadID, partsList)); assertEquals(ex.getCode(), S3ErrorTable.INVALID_PART.getCode()); } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java index 702c32d1abab..8bb2cf3fa623 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java @@ -18,6 +18,11 @@ package org.apache.hadoop.ozone.s3.endpoint; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.completeMultipartUpload; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.uploadPart; import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER_RANGE; import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_IF_MODIFIED_SINCE; @@ -30,7 +35,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -132,14 +136,15 @@ public static void setUp() throws Exception { @Test public void testMultipart() throws Exception { // Initiate multipart upload - String uploadID = initiateMultipartUpload(KEY); + String uploadID = initiateMultipartUpload(endpoint, OzoneConsts.S3_BUCKET, KEY); List partsList = new ArrayList<>(); // Upload parts String content = "Multipart Upload 1"; - Part part1 = uploadPart(KEY, uploadID, 1, content); + setHeaders(); + Part part1 = uploadPart(endpoint, OzoneConsts.S3_BUCKET, KEY, 1, uploadID, content); partsList.add(part1); Part part2 = @@ -160,13 +165,8 @@ public void testMultipart() throws Exception { ); partsList.add(part4); - // complete multipart upload - CompleteMultipartUploadRequest completeMultipartUploadRequest = new - CompleteMultipartUploadRequest(); - completeMultipartUploadRequest.setPartList(partsList); - - completeMultipartUpload(KEY, completeMultipartUploadRequest, - uploadID); + setHeaders(); + completeMultipartUpload(endpoint, OzoneConsts.S3_BUCKET, TestMultipartUploadWithCopy.KEY, uploadID, partsList); OzoneBucket bucket = client.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET); @@ -309,38 +309,6 @@ public void testMultipartTSHeaders() throws Exception { } } - private String initiateMultipartUpload(String key) throws IOException, - OS3Exception { - setHeaders(); - Response response = endpoint.initializeMultipartUpload(OzoneConsts.S3_BUCKET, - key); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); - - assertEquals(200, response.getStatus()); - - return uploadID; - - } - - private Part uploadPart(String key, String uploadID, int partNumber, String - content) throws IOException, OS3Exception { - setHeaders(); - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - Response response = endpoint.put(OzoneConsts.S3_BUCKET, key, content.length(), - partNumber, uploadID, null, null, body); - assertEquals(200, response.getStatus()); - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); - Part part = new Part(); - part.setETag(response.getHeaderString(OzoneConsts.ETAG)); - part.setPartNumber(partNumber); - - return part; - } - private Part uploadPartWithCopy(String key, String uploadID, int partNumber, String keyOrigin, String range) throws IOException, OS3Exception { return uploadPartWithCopy(key, uploadID, partNumber, keyOrigin, @@ -350,7 +318,8 @@ private Part uploadPartWithCopy(String key, String uploadID, int partNumber, private Part uploadPartWithCopy(String ifModifiedSinceStr, String ifUnmodifiedSinceStr) throws IOException, OS3Exception { // Initiate multipart upload - String uploadID = initiateMultipartUpload(KEY); + setHeaders(); + String uploadID = initiateMultipartUpload(endpoint, OzoneConsts.S3_BUCKET, KEY); return uploadPartWithCopy(KEY, uploadID, 1, OzoneConsts.S3_BUCKET + "/" + EXISTING_KEY, "bytes=0-3", @@ -374,19 +343,18 @@ private Part uploadPartWithCopy(String key, String uploadID, int partNumber, } setHeaders(additionalHeaders); - ByteArrayInputStream body = new ByteArrayInputStream("".getBytes(UTF_8)); - Response response = endpoint.put(OzoneConsts.S3_BUCKET, key, 0, partNumber, - uploadID, null, null, body); - assertEquals(200, response.getStatus()); + try (Response response = put(endpoint, OzoneConsts.S3_BUCKET, key, partNumber, uploadID, "")) { + assertEquals(200, response.getStatus()); - CopyPartResult result = (CopyPartResult) response.getEntity(); - assertNotNull(result.getETag()); - assertNotNull(result.getLastModified()); - Part part = new Part(); - part.setETag(result.getETag()); - part.setPartNumber(partNumber); + CopyPartResult result = (CopyPartResult) response.getEntity(); + assertNotNull(result.getETag()); + assertNotNull(result.getLastModified()); + Part part = new Part(); + part.setETag(result.getETag()); + part.setPartNumber(partNumber); - return part; + return part; + } } @Test @@ -396,14 +364,13 @@ public void testUploadWithRangeCopyContentLength() // the Content-Length, the key Commit will compare the Content-Length with // the actual length of the data written. - String uploadID = initiateMultipartUpload(KEY); - ByteArrayInputStream body = new ByteArrayInputStream("".getBytes(UTF_8)); + String uploadID = initiateMultipartUpload(endpoint, OzoneConsts.S3_BUCKET, KEY); Map additionalHeaders = new HashMap<>(); additionalHeaders.put(COPY_SOURCE_HEADER, OzoneConsts.S3_BUCKET + "/" + EXISTING_KEY); additionalHeaders.put(COPY_SOURCE_HEADER_RANGE, "bytes=0-3"); setHeaders(additionalHeaders); - endpoint.put(OzoneConsts.S3_BUCKET, KEY, 0, 1, uploadID, null, null, body); + assertSucceeds(() -> put(endpoint, OzoneConsts.S3_BUCKET, KEY, 1, uploadID, "")); OzoneMultipartUploadPartListParts parts = client.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET) .listParts(KEY, uploadID, 0, 100); @@ -411,26 +378,6 @@ public void testUploadWithRangeCopyContentLength() assertEquals(4, parts.getPartInfoList().get(0).getSize()); } - private void completeMultipartUpload(String key, - CompleteMultipartUploadRequest completeMultipartUploadRequest, - String uploadID) throws IOException, OS3Exception { - setHeaders(); - Response response = endpoint.completeMultipartUpload(OzoneConsts.S3_BUCKET, key, - uploadID, completeMultipartUploadRequest); - - assertEquals(200, response.getStatus()); - - CompleteMultipartUploadResponse completeMultipartUploadResponse = - (CompleteMultipartUploadResponse) response.getEntity(); - - assertEquals(OzoneConsts.S3_BUCKET, - completeMultipartUploadResponse.getBucket()); - assertEquals(KEY, completeMultipartUploadResponse.getKey()); - assertEquals(OzoneConsts.S3_BUCKET, - completeMultipartUploadResponse.getLocation()); - assertNotNull(completeMultipartUploadResponse.getETag()); - } - private void setHeaders(Map additionalHeaders) { HttpHeaders headers = mock(HttpHeaders.class); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index 7c1352e59dd2..ee2158d73340 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -17,8 +17,9 @@ package org.apache.hadoop.ozone.s3.endpoint; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER; @@ -31,18 +32,16 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.time.format.DateTimeFormatter; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; -import org.apache.commons.io.IOUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.client.OzoneClientTestUtils; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -72,13 +71,14 @@ public class TestObjectGet { private HttpHeaders headers; private ObjectEndpoint rest; - private OzoneClient client; + private OzoneBucket bucket; @BeforeEach public void init() throws OS3Exception, IOException { //GIVEN - client = new OzoneClientStub(); + OzoneClient client = new OzoneClientStub(); client.getObjectStore().createS3Bucket(BUCKET_NAME); + bucket = client.getObjectStore().getS3Bucket(BUCKET_NAME); headers = mock(HttpHeaders.class); when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("UNSIGNED-PAYLOAD"); @@ -88,13 +88,11 @@ public void init() throws OS3Exception, IOException { .setHeaders(headers) .build(); - ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); - rest.put(BUCKET_NAME, KEY_NAME, CONTENT.length(), - 1, null, null, null, body); + assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_NAME, CONTENT)); + // Create a key with object tags when(headers.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag2=value2"); - rest.put(BUCKET_NAME, KEY_WITH_TAG, CONTENT.length(), - 1, null, null, null, body); + assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_WITH_TAG, CONTENT)); } @Test @@ -103,14 +101,8 @@ public void get() throws IOException, OS3Exception { Response response = rest.get(BUCKET_NAME, KEY_NAME, 0, null, 0, null, null); //THEN - OzoneInputStream ozoneInputStream = - client.getObjectStore().getS3Bucket(BUCKET_NAME) - .readKey(KEY_NAME); - String keyContent = - IOUtils.toString(ozoneInputStream, UTF_8); - - assertEquals(CONTENT, keyContent); - assertEquals(String.valueOf(keyContent.length()), + OzoneClientTestUtils.assertKeyContent(bucket, KEY_NAME, CONTENT); + assertEquals(String.valueOf(CONTENT.length()), response.getHeaderString("Content-Length")); DateTimeFormatter.RFC_1123_DATE_TIME @@ -125,14 +117,8 @@ public void getKeyWithTag() throws IOException, OS3Exception { Response response = rest.get(BUCKET_NAME, KEY_WITH_TAG, 0, null, 0, null, null); //THEN - OzoneInputStream ozoneInputStream = - client.getObjectStore().getS3Bucket(BUCKET_NAME) - .readKey(KEY_NAME); - String keyContent = - IOUtils.toString(ozoneInputStream, UTF_8); - - assertEquals(CONTENT, keyContent); - assertEquals(String.valueOf(keyContent.length()), + OzoneClientTestUtils.assertKeyContent(bucket, KEY_WITH_TAG, CONTENT); + assertEquals(String.valueOf(CONTENT.length()), response.getHeaderString("Content-Length")); DateTimeFormatter.RFC_1123_DATE_TIME @@ -251,7 +237,6 @@ public void testGetWhenKeyIsDirectoryAndDoesNotEndWithASlash() OzoneConfiguration config = new OzoneConfiguration(); config.set(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, "true"); rest.setOzoneConfiguration(config); - OzoneBucket bucket = client.getObjectStore().getS3Bucket(BUCKET_NAME); bucket.createDirectory(keyPath); // WHEN diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java index 2da2a42a05b6..0ac2508565ce 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java @@ -17,17 +17,19 @@ package org.apache.hadoop.ozone.s3.endpoint; -import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; import static org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; @@ -36,7 +38,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -45,211 +46,131 @@ import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import org.apache.commons.io.IOUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.MockedStatic; /** * This class tests Upload part request. */ +@ParameterizedClass +@ValueSource(booleans = {false, true}) public class TestPartUpload { private ObjectEndpoint rest; private OzoneClient client; + private HttpHeaders headers; + + @Parameter + private boolean enableDataStream; + @BeforeEach public void setUp() throws Exception { - client = new OzoneClientStub(); client.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET); - - HttpHeaders headers = mock(HttpHeaders.class); + headers = mock(HttpHeaders.class); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( "STANDARD"); when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) .thenReturn("mockSignature"); - rest = EndpointBuilder.newObjectEndpointBuilder() + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, enableDataStream); + + rest = spy(EndpointBuilder.newObjectEndpointBuilder() .setHeaders(headers) .setClient(client) - .build(); + .setConfig(conf) + .build()); + assertEquals(enableDataStream, rest.isDatastreamEnabled()); } @Test public void testPartUpload() throws Exception { - - Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET, - OzoneConsts.KEY); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); - - assertEquals(200, response.getStatus()); + String uploadID = initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY); String content = "Multipart Upload"; - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, - content.length(), 1, uploadID, null, null, body); - - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); - - } - - @Test - public void testPartUploadWithOverride() throws Exception { - - Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET, - OzoneConsts.KEY); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); - - assertEquals(200, response.getStatus()); - - String content = "Multipart Upload"; - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, - content.length(), 1, uploadID, null, null, body); - - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); - - String eTag = response.getHeaderString(OzoneConsts.ETAG); + String eTag; + try (Response response = put(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 1, uploadID, content)) { + eTag = response.getHeaderString(OzoneConsts.ETAG); + assertNotNull(eTag); + } + assertContentLength(uploadID, OzoneConsts.KEY, content.length()); // Upload part again with same part Number, the ETag should be changed. - content = "Multipart Upload Changed"; - response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, - content.length(), 1, uploadID, null, null, body); - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); - assertNotEquals(eTag, response.getHeaderString(OzoneConsts.ETAG)); - + String newContent = "Multipart Upload Changed"; + try (Response response = put(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 1, uploadID, newContent)) { + String newETag = response.getHeaderString(OzoneConsts.ETAG); + assertNotNull(newETag); + assertNotEquals(eTag, newETag); + } } @Test - public void testPartUploadWithIncorrectUploadID() throws Exception { - OS3Exception ex = assertThrows(OS3Exception.class, () -> { - String content = "Multipart Upload With Incorrect uploadID"; - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, content.length(), 1, - "random", null, null, body); - }); - assertEquals("NoSuchUpload", ex.getCode()); - assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); + public void testPartUploadWithIncorrectUploadID() { + assertErrorResponse(S3ErrorTable.NO_SUCH_UPLOAD, + () -> put(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 1, "random", "any")); } @Test public void testPartUploadStreamContentLength() throws IOException, OS3Exception { - HttpHeaders headers = mock(HttpHeaders.class); - when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) - .thenReturn("mockSignature"); - ObjectEndpoint objectEndpoint = EndpointBuilder.newObjectEndpointBuilder() - .setHeaders(headers) - .setClient(client) - .build(); String keyName = UUID.randomUUID().toString(); + int contentLength = 15; String chunkedContent = "0a;chunk-signature=signature\r\n" + "1234567890\r\n" + "05;chunk-signature=signature\r\n" + "abcde\r\n"; - when(headers.getHeaderString("x-amz-content-sha256")) + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) .thenReturn("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"); when(headers.getHeaderString(DECODED_CONTENT_LENGTH_HEADER)) - .thenReturn("15"); + .thenReturn(String.valueOf(contentLength)); - Response response = objectEndpoint.initializeMultipartUpload( - OzoneConsts.S3_BUCKET, keyName); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); - long contentLength = chunkedContent.length(); + String uploadID = initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET, keyName); - objectEndpoint.put(OzoneConsts.S3_BUCKET, keyName, contentLength, 1, - uploadID, null, null, new ByteArrayInputStream(chunkedContent.getBytes(UTF_8))); - assertContentLength(uploadID, keyName, 15); - } - - @Test - public void testPartUploadContentLength() throws IOException, OS3Exception { - // The contentLength specified when creating the Key should be the same as - // the Content-Length, the key Commit will compare the Content-Length with - // the actual length of the data written. - - String keyName = UUID.randomUUID().toString(); - Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET, - keyName); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); - String content = "Multipart Upload"; - long contentLength = content.length(); - - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - rest.put(OzoneConsts.S3_BUCKET, keyName, - contentLength, 1, uploadID, null, null, body); - assertContentLength(uploadID, keyName, content.length()); + assertSucceeds(() -> put(rest, OzoneConsts.S3_BUCKET, keyName, 1, uploadID, chunkedContent)); + assertContentLength(uploadID, keyName, contentLength); } @Test public void testPartUploadMessageDigestResetDuringException() throws IOException, OS3Exception { - OzoneClient clientStub = new OzoneClientStub(); - clientStub.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET); - - - HttpHeaders headers = mock(HttpHeaders.class); - when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) - .thenReturn("mockSignature"); - when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( - "STANDARD"); - - ObjectEndpoint objectEndpoint = EndpointBuilder.newObjectEndpointBuilder() - .setHeaders(headers) - .setClient(clientStub) - .build(); - - objectEndpoint = spy(objectEndpoint); - - Response response = objectEndpoint.initializeMultipartUpload(OzoneConsts.S3_BUCKET, - OzoneConsts.KEY); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); - - assertEquals(200, response.getStatus()); + String uploadID = initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY); MessageDigest messageDigest = mock(MessageDigest.class); when(messageDigest.getAlgorithm()).thenReturn("MD5"); MessageDigest sha256Digest = mock(MessageDigest.class); when(sha256Digest.getAlgorithm()).thenReturn("SHA-256"); - try (MockedStatic mocked = mockStatic(IOUtils.class)) { - // Add the mocked methods only during the copy request - when(objectEndpoint.getMessageDigestInstance()).thenReturn(messageDigest); - when(objectEndpoint.getSha256DigestInstance()).thenReturn(sha256Digest); - mocked.when(() -> IOUtils.copyLarge(any(InputStream.class), any(OutputStream.class), anyLong(), - anyLong(), any(byte[].class))) - .thenThrow(IOException.class); + try (MockedStatic ioutils = mockStatic(IOUtils.class); + MockedStatic streaming = mockStatic(ObjectEndpointStreaming.class)) { + // Add the mocked methods only during part upload + when(rest.getMessageDigestInstance()).thenReturn(messageDigest); + when(rest.getSha256DigestInstance()).thenReturn(sha256Digest); + if (enableDataStream) { + streaming.when(() -> ObjectEndpointStreaming.createMultipartKey(any(), any(), anyLong(), anyInt(), any(), + anyInt(), any(), any())) + .thenThrow(IOException.class); + } else { + ioutils.when(() -> IOUtils.copyLarge(any(InputStream.class), any(OutputStream.class), anyLong(), + anyLong(), any(byte[].class))) + .thenThrow(IOException.class); + } String content = "Multipart Upload"; - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - try { - objectEndpoint.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, - content.length(), 1, uploadID, null, null, body); + try (Response ignored = put(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 1, uploadID, content)) { fail("Should throw IOException"); } catch (IOException ignored) { // Verify that the message digest is reset so that the instance can be reused for the diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java deleted file mode 100644 index 736660073d57..000000000000 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.s3.endpoint; - -import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; -import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.ByteArrayInputStream; -import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientStub; -import org.apache.hadoop.ozone.s3.exception.OS3Exception; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * This class tests Upload part request. - */ -public class TestPartUploadWithStream { - - private ObjectEndpoint rest; - - private static final String S3BUCKET = "streampartb1"; - private static final String S3KEY = "testkey"; - - @BeforeEach - public void setUp() throws Exception { - OzoneClient client = new OzoneClientStub(); - client.getObjectStore().createS3Bucket(S3BUCKET); - - - HttpHeaders headers = mock(HttpHeaders.class); - when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD"); - when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) - .thenReturn("mockSignature"); - - - OzoneConfiguration conf = new OzoneConfiguration(); - conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, - true); - - rest = EndpointBuilder.newObjectEndpointBuilder() - .setHeaders(headers) - .setClient(client) - .setConfig(conf) - .build(); - } - - @Test - public void testEnableStream() { - assertTrue(rest.isDatastreamEnabled()); - } - - @Test - public void testPartUpload() throws Exception { - - Response response = rest.initializeMultipartUpload(S3BUCKET, S3KEY); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); - - assertEquals(200, response.getStatus()); - - String content = "Multipart Upload"; - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - response = rest.put(S3BUCKET, S3KEY, - content.length(), 1, uploadID, null, null, body); - - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); - - } - - @Test - public void testPartUploadWithOverride() throws Exception { - - Response response = rest.initializeMultipartUpload(S3BUCKET, S3KEY); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - assertNotNull(multipartUploadInitiateResponse.getUploadID()); - String uploadID = multipartUploadInitiateResponse.getUploadID(); - - assertEquals(200, response.getStatus()); - - String content = "Multipart Upload"; - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - response = rest.put(S3BUCKET, S3KEY, - content.length(), 1, uploadID, null, null, body); - - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); - - String eTag = response.getHeaderString(OzoneConsts.ETAG); - - // Upload part again with same part Number, the ETag should be changed. - content = "Multipart Upload Changed"; - response = rest.put(S3BUCKET, S3KEY, - content.length(), 1, uploadID, null, null, body); - assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); - assertNotEquals(eTag, response.getHeaderString(OzoneConsts.ETAG)); - - } - - @Test - public void testPartUploadWithIncorrectUploadID() throws Exception { - OS3Exception ex = assertThrows(OS3Exception.class, () -> { - String content = "Multipart Upload With Incorrect uploadID"; - ByteArrayInputStream body = - new ByteArrayInputStream(content.getBytes(UTF_8)); - rest.put(S3BUCKET, S3KEY, content.length(), 1, - "random", null, null, body); - }); - assertEquals("NoSuchUpload", ex.getCode()); - assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); - } -} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java index e9d70f679824..66d4a4cbef66 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java @@ -19,6 +19,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; @@ -27,12 +29,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; import java.io.OutputStream; import java.util.HashMap; import java.util.Map; import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.Response; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; @@ -89,12 +89,7 @@ public void testEnableStream() { @Test public void testUpload() throws Exception { - byte[] keyContent = S3_COPY_EXISTING_KEY_CONTENT.getBytes(UTF_8); - ByteArrayInputStream body = - new ByteArrayInputStream(keyContent); - Response response = rest.put(S3BUCKET, S3KEY, 0, 0, null, null, null, body); - - assertEquals(200, response.getStatus()); + assertSucceeds(() -> put(rest, S3BUCKET, S3KEY, S3_COPY_EXISTING_KEY_CONTENT)); } @Test @@ -126,9 +121,7 @@ public void testUploadWithCopy() throws Exception { .forEach((k, v) -> when(headers.getHeaderString(k)).thenReturn(v)); rest.setHeaders(headers); - Response response = rest.put(S3BUCKET, S3KEY, 0, 0, null, null, null, null); - - assertEquals(200, response.getStatus()); + assertSucceeds(() -> put(rest, S3BUCKET, S3KEY, null)); final long newDataSize = bucket.getKey(S3KEY).getDataSize(); assertEquals(dataSize, newDataSize); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java index 8aa4ba707efa..d01acbe708b9 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java @@ -51,7 +51,7 @@ import org.apache.hadoop.ozone.s3.endpoint.BucketEndpoint; import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest; import org.apache.hadoop.ozone.s3.endpoint.EndpointBuilder; -import org.apache.hadoop.ozone.s3.endpoint.MultipartUploadInitiateResponse; +import org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils; import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint; import org.apache.hadoop.ozone.s3.endpoint.RootEndpoint; import org.apache.hadoop.ozone.s3.endpoint.TestBucketAcl; @@ -357,7 +357,7 @@ public void testGetKeyFailure() { public void testInitMultiPartUploadSuccess() throws Exception { long oriMetric = metrics.getInitMultiPartUploadSuccess(); - keyEndpoint.initializeMultipartUpload(bucketName, keyName); + EndpointTestUtils.initiateMultipartUpload(keyEndpoint, bucketName, keyName); long curMetric = metrics.getInitMultiPartUploadSuccess(); assertEquals(1L, curMetric - oriMetric); } @@ -560,20 +560,8 @@ public void testDeleteObjectTaggingFailure() { assertEquals(1L, curMetric - oriMetric); } - private String initiateMultipartUpload(String bktName, String key) - throws IOException, - OS3Exception { - // Initiate the Upload - Response response = - keyEndpoint.initializeMultipartUpload(bktName, key); - MultipartUploadInitiateResponse multipartUploadInitiateResponse = - (MultipartUploadInitiateResponse) response.getEntity(); - if (response.getStatus() == 200) { - // Fetch the Upload-Id - String uploadID = multipartUploadInitiateResponse.getUploadID(); - return uploadID; - } - return "Invalid-Id"; + private String initiateMultipartUpload(String bktName, String key) throws IOException, OS3Exception { + return EndpointTestUtils.initiateMultipartUpload(keyEndpoint, bktName, key); } private static String getPutTaggingBody() {