diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index db5d12e52f6a..1177c01e5dee 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -104,32 +104,30 @@ public class BucketEndpoint extends EndpointBase { * for more details. */ @GET - @SuppressWarnings({"parameternumber", "methodlength"}) + @SuppressWarnings("methodlength") public Response get( @PathParam(BUCKET) String bucketName, - @QueryParam(QueryParams.DELIMITER) String delimiter, - @QueryParam(QueryParams.ENCODING_TYPE) String encodingType, - @QueryParam(QueryParams.MARKER) String marker, @DefaultValue("1000") @QueryParam(QueryParams.MAX_KEYS) int maxKeys, - @QueryParam(QueryParams.PREFIX) String prefix, - @QueryParam(QueryParams.CONTINUATION_TOKEN) String continueToken, - @QueryParam(QueryParams.START_AFTER) String startAfter, - @QueryParam(QueryParams.UPLOADS) String uploads, - @QueryParam(QueryParams.ACL) String aclMarker, - @QueryParam(QueryParams.KEY_MARKER) String keyMarker, - @QueryParam(QueryParams.UPLOAD_ID_MARKER) String uploadIdMarker, @DefaultValue("1000") @QueryParam(QueryParams.MAX_UPLOADS) int maxUploads ) throws OS3Exception, IOException { long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.GET_BUCKET; PerformanceStringBuilder perf = new PerformanceStringBuilder(); + final String continueToken = getQueryParam(QueryParams.CONTINUATION_TOKEN); + final String delimiter = getQueryParam(QueryParams.DELIMITER); + final String encodingType = getQueryParam(QueryParams.ENCODING_TYPE); + final String marker = getQueryParam(QueryParams.MARKER); + String prefix = getQueryParam(QueryParams.PREFIX); + String startAfter = getQueryParam(QueryParams.START_AFTER); + Iterator ozoneKeyIterator = null; ContinueToken decodedToken = ContinueToken.decodeFromString(continueToken); OzoneBucket bucket = null; try { + final String aclMarker = getQueryParam(QueryParams.ACL); if (aclMarker != null) { s3GAction = S3GAction.GET_ACL; S3BucketAcl result = getAcl(bucketName); @@ -138,8 +136,11 @@ public Response get( return Response.ok(result, MediaType.APPLICATION_XML_TYPE).build(); } + final String uploads = getQueryParam(QueryParams.UPLOADS); if (uploads != null) { s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; + final String uploadIdMarker = getQueryParam(QueryParams.UPLOAD_ID_MARKER); + final String keyMarker = getQueryParam(QueryParams.KEY_MARKER); return listMultipartUploads(bucketName, prefix, keyMarker, uploadIdMarker, maxUploads); } @@ -307,13 +308,13 @@ private int validateMaxKeys(int maxKeys) throws OS3Exception { @PUT public Response put( @PathParam(BUCKET) String bucketName, - @QueryParam(QueryParams.ACL) String aclMarker, InputStream body ) throws IOException, OS3Exception { long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.CREATE_BUCKET; try { + final String aclMarker = getQueryParam(QueryParams.ACL); if (aclMarker != null) { s3GAction = S3GAction.PUT_ACL; Response response = putAcl(bucketName, body); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 6f41e2bee06d..99d7adc3042f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -114,6 +114,14 @@ public abstract class EndpointBase { protected static final AuditLogger AUDIT = new AuditLogger(AuditLoggerType.S3GLOGGER); + protected String getQueryParam(String key) { + return getQueryParameters().getFirst(key); + } + + public MultivaluedMap getQueryParameters() { + return context.getUriInfo().getQueryParameters(); + } + protected OzoneBucket getBucket(OzoneVolume volume, String bucketName) throws OS3Exception, IOException { OzoneBucket bucket; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java index ffba1be40359..62af30219d33 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -68,6 +69,7 @@ public void setup() throws IOException { .setClient(client) .setHeaders(headers) .build(); + bucketEndpoint.getQueryParameters().add(QueryParams.ACL, ACL_MARKER); } @AfterEach @@ -81,8 +83,7 @@ public void clean() throws IOException { public void testGetAcl() throws Exception { when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); Response response = - bucketEndpoint.get(BUCKET_NAME, null, null, null, 0, null, - null, null, null, ACL_MARKER, null, null, 0); + bucketEndpoint.get(BUCKET_NAME, 0, 0); assertEquals(HTTP_OK, response.getStatus()); System.out.println(response.getEntity()); } @@ -93,7 +94,7 @@ public void testSetAclWithNotSupportedGranteeType() throws Exception { .thenReturn(S3Acl.ACLIdentityType.GROUP.getHeaderType() + "=root"); when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); OS3Exception e = assertThrows(OS3Exception.class, () -> - bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, null)); + bucketEndpoint.put(BUCKET_NAME, null)); assertEquals(e.getHttpCode(), HTTP_NOT_IMPLEMENTED); } @@ -103,7 +104,7 @@ public void testRead() throws Exception { when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, null); + bucketEndpoint.put(BUCKET_NAME, null); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); @@ -117,7 +118,7 @@ public void testWrite() throws Exception { when(headers.getHeaderString(S3Acl.GRANT_WRITE)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, null); + bucketEndpoint.put(BUCKET_NAME, null); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); @@ -131,7 +132,7 @@ public void testReadACP() throws Exception { when(headers.getHeaderString(S3Acl.GRANT_READ_CAP)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, null); + bucketEndpoint.put(BUCKET_NAME, null); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); @@ -146,7 +147,7 @@ public void testWriteACP() throws Exception { when(headers.getHeaderString(S3Acl.GRANT_WRITE_CAP)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, null); + bucketEndpoint.put(BUCKET_NAME, null); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); @@ -160,7 +161,7 @@ public void testFullControl() throws Exception { when(headers.getHeaderString(S3Acl.GRANT_FULL_CONTROL)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, null); + bucketEndpoint.put(BUCKET_NAME, null); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); @@ -182,7 +183,7 @@ public void testCombination() throws Exception { when(headers.getHeaderString(S3Acl.GRANT_FULL_CONTROL)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, null); + bucketEndpoint.put(BUCKET_NAME, null); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(5, getResponse.getAclList().getGrantList().size()); @@ -195,7 +196,7 @@ public void testPutClearOldAcls() throws Exception { .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); // Put READ Response response = - bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, null); + bucketEndpoint.put(BUCKET_NAME, null); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); @@ -212,7 +213,7 @@ public void testPutClearOldAcls() throws Exception { .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); //Put WRITE response = - bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, null); + bucketEndpoint.put(BUCKET_NAME, null); assertEquals(HTTP_OK, response.getStatus()); getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); @@ -230,7 +231,7 @@ public void testAclInBodyWithGroupUser() { .getResourceAsStream("groupAccessControlList.xml"); when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); assertThrows(OS3Exception.class, () -> bucketEndpoint.put( - BUCKET_NAME, ACL_MARKER, inputBody)); + BUCKET_NAME, inputBody)); } @Test @@ -239,7 +240,7 @@ public void testAclInBody() throws Exception { .getResourceAsStream("userAccessControlList.xml"); when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); Response response = - bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, inputBody); + bucketEndpoint.put(BUCKET_NAME, inputBody); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(2, getResponse.getAclList().getGrantList().size()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java index 0d2f087a75b5..332b6eb36eb7 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.s3.commontypes.EncodingTypeObject; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; +import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams; import org.apache.hadoop.security.UserGroupInformation; import org.junit.jupiter.api.Test; @@ -53,9 +54,10 @@ public void listRoot() throws OS3Exception, IOException { .setClient(client) .build(); + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, ""); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, "", - null, null, null, null, null, null, 0) + (ListObjectResponse) endpoint.get("b1", 100, 0) .getEntity(); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -72,9 +74,10 @@ public void listDir() throws OS3Exception, IOException { OzoneClient client = createClientWithKeys("dir1/file2", "dir1/dir2/file2"); BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir1"); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, - "dir1", null, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); assertEquals("dir1/", @@ -91,10 +94,10 @@ public void listSubDir() throws OS3Exception, IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir1/"); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint - .get("b1", "/", null, null, 100, "dir1/", null, - null, null, null, null, null, 0) + (ListObjectResponse) endpoint.get("b1", 100, 0) .getEntity(); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -124,9 +127,10 @@ public void listObjectOwner() throws OS3Exception, IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "key"); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, - "key", null, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); assertEquals(2, getBucketResponse.getContents().size()); assertEquals(user1.getShortUserName(), @@ -143,9 +147,10 @@ public void listWithPrefixAndDelimiter() throws OS3Exception, IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir1"); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, - "dir1", null, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); assertEquals(3, getBucketResponse.getCommonPrefixes().size()); } @@ -158,9 +163,10 @@ public void listWithPrefixAndDelimiter1() throws OS3Exception, IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, ""); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, - "", null, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); assertEquals(3, getBucketResponse.getCommonPrefixes().size()); assertEquals("file2", getBucketResponse.getContents().get(0) @@ -175,9 +181,11 @@ public void listWithPrefixAndDelimiter2() throws OS3Exception, IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir1bh"); + endpoint.getQueryParameters().putSingle(QueryParams.START_AFTER, "dir1/dir2/file2"); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, "dir1bh", - null, "dir1/dir2/file2", null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); } @@ -192,9 +200,10 @@ public void listWithPrefixAndEmptyStrDelimiter() BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); // Should behave the same if delimiter is null + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, ""); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir1/"); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "", null, null, 100, "dir1/", - null, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); assertEquals(0, getBucketResponse.getCommonPrefixes().size()); assertEquals(4, getBucketResponse.getContents().size()); @@ -220,28 +229,24 @@ public void listWithContinuationToken() throws OS3Exception, IOException { // As we have 5 keys, with max keys 2 we should call list 3 times. // First time + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, ""); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", null, null, null, maxKeys, - "", null, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getContents().size()); // 2nd time - String continueToken = getBucketResponse.getNextToken(); + endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, getBucketResponse.getNextToken()); getBucketResponse = - (ListObjectResponse) endpoint.get("b1", null, null, null, maxKeys, - "", continueToken, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getContents().size()); - - continueToken = getBucketResponse.getNextToken(); - //3rd time + endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, getBucketResponse.getNextToken()); getBucketResponse = - (ListObjectResponse) endpoint.get("b1", null, null, null, maxKeys, - "", continueToken, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(1, getBucketResponse.getContents().size()); @@ -267,9 +272,10 @@ public void listWithContinuationTokenDirBreak() ListObjectResponse getBucketResponse; + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "test/"); getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, maxKeys, - "test/", null, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); assertEquals(0, getBucketResponse.getContents().size()); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); @@ -278,10 +284,9 @@ public void listWithContinuationTokenDirBreak() assertEquals("test/dir2/", getBucketResponse.getCommonPrefixes().get(1).getPrefix().getName()); + endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, getBucketResponse.getNextToken()); getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, maxKeys, - "test/", getBucketResponse.getNextToken(), null, null, null, - null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); assertEquals(1, getBucketResponse.getContents().size()); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); assertEquals("test/dir3/", @@ -306,26 +311,25 @@ public void listWithContinuationToken1() throws OS3Exception, IOException { // As we have 5 keys, with max keys 2 we should call list 3 times. // First time + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir"); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, maxKeys, - "dir", null, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); // 2nd time - String continueToken = getBucketResponse.getNextToken(); + endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, getBucketResponse.getNextToken()); getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, maxKeys, - "dir", continueToken, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); //3rd time - continueToken = getBucketResponse.getNextToken(); + endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, getBucketResponse.getNextToken()); getBucketResponse = - (ListObjectResponse) endpoint.get("b1", "/", null, null, maxKeys, - "dir", continueToken, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -339,9 +343,10 @@ public void listWithContinuationTokenFail() throws IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); - OS3Exception e = assertThrows(OS3Exception.class, () -> endpoint.get("b1", - "/", null, null, 2, "dir", "random", null, null, null, null, null, 1000) - .getEntity(), "listWithContinuationTokenFail"); + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir"); + endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, "random"); + OS3Exception e = assertThrows(OS3Exception.class, () -> endpoint.get("b1", 2, 1000).getEntity()); assertEquals("random", e.getResource()); assertEquals("Invalid Argument", e.getErrorMessage()); } @@ -355,8 +360,7 @@ public void testStartAfter() throws IOException, OS3Exception { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", null, null, null, 1000, - null, null, null, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", 1000, 0).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(5, getBucketResponse.getContents().size()); @@ -365,16 +369,16 @@ public void testStartAfter() throws IOException, OS3Exception { // have 4 keys. String startAfter = "dir0/file1"; + endpoint.getQueryParameters().putSingle(QueryParams.START_AFTER, startAfter); getBucketResponse = - (ListObjectResponse) endpoint.get("b1", null, null, null, - 1000, null, null, startAfter, null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", 1000, 0).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(4, getBucketResponse.getContents().size()); + endpoint.getQueryParameters().putSingle(QueryParams.START_AFTER, "random"); getBucketResponse = - (ListObjectResponse) endpoint.get("b1", null, null, null, - 1000, null, null, "random", null, null, null, null, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1", 1000, 0).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(0, getBucketResponse.getContents().size()); @@ -415,9 +419,11 @@ public void testEncodingType() throws IOException, OS3Exception { String startAfter = "data="; String encodingType = ENCODING_TYPE; - ListObjectResponse response = (ListObjectResponse) endpoint.get( - "b1", delimiter, encodingType, null, 1000, prefix, - null, startAfter, null, null, null, null, 0).getEntity(); + endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, delimiter); + endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, prefix); + endpoint.getQueryParameters().putSingle(QueryParams.ENCODING_TYPE, encodingType); + endpoint.getQueryParameters().putSingle(QueryParams.START_AFTER, startAfter); + ListObjectResponse response = (ListObjectResponse) endpoint.get("b1", 1000, 0).getEntity(); // Assert encodingType == url. // The Object name will be encoded by ObjectKeyNameAdapter @@ -433,9 +439,8 @@ public void testEncodingType() throws IOException, OS3Exception { assertEquals(encodingType, response.getContents().get(0).getKey().getEncodingType()); - response = (ListObjectResponse) endpoint.get( - "b1", delimiter, null, null, 1000, prefix, - null, startAfter, null, null, null, null, 0).getEntity(); + endpoint.getQueryParameters().remove(QueryParams.ENCODING_TYPE); + response = (ListObjectResponse) endpoint.get("b1", 1000, 0).getEntity(); // Assert encodingType == null. // The Object name will not be encoded by ObjectKeyNameAdapter @@ -455,9 +460,10 @@ public void testEncodingTypeException() throws IOException { OzoneClient client = new OzoneClientStub(); client.getObjectStore().createS3Bucket("b1"); BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); + + endpoint.getQueryParameters().putSingle(QueryParams.ENCODING_TYPE, "unSupportType"); OS3Exception e = assertThrows(OS3Exception.class, () -> endpoint.get( - "b1", null, "unSupportType", null, 1000, null, - null, null, null, null, null, null, 0).getEntity()); + "b1", 1000, 0).getEntity()); assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e.getCode()); } @@ -471,8 +477,7 @@ public void testListObjectsWithNegativeMaxKeys() throws Exception { // maxKeys < 0 should throw InvalidArgument OS3Exception e1 = assertThrows(OS3Exception.class, () -> - bucketEndpoint.get("bucket", null, null, null, -1, null, - null, null, null, null, null, null, 1000) + bucketEndpoint.get("bucket", -1, 1000) ); assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e1.getCode()); } @@ -487,8 +492,7 @@ public void testListObjectsWithZeroMaxKeys() throws Exception { // maxKeys = 0, should return empty list and not throw. ListObjectResponse response = (ListObjectResponse) bucketEndpoint.get( - "bucket", null, null, null, 0, null, - null, null, null, null, null, null, 1000).getEntity(); + "bucket", 0, 1000).getEntity(); assertEquals(0, response.getContents().size()); assertFalse(response.isTruncated()); @@ -502,16 +506,14 @@ public void testListObjectsWithZeroMaxKeysInNonEmptyBucket() throws Exception { .build(); ListObjectResponse response = (ListObjectResponse) bucketEndpoint.get( - "b1", null, null, null, 0, null, - null, null, null, null, null, null, 1000).getEntity(); + "b1", 0, 1000).getEntity(); // Should return empty list and not throw. assertEquals(0, response.getContents().size()); assertFalse(response.isTruncated()); ListObjectResponse fullResponse = (ListObjectResponse) bucketEndpoint.get( - "b1", null, null, null, 1000, null, - null, null, null, null, null, null, 1000).getEntity(); + "b1", 1000, 1000).getEntity(); assertEquals(5, fullResponse.getContents().size()); } @@ -539,8 +541,7 @@ public void testListObjectsRespectsConfiguredMaxKeysLimit() throws Exception { // Act: Request more keys than the configured max-keys limit final int requestedMaxKeys = Integer.parseInt(configuredMaxKeysLimit) + 1; ListObjectResponse response = (ListObjectResponse) - bucketEndpoint.get("b1", null, null, null, requestedMaxKeys, - null, null, null, null, null, null, null, + bucketEndpoint.get("b1", requestedMaxKeys, 1000).getEntity(); // Assert: The number of returned keys should be capped at the configured limit diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java index 0464ff54edca..96eea22d9ebd 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java @@ -56,7 +56,7 @@ public void setup() throws Exception { @Test public void testBucketFailWithAuthHeaderMissing() throws Exception { try { - bucketEndpoint.put(bucketName, null, null); + bucketEndpoint.put(bucketName, null); } catch (OS3Exception ex) { assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); assertEquals(MALFORMED_HEADER.getCode(), ex.getCode()); @@ -65,13 +65,13 @@ public void testBucketFailWithAuthHeaderMissing() throws Exception { @Test public void testBucketPut() throws Exception { - Response response = bucketEndpoint.put(bucketName, null, null); + Response response = bucketEndpoint.put(bucketName, null); assertEquals(200, response.getStatus()); assertNotNull(response.getLocation()); // Create-bucket on an existing bucket fails OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.put( - bucketName, null, null)); + bucketName, null)); assertEquals(HTTP_CONFLICT, e.getHttpCode()); assertEquals(BUCKET_ALREADY_EXISTS.getCode(), e.getCode()); } @@ -79,7 +79,7 @@ public void testBucketPut() throws Exception { @Test public void testBucketFailWithInvalidHeader() throws Exception { try { - bucketEndpoint.put(bucketName, null, null); + bucketEndpoint.put(bucketName, null); } catch (OS3Exception ex) { assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); assertEquals(MALFORMED_HEADER.getCode(), ex.getCode()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index e422e4920792..9872a711c639 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -55,6 +55,7 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; import org.apache.hadoop.ozone.s3.util.S3Consts; +import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -131,7 +132,7 @@ public void testCreateBucket() throws IOException { .setClient(client) .build(); OS3Exception e = assertThrows(OS3Exception.class, () -> - bucketEndpoint.put("bucketName", null, null)); + bucketEndpoint.put("bucketName", null)); assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); } @@ -169,8 +170,7 @@ public void testListKey() throws IOException { .setClient(client) .build(); OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( - "bucketName", null, null, null, 1000, - null, null, null, null, null, null, null, 0)); + "bucketName", 1000, 0)); assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); } @@ -214,9 +214,9 @@ public void testGetAcl() throws Exception { .setClient(client) .setHeaders(headers) .build(); + bucketEndpoint.getQueryParameters().putSingle(QueryParams.ACL, "acl"); OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( - "bucketName", null, null, null, 1000, null, null, null, null, "acl", - null, null, 0), "Expected OS3Exception with FORBIDDEN http code."); + "bucketName", 1000, 0)); assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); } @@ -237,8 +237,9 @@ public void testSetAcl() throws Exception { .setClient(client) .setHeaders(headers) .build(); + bucketEndpoint.getQueryParameters().putSingle(QueryParams.ACL, "acl"); try { - bucketEndpoint.put("bucketName", "acl", null); + bucketEndpoint.put("bucketName", null); } catch (Exception e) { assertTrue(e instanceof OS3Exception && ((OS3Exception)e).getHttpCode() == HTTP_FORBIDDEN); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java index cae9a9422885..018ad0f1f5e2 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java @@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.hadoop.ozone.s3.util.S3Consts; +import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -133,10 +134,7 @@ public void testListBucketSuccess() throws Exception { public void testGetBucketSuccess() throws Exception { long oriMetric = metrics.getGetBucketSuccess(); - bucketEndpoint.get(bucketName, null, - null, null, 1000, null, - null, "random", null, - null, null, null, 0).getEntity(); + bucketEndpoint.get(bucketName, 1000, 0).getEntity(); long curMetric = metrics.getGetBucketSuccess(); assertEquals(1L, curMetric - oriMetric); @@ -148,8 +146,7 @@ public void testGetBucketFailure() throws Exception { // Searching for a bucket that does not exist OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( - "newBucket", null, null, null, 1000, null, null, "random", null, - null, null, null, 0)); + "newBucket", 1000, 0)); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), e.getCode()); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getErrorMessage(), e.getErrorMessage()); @@ -161,7 +158,7 @@ public void testGetBucketFailure() throws Exception { public void testCreateBucketSuccess() throws Exception { long oriMetric = metrics.getCreateBucketSuccess(); - assertDoesNotThrow(() -> bucketEndpoint.put("newBucket", null, null)); + assertDoesNotThrow(() -> bucketEndpoint.put("newBucket", null)); long curMetric = metrics.getCreateBucketSuccess(); assertEquals(1L, curMetric - oriMetric); } @@ -172,7 +169,7 @@ public void testCreateBucketFailure() throws Exception { // Creating an error by trying to create a bucket that already exists OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.put( - bucketName, null, null)); + bucketName, null)); assertEquals(HTTP_CONFLICT, e.getHttpCode()); assertEquals(BUCKET_ALREADY_EXISTS.getCode(), e.getCode()); @@ -210,10 +207,9 @@ public void testDeleteBucketFailure() throws Exception { public void testGetAclSuccess() throws Exception { long oriMetric = metrics.getGetAclSuccess(); + bucketEndpoint.getQueryParameters().add(QueryParams.ACL, ACL_MARKER); Response response = - bucketEndpoint.get(bucketName, null, null, - null, 0, null, null, - null, null, "acl", null, null, 0); + bucketEndpoint.get(bucketName, 0, 0); long curMetric = metrics.getGetAclSuccess(); assertEquals(HTTP_OK, response.getStatus()); assertEquals(1L, curMetric - oriMetric); @@ -223,10 +219,9 @@ public void testGetAclSuccess() throws Exception { public void testGetAclFailure() throws Exception { long oriMetric = metrics.getGetAclFailure(); + bucketEndpoint.getQueryParameters().add(QueryParams.ACL, ACL_MARKER); // Failing the getACL endpoint by applying ACL on a non-Existent Bucket - OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( - "random_bucket", null, null, null, 0, null, - null, null, null, "acl", null, null, 0)); + OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get("random_bucket", 0, 0)); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), e.getCode()); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getErrorMessage(), e.getErrorMessage()); @@ -242,7 +237,8 @@ public void testPutAclSuccess() throws Exception { InputStream inputBody = TestBucketAcl.class.getClassLoader() .getResourceAsStream("userAccessControlList.xml"); - bucketEndpoint.put("b1", ACL_MARKER, inputBody); + bucketEndpoint.getQueryParameters().add(QueryParams.ACL, ACL_MARKER); + bucketEndpoint.put("b1", inputBody); inputBody.close(); long curMetric = metrics.getPutAclSuccess(); assertEquals(1L, curMetric - oriMetric); @@ -255,9 +251,9 @@ public void testPutAclFailure() throws Exception { InputStream inputBody = TestBucketAcl.class.getClassLoader() .getResourceAsStream("userAccessControlList.xml"); + bucketEndpoint.getQueryParameters().add(QueryParams.ACL, ACL_MARKER); try { - assertThrows(OS3Exception.class, () -> bucketEndpoint.put("unknown_bucket", ACL_MARKER, - inputBody)); + assertThrows(OS3Exception.class, () -> bucketEndpoint.put("unknown_bucket", inputBody)); } finally { inputBody.close(); }