diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java new file mode 100644 index 000000000000..7b5e2d0d6e82 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.commontypes; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MultivaluedMap; +import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; + +/** Allow looking up query parameters as primitive types. */ +public interface RequestParameters { + + String get(String key); + + static MultivaluedMapImpl of(MultivaluedMap params) { + return new MultivaluedMapImpl(params); + } + + default String get(String key, String defaultValue) { + final String value = get(key); + return value != null ? value : defaultValue; + } + + default int getInt(String key, int defaultValue) { + final String value = get(key); + if (value == null) { + return defaultValue; + } + + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw translateException(e); + } + } + + default WebApplicationException translateException(RuntimeException e) { + return new WebApplicationException(e.getMessage(), S3ErrorTable.INVALID_ARGUMENT.getHttpCode()); + } + + /** Additional methods for tests. */ + interface Mutable extends RequestParameters { + + void set(String key, String value); + + void unset(String key); + + default void setInt(String key, int value) { + set(key, String.valueOf(value)); + } + } + + /** Mutable implementation based on {@link MultivaluedMap}. */ + final class MultivaluedMapImpl implements Mutable { + private final MultivaluedMap params; + + private MultivaluedMapImpl(MultivaluedMap params) { + this.params = params; + } + + @Override + public String get(String key) { + return params.getFirst(key); + } + + @Override + public void set(String key, String value) { + params.putSingle(key, value); + } + + @Override + public void unset(String key) { + params.remove(key); + } + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 1177c01e5dee..6b888ce12fcb 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -42,7 +42,6 @@ import java.util.Set; import javax.annotation.PostConstruct; import javax.ws.rs.DELETE; -import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.HEAD; import javax.ws.rs.POST; @@ -106,20 +105,20 @@ public class BucketEndpoint extends EndpointBase { @GET @SuppressWarnings("methodlength") public Response get( - @PathParam(BUCKET) String bucketName, - @DefaultValue("1000") @QueryParam(QueryParams.MAX_KEYS) int maxKeys, - @DefaultValue("1000") @QueryParam(QueryParams.MAX_UPLOADS) int maxUploads + @PathParam(BUCKET) String bucketName ) throws OS3Exception, IOException { long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.GET_BUCKET; PerformanceStringBuilder perf = new PerformanceStringBuilder(); - final String continueToken = getQueryParam(QueryParams.CONTINUATION_TOKEN); - final String delimiter = getQueryParam(QueryParams.DELIMITER); - final String encodingType = getQueryParam(QueryParams.ENCODING_TYPE); - final String marker = getQueryParam(QueryParams.MARKER); - String prefix = getQueryParam(QueryParams.PREFIX); - String startAfter = getQueryParam(QueryParams.START_AFTER); + final String continueToken = queryParams().get(QueryParams.CONTINUATION_TOKEN); + final String delimiter = queryParams().get(QueryParams.DELIMITER); + final String encodingType = queryParams().get(QueryParams.ENCODING_TYPE); + final String marker = queryParams().get(QueryParams.MARKER); + int maxKeys = queryParams().getInt(QueryParams.MAX_KEYS, 1000); + final int maxUploads = queryParams().getInt(QueryParams.MAX_UPLOADS, 1000); + String prefix = queryParams().get(QueryParams.PREFIX); + String startAfter = queryParams().get(QueryParams.START_AFTER); Iterator ozoneKeyIterator = null; ContinueToken decodedToken = @@ -127,7 +126,7 @@ public Response get( OzoneBucket bucket = null; try { - final String aclMarker = getQueryParam(QueryParams.ACL); + final String aclMarker = queryParams().get(QueryParams.ACL); if (aclMarker != null) { s3GAction = S3GAction.GET_ACL; S3BucketAcl result = getAcl(bucketName); @@ -136,11 +135,11 @@ public Response get( return Response.ok(result, MediaType.APPLICATION_XML_TYPE).build(); } - final String uploads = getQueryParam(QueryParams.UPLOADS); + final String uploads = queryParams().get(QueryParams.UPLOADS); if (uploads != null) { s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; - final String uploadIdMarker = getQueryParam(QueryParams.UPLOAD_ID_MARKER); - final String keyMarker = getQueryParam(QueryParams.KEY_MARKER); + final String uploadIdMarker = queryParams().get(QueryParams.UPLOAD_ID_MARKER); + final String keyMarker = queryParams().get(QueryParams.KEY_MARKER); return listMultipartUploads(bucketName, prefix, keyMarker, uploadIdMarker, maxUploads); } @@ -314,7 +313,7 @@ public Response put( S3GAction s3GAction = S3GAction.CREATE_BUCKET; try { - final String aclMarker = getQueryParam(QueryParams.ACL); + final String aclMarker = queryParams().get(QueryParams.ACL); if (aclMarker != null) { s3GAction = S3GAction.PUT_ACL; Response response = putAcl(bucketName, body); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 99d7adc3042f..dbc91c1e55e7 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -69,6 +69,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.om.protocol.S3Auth; import org.apache.hadoop.ozone.s3.RequestIdentifier; +import org.apache.hadoop.ozone.s3.commontypes.RequestParameters; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; @@ -106,6 +107,9 @@ public abstract class EndpointBase { @Context private HttpHeaders headers; + // initialized in @PostConstruct + private RequestParameters.MultivaluedMapImpl queryParams; + private final Set excludeMetadataFields = new HashSet<>(Arrays.asList(OzoneConsts.GDPR_FLAG, STORAGE_CONFIG_HEADER)); private static final Logger LOG = @@ -114,12 +118,14 @@ public abstract class EndpointBase { protected static final AuditLogger AUDIT = new AuditLogger(AuditLoggerType.S3GLOGGER); - protected String getQueryParam(String key) { - return getQueryParameters().getFirst(key); + /** Read-only access to query parameters. */ + protected RequestParameters queryParams() { + return queryParams; } - public MultivaluedMap getQueryParameters() { - return context.getUriInfo().getQueryParameters(); + /** For setting multiple values use {@link #getContext()}. */ + public RequestParameters.Mutable queryParamsForTest() { + return queryParams; } protected OzoneBucket getBucket(OzoneVolume volume, String bucketName) @@ -149,6 +155,7 @@ protected OzoneBucket getBucket(OzoneVolume volume, String bucketName) */ @PostConstruct public void initialization() { + queryParams = RequestParameters.of(context.getUriInfo().getQueryParameters()); // Note: userPrincipal is initialized to be the same value as accessId, // could be updated later in RpcClient#getS3Volume s3Auth = new S3Auth(signatureInfo.getStringToSign(), diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java index 62af30219d33..1598a48a9892 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -69,7 +69,7 @@ public void setup() throws IOException { .setClient(client) .setHeaders(headers) .build(); - bucketEndpoint.getQueryParameters().add(QueryParams.ACL, ACL_MARKER); + bucketEndpoint.queryParamsForTest().set(QueryParams.ACL, ACL_MARKER); } @AfterEach @@ -82,8 +82,7 @@ public void clean() throws IOException { @Test public void testGetAcl() throws Exception { when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); - Response response = - bucketEndpoint.get(BUCKET_NAME, 0, 0); + Response response = bucketEndpoint.get(BUCKET_NAME); assertEquals(HTTP_OK, response.getStatus()); System.out.println(response.getEntity()); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java index 332b6eb36eb7..c62a7e8da1c8 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java @@ -54,11 +54,9 @@ public void listRoot() throws OS3Exception, IOException { .setClient(client) .build(); - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, ""); - ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 100, 0) - .getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, ""); + ListObjectResponse getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); assertEquals("dir1/", @@ -74,10 +72,9 @@ public void listDir() throws OS3Exception, IOException { OzoneClient client = createClientWithKeys("dir1/file2", "dir1/dir2/file2"); BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir1"); - ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, "dir1"); + ListObjectResponse getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); assertEquals("dir1/", @@ -94,11 +91,9 @@ public void listSubDir() throws OS3Exception, IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir1/"); - ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 100, 0) - .getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, "dir1/"); + ListObjectResponse getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); assertEquals("dir1/dir2/", @@ -127,10 +122,9 @@ public void listObjectOwner() throws OS3Exception, IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "key"); - ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, "key"); + ListObjectResponse getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertEquals(2, getBucketResponse.getContents().size()); assertEquals(user1.getShortUserName(), @@ -147,10 +141,9 @@ public void listWithPrefixAndDelimiter() throws OS3Exception, IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir1"); - ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, "dir1"); + ListObjectResponse getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertEquals(3, getBucketResponse.getCommonPrefixes().size()); } @@ -163,10 +156,9 @@ public void listWithPrefixAndDelimiter1() throws OS3Exception, IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, ""); - ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, ""); + ListObjectResponse getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertEquals(3, getBucketResponse.getCommonPrefixes().size()); assertEquals("file2", getBucketResponse.getContents().get(0) @@ -181,11 +173,10 @@ public void listWithPrefixAndDelimiter2() throws OS3Exception, IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir1bh"); - endpoint.getQueryParameters().putSingle(QueryParams.START_AFTER, "dir1/dir2/file2"); - ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, "dir1bh"); + endpoint.queryParamsForTest().set(QueryParams.START_AFTER, "dir1/dir2/file2"); + ListObjectResponse getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); } @@ -200,10 +191,9 @@ public void listWithPrefixAndEmptyStrDelimiter() BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); // Should behave the same if delimiter is null - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, ""); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir1/"); - ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 100, 0).getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, ""); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, "dir1/"); + ListObjectResponse getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertEquals(0, getBucketResponse.getCommonPrefixes().size()); assertEquals(4, getBucketResponse.getContents().size()); @@ -229,24 +219,24 @@ public void listWithContinuationToken() throws OS3Exception, IOException { // As we have 5 keys, with max keys 2 we should call list 3 times. // First time - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, ""); - ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, ""); + endpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, maxKeys); + ListObjectResponse getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getContents().size()); // 2nd time - endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, getBucketResponse.getNextToken()); - getBucketResponse = - (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); + String value1 = getBucketResponse.getNextToken(); + endpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN, value1); + getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getContents().size()); //3rd time - endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, getBucketResponse.getNextToken()); - getBucketResponse = - (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); + String value = getBucketResponse.getNextToken(); + endpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN, value); + getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(1, getBucketResponse.getContents().size()); @@ -272,10 +262,10 @@ public void listWithContinuationTokenDirBreak() ListObjectResponse getBucketResponse; - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "test/"); - getBucketResponse = - (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, "test/"); + endpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, maxKeys); + getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertEquals(0, getBucketResponse.getContents().size()); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); @@ -284,9 +274,9 @@ public void listWithContinuationTokenDirBreak() assertEquals("test/dir2/", getBucketResponse.getCommonPrefixes().get(1).getPrefix().getName()); - endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, getBucketResponse.getNextToken()); - getBucketResponse = - (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); + String value = getBucketResponse.getNextToken(); + endpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN, value); + getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertEquals(1, getBucketResponse.getContents().size()); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); assertEquals("test/dir3/", @@ -311,25 +301,25 @@ public void listWithContinuationToken1() throws OS3Exception, IOException { // As we have 5 keys, with max keys 2 we should call list 3 times. // First time - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir"); - ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, "dir"); + endpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, maxKeys); + ListObjectResponse getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); // 2nd time - endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, getBucketResponse.getNextToken()); - getBucketResponse = - (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); + String value1 = getBucketResponse.getNextToken(); + endpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN, value1); + getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); //3rd time - endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, getBucketResponse.getNextToken()); - getBucketResponse = - (ListObjectResponse) endpoint.get("b1", maxKeys, 0).getEntity(); + String value = getBucketResponse.getNextToken(); + endpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN, value); + getBucketResponse = (ListObjectResponse) endpoint.get("b1").getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -343,10 +333,11 @@ public void listWithContinuationTokenFail() throws IOException { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, "/"); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, "dir"); - endpoint.getQueryParameters().putSingle(QueryParams.CONTINUATION_TOKEN, "random"); - OS3Exception e = assertThrows(OS3Exception.class, () -> endpoint.get("b1", 2, 1000).getEntity()); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, "dir"); + endpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN, "random"); + endpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, 2); + OS3Exception e = assertThrows(OS3Exception.class, () -> endpoint.get("b1").getEntity()); assertEquals("random", e.getResource()); assertEquals("Invalid Argument", e.getErrorMessage()); } @@ -360,7 +351,7 @@ public void testStartAfter() throws IOException, OS3Exception { BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); ListObjectResponse getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 1000, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1").getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(5, getBucketResponse.getContents().size()); @@ -369,16 +360,16 @@ public void testStartAfter() throws IOException, OS3Exception { // have 4 keys. String startAfter = "dir0/file1"; - endpoint.getQueryParameters().putSingle(QueryParams.START_AFTER, startAfter); + endpoint.queryParamsForTest().set(QueryParams.START_AFTER, startAfter); getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 1000, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1").getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(4, getBucketResponse.getContents().size()); - endpoint.getQueryParameters().putSingle(QueryParams.START_AFTER, "random"); + endpoint.queryParamsForTest().set(QueryParams.START_AFTER, "random"); getBucketResponse = - (ListObjectResponse) endpoint.get("b1", 1000, 0).getEntity(); + (ListObjectResponse) endpoint.get("b1").getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(0, getBucketResponse.getContents().size()); @@ -419,11 +410,11 @@ public void testEncodingType() throws IOException, OS3Exception { String startAfter = "data="; String encodingType = ENCODING_TYPE; - endpoint.getQueryParameters().putSingle(QueryParams.DELIMITER, delimiter); - endpoint.getQueryParameters().putSingle(QueryParams.PREFIX, prefix); - endpoint.getQueryParameters().putSingle(QueryParams.ENCODING_TYPE, encodingType); - endpoint.getQueryParameters().putSingle(QueryParams.START_AFTER, startAfter); - ListObjectResponse response = (ListObjectResponse) endpoint.get("b1", 1000, 0).getEntity(); + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, delimiter); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, prefix); + endpoint.queryParamsForTest().set(QueryParams.ENCODING_TYPE, encodingType); + endpoint.queryParamsForTest().set(QueryParams.START_AFTER, startAfter); + ListObjectResponse response = (ListObjectResponse) endpoint.get("b1").getEntity(); // Assert encodingType == url. // The Object name will be encoded by ObjectKeyNameAdapter @@ -439,8 +430,8 @@ public void testEncodingType() throws IOException, OS3Exception { assertEquals(encodingType, response.getContents().get(0).getKey().getEncodingType()); - endpoint.getQueryParameters().remove(QueryParams.ENCODING_TYPE); - response = (ListObjectResponse) endpoint.get("b1", 1000, 0).getEntity(); + endpoint.queryParamsForTest().unset(QueryParams.ENCODING_TYPE); + response = (ListObjectResponse) endpoint.get("b1").getEntity(); // Assert encodingType == null. // The Object name will not be encoded by ObjectKeyNameAdapter @@ -461,9 +452,8 @@ public void testEncodingTypeException() throws IOException { client.getObjectStore().createS3Bucket("b1"); BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); - endpoint.getQueryParameters().putSingle(QueryParams.ENCODING_TYPE, "unSupportType"); - OS3Exception e = assertThrows(OS3Exception.class, () -> endpoint.get( - "b1", 1000, 0).getEntity()); + endpoint.queryParamsForTest().set(QueryParams.ENCODING_TYPE, "unSupportType"); + OS3Exception e = assertThrows(OS3Exception.class, () -> endpoint.get("b1").getEntity()); assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e.getCode()); } @@ -476,9 +466,8 @@ public void testListObjectsWithNegativeMaxKeys() throws Exception { .build(); // maxKeys < 0 should throw InvalidArgument - OS3Exception e1 = assertThrows(OS3Exception.class, () -> - bucketEndpoint.get("bucket", -1, 1000) - ); + bucketEndpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, -1); + OS3Exception e1 = assertThrows(OS3Exception.class, () -> bucketEndpoint.get("bucket")); assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e1.getCode()); } @@ -491,8 +480,8 @@ public void testListObjectsWithZeroMaxKeys() throws Exception { .build(); // maxKeys = 0, should return empty list and not throw. - ListObjectResponse response = (ListObjectResponse) bucketEndpoint.get( - "bucket", 0, 1000).getEntity(); + bucketEndpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, 0); + ListObjectResponse response = (ListObjectResponse) bucketEndpoint.get("bucket").getEntity(); assertEquals(0, response.getContents().size()); assertFalse(response.isTruncated()); @@ -505,15 +494,15 @@ public void testListObjectsWithZeroMaxKeysInNonEmptyBucket() throws Exception { .setClient(client) .build(); - ListObjectResponse response = (ListObjectResponse) bucketEndpoint.get( - "b1", 0, 1000).getEntity(); + bucketEndpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, 0); + ListObjectResponse response = (ListObjectResponse) bucketEndpoint.get("b1").getEntity(); // Should return empty list and not throw. assertEquals(0, response.getContents().size()); assertFalse(response.isTruncated()); - ListObjectResponse fullResponse = (ListObjectResponse) bucketEndpoint.get( - "b1", 1000, 1000).getEntity(); + bucketEndpoint.queryParamsForTest().unset(QueryParams.MAX_KEYS); + ListObjectResponse fullResponse = (ListObjectResponse) bucketEndpoint.get("b1").getEntity(); assertEquals(5, fullResponse.getContents().size()); } @@ -540,9 +529,8 @@ public void testListObjectsRespectsConfiguredMaxKeysLimit() throws Exception { // Act: Request more keys than the configured max-keys limit final int requestedMaxKeys = Integer.parseInt(configuredMaxKeysLimit) + 1; - ListObjectResponse response = (ListObjectResponse) - bucketEndpoint.get("b1", requestedMaxKeys, - 1000).getEntity(); + bucketEndpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, requestedMaxKeys); + ListObjectResponse response = (ListObjectResponse) bucketEndpoint.get("b1").getEntity(); // Assert: The number of returned keys should be capped at the configured limit assertEquals(Integer.parseInt(configuredMaxKeysLimit), response.getContents().size()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index 9872a711c639..8e9eef2d974d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -169,8 +169,7 @@ public void testListKey() throws IOException { BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder() .setClient(client) .build(); - OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( - "bucketName", 1000, 0)); + OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get("bucketName")); assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); } @@ -214,9 +213,8 @@ public void testGetAcl() throws Exception { .setClient(client) .setHeaders(headers) .build(); - bucketEndpoint.getQueryParameters().putSingle(QueryParams.ACL, "acl"); - OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( - "bucketName", 1000, 0)); + bucketEndpoint.queryParamsForTest().set(QueryParams.ACL, "acl"); + OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get("bucketName")); assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); } @@ -237,7 +235,7 @@ public void testSetAcl() throws Exception { .setClient(client) .setHeaders(headers) .build(); - bucketEndpoint.getQueryParameters().putSingle(QueryParams.ACL, "acl"); + bucketEndpoint.queryParamsForTest().set(QueryParams.ACL, "acl"); try { bucketEndpoint.put("bucketName", null); } catch (Exception e) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java index 018ad0f1f5e2..bfc471e22d5d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java @@ -134,7 +134,7 @@ public void testListBucketSuccess() throws Exception { public void testGetBucketSuccess() throws Exception { long oriMetric = metrics.getGetBucketSuccess(); - bucketEndpoint.get(bucketName, 1000, 0).getEntity(); + bucketEndpoint.get(bucketName).getEntity(); long curMetric = metrics.getGetBucketSuccess(); assertEquals(1L, curMetric - oriMetric); @@ -145,8 +145,7 @@ public void testGetBucketFailure() throws Exception { long oriMetric = metrics.getGetBucketFailure(); // Searching for a bucket that does not exist - OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( - "newBucket", 1000, 0)); + OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get("newBucket")); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), e.getCode()); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getErrorMessage(), e.getErrorMessage()); @@ -207,9 +206,8 @@ public void testDeleteBucketFailure() throws Exception { public void testGetAclSuccess() throws Exception { long oriMetric = metrics.getGetAclSuccess(); - bucketEndpoint.getQueryParameters().add(QueryParams.ACL, ACL_MARKER); - Response response = - bucketEndpoint.get(bucketName, 0, 0); + bucketEndpoint.queryParamsForTest().set(QueryParams.ACL, ACL_MARKER); + Response response = bucketEndpoint.get(bucketName); long curMetric = metrics.getGetAclSuccess(); assertEquals(HTTP_OK, response.getStatus()); assertEquals(1L, curMetric - oriMetric); @@ -219,9 +217,9 @@ public void testGetAclSuccess() throws Exception { public void testGetAclFailure() throws Exception { long oriMetric = metrics.getGetAclFailure(); - bucketEndpoint.getQueryParameters().add(QueryParams.ACL, ACL_MARKER); + bucketEndpoint.queryParamsForTest().set(QueryParams.ACL, ACL_MARKER); // Failing the getACL endpoint by applying ACL on a non-Existent Bucket - OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get("random_bucket", 0, 0)); + OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get("random_bucket")); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), e.getCode()); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getErrorMessage(), e.getErrorMessage()); @@ -237,7 +235,7 @@ public void testPutAclSuccess() throws Exception { InputStream inputBody = TestBucketAcl.class.getClassLoader() .getResourceAsStream("userAccessControlList.xml"); - bucketEndpoint.getQueryParameters().add(QueryParams.ACL, ACL_MARKER); + bucketEndpoint.queryParamsForTest().set(QueryParams.ACL, ACL_MARKER); bucketEndpoint.put("b1", inputBody); inputBody.close(); long curMetric = metrics.getPutAclSuccess(); @@ -251,7 +249,7 @@ public void testPutAclFailure() throws Exception { InputStream inputBody = TestBucketAcl.class.getClassLoader() .getResourceAsStream("userAccessControlList.xml"); - bucketEndpoint.getQueryParameters().add(QueryParams.ACL, ACL_MARKER); + bucketEndpoint.queryParamsForTest().set(QueryParams.ACL, ACL_MARKER); try { assertThrows(OS3Exception.class, () -> bucketEndpoint.put("unknown_bucket", inputBody)); } finally {