diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java index 42671e2c294a..7dad6d600f0b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_EMPTY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; +import static org.mockito.Mockito.mock; import java.io.IOException; import java.util.HashMap; @@ -44,7 +45,7 @@ public class ObjectStoreStub extends ObjectStore { private Map bucketEmptyStatus = new HashMap<>(); public ObjectStoreStub() { - super(); + super(conf, mock(ClientProtocol.class)); } public ObjectStoreStub(ConfigurationSource conf, ClientProtocol proxy) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBuilder.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBuilder.java index 9c4cade64079..13db3962a89b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBuilder.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBuilder.java @@ -27,6 +27,7 @@ import javax.ws.rs.core.UriInfo; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.s3.RequestIdentifier; import org.apache.hadoop.ozone.s3.signature.SignatureInfo; @@ -97,8 +98,8 @@ public EndpointBuilder setSignatureInfo(SignatureInfo newSignatureInfo) { public T build() { T endpoint = base != null ? base : constructor.get(); - if (ozoneClient != null) { - endpoint.setClient(ozoneClient); + if (endpoint.getClient() == null) { + endpoint.setClient(getClient()); } final OzoneConfiguration config = getConfig(); @@ -109,10 +110,15 @@ public T build() { endpoint.setRequestIdentifier(identifier); endpoint.setSignatureInfo(signatureInfo); + endpoint.initialization(); + return endpoint; } protected OzoneClient getClient() { + if (ozoneClient == null) { + ozoneClient = new OzoneClientStub(); + } return ozoneClient; } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java index f23f8f81b60d..0d2f087a75b5 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.s3.endpoint; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_MAX_KEYS_LIMIT; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointBuilder.newBucketEndpointBuilder; import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -32,7 +33,6 @@ import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; -import org.apache.hadoop.ozone.s3.RequestIdentifier; import org.apache.hadoop.ozone.s3.commontypes.EncodingTypeObject; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; @@ -48,15 +48,13 @@ public class TestBucketList { @Test public void listRoot() throws OS3Exception, IOException { - OzoneClient client = createClientWithKeys("file1", "dir1/file2"); - - BucketEndpoint getBucket = EndpointBuilder.newBucketEndpointBuilder() + BucketEndpoint endpoint = newBucketEndpointBuilder() .setClient(client) .build(); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, "", + (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, "", null, null, null, null, null, null, 0) .getEntity(); @@ -67,21 +65,15 @@ public void listRoot() throws OS3Exception, IOException { assertEquals(1, getBucketResponse.getContents().size()); assertEquals("file1", getBucketResponse.getContents().get(0).getKey().getName()); - } @Test public void listDir() throws OS3Exception, IOException { - - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient client = createClientWithKeys("dir1/file2", "dir1/dir2/file2"); - - getBucket.setClient(client); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, + (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, "dir1", null, null, null, null, null, null, 0).getEntity(); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -89,22 +81,18 @@ public void listDir() throws OS3Exception, IOException { getBucketResponse.getCommonPrefixes().get(0).getPrefix().getName()); assertEquals(0, getBucketResponse.getContents().size()); - } @Test public void listSubDir() throws OS3Exception, IOException { - - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient ozoneClient = createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file", "dir1bha/file2"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); + ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket + (ListObjectResponse) endpoint .get("b1", "/", null, null, 100, "dir1/", null, null, null, null, null, null, 0) .getEntity(); @@ -116,18 +104,15 @@ public void listSubDir() throws OS3Exception, IOException { assertEquals(1, getBucketResponse.getContents().size()); assertEquals("dir1/file2", getBucketResponse.getContents().get(0).getKey().getName()); - } @Test public void listObjectOwner() throws OS3Exception, IOException { - UserGroupInformation user1 = UserGroupInformation .createUserForTesting("user1", new String[] {"user1"}); UserGroupInformation user2 = UserGroupInformation .createUserForTesting("user2", new String[] {"user2"}); - BucketEndpoint getBucket = new BucketEndpoint(); OzoneClient client = new OzoneClientStub(); client.getObjectStore().createS3Bucket("b1"); OzoneBucket bucket = client.getObjectStore().getS3Bucket("b1"); @@ -137,10 +122,10 @@ public void listObjectOwner() throws OS3Exception, IOException { UserGroupInformation.setLoginUser(user2); bucket.createKey("key2", 0).close(); - getBucket.setClient(client); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); + ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, + (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, "key", null, null, null, null, null, null, 0).getEntity(); assertEquals(2, getBucketResponse.getContents().size()); @@ -148,84 +133,67 @@ public void listObjectOwner() throws OS3Exception, IOException { getBucketResponse.getContents().get(0).getOwner().getDisplayName()); assertEquals(user2.getShortUserName(), getBucketResponse.getContents().get(1).getOwner().getDisplayName()); - } @Test public void listWithPrefixAndDelimiter() throws OS3Exception, IOException { - - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient ozoneClient = createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file", "dir1bha/file2", "file2"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, + (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, "dir1", null, null, null, null, null, null, 0).getEntity(); assertEquals(3, getBucketResponse.getCommonPrefixes().size()); - } @Test public void listWithPrefixAndDelimiter1() throws OS3Exception, IOException { - - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient ozoneClient = createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file", "dir1bha/file2", "file2"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, + (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, "", null, null, null, null, null, null, 0).getEntity(); assertEquals(3, getBucketResponse.getCommonPrefixes().size()); assertEquals("file2", getBucketResponse.getContents().get(0) .getKey().getName()); - } @Test public void listWithPrefixAndDelimiter2() throws OS3Exception, IOException { - - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient ozoneClient = createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file", "dir1bha/file2", "file2"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); + ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, "dir1bh", + (ListObjectResponse) endpoint.get("b1", "/", null, null, 100, "dir1bh", null, "dir1/dir2/file2", null, null, null, null, 0).getEntity(); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); - } @Test public void listWithPrefixAndEmptyStrDelimiter() throws OS3Exception, IOException { - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient ozoneClient = createClientWithKeys("dir1/", "dir1/dir2/", "dir1/dir2/file1", "dir1/dir2/file2"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); + // Should behave the same if delimiter is null ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "", null, null, 100, "dir1/", + (ListObjectResponse) endpoint.get("b1", "", null, null, 100, "dir1/", null, null, null, null, null, null, 0).getEntity(); assertEquals(0, getBucketResponse.getCommonPrefixes().size()); @@ -238,26 +206,22 @@ public void listWithPrefixAndEmptyStrDelimiter() getBucketResponse.getContents().get(2).getKey().getName()); assertEquals("dir1/dir2/file2", getBucketResponse.getContents().get(3).getKey().getName()); - } @Test public void listWithContinuationToken() throws OS3Exception, IOException { - - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient ozoneClient = createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file", "dir1bha/file2", "file2"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); + int maxKeys = 2; // As we have 5 keys, with max keys 2 we should call list 3 times. // First time ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.get("b1", null, null, null, maxKeys, + (ListObjectResponse) endpoint.get("b1", null, null, null, maxKeys, "", null, null, null, null, null, null, 0).getEntity(); assertTrue(getBucketResponse.isTruncated()); @@ -266,7 +230,7 @@ public void listWithContinuationToken() throws OS3Exception, IOException { // 2nd time String continueToken = getBucketResponse.getNextToken(); getBucketResponse = - (ListObjectResponse) getBucket.get("b1", null, null, null, maxKeys, + (ListObjectResponse) endpoint.get("b1", null, null, null, maxKeys, "", continueToken, null, null, null, null, null, 0).getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getContents().size()); @@ -276,20 +240,16 @@ public void listWithContinuationToken() throws OS3Exception, IOException { //3rd time getBucketResponse = - (ListObjectResponse) getBucket.get("b1", null, null, null, maxKeys, + (ListObjectResponse) endpoint.get("b1", null, null, null, maxKeys, "", continueToken, null, null, null, null, null, 0).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(1, getBucketResponse.getContents().size()); - } @Test public void listWithContinuationTokenDirBreak() throws OS3Exception, IOException { - - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient ozoneClient = createClientWithKeys( "test/dir1/file1", @@ -301,15 +261,14 @@ public void listWithContinuationTokenDirBreak() "test/dir3/file7", "test/file8"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); int maxKeys = 2; ListObjectResponse getBucketResponse; getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, + (ListObjectResponse) endpoint.get("b1", "/", null, null, maxKeys, "test/", null, null, null, null, null, null, 0).getEntity(); assertEquals(0, getBucketResponse.getContents().size()); @@ -320,7 +279,7 @@ public void listWithContinuationTokenDirBreak() getBucketResponse.getCommonPrefixes().get(1).getPrefix().getName()); getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, + (ListObjectResponse) endpoint.get("b1", "/", null, null, maxKeys, "test/", getBucketResponse.getNextToken(), null, null, null, null, null, 0).getEntity(); assertEquals(1, getBucketResponse.getContents().size()); @@ -329,7 +288,6 @@ public void listWithContinuationTokenDirBreak() getBucketResponse.getCommonPrefixes().get(0).getPrefix().getName()); assertEquals("test/file8", getBucketResponse.getContents().get(0).getKey().getName()); - } /** @@ -338,22 +296,18 @@ public void listWithContinuationTokenDirBreak() */ @Test public void listWithContinuationToken1() throws OS3Exception, IOException { - - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient ozoneClient = createClientWithKeys("dir1/file1", "dir1bh/file1", "dir1bha/file1", "dir0/file1", "dir2/file1"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); int maxKeys = 2; // As we have 5 keys, with max keys 2 we should call list 3 times. // First time ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, + (ListObjectResponse) endpoint.get("b1", "/", null, null, maxKeys, "dir", null, null, null, null, null, null, 0).getEntity(); assertTrue(getBucketResponse.isTruncated()); @@ -362,7 +316,7 @@ public void listWithContinuationToken1() throws OS3Exception, IOException { // 2nd time String continueToken = getBucketResponse.getNextToken(); getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, + (ListObjectResponse) endpoint.get("b1", "/", null, null, maxKeys, "dir", continueToken, null, null, null, null, null, 0).getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); @@ -370,27 +324,22 @@ public void listWithContinuationToken1() throws OS3Exception, IOException { //3rd time continueToken = getBucketResponse.getNextToken(); getBucketResponse = - (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, + (ListObjectResponse) endpoint.get("b1", "/", null, null, maxKeys, "dir", continueToken, null, null, null, null, null, 0).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); - } @Test public void listWithContinuationTokenFail() throws IOException { - - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient ozoneClient = createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file", "dir1bha/file2", "dir1", "dir2", "dir3"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); - OS3Exception e = assertThrows(OS3Exception.class, () -> getBucket.get("b1", + OS3Exception e = assertThrows(OS3Exception.class, () -> endpoint.get("b1", "/", null, null, 2, "dir", "random", null, null, null, null, null, 1000) .getEntity(), "listWithContinuationTokenFail"); assertEquals("random", e.getResource()); @@ -399,17 +348,14 @@ public void listWithContinuationTokenFail() throws IOException { @Test public void testStartAfter() throws IOException, OS3Exception { - BucketEndpoint getBucket = new BucketEndpoint(); - OzoneClient ozoneClient = createClientWithKeys("dir1/file1", "dir1bh/file1", "dir1bha/file1", "dir0/file1", "dir2/file1"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.get("b1", null, null, null, 1000, + (ListObjectResponse) endpoint.get("b1", null, null, null, 1000, null, null, null, null, null, null, null, 0).getEntity(); assertFalse(getBucketResponse.isTruncated()); @@ -420,20 +366,18 @@ public void testStartAfter() throws IOException, OS3Exception { String startAfter = "dir0/file1"; getBucketResponse = - (ListObjectResponse) getBucket.get("b1", null, null, null, + (ListObjectResponse) endpoint.get("b1", null, null, null, 1000, null, null, startAfter, null, null, null, null, 0).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(4, getBucketResponse.getContents().size()); getBucketResponse = - (ListObjectResponse) getBucket.get("b1", null, null, null, + (ListObjectResponse) endpoint.get("b1", null, null, null, 1000, null, null, "random", null, null, null, null, 0).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(0, getBucketResponse.getContents().size()); - - } @Test @@ -462,18 +406,16 @@ public void testEncodingType() throws IOException, OS3Exception { if encodingType == null , the = will not be encoded to "%3D" * */ - BucketEndpoint getBucket = new BucketEndpoint(); OzoneClient ozoneClient = createClientWithKeys("data=1970", "data==1970"); - getBucket.setClient(ozoneClient); - getBucket.setRequestIdentifier(new RequestIdentifier()); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(ozoneClient).build(); String delimiter = "="; String prefix = "data="; String startAfter = "data="; String encodingType = ENCODING_TYPE; - ListObjectResponse response = (ListObjectResponse) getBucket.get( + ListObjectResponse response = (ListObjectResponse) endpoint.get( "b1", delimiter, encodingType, null, 1000, prefix, null, startAfter, null, null, null, null, 0).getEntity(); @@ -491,7 +433,7 @@ public void testEncodingType() throws IOException, OS3Exception { assertEquals(encodingType, response.getContents().get(0).getKey().getEncodingType()); - response = (ListObjectResponse) getBucket.get( + response = (ListObjectResponse) endpoint.get( "b1", delimiter, null, null, 1000, prefix, null, startAfter, null, null, null, null, 0).getEntity(); @@ -506,17 +448,14 @@ public void testEncodingType() throws IOException, OS3Exception { assertEncodingTypeObject(prefix + delimiter, null, response.getCommonPrefixes().get(0).getPrefix()); assertNull(response.getContents().get(0).getKey().getEncodingType()); - } @Test public void testEncodingTypeException() throws IOException { - BucketEndpoint getBucket = new BucketEndpoint(); OzoneClient client = new OzoneClientStub(); client.getObjectStore().createS3Bucket("b1"); - getBucket.setClient(client); - getBucket.setRequestIdentifier(new RequestIdentifier()); - OS3Exception e = assertThrows(OS3Exception.class, () -> getBucket.get( + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); + OS3Exception e = assertThrows(OS3Exception.class, () -> endpoint.get( "b1", null, "unSupportType", null, 1000, null, null, null, null, null, null, null, 0).getEntity()); assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e.getCode()); @@ -526,7 +465,7 @@ public void testEncodingTypeException() throws IOException { public void testListObjectsWithNegativeMaxKeys() throws Exception { OzoneClient client = new OzoneClientStub(); client.getObjectStore().createS3Bucket("bucket"); - BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder() + BucketEndpoint bucketEndpoint = newBucketEndpointBuilder() .setClient(client) .build(); @@ -542,7 +481,7 @@ public void testListObjectsWithNegativeMaxKeys() throws Exception { public void testListObjectsWithZeroMaxKeys() throws Exception { OzoneClient client = new OzoneClientStub(); client.getObjectStore().createS3Bucket("bucket"); - BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder() + BucketEndpoint bucketEndpoint = newBucketEndpointBuilder() .setClient(client) .build(); @@ -558,7 +497,7 @@ public void testListObjectsWithZeroMaxKeys() throws Exception { @Test public void testListObjectsWithZeroMaxKeysInNonEmptyBucket() throws Exception { OzoneClient client = createClientWithKeys("file1", "file2", "file3", "file4", "file5"); - BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder() + BucketEndpoint bucketEndpoint = newBucketEndpointBuilder() .setClient(client) .build(); @@ -588,11 +527,10 @@ public void testListObjectsRespectsConfiguredMaxKeysLimit() throws Exception { config.set(OZONE_S3G_LIST_MAX_KEYS_LIMIT, configuredMaxKeysLimit); // Arrange: Build and initialize the BucketEndpoint with the config - BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder() + BucketEndpoint bucketEndpoint = newBucketEndpointBuilder() .setClient(client) .setConfig(config) .build(); - bucketEndpoint.init(); // Assert: Ensure the config value is correctly set in the endpoint assertEquals(configuredMaxKeysLimit, diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java index fd83523214ca..702c32d1abab 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java @@ -45,17 +45,14 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; -import org.apache.hadoop.ozone.s3.RequestIdentifier; import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; -import org.apache.hadoop.ozone.s3.signature.SignatureInfo; import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -66,12 +63,9 @@ public class TestMultipartUploadWithCopy { - private static final ObjectEndpoint REST = new ObjectEndpoint(); - private static final String KEY = "key2"; private static final String EXISTING_KEY = "key1"; private static final String EXISTING_KEY_CONTENT = "testkey"; - private static final OzoneClient CLIENT = new OzoneClientStub(); private static final long DELAY_MS = 2000; private static String beforeSourceKeyModificationTimeStr; private static String afterSourceKeyModificationTimeStr; @@ -80,12 +74,16 @@ public class TestMultipartUploadWithCopy { private static final String ERROR_CODE = S3ErrorTable.PRECOND_FAILED.getCode(); + private static ObjectEndpoint endpoint; + private static OzoneClient client; + @BeforeAll public static void setUp() throws Exception { - CLIENT.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET); + client = new OzoneClientStub(); + client.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET); OzoneBucket bucket = - CLIENT.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET); + client.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET); byte[] keyContent = EXISTING_KEY_CONTENT.getBytes(UTF_8); try (OutputStream stream = bucket @@ -99,7 +97,7 @@ public static void setUp() throws Exception { stream.write(keyContent); } - long sourceKeyLastModificationTime = CLIENT.getObjectStore() + long sourceKeyLastModificationTime = client.getObjectStore() .getS3Bucket(OzoneConsts.S3_BUCKET) .getKey(EXISTING_KEY) .getModificationTime().toEpochMilli(); @@ -125,13 +123,10 @@ public static void setUp() throws Exception { when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) .thenReturn("mockSignature"); - REST.setHeaders(headers); - REST.setClient(CLIENT); - REST.setOzoneConfiguration(new OzoneConfiguration()); - REST.setRequestIdentifier(new RequestIdentifier()); - SignatureInfo signatureInfo = mock(SignatureInfo.class); - when(signatureInfo.isSignPayload()).thenReturn(true); - REST.setSignatureInfo(signatureInfo); + endpoint = EndpointBuilder.newObjectEndpointBuilder() + .setHeaders(headers) + .setClient(client) + .build(); } @Test @@ -174,7 +169,7 @@ public void testMultipart() throws Exception { uploadID); OzoneBucket bucket = - CLIENT.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET); + client.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET); try (InputStream is = bucket.readKey(KEY)) { String keyContent = new Scanner(is, UTF_8.name()) .useDelimiter("\\A").next(); @@ -317,7 +312,7 @@ public void testMultipartTSHeaders() throws Exception { private String initiateMultipartUpload(String key) throws IOException, OS3Exception { setHeaders(); - Response response = REST.initializeMultipartUpload(OzoneConsts.S3_BUCKET, + Response response = endpoint.initializeMultipartUpload(OzoneConsts.S3_BUCKET, key); MultipartUploadInitiateResponse multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); @@ -335,7 +330,7 @@ private Part uploadPart(String key, String uploadID, int partNumber, String setHeaders(); ByteArrayInputStream body = new ByteArrayInputStream(content.getBytes(UTF_8)); - Response response = REST.put(OzoneConsts.S3_BUCKET, key, content.length(), + Response response = endpoint.put(OzoneConsts.S3_BUCKET, key, content.length(), partNumber, uploadID, null, null, body); assertEquals(200, response.getStatus()); assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); @@ -380,7 +375,7 @@ private Part uploadPartWithCopy(String key, String uploadID, int partNumber, setHeaders(additionalHeaders); ByteArrayInputStream body = new ByteArrayInputStream("".getBytes(UTF_8)); - Response response = REST.put(OzoneConsts.S3_BUCKET, key, 0, partNumber, + Response response = endpoint.put(OzoneConsts.S3_BUCKET, key, 0, partNumber, uploadID, null, null, body); assertEquals(200, response.getStatus()); @@ -408,9 +403,9 @@ public void testUploadWithRangeCopyContentLength() OzoneConsts.S3_BUCKET + "/" + EXISTING_KEY); additionalHeaders.put(COPY_SOURCE_HEADER_RANGE, "bytes=0-3"); setHeaders(additionalHeaders); - REST.put(OzoneConsts.S3_BUCKET, KEY, 0, 1, uploadID, null, null, body); + endpoint.put(OzoneConsts.S3_BUCKET, KEY, 0, 1, uploadID, null, null, body); OzoneMultipartUploadPartListParts parts = - CLIENT.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET) + client.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET) .listParts(KEY, uploadID, 0, 100); assertEquals(1, parts.getPartInfoList().size()); assertEquals(4, parts.getPartInfoList().get(0).getSize()); @@ -420,7 +415,7 @@ private void completeMultipartUpload(String key, CompleteMultipartUploadRequest completeMultipartUploadRequest, String uploadID) throws IOException, OS3Exception { setHeaders(); - Response response = REST.completeMultipartUpload(OzoneConsts.S3_BUCKET, key, + Response response = endpoint.completeMultipartUpload(OzoneConsts.S3_BUCKET, key, uploadID, completeMultipartUploadRequest); assertEquals(200, response.getStatus()); @@ -445,7 +440,7 @@ private void setHeaders(Map additionalHeaders) { additionalHeaders .forEach((k, v) -> when(headers.getHeaderString(k)).thenReturn(v)); - REST.setHeaders(headers); + endpoint.setHeaders(headers); } private void setHeaders() { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java index cb3e24472b05..488474e30390 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java @@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.s3.exception.OS3Exception; @@ -124,6 +125,7 @@ public void testDeleteObjectTaggingNotImplemented() throws Exception { when(mockClient.getObjectStore()).thenReturn(mockObjectStore); when(mockObjectStore.getS3Volume()).thenReturn(mockVolume); + when(mockObjectStore.getClientProxy()).thenReturn(mock(ClientProtocol.class)); when(mockVolume.getBucket("fsoBucket")).thenReturn(mockBucket); ObjectEndpoint endpoint = EndpointBuilder.newObjectEndpointBuilder() diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java index 02b71e8772c4..d1651d6b59c0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java @@ -47,6 +47,7 @@ import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.s3.exception.OS3Exception; @@ -170,6 +171,7 @@ public void testPutObjectTaggingNotImplemented() throws Exception { when(mockClient.getObjectStore()).thenReturn(mockObjectStore); when(mockObjectStore.getS3Volume()).thenReturn(mockVolume); + when(mockObjectStore.getClientProxy()).thenReturn(mock(ClientProtocol.class)); when(mockVolume.getBucket("fsoBucket")).thenReturn(mockBucket); ObjectEndpoint endpoint = EndpointBuilder.newObjectEndpointBuilder() diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java index 4b2d8a49efb9..736660073d57 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java @@ -72,8 +72,6 @@ public void setUp() throws Exception { .setClient(client) .setConfig(conf) .build(); - - rest.init(); } @Test diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index 81f6853bf73f..e422e4920792 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -93,6 +93,7 @@ public void setup() { clientProtocol = mock(ClientProtocol.class); S3GatewayMetrics.create(conf); when(client.getProxy()).thenReturn(clientProtocol); + when(objectStore.getClientProxy()).thenReturn(clientProtocol); } /** diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java index 992b0dc2dc9d..dbe21601dbd3 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java @@ -80,8 +80,6 @@ public void setUp() throws Exception { .setHeaders(headers) .setConfig(conf) .build(); - - rest.init(); } @Test diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java index 5a72fad11b2e..cae9a9422885 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java @@ -78,6 +78,7 @@ public void setup() throws Exception { clientStub = new OzoneClientStub(); clientStub.getObjectStore().createS3Bucket(bucketName); bucket = clientStub.getObjectStore().getS3Bucket(bucketName); + bucket.createKey("file1", 0).close(); headers = mock(HttpHeaders.class); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( @@ -132,8 +133,6 @@ public void testListBucketSuccess() throws Exception { public void testGetBucketSuccess() throws Exception { long oriMetric = metrics.getGetBucketSuccess(); - clientStub = createClientWithKeys("file1"); - bucketEndpoint.setClient(clientStub); bucketEndpoint.get(bucketName, null, null, null, 1000, null, null, "random", null, @@ -637,13 +636,6 @@ public void testDeleteObjectTaggingFailure() throws Exception { assertEquals(1L, curMetric - oriMetric); } - private OzoneClient createClientWithKeys(String... keys) throws IOException { - for (String key : keys) { - bucket.createKey(key, 0).close(); - } - return clientStub; - } - private String initiateMultipartUpload(String bktName, String key) throws IOException, OS3Exception {