Skip to content

Commit 4501218

Browse files
adoroszlaiadoroszlai
authored
HDDS-15003. Inline EndpointBase#createS3Bucket/deleteS3Bucket (#10060)
1 parent 3c1e005 commit 4501218

12 files changed

Lines changed: 102 additions & 158 deletions

File tree

‎hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketAclHandler.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ Response handleGetRequest(S3RequestContext context, String bucketName)
8888
context.setAction(S3GAction.GET_ACL);
8989

9090
try {
91-
OzoneBucket bucket = getBucket(bucketName);
91+
OzoneBucket bucket = context.getVolume().getBucket(bucketName);
9292
S3Owner.verifyBucketOwnerCondition(getHeaders(), bucketName, bucket.getOwner());
9393
S3Owner owner = S3Owner.of(bucket.getOwner());
9494

@@ -139,9 +139,9 @@ Response handlePutRequest(S3RequestContext context, String bucketName, InputStre
139139
String grantFull = getHeaders().getHeaderString(S3Acl.GRANT_FULL_CONTROL);
140140

141141
try {
142-
OzoneBucket bucket = getBucket(bucketName);
142+
OzoneVolume volume = context.getVolume();
143+
OzoneBucket bucket = volume.getBucket(bucketName);
143144
S3Owner.verifyBucketOwnerCondition(getHeaders(), bucketName, bucket.getOwner());
144-
OzoneVolume volume = getVolume();
145145

146146
List<OzoneAcl> ozoneAclListOnBucket = new ArrayList<>();
147147
List<OzoneAcl> ozoneAclListOnVolume = new ArrayList<>();

‎hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import java.io.IOException;
2121
import java.io.InputStream;
22+
import javax.ws.rs.core.HttpHeaders;
2223
import javax.ws.rs.core.Response;
2324
import org.apache.hadoop.ozone.audit.S3GAction;
2425
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -62,10 +63,16 @@ Response handlePutRequest(S3RequestContext context, String bucketName, InputStre
6263

6364
context.setAction(S3GAction.CREATE_BUCKET);
6465

65-
String location = createS3Bucket(bucketName);
66-
getMetrics().updateCreateBucketSuccessStats(context.getStartNanos());
67-
return Response.status(HttpStatus.SC_OK).header("Location", location)
68-
.build();
66+
try {
67+
getClient().getObjectStore().createS3Bucket(bucketName);
68+
getMetrics().updateCreateBucketSuccessStats(context.getStartNanos());
69+
return Response.status(HttpStatus.SC_OK)
70+
.header(HttpHeaders.LOCATION, "/" + bucketName)
71+
.build();
72+
} catch (Exception e) {
73+
getMetrics().updateCreateBucketFailureStats(context.getStartNanos());
74+
throw e;
75+
}
6976
}
7077

7178
/**
@@ -83,10 +90,10 @@ Response handleDeleteRequest(S3RequestContext context, String bucketName)
8390

8491
try {
8592
if (S3Owner.hasBucketOwnershipVerificationConditions(getHeaders())) {
86-
OzoneBucket bucket = getBucket(bucketName);
93+
OzoneBucket bucket = context.getVolume().getBucket(bucketName);
8794
S3Owner.verifyBucketOwnerCondition(getHeaders(), bucketName, bucket.getOwner());
8895
}
89-
deleteS3Bucket(bucketName);
96+
context.getVolume().deleteBucket(bucketName);
9097
} catch (Exception ex) {
9198
getMetrics().updateDeleteBucketFailureStats(context.getStartNanos());
9299
throw ex;

‎hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ Response handleGetRequest(S3RequestContext context, String bucketName) throws IO
136136
boolean shallow = listKeysShallowEnabled
137137
&& OZONE_URI_DELIMITER.equals(delimiter);
138138

139-
bucket = getBucket(bucketName);
139+
bucket = context.getVolume().getBucket(bucketName);
140140
S3Owner.verifyBucketOwnerCondition(getHeaders(), bucketName, bucket.getOwner());
141141

142142
ozoneKeyIterator = bucket.listKeys(prefix, prevKey, shallow);
@@ -293,11 +293,14 @@ public Response head(@PathParam(BUCKET) String bucketName)
293293
long startNanos = Time.monotonicNowNanos();
294294
S3GAction s3GAction = S3GAction.HEAD_BUCKET;
295295
try {
296-
OzoneBucket bucket = getBucket(bucketName);
296+
OzoneBucket bucket = getVolume().getBucket(bucketName);
297297
S3Owner.verifyBucketOwnerCondition(getHeaders(), bucketName, bucket.getOwner());
298298
auditReadSuccess(s3GAction);
299299
getMetrics().updateHeadBucketSuccessStats(startNanos);
300300
return Response.ok().build();
301+
} catch (OMException e) {
302+
auditReadFailure(s3GAction, e);
303+
throw newError(translateException(e), bucketName, e);
301304
} catch (Exception e) {
302305
auditReadFailure(s3GAction, e);
303306
throw e;
@@ -342,7 +345,7 @@ public MultiDeleteResponse multiDelete(
342345
) throws OS3Exception, IOException {
343346
S3GAction s3GAction = S3GAction.MULTI_DELETE;
344347

345-
OzoneBucket bucket = getBucket(bucketName);
348+
OzoneBucket bucket = getVolume().getBucket(bucketName);
346349
MultiDeleteResponse result = new MultiDeleteResponse();
347350
List<String> deleteKeys = new ArrayList<>();
348351

@@ -429,15 +432,21 @@ protected void init() {
429432
}
430433

431434
private static OS3Exception translateException(OMException ex) {
432-
if (isAccessDenied(ex)) {
435+
switch (ex.getResult()) {
436+
case ACCESS_DENIED:
437+
case INVALID_TOKEN:
438+
case PERMISSION_DENIED:
433439
return S3ErrorTable.ACCESS_DENIED;
434-
} else if (ex.getResult() == ResultCodes.BUCKET_NOT_EMPTY) {
440+
case BUCKET_ALREADY_EXISTS:
441+
return S3ErrorTable.BUCKET_ALREADY_EXISTS;
442+
case BUCKET_NOT_EMPTY:
435443
return S3ErrorTable.BUCKET_NOT_EMPTY;
436-
} else if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
444+
case BUCKET_NOT_FOUND:
445+
case VOLUME_NOT_FOUND:
437446
return S3ErrorTable.NO_SUCH_BUCKET;
438-
} else if (ex.getResult() == ResultCodes.INVALID_BUCKET_NAME) {
447+
case INVALID_BUCKET_NAME:
439448
return S3ErrorTable.INVALID_BUCKET_NAME;
440-
} else {
449+
default:
441450
return INTERNAL_ERROR;
442451
}
443452
}

‎hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java‎

Lines changed: 0 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,6 @@
108108
import org.apache.hadoop.ozone.s3.util.AuditUtils;
109109
import org.apache.hadoop.ozone.s3.util.S3Utils;
110110
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
111-
import org.apache.hadoop.util.Time;
112111
import org.apache.http.NameValuePair;
113112
import org.apache.http.client.utils.URLEncodedUtils;
114113
import org.slf4j.Logger;
@@ -186,27 +185,6 @@ public RequestParameters.Mutable queryParamsForTest() {
186185
return queryParams;
187186
}
188187

189-
protected OzoneBucket getBucket(OzoneVolume volume, String bucketName)
190-
throws OS3Exception, IOException {
191-
OzoneBucket bucket;
192-
try {
193-
bucket = volume.getBucket(bucketName);
194-
} catch (OMException ex) {
195-
if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
196-
throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex);
197-
} else if (ex.getResult() == ResultCodes.INVALID_TOKEN) {
198-
throw newError(S3ErrorTable.ACCESS_DENIED,
199-
s3Auth.getAccessID(), ex);
200-
} else if (ex.getResult() == ResultCodes.TIMEOUT ||
201-
ex.getResult() == ResultCodes.INTERNAL_ERROR) {
202-
throw newError(S3ErrorTable.INTERNAL_ERROR, bucketName, ex);
203-
} else {
204-
throw ex;
205-
}
206-
}
207-
return bucket;
208-
}
209-
210188
/**
211189
* Initializes the object post construction. Calls init() from any
212190
* child classes to work around the issue of only one method can be annotated.
@@ -274,62 +252,6 @@ protected OzoneVolume getVolume() throws IOException {
274252
return client.getObjectStore().getS3Volume();
275253
}
276254

277-
/**
278-
* Create an S3Bucket, and also it creates mapping needed to access via
279-
* ozone and S3.
280-
* @param bucketName
281-
* @return location of the S3Bucket.
282-
* @throws IOException
283-
*/
284-
protected String createS3Bucket(String bucketName) throws
285-
IOException, OS3Exception {
286-
long startNanos = Time.monotonicNowNanos();
287-
try {
288-
client.getObjectStore().createS3Bucket(bucketName);
289-
} catch (OMException ex) {
290-
getMetrics().updateCreateBucketFailureStats(startNanos);
291-
if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
292-
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
293-
} else if (ex.getResult() == ResultCodes.INVALID_TOKEN) {
294-
throw newError(S3ErrorTable.ACCESS_DENIED,
295-
s3Auth.getAccessID(), ex);
296-
} else if (ex.getResult() == ResultCodes.TIMEOUT ||
297-
ex.getResult() == ResultCodes.INTERNAL_ERROR) {
298-
throw newError(S3ErrorTable.INTERNAL_ERROR, bucketName, ex);
299-
} else if (ex.getResult() == ResultCodes.BUCKET_ALREADY_EXISTS) {
300-
throw newError(S3ErrorTable.BUCKET_ALREADY_EXISTS, bucketName, ex);
301-
} else {
302-
throw ex;
303-
}
304-
}
305-
return "/" + bucketName;
306-
}
307-
308-
/**
309-
* Deletes an s3 bucket and removes mapping of Ozone volume/bucket.
310-
* @param s3BucketName - S3 Bucket Name.
311-
* @throws IOException in case the bucket cannot be deleted.
312-
*/
313-
protected void deleteS3Bucket(String s3BucketName)
314-
throws IOException, OS3Exception {
315-
try {
316-
client.getObjectStore().deleteS3Bucket(s3BucketName);
317-
} catch (OMException ex) {
318-
if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
319-
throw newError(S3ErrorTable.ACCESS_DENIED,
320-
s3BucketName, ex);
321-
} else if (ex.getResult() == ResultCodes.INVALID_TOKEN) {
322-
throw newError(S3ErrorTable.ACCESS_DENIED,
323-
s3Auth.getAccessID(), ex);
324-
} else if (ex.getResult() == ResultCodes.TIMEOUT ||
325-
ex.getResult() == ResultCodes.INTERNAL_ERROR) {
326-
throw newError(S3ErrorTable.INTERNAL_ERROR, s3BucketName, ex);
327-
} else {
328-
throw ex;
329-
}
330-
}
331-
}
332-
333255
/**
334256
* Returns Iterator to iterate over all buckets for a specific user.
335257
* The result can be restricted using bucket prefix, will return all

‎hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ Response handleGetRequest(S3RequestContext context, String bucketName)
5656

5757
long startNanos = context.getStartNanos();
5858

59-
OzoneBucket bucket = getBucket(bucketName);
59+
OzoneBucket bucket = context.getVolume().getBucket(bucketName);
6060

6161
try {
6262
S3Owner.verifyBucketOwnerCondition(getHeaders(), bucketName, bucket.getOwner());

‎hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java‎

Lines changed: 17 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,6 @@
1717

1818
package org.apache.hadoop.ozone.client;
1919

20-
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_ALREADY_EXISTS;
21-
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_EMPTY;
22-
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
2320
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
2421
import static org.mockito.Mockito.mock;
2522

@@ -40,20 +37,20 @@
4037
*/
4138
public class ObjectStoreStub extends ObjectStore {
4239

43-
private static OzoneConfiguration conf = new OzoneConfiguration();
44-
private Map<String, OzoneVolumeStub> volumes = new HashMap<>();
45-
private Map<String, Boolean> bucketEmptyStatus = new HashMap<>();
40+
private final Map<String, OzoneVolumeStub> volumes = new HashMap<>();
41+
private final String s3VolumeName;
4642

4743
public ObjectStoreStub() {
48-
super(conf, mock(ClientProtocol.class));
44+
this(new OzoneConfiguration(), mock(ClientProtocol.class));
4945
}
5046

5147
public ObjectStoreStub(ConfigurationSource conf, ClientProtocol proxy) {
5248
super(conf, proxy);
49+
s3VolumeName = HddsClientUtils.getDefaultS3VolumeName(conf);
5350
}
5451

5552
@Override
56-
public void createVolume(String volumeName) throws IOException {
53+
public void createVolume(String volumeName) {
5754
createVolume(volumeName,
5855
VolumeArgs.newBuilder()
5956
.setAdmin("root")
@@ -86,8 +83,7 @@ public OzoneVolume getVolume(String volumeName) throws IOException {
8683
}
8784

8885
@Override
89-
public Iterator<? extends OzoneVolume> listVolumes(String volumePrefix)
90-
throws IOException {
86+
public Iterator<? extends OzoneVolume> listVolumes(String volumePrefix) {
9187
return volumes.values()
9288
.stream()
9389
.filter(volume -> volume.getName().startsWith(volumePrefix))
@@ -98,7 +94,7 @@ public Iterator<? extends OzoneVolume> listVolumes(String volumePrefix)
9894

9995
@Override
10096
public Iterator<? extends OzoneVolume> listVolumes(String volumePrefix,
101-
String prevVolume) throws IOException {
97+
String prevVolume) {
10298
return volumes.values()
10399
.stream()
104100
.filter(volume -> volume.getName().compareTo(prevVolume) > 0)
@@ -109,7 +105,7 @@ public Iterator<? extends OzoneVolume> listVolumes(String volumePrefix,
109105

110106
@Override
111107
public Iterator<? extends OzoneVolume> listVolumesByUser(String user,
112-
String volumePrefix, String prevVolume) throws IOException {
108+
String volumePrefix, String prevVolume) {
113109
return volumes.values()
114110
.stream()
115111
.filter(volume -> volume.getOwner().equals(user))
@@ -120,49 +116,33 @@ public Iterator<? extends OzoneVolume> listVolumesByUser(String user,
120116
}
121117

122118
@Override
123-
public void deleteVolume(String volumeName) throws IOException {
119+
public void deleteVolume(String volumeName) {
124120
volumes.remove(volumeName);
125121
}
126122

127123
@Override
128124
public OzoneVolume getS3Volume() throws IOException {
129125
// Always return default S3 volume. This class will not be used for
130126
// multitenant testing.
131-
String volumeName = HddsClientUtils.getDefaultS3VolumeName(conf);
132-
return getVolume(volumeName);
127+
return getVolume(s3VolumeName);
133128
}
134129

135130
@Override
136131
public void createS3Bucket(String s3BucketName) throws
137132
IOException {
138-
if (!bucketEmptyStatus.containsKey(s3BucketName)) {
139-
String volumeName = HddsClientUtils.getDefaultS3VolumeName(conf);
140-
bucketEmptyStatus.put(s3BucketName, true);
141-
if (!volumes.containsKey(volumeName)) {
142-
createVolume(volumeName);
143-
}
144-
volumes.get(volumeName).createBucket(s3BucketName);
145-
} else {
146-
throw new OMException("", BUCKET_ALREADY_EXISTS);
133+
if (!volumes.containsKey(s3VolumeName)) {
134+
createVolume(s3VolumeName);
147135
}
136+
volumes.get(s3VolumeName).createBucket(s3BucketName);
148137
}
149138

150139
@Override
151140
public void deleteS3Bucket(String s3BucketName) throws
152141
IOException {
153-
if (bucketEmptyStatus.containsKey(s3BucketName)) {
154-
if (bucketEmptyStatus.get(s3BucketName)) {
155-
bucketEmptyStatus.remove(s3BucketName);
156-
} else {
157-
throw new OMException("", BUCKET_NOT_EMPTY);
158-
}
159-
} else {
160-
throw new OMException("", BUCKET_NOT_FOUND);
142+
OzoneVolume volume = getS3Volume();
143+
if (volume == null) {
144+
throw new OMException("", VOLUME_NOT_FOUND);
161145
}
146+
volume.deleteBucket(s3BucketName);
162147
}
163-
164-
public void setBucketEmptyStatus(String bucketName, boolean status) {
165-
bucketEmptyStatus.computeIfPresent(bucketName, (k, v) -> status);
166-
}
167-
168148
}

‎hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,10 @@ public OzoneBucketStub build() {
105105
}
106106
}
107107

108+
boolean isEmpty() {
109+
return keyDetails.isEmpty();
110+
}
111+
108112
@Override
109113
public OzoneOutputStream createKey(String key, long size) throws IOException {
110114
return createKey(key, size,

0 commit comments

Comments
 (0)