diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java index b5fd9656b7dc..d50ec6ac58ac 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java @@ -101,6 +101,10 @@ public Builder withParams(Map args) { return this; } + public Map getParams() { + return params; + } + public Builder withResult(AuditEventStatus result) { this.ret = result.getStatus(); return this; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index e367fa46cf49..439aa27d1d0c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -55,6 +55,8 @@ import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.audit.AuditEventStatus; +import org.apache.hadoop.ozone.audit.AuditMessage; import org.apache.hadoop.ozone.audit.S3GAction; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneKey; @@ -128,8 +130,7 @@ public Response get( s3GAction = S3GAction.GET_ACL; S3BucketAcl result = getAcl(bucketName); getMetrics().updateGetAclSuccessStats(startNanos); - AUDIT.logReadSuccess( - buildAuditMessageForSuccess(s3GAction, getAuditParameters())); + auditReadSuccess(s3GAction); return Response.ok(result, MediaType.APPLICATION_XML_TYPE).build(); } @@ -165,8 +166,7 @@ public Response get( ozoneKeyIterator = bucket.listKeys(prefix, prevKey, shallow); } catch (OMException ex) { - AUDIT.logReadFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditReadFailure(s3GAction, ex); getMetrics().updateGetBucketFailureStats(startNanos); if (isAccessDenied(ex)) { throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex); @@ -178,8 +178,7 @@ public Response get( } } catch (Exception ex) { getMetrics().updateGetBucketFailureStats(startNanos); - AUDIT.logReadFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditReadFailure(s3GAction, ex); throw ex; } @@ -288,8 +287,7 @@ public Response get( getMetrics().incListKeyCount(keyCount); perf.appendCount(keyCount); perf.appendOpLatencyNanos(opLatencyNs); - AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters(), perf)); + auditReadSuccess(s3GAction, perf); response.setKeyCount(keyCount); return Response.ok(response).build(); } @@ -313,13 +311,11 @@ public Response put(@PathParam("bucket") String bucketName, if (aclMarker != null) { s3GAction = S3GAction.PUT_ACL; Response response = putAcl(bucketName, body); - AUDIT.logWriteSuccess( - buildAuditMessageForSuccess(s3GAction, getAuditParameters())); + auditWriteSuccess(s3GAction); return response; } String location = createS3Bucket(bucketName); - AUDIT.logWriteSuccess( - buildAuditMessageForSuccess(s3GAction, getAuditParameters())); + auditWriteSuccess(s3GAction); getMetrics().updateCreateBucketSuccessStats(startNanos); return Response.status(HttpStatus.SC_OK).header("Location", location) .build(); @@ -331,8 +327,7 @@ public Response put(@PathParam("bucket") String bucketName, } throw exception; } catch (Exception ex) { - AUDIT.logWriteFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditWriteFailure(s3GAction, ex); throw ex; } } @@ -379,22 +374,18 @@ public Response listMultipartUploads( upload.getCreationTime(), S3StorageType.fromReplicationConfig(upload.getReplicationConfig()) ))); - AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters())); + auditReadSuccess(s3GAction); getMetrics().updateListMultipartUploadsSuccessStats(startNanos); return Response.ok(result).build(); } catch (OMException exception) { - AUDIT.logReadFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), - exception)); + auditReadFailure(s3GAction, exception); getMetrics().updateListMultipartUploadsFailureStats(startNanos); if (isAccessDenied(exception)) { throw newError(S3ErrorTable.ACCESS_DENIED, prefix, exception); } throw exception; } catch (Exception ex) { - AUDIT.logReadFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditReadFailure(s3GAction, ex); throw ex; } } @@ -413,13 +404,11 @@ public Response head(@PathParam("bucket") String bucketName) try { OzoneBucket bucket = getBucket(bucketName); S3Owner.verifyBucketOwnerCondition(getHeaders(), bucketName, bucket.getOwner()); - AUDIT.logReadSuccess( - buildAuditMessageForSuccess(s3GAction, getAuditParameters())); + auditReadSuccess(s3GAction); getMetrics().updateHeadBucketSuccessStats(startNanos); return Response.ok().build(); } catch (Exception e) { - AUDIT.logReadFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), e)); + auditReadFailure(s3GAction, e); throw e; } } @@ -443,8 +432,7 @@ public Response delete(@PathParam("bucket") String bucketName) } deleteS3Bucket(bucketName); } catch (OMException ex) { - AUDIT.logWriteFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditWriteFailure(s3GAction, ex); getMetrics().updateDeleteBucketFailureStats(startNanos); if (ex.getResult() == ResultCodes.BUCKET_NOT_EMPTY) { throw newError(S3ErrorTable.BUCKET_NOT_EMPTY, bucketName, ex); @@ -456,13 +444,11 @@ public Response delete(@PathParam("bucket") String bucketName) throw ex; } } catch (Exception ex) { - AUDIT.logWriteFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditWriteFailure(s3GAction, ex); throw ex; } - AUDIT.logWriteSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters())); + auditWriteSuccess(s3GAction); getMetrics().updateDeleteBucketSuccessStats(startNanos); return Response .status(HttpStatus.SC_NO_CONTENT) @@ -521,15 +507,16 @@ public MultiDeleteResponse multiDelete(@PathParam("bucket") String bucketName, } } - Map auditMap = getAuditParameters(); - auditMap.put("failedDeletes", deleteKeys.toString()); + AuditMessage.Builder message = auditMessageFor(s3GAction); + message.getParams().put("failedDeletes", deleteKeys.toString()); + if (!result.getErrors().isEmpty()) { - AUDIT.logWriteFailure(buildAuditMessageForFailure(s3GAction, - auditMap, new Exception("MultiDelete Exception"))); + AUDIT.logWriteFailure(message.withResult(AuditEventStatus.FAILURE) + .withException(new Exception("MultiDelete Exception")).build()); } else { - AUDIT.logWriteSuccess( - buildAuditMessageForSuccess(s3GAction, auditMap)); + AUDIT.logWriteSuccess(message.withResult(AuditEventStatus.SUCCESS).build()); } + return result; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 42ad2c6b38ab..6f41e2bee06d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -60,7 +60,6 @@ import org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder; import org.apache.hadoop.ozone.audit.AuditLoggerType; import org.apache.hadoop.ozone.audit.AuditMessage; -import org.apache.hadoop.ozone.audit.Auditor; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKey; @@ -84,7 +83,7 @@ /** * Basic helpers for all the REST endpoints. */ -public abstract class EndpointBase implements Auditor { +public abstract class EndpointBase { protected static final String ETAG_CUSTOM = "etag-custom"; @@ -107,7 +106,7 @@ public abstract class EndpointBase implements Auditor { @Context private HttpHeaders headers; - private Set excludeMetadataFields = + private final Set excludeMetadataFields = new HashSet<>(Arrays.asList(OzoneConsts.GDPR_FLAG, STORAGE_CONFIG_HEADER)); private static final Logger LOG = LoggerFactory.getLogger(EndpointBase.class); @@ -456,8 +455,8 @@ protected static Map validateAndGetTagging( return Collections.unmodifiableMap(tags); } - private AuditMessage.Builder auditMessageBaseBuilder(AuditAction op, - Map auditMap) { + protected AuditMessage.Builder auditMessageFor(AuditAction op) { + Map auditMap = getAuditParameters(); auditMap.put("x-amz-request-id", requestIdentifier.getRequestId()); auditMap.put("x-amz-id-2", requestIdentifier.getAmzId()); @@ -475,29 +474,15 @@ private AuditMessage.Builder auditMessageBaseBuilder(AuditAction op, return builder; } - @Override - public AuditMessage buildAuditMessageForSuccess(AuditAction op, - Map auditMap) { - AuditMessage.Builder builder = auditMessageBaseBuilder(op, auditMap) + protected AuditMessage.Builder auditMessageForSuccess(AuditAction op) { + return auditMessageFor(op) .withResult(AuditEventStatus.SUCCESS); - return builder.build(); } - public AuditMessage buildAuditMessageForSuccess(AuditAction op, - Map auditMap, PerformanceStringBuilder performance) { - AuditMessage.Builder builder = auditMessageBaseBuilder(op, auditMap) - .withResult(AuditEventStatus.SUCCESS); - builder.setPerformance(performance); - return builder.build(); - } - - @Override - public AuditMessage buildAuditMessageForFailure(AuditAction op, - Map auditMap, Throwable throwable) { - AuditMessage.Builder builder = auditMessageBaseBuilder(op, auditMap) + protected AuditMessage.Builder auditMessageForFailure(AuditAction op, Throwable throwable) { + return auditMessageFor(op) .withResult(AuditEventStatus.FAILURE) .withException(throwable); - return builder.build(); } @VisibleForTesting @@ -556,14 +541,28 @@ protected Map getAuditParameters() { return AuditUtils.getAuditParameters(context); } + protected void auditWriteSuccess(AuditAction action, PerformanceStringBuilder perf) { + AUDIT.logWriteSuccess(auditMessageForSuccess(action).setPerformance(perf).build()); + } + + protected void auditWriteSuccess(AuditAction action) { + AUDIT.logWriteSuccess(auditMessageForSuccess(action).build()); + } + + protected void auditReadSuccess(AuditAction action, PerformanceStringBuilder perf) { + AUDIT.logReadSuccess(auditMessageForSuccess(action).setPerformance(perf).build()); + } + + protected void auditReadSuccess(AuditAction action) { + AUDIT.logReadSuccess(auditMessageForSuccess(action).build()); + } + protected void auditWriteFailure(AuditAction action, Throwable ex) { - AUDIT.logWriteFailure( - buildAuditMessageForFailure(action, getAuditParameters(), ex)); + AUDIT.logWriteFailure(auditMessageForFailure(action, ex).build()); } protected void auditReadFailure(AuditAction action, Exception ex) { - AUDIT.logReadFailure( - buildAuditMessageForFailure(action, getAuditParameters(), ex)); + AUDIT.logReadFailure(auditMessageForFailure(action, ex).build()); } protected boolean isAccessDenied(OMException ex) { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 4799570a69e1..3154ad74d017 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -387,8 +387,7 @@ public Response put( if (auditSuccess) { long opLatencyNs = getMetrics().updateCreateKeySuccessStats(startNanos); perf.appendOpLatencyNanos(opLatencyNs); - AUDIT.logWriteSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters(), perf)); + auditWriteSuccess(s3GAction, perf); } } } @@ -430,8 +429,7 @@ public Response get( int partMarker = parsePartNumberMarker(partNumberMarker); Response response = listParts(bucket, keyPath, uploadId, partMarker, maxParts, perf); - AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters(), perf)); + auditReadSuccess(s3GAction, perf); return response; } @@ -469,8 +467,7 @@ public Response get( } long opLatencyNs = getMetrics().updateGetKeySuccessStats(startNanos); perf.appendOpLatencyNanos(opLatencyNs); - AUDIT.logReadSuccess(buildAuditMessageForSuccess(S3GAction.GET_KEY, - getAuditParameters(), perf)); + auditReadSuccess(S3GAction.GET_KEY, perf); }; responseBuilder = Response .ok(output) @@ -493,8 +490,7 @@ public Response get( } long opLatencyNs = getMetrics().updateGetKeySuccessStats(startNanos); perf.appendOpLatencyNanos(opLatencyNs); - AUDIT.logReadSuccess(buildAuditMessageForSuccess(S3GAction.GET_KEY, - getAuditParameters(), perf)); + auditReadSuccess(S3GAction.GET_KEY, perf); }; responseBuilder = Response .status(Status.PARTIAL_CONTENT) @@ -549,9 +545,7 @@ public Response get( perf.appendMetaLatencyNanos(metadataLatencyNs); return responseBuilder.build(); } catch (OMException ex) { - AUDIT.logReadFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex) - ); + auditReadFailure(s3GAction, ex); if (taggingMarker != null) { getMetrics().updateGetObjectTaggingFailureStats(startNanos); } else if (uploadId != null) { @@ -569,9 +563,7 @@ public Response get( throw ex; } } catch (Exception ex) { - AUDIT.logReadFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex) - ); + auditReadFailure(s3GAction, ex); throw ex; } } @@ -622,8 +614,7 @@ public Response head( isFile(keyPath, key); // TODO: return the specified range bytes of this object. } catch (OMException ex) { - AUDIT.logReadFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditReadFailure(s3GAction, ex); getMetrics().updateHeadKeyFailureStats(startNanos); if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) { // Just return 404 with no content @@ -636,8 +627,7 @@ public Response head( throw ex; } } catch (Exception ex) { - AUDIT.logReadFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditReadFailure(s3GAction, ex); throw ex; } @@ -665,8 +655,7 @@ public Response head( addLastModifiedDate(response, key); addCustomMetadataHeaders(response, key); getMetrics().updateHeadKeySuccessStats(startNanos); - AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters())); + auditReadSuccess(s3GAction); return response.build(); } @@ -754,8 +743,7 @@ public Response delete( getClientProtocol().deleteKey(volume.getName(), bucketName, keyPath, false); } catch (OMException ex) { - AUDIT.logWriteFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditWriteFailure(s3GAction, ex); if (uploadId != null && !uploadId.equals("")) { getMetrics().updateAbortMultipartUploadFailureStats(startNanos); } else { @@ -780,8 +768,7 @@ public Response delete( throw ex; } } catch (Exception ex) { - AUDIT.logWriteFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditWriteFailure(s3GAction, ex); if (taggingMarker != null) { getMetrics().updateDeleteObjectTaggingFailureStats(startNanos); } else if (uploadId != null && !uploadId.equals("")) { @@ -792,8 +779,7 @@ public Response delete( throw ex; } getMetrics().updateDeleteKeySuccessStats(startNanos); - AUDIT.logWriteSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters())); + auditWriteSuccess(s3GAction); return Response .status(Status.NO_CONTENT) .build(); @@ -839,8 +825,7 @@ public Response initializeMultipartUpload( multipartUploadInitiateResponse.setKey(key); multipartUploadInitiateResponse.setUploadID(multipartInfo.getUploadID()); - AUDIT.logWriteSuccess( - buildAuditMessageForSuccess(s3GAction, getAuditParameters())); + auditWriteSuccess(s3GAction); getMetrics().updateInitMultipartUploadSuccessStats(startNanos); return Response.status(Status.OK).entity( multipartUploadInitiateResponse).build(); @@ -852,8 +837,7 @@ public Response initializeMultipartUpload( } throw ex; } catch (Exception ex) { - AUDIT.logWriteFailure( - buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex)); + auditWriteFailure(s3GAction, ex); getMetrics().updateInitMultipartUploadFailureStats(startNanos); throw ex; } @@ -908,8 +892,7 @@ public Response completeMultipartUpload(@PathParam("bucket") String bucket, wrapInQuotes(omMultipartUploadCompleteInfo.getHash())); // Location also setting as bucket name. completeMultipartUploadResponse.setLocation(bucket); - AUDIT.logWriteSuccess( - buildAuditMessageForSuccess(s3GAction, getAuditParameters())); + auditWriteSuccess(s3GAction); getMetrics().updateCompleteMultipartUploadSuccessStats(startNanos); return Response.status(Status.OK).entity(completeMultipartUploadResponse) .build(); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java index 9ff2e945cda3..6c64df079cde 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java @@ -73,17 +73,11 @@ public Response get() return Response.ok(response).build(); } catch (Exception ex) { auditSuccess = false; - AUDIT.logReadFailure( - buildAuditMessageForFailure(S3GAction.LIST_S3_BUCKETS, - getAuditParameters(), ex) - ); + auditReadFailure(S3GAction.LIST_S3_BUCKETS, ex); throw ex; } finally { if (auditSuccess) { - AUDIT.logReadSuccess( - buildAuditMessageForSuccess(S3GAction.LIST_S3_BUCKETS, - getAuditParameters()) - ); + auditReadSuccess(S3GAction.LIST_S3_BUCKETS); } } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java index 411452dec624..8709fa1f6336 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java @@ -120,7 +120,7 @@ public void testHeadBucket() throws Exception { bucketEndpoint.head(bucketName); - String expected = "INFO | S3GAudit | org.apache.hadoop.ozone.s3.endpoint.BucketEndpoint | user=null | ip=null | " + + String expected = "INFO | S3GAudit | org.apache.hadoop.ozone.s3.endpoint.EndpointBase | user=null | ip=null | " + "op=HEAD_BUCKET {\"bucket\":\"[bucket]\",\"x-amz-request-id\":\"" + requestIdentifier.getRequestId() + "\",\"x-amz-id-2\":\"" + requestIdentifier.getAmzId() + "\"} | ret=SUCCESS"; @@ -131,7 +131,7 @@ public void testHeadBucket() throws Exception { public void testListBucket() throws Exception { rootEndpoint.get().getEntity(); - String expected = "INFO | S3GAudit | org.apache.hadoop.ozone.s3.endpoint.RootEndpoint | user=null | ip=null | " + + String expected = "INFO | S3GAudit | org.apache.hadoop.ozone.s3.endpoint.EndpointBase | user=null | ip=null | " + "op=LIST_S3_BUCKETS {\"x-amz-request-id\":\"" + requestIdentifier.getRequestId() + "\",\"x-amz-id-2\":\"" + requestIdentifier.getAmzId() + "\"} | ret=SUCCESS"; @@ -152,7 +152,7 @@ public void testHeadObject() throws Exception { parametersMap.put("path", "[key1]"); keyEndpoint.head(bucketName, "key1"); - String expected = "INFO | S3GAudit | org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint | user=null | ip=null | " + + String expected = "INFO | S3GAudit | org.apache.hadoop.ozone.s3.endpoint.EndpointBase | user=null | ip=null | " + "op=HEAD_KEY {\"bucket\":\"[bucket]\",\"path\":\"[key1]\",\"x-amz-request-id\":\"" + requestIdentifier.getRequestId() + "\",\"x-amz-id-2\":\"" + requestIdentifier.getAmzId() + "\"} | ret=SUCCESS";