HDDS-14935. [STS] Handle Latent Inconsistencies in S3 API Acl Checks

hadoop-hdds/dev-support/checkstyle/checkstyle.xml

@@ -83,6 +83,12 @@
         <property name="matchAcrossLines" value="true"/>
         <property name="message" value="Use Objects.requireNonNull instead of Preconditions.checkNotNull"/>
     </module>
+    <module name="RegexpMultiline">
+        <property name="format"
+                  value="@Override\s+public\s+(?:(?:final|synchronized)\s+)*OMRequest\s+preExecute\s*\([^\)]*\)\s*(?:throws\s+[^{]+)?\s*\{(?!(?:(?!\/\*|\*\/)[\s\S]|\/\*(?:(?!\*\/)[\s\S])*\*\/){0,400}?(?:^|\r?\n|(?&lt;=\{))[ \t]*(?!\*|\/)(?:return\s+|final\s+(?:[a-zA-Z0-9_]+\s+)*[a-zA-Z0-9_]+[ \t]*=[ \t]*)?(?:\([a-zA-Z0-9_]+\)\s*)?super\s*\.\s*preExecute\s*\()"/>
+        <property name="matchAcrossLines" value="true"/>
+        <property name="message" value="preExecute() overrides must call super.preExecute(ozoneManager) and its result must be final or returned"/>
+    </module>
     <module name="TreeWalker">
         <module name="SuppressWarningsHolder"/>
         <module name="SuppressionCommentFilter"/>

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -2338,6 +2338,14 @@ message S3Authentication {
     // If present, indicates this request uses STS temporary credentials
     // and carries the base64-encoded session token for validation.
     optional string sessionToken = 4;
+    // The following fields are resolved from the STS session token by OM.
+    // They are used to enforce STS session policies during Ratis apply.
+    // They must be written or cleared by the OM leader when the token is validated.
+    optional string resolvedStsSessionPolicy = 5;
+    optional string resolvedStsRoleArn = 6;
+    optional string resolvedStsOriginalAccessKeyId = 7;
+    optional string resolvedStsTempAccessKeyId = 8;
+    optional string resolvedStsSecretKeyId = 9;
 }
 
 message RecoverLeaseRequest {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java

@@ -57,6 +57,8 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
 import org.apache.hadoop.ozone.protocolPB.OzoneManagerRequestHandler;
 import org.apache.hadoop.ozone.protocolPB.RequestHandler;
+import org.apache.hadoop.ozone.security.STSSecurityUtil;
+import org.apache.hadoop.ozone.security.STSTokenIdentifier;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
@@ -662,7 +664,29 @@ public void close() {
    */
   @VisibleForTesting
   OMResponse runCommand(OMRequest request, TermIndex termIndex) {
+    boolean isStsThreadLocalSet = false;
     try {
+      if (ozoneManager.isSecurityEnabled() && request.hasS3Authentication()) {
+        // STS token verification runs on the leader RPC path so we don't need to recheck here on the apply
+        // after the log is committed
+        STSSecurityUtil.ensureResolvedStsFieldsInvariants(request);
+
+        final OzoneManagerProtocolProtos.S3Authentication s3Auth = request.getS3Authentication();
+        if (s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty()) {
+          // ThreadLocal carries session policy for OmMetadataReader
+          final STSTokenIdentifier rehydratedTokenIdentifier = new STSTokenIdentifier(
+                  s3Auth.hasResolvedStsTempAccessKeyId() ? s3Auth.getResolvedStsTempAccessKeyId() : "",
+                  s3Auth.hasResolvedStsOriginalAccessKeyId() ? s3Auth.getResolvedStsOriginalAccessKeyId() : "",
+                  s3Auth.hasResolvedStsRoleArn() ? s3Auth.getResolvedStsRoleArn() : "",
+                  java.time.Instant.MAX, // ensure it deterministically is not expired
+                  "", // no secretAccessKey needed
+                  s3Auth.hasResolvedStsSessionPolicy() ? s3Auth.getResolvedStsSessionPolicy() : "",
+                  null // no encryption key needed
+              );
+          OzoneManager.setStsTokenIdentifier(rehydratedTokenIdentifier);
+          isStsThreadLocalSet = true;
+        }
+      }
       ExecutionContext context = ExecutionContext.of(termIndex.getIndex(), termIndex);
       final OMClientResponse omClientResponse = handler.handleWriteRequest(
           request, context, ozoneManagerDoubleBuffer);
@@ -681,6 +705,10 @@ OMResponse runCommand(OMRequest request, TermIndex termIndex) {
       // For any Runtime exceptions, terminate OM.
       String errorMessage = "Request " + request + " failed with exception";
       ExitUtils.terminate(1, errorMessage, e, LOG);
+    } finally {
+      if (isStsThreadLocalSet) {
+        OzoneManager.setStsTokenIdentifier(null);
+      }
     }
     return null;
   }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java

@@ -27,6 +27,7 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.UUID;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hdds.utils.TransactionInfo;
 import org.apache.hadoop.ipc_.ProtobufRpcEngine;
@@ -50,10 +51,12 @@
 import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
 import org.apache.hadoop.ozone.om.request.s3.security.S3AssumeRoleRequest;
 import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LayoutVersion;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.security.STSTokenIdentifier;
 import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
 import org.apache.hadoop.ozone.security.acl.OzoneObj;
 import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
@@ -112,15 +115,66 @@ public OMClientRequest(OMRequest omRequest) {
    */
   public OMRequest preExecute(OzoneManager ozoneManager)
       throws IOException {
-    LayoutVersion layoutVersion = LayoutVersion.newBuilder()
-        .setVersion(ozoneManager.getVersionManager().getMetadataLayoutVersion())
-        .build();
-    omRequest = getOmRequest().toBuilder()
-        .setUserInfo(getUserIfNotExists(ozoneManager))
-        .setLayoutVersion(layoutVersion).build();
+    final OMRequest.Builder requestBuilder = getOmRequest().toBuilder()
+        .setUserInfo(getUserIfNotExists(ozoneManager));
+
+    // VersionManager is always expected in production OzoneManager instances.
+    // Some unit tests use a minimal mocked OzoneManager, so perform null check here.
+    final OMLayoutVersionManager versionManager = ozoneManager.getVersionManager();
+    if (versionManager != null) {
+      final LayoutVersion layoutVersion = LayoutVersion.newBuilder()
+          .setVersion(versionManager.getMetadataLayoutVersion())
+          .build();
+      requestBuilder.setLayoutVersion(layoutVersion);
+    }
+
+    if (requestBuilder.hasS3Authentication()) {
+      final OzoneManagerProtocolProtos.S3Authentication s3Auth = requestBuilder.getS3Authentication();
+      final boolean hasSessionToken = s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty();
+      final STSTokenIdentifier stsTokenIdentifier = OzoneManager.getStsTokenIdentifier();
+
+      // This should not happen, so explicitly throw an error.  An existing sessionToken
+      // implies prior STS validation must have populated the ThreadLocal.
+      if (ozoneManager.isSecurityEnabled() && hasSessionToken && stsTokenIdentifier == null) {
+        throw new OMException(
+            "S3Authentication has session token but no STS token identifier in OzoneManager ThreadLocal",
+            OMException.ResultCodes.INVALID_REQUEST);
+      }
+
+      requestBuilder.setS3Authentication(resolveS3Authentication(s3Auth, stsTokenIdentifier));
+    }
+
+    omRequest = requestBuilder.build();
     return omRequest;
   }
 
+  private static OzoneManagerProtocolProtos.S3Authentication resolveS3Authentication(
+      OzoneManagerProtocolProtos.S3Authentication s3Auth, STSTokenIdentifier stsTokenIdentifier) {
+    final OzoneManagerProtocolProtos.S3Authentication.Builder s3AuthBuilder = s3Auth.toBuilder();
+
+    if (s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty() && stsTokenIdentifier != null) {
+      s3AuthBuilder.setResolvedStsSessionPolicy(
+          StringUtils.defaultString(stsTokenIdentifier.getSessionPolicy()));
+      s3AuthBuilder.setResolvedStsRoleArn(
+          StringUtils.defaultString(stsTokenIdentifier.getRoleArn()));
+      s3AuthBuilder.setResolvedStsOriginalAccessKeyId(
+          StringUtils.defaultString(stsTokenIdentifier.getOriginalAccessKeyId()));
+      s3AuthBuilder.setResolvedStsTempAccessKeyId(
+          StringUtils.defaultString(stsTokenIdentifier.getTempAccessKeyId()));
+      final UUID secretKeyId = stsTokenIdentifier.getSecretKeyId();
+      s3AuthBuilder.setResolvedStsSecretKeyId(
+          secretKeyId != null ? secretKeyId.toString() : "");
+    } else {
+      s3AuthBuilder.clearResolvedStsSessionPolicy();
+      s3AuthBuilder.clearResolvedStsRoleArn();
+      s3AuthBuilder.clearResolvedStsOriginalAccessKeyId();
+      s3AuthBuilder.clearResolvedStsTempAccessKeyId();
+      s3AuthBuilder.clearResolvedStsSecretKeyId();
+    }
+
+    return s3AuthBuilder.build();
+  }
+
   /**
    * Performs any request specific failure handling during request
    * submission. An example of this would be an undo of any steps

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetOwnerRequest.java

@@ -63,15 +63,16 @@ public OMBucketSetOwnerRequest(OMRequest omRequest) {
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager)
       throws IOException {
+    final OMRequest omRequest = super.preExecute(ozoneManager);
+
     long modificationTime = Time.now();
     OzoneManagerProtocolProtos.SetBucketPropertyRequest.Builder
-        setBucketPropertyRequestBuilder = getOmRequest()
+        setBucketPropertyRequestBuilder = omRequest
         .getSetBucketPropertyRequest().toBuilder()
         .setModificationTime(modificationTime);
 
-    return getOmRequest().toBuilder()
+    return omRequest.toBuilder()
         .setSetBucketPropertyRequest(setBucketPropertyRequestBuilder)
-        .setUserInfo(getUserInfo())
         .build();
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java

@@ -79,14 +79,16 @@ public OMBucketSetPropertyRequest(OMRequest omRequest) {
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager)
       throws IOException {
+    final OMRequest omRequest = super.preExecute(ozoneManager);
+
     long modificationTime = Time.now();
     OzoneManagerProtocolProtos.SetBucketPropertyRequest.Builder
-        setBucketPropertyRequestBuilder = getOmRequest()
+        setBucketPropertyRequestBuilder = omRequest
         .getSetBucketPropertyRequest().toBuilder()
         .setModificationTime(modificationTime);
 
     BucketArgs bucketArgs =
-        getOmRequest().getSetBucketPropertyRequest().getBucketArgs();
+        omRequest.getSetBucketPropertyRequest().getBucketArgs();
 
     if (bucketArgs.hasBekInfo()) {
       KeyProviderCryptoExtension kmsProvider = ozoneManager.getKmsProvider();
@@ -97,9 +99,8 @@ public OMRequest preExecute(OzoneManager ozoneManager)
       setBucketPropertyRequestBuilder.setBucketArgs(bucketArgsBuilder.build());
     }
 
-    return getOmRequest().toBuilder()
+    return omRequest.toBuilder()
         .setSetBucketPropertyRequest(setBucketPropertyRequestBuilder)
-        .setUserInfo(getUserInfo())
         .build();
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java

@@ -56,14 +56,15 @@ public class OMBucketAddAclRequest extends OMBucketAclRequest {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    long modificationTime = Time.now();
-    OzoneManagerProtocolProtos.AddAclRequest.Builder addAclRequestBuilder =
-        getOmRequest().getAddAclRequest().toBuilder()
+    final OMRequest omRequest = super.preExecute(ozoneManager);
+
+    final long modificationTime = Time.now();
+    final OzoneManagerProtocolProtos.AddAclRequest.Builder addAclRequestBuilder =
+        omRequest.getAddAclRequest().toBuilder()
             .setModificationTime(modificationTime);
 
-    return getOmRequest().toBuilder()
+    return omRequest.toBuilder()
         .setAddAclRequest(addAclRequestBuilder)
-        .setUserInfo(getUserInfo())
         .build();
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java

@@ -55,14 +55,15 @@ public class OMBucketRemoveAclRequest extends OMBucketAclRequest {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    long modificationTime = Time.now();
-    OzoneManagerProtocolProtos.RemoveAclRequest.Builder removeAclRequestBuilder
-        = getOmRequest().getRemoveAclRequest().toBuilder()
+    final OMRequest omRequest = super.preExecute(ozoneManager);
+
+    final long modificationTime = Time.now();
+    final OzoneManagerProtocolProtos.RemoveAclRequest.Builder removeAclRequestBuilder =
+        omRequest.getRemoveAclRequest().toBuilder()
             .setModificationTime(modificationTime);
 
-    return getOmRequest().toBuilder()
+    return omRequest.toBuilder()
         .setRemoveAclRequest(removeAclRequestBuilder)
-        .setUserInfo(getUserInfo())
         .build();
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java

@@ -54,14 +54,15 @@ public class OMBucketSetAclRequest extends OMBucketAclRequest {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    long modificationTime = Time.now();
-    OzoneManagerProtocolProtos.SetAclRequest.Builder setAclRequestBuilder =
+    final long modificationTime = Time.now();
+    final OzoneManagerProtocolProtos.SetAclRequest.Builder setAclRequestBuilder =
         getOmRequest().getSetAclRequest().toBuilder()
             .setModificationTime(modificationTime);
 
-    return getOmRequest().toBuilder()
+    // super.preExecute resolves S3Authentication (STS) for Ratis apply.  Merge SetAclRequest changes on top.
+    final OMRequest request = super.preExecute(ozoneManager);
+    return request.toBuilder()
         .setSetAclRequest(setAclRequestBuilder)
-        .setUserInfo(getUserInfo())
         .build();
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java

@@ -82,7 +82,7 @@ public OMFileCreateRequest(OMRequest omRequest, BucketLayout bucketLayout) {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    CreateFileRequest createFileRequest = super.preExecute(ozoneManager)
+    final CreateFileRequest createFileRequest = super.preExecute(ozoneManager)
         .getCreateFileRequest();
     Objects.requireNonNull(createFileRequest, "createFileRequest == null");
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java

@@ -96,7 +96,7 @@ public OMRecoverLeaseRequest(OMRequest omRequest) {
   @Override
   @DisallowedUntilLayoutVersion(HBASE_SUPPORT)
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    OMRequest request = super.preExecute(ozoneManager);
+    final OMRequest request = super.preExecute(ozoneManager);
     RecoverLeaseRequest recoverLeaseRequest = request.getRecoverLeaseRequest();
 
     String keyPath = recoverLeaseRequest.getKeyName();

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java

@@ -87,7 +87,7 @@ public OMKeyCommitRequest(OMRequest omRequest, BucketLayout bucketLayout) {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    OMRequest request = super.preExecute(ozoneManager);
+    final OMRequest request = super.preExecute(ozoneManager);
     CommitKeyRequest commitKeyRequest = request.getCommitKeyRequest();
     Objects.requireNonNull(commitKeyRequest, "commitKeyRequest == null");
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java

@@ -87,7 +87,7 @@ public OMKeyCreateRequest(OMRequest omRequest, BucketLayout bucketLayout) {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    CreateKeyRequest createKeyRequest = super.preExecute(ozoneManager)
+    final CreateKeyRequest createKeyRequest = super.preExecute(ozoneManager)
         .getCreateKeyRequest();
     Objects.requireNonNull(createKeyRequest, "createKeyRequest == null");
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java

@@ -75,7 +75,7 @@ public OMKeyDeleteRequest(OMRequest omRequest, BucketLayout bucketLayout) {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    DeleteKeyRequest deleteKeyRequest = super.preExecute(ozoneManager)
+    final DeleteKeyRequest deleteKeyRequest = super.preExecute(ozoneManager)
         .getDeleteKeyRequest();
     Objects.requireNonNull(deleteKeyRequest, "deleteKeyRequest == null");
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java

@@ -72,7 +72,7 @@ public OMKeyRenameRequest(OMRequest omRequest, BucketLayout bucketLayout) {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    RenameKeyRequest renameKeyRequest = super.preExecute(ozoneManager)
+    final RenameKeyRequest renameKeyRequest = super.preExecute(ozoneManager)
         .getRenameKeyRequest();
     Objects.requireNonNull(renameKeyRequest, "renameKeyRequest == null");
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequest.java

@@ -62,7 +62,7 @@ public class OMKeySetTimesRequest extends OMKeyRequest {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    OMRequest request = super.preExecute(ozoneManager);
+    final OMRequest request = super.preExecute(ozoneManager);
     SetTimesRequest setTimesRequest = request.getSetTimesRequest();
     String keyPath = setTimesRequest.getKeyArgs().getKeyName();
     String normalizedKeyPath =

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java

@@ -63,14 +63,15 @@ public class OMKeyAddAclRequest extends OMKeyAclRequest {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    long modificationTime = Time.now();
-    OzoneManagerProtocolProtos.AddAclRequest.Builder addAclRequestBuilder =
-        getOmRequest().getAddAclRequest().toBuilder()
+    final OMRequest omRequest = super.preExecute(ozoneManager);
+
+    final long modificationTime = Time.now();
+    final OzoneManagerProtocolProtos.AddAclRequest.Builder addAclRequestBuilder =
+        omRequest.getAddAclRequest().toBuilder()
             .setModificationTime(modificationTime);
 
-    return getOmRequest().toBuilder()
+    return omRequest.toBuilder()
         .setAddAclRequest(addAclRequestBuilder)
-        .setUserInfo(getUserInfo())
         .build();
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java

@@ -63,14 +63,15 @@ public class OMKeyRemoveAclRequest extends OMKeyAclRequest {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    long modificationTime = Time.now();
-    OzoneManagerProtocolProtos.RemoveAclRequest.Builder removeAclRequestBuilder
-        = getOmRequest().getRemoveAclRequest().toBuilder()
+    final OMRequest omRequest = super.preExecute(ozoneManager);
+
+    final long modificationTime = Time.now();
+    final OzoneManagerProtocolProtos.RemoveAclRequest.Builder removeAclRequestBuilder =
+        omRequest.getRemoveAclRequest().toBuilder()
             .setModificationTime(modificationTime);
 
-    return getOmRequest().toBuilder()
+    return omRequest.toBuilder()
         .setRemoveAclRequest(removeAclRequestBuilder)
-        .setUserInfo(getUserInfo())
         .build();
   }