handle latent inconsistencies in s3 api acl checks

Author: Fabian Morgan

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -2311,6 +2311,14 @@ message S3Authentication {
     // If present, indicates this request uses STS temporary credentials
     // and carries the base64-encoded session token for validation.
     optional string sessionToken = 4;
+    // The following fields are resolved from the STS session token by OM.
+    // They are used to enforce STS session policies during Ratis apply.
+    // They must be written or cleared by the OM leader when the token is validated.
+    optional string resolvedStsSessionPolicy = 5;
+    optional string resolvedStsRoleArn = 6;
+    optional string resolvedStsOriginalAccessKeyId = 7;
+    optional string resolvedStsTempAccessKeyId = 8;
+    optional string resolvedStsSecretKeyId = 9;
 }
 
 message RecoverLeaseRequest {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java

@@ -55,6 +55,8 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
 import org.apache.hadoop.ozone.protocolPB.OzoneManagerRequestHandler;
 import org.apache.hadoop.ozone.protocolPB.RequestHandler;
+import org.apache.hadoop.ozone.security.STSSecurityUtil;
+import org.apache.hadoop.ozone.security.STSTokenIdentifier;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
@@ -617,7 +619,29 @@ public void close() {
    */
   @VisibleForTesting
   OMResponse runCommand(OMRequest request, TermIndex termIndex) {
+    boolean isStsThreadLocalSet = false;
     try {
+      if (ozoneManager.isSecurityEnabled() && request.hasS3Authentication()) {
+        // STS token verification runs on the leader RPC path so we don't need to recheck here on the apply
+        // after the log is committed
+        STSSecurityUtil.ensureResolvedStsFieldsInvariants(request);
+
+        final OzoneManagerProtocolProtos.S3Authentication s3Auth = request.getS3Authentication();
+        if (s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty()) {
+          // ThreadLocal carries session policy for OmMetadataReader
+          final STSTokenIdentifier rehydratedTokenIdentifier = new STSTokenIdentifier(
+                  s3Auth.hasResolvedStsTempAccessKeyId() ? s3Auth.getResolvedStsTempAccessKeyId() : "",
+                  s3Auth.hasResolvedStsOriginalAccessKeyId() ? s3Auth.getResolvedStsOriginalAccessKeyId() : "",
+                  s3Auth.hasResolvedStsRoleArn() ? s3Auth.getResolvedStsRoleArn() : "",
+                  java.time.Instant.MAX, // ensure it deterministically is not expired
+                  "", // no secretAccessKey needed
+                  s3Auth.hasResolvedStsSessionPolicy() ? s3Auth.getResolvedStsSessionPolicy() : "",
+                  null // no encryption key needed
+              );
+          OzoneManager.setStsTokenIdentifier(rehydratedTokenIdentifier);
+          isStsThreadLocalSet = true;
+        }
+      }
       ExecutionContext context = ExecutionContext.of(termIndex.getIndex(), termIndex);
       final OMClientResponse omClientResponse = handler.handleWriteRequest(
           request, context, ozoneManagerDoubleBuffer);
@@ -636,6 +660,10 @@ OMResponse runCommand(OMRequest request, TermIndex termIndex) {
       // For any Runtime exceptions, terminate OM.
       String errorMessage = "Request " + request + " failed with exception";
       ExitUtils.terminate(1, errorMessage, e, LOG);
+    } finally {
+      if (isStsThreadLocalSet) {
+        OzoneManager.setStsTokenIdentifier(null);
+      }
     }
     return null;
   }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java

@@ -27,6 +27,7 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.UUID;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hdds.utils.TransactionInfo;
 import org.apache.hadoop.ipc_.ProtobufRpcEngine;
@@ -54,6 +55,7 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LayoutVersion;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.security.STSTokenIdentifier;
 import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
 import org.apache.hadoop.ozone.security.acl.OzoneObj;
 import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
@@ -112,15 +114,50 @@ public OMClientRequest(OMRequest omRequest) {
    */
   public OMRequest preExecute(OzoneManager ozoneManager)
       throws IOException {
-    LayoutVersion layoutVersion = LayoutVersion.newBuilder()
+    final LayoutVersion layoutVersion = LayoutVersion.newBuilder()
         .setVersion(ozoneManager.getVersionManager().getMetadataLayoutVersion())
         .build();
-    omRequest = getOmRequest().toBuilder()
+
+    final OMRequest.Builder requestBuilder = getOmRequest().toBuilder()
         .setUserInfo(getUserIfNotExists(ozoneManager))
-        .setLayoutVersion(layoutVersion).build();
+        .setLayoutVersion(layoutVersion);
+
+    if (requestBuilder.hasS3Authentication()) {
+      requestBuilder.setS3Authentication(
+          resolveS3Authentication(requestBuilder.getS3Authentication(), OzoneManager.getStsTokenIdentifier()));
+    }
+
+    omRequest = requestBuilder.build();
     return omRequest;
   }
 
+  private static OzoneManagerProtocolProtos.S3Authentication resolveS3Authentication(
+      OzoneManagerProtocolProtos.S3Authentication s3Auth, STSTokenIdentifier stsTokenIdentifier) {
+    final OzoneManagerProtocolProtos.S3Authentication.Builder s3AuthBuilder = s3Auth.toBuilder();
+
+    if (s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty() && stsTokenIdentifier != null) {
+      s3AuthBuilder.setResolvedStsSessionPolicy(
+          StringUtils.defaultString(stsTokenIdentifier.getSessionPolicy()));
+      s3AuthBuilder.setResolvedStsRoleArn(
+          StringUtils.defaultString(stsTokenIdentifier.getRoleArn()));
+      s3AuthBuilder.setResolvedStsOriginalAccessKeyId(
+          StringUtils.defaultString(stsTokenIdentifier.getOriginalAccessKeyId()));
+      s3AuthBuilder.setResolvedStsTempAccessKeyId(
+          StringUtils.defaultString(stsTokenIdentifier.getTempAccessKeyId()));
+      final UUID secretKeyId = stsTokenIdentifier.getSecretKeyId();
+      s3AuthBuilder.setResolvedStsSecretKeyId(
+          secretKeyId != null ? secretKeyId.toString() : "");
+    } else {
+      s3AuthBuilder.clearResolvedStsSessionPolicy();
+      s3AuthBuilder.clearResolvedStsRoleArn();
+      s3AuthBuilder.clearResolvedStsOriginalAccessKeyId();
+      s3AuthBuilder.clearResolvedStsTempAccessKeyId();
+      s3AuthBuilder.clearResolvedStsSecretKeyId();
+    }
+
+    return s3AuthBuilder.build();
+  }
+
   /**
    * Performs any request specific failure handling during request
    * submission. An example of this would be an undo of any steps

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java

@@ -54,14 +54,15 @@ public class OMBucketSetAclRequest extends OMBucketAclRequest {
 
   @Override
   public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
-    long modificationTime = Time.now();
-    OzoneManagerProtocolProtos.SetAclRequest.Builder setAclRequestBuilder =
+    final long modificationTime = Time.now();
+    final OzoneManagerProtocolProtos.SetAclRequest.Builder setAclRequestBuilder =
         getOmRequest().getSetAclRequest().toBuilder()
             .setModificationTime(modificationTime);
 
-    return getOmRequest().toBuilder()
+    // super.preExecute resolves S3Authentication (STS) for Ratis apply.  Merge SetAclRequest changes on top.
+    final OMRequest request = super.preExecute(ozoneManager);
+    return request.toBuilder()
         .setSetAclRequest(setAclRequestBuilder)
-        .setUserInfo(getUserInfo())
         .build();
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java

@@ -31,7 +31,9 @@
 import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
 import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.security.token.Token;
 
@@ -175,5 +177,44 @@ static void ensureEssentialFieldsArePresentInToken(STSTokenIdentifier stsTokenId
       throw new SecretManager.InvalidToken("Invalid STS token - secretAccessKey is null/empty");
     }
   }
+
+  /**
+   * Ensures STS-related {@link S3Authentication} fields are structurally consistent on the Ratis
+   * apply path. Cryptographic validation (signature, expiry, secret key lookup) runs on the leader
+   * RPC path (e.g. {@code S3SecurityUtil.validateS3Credential}).  This method performs no crypto and does
+   * not contact {@link SecretKeyClient}, keeping the apply thread deterministic and lightweight.
+   *
+   * @param request OM request possibly containing S3 authentication
+   * @throws OMException if resolved fields and session token presence are inconsistent
+   */
+  public static void ensureResolvedStsFieldsInvariants(OzoneManagerProtocolProtos.OMRequest request)
+      throws OMException {
+    if (!request.hasS3Authentication()) {
+      return;
+    }
+
+    final S3Authentication s3Auth = request.getS3Authentication();
+    final boolean hasSessionToken = s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty();
+
+    if (!hasSessionToken) {
+      // If sessionToken is missing/empty, resolved fields must be empty.
+      if (s3Auth.hasResolvedStsSessionPolicy() || s3Auth.hasResolvedStsRoleArn() ||
+          s3Auth.hasResolvedStsOriginalAccessKeyId() || s3Auth.hasResolvedStsTempAccessKeyId() ||
+          s3Auth.hasResolvedStsSecretKeyId()) {
+        throw new OMException("Resolved STS fields must be empty when sessionToken is not present", INVALID_TOKEN);
+      }
+      return;
+    }
+
+    ensureResolvedFieldsArePresent(s3Auth);
+  }
+
+  private static void ensureResolvedFieldsArePresent(S3Authentication s3Auth) throws OMException {
+    if (!s3Auth.hasResolvedStsSessionPolicy() || !s3Auth.hasResolvedStsRoleArn() ||
+        !s3Auth.hasResolvedStsOriginalAccessKeyId() || !s3Auth.hasResolvedStsTempAccessKeyId() ||
+        !s3Auth.hasResolvedStsSecretKeyId()) {
+      throw new OMException("Resolved STS fields must be present when sessionToken is present", INVALID_TOKEN);
+    }
+  }
 }
 

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java

@@ -210,6 +210,72 @@ public void testUserInfoWithSTSToken() throws IOException {
     }
   }
 
+  @Test
+  public void testPreExecuteOverwritesResolvedStsFields() throws Exception {
+    try (MockedStatic<Server> mockedRpcServer = mockStatic(Server.class)) {
+      mockedRpcServer.when(Server::getRemoteUser).thenReturn(userGroupInformation);
+      mockedRpcServer.when(Server::getRemoteIp).thenReturn(inetAddress);
+      mockedRpcServer.when(Server::getRemoteAddress).thenReturn(inetAddress.toString());
+
+      final String accessId = "ASIA12345";
+      final String signature = "Signature";
+      final String stringToSign = "StringToSign";
+      final String sessionToken = "SessionToken";
+      final String originalAccessKeyId = "AKIAORIGINAL";
+      final String roleArn = "arn:aws:iam::123456789012:role/test-role";
+      final String sessionPolicy = "test-session-policy";
+      final UUID secretKeyId = UUID.randomUUID();
+
+      final STSTokenIdentifier stsTokenIdentifier = mock(STSTokenIdentifier.class);
+      when(stsTokenIdentifier.getSessionPolicy()).thenReturn(sessionPolicy);
+      when(stsTokenIdentifier.getRoleArn()).thenReturn(roleArn);
+      when(stsTokenIdentifier.getOriginalAccessKeyId()).thenReturn(originalAccessKeyId);
+      when(stsTokenIdentifier.getTempAccessKeyId()).thenReturn(accessId);
+      when(stsTokenIdentifier.getSecretKeyId()).thenReturn(secretKeyId);
+
+      final S3Authentication s3Authentication = S3Authentication.newBuilder()
+          .setAccessId(accessId)
+          .setSignature(signature)
+          .setStringToSign(stringToSign)
+          .setSessionToken(sessionToken)
+          .setResolvedStsSessionPolicy("client-session-policy")
+          .setResolvedStsRoleArn("client-role")
+          .setResolvedStsOriginalAccessKeyId("client-original-access-key-id")
+          .setResolvedStsTempAccessKeyId("client-temp-access-key-id")
+          .setResolvedStsSecretKeyId("client-secret-key-id")
+          .build();
+
+      OzoneManager.setS3Auth(s3Authentication);
+      OzoneManager.setStsTokenIdentifier(stsTokenIdentifier);
+
+      try {
+        final String bucketName = UUID.randomUUID().toString();
+        final String volumeName = UUID.randomUUID().toString();
+        final BucketInfo.Builder bucketInfo =
+            newBucketInfoBuilder(bucketName, volumeName)
+                .setIsVersionEnabled(true)
+                .setStorageType(StorageTypeProto.DISK);
+
+        final OMRequest omRequest = newCreateBucketRequest(bucketInfo)
+            .setS3Authentication(s3Authentication)
+            .build();
+
+        final OMBucketCreateRequest omBucketCreateRequest = new OMBucketCreateRequest(omRequest);
+        final OMRequest modifiedRequest = omBucketCreateRequest.preExecute(ozoneManager);
+        final S3Authentication modifiedS3Auth = modifiedRequest.getS3Authentication();
+
+        assertEquals(sessionPolicy, modifiedS3Auth.getResolvedStsSessionPolicy());
+        assertEquals(roleArn, modifiedS3Auth.getResolvedStsRoleArn());
+        assertEquals(originalAccessKeyId, modifiedS3Auth.getResolvedStsOriginalAccessKeyId());
+        assertEquals(accessId, modifiedS3Auth.getResolvedStsTempAccessKeyId());
+        assertEquals(secretKeyId.toString(), modifiedS3Auth.getResolvedStsSecretKeyId());
+      } finally {
+        OzoneManager.setStsTokenIdentifier(null);
+        OzoneManager.setS3Auth(null);
+      }
+    }
+  }
+
   @Test
   public void testUserInfoWithSTSAccessKeyMissingSessionToken() {
     final String accessId = "ASIA12345";

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java

@@ -35,7 +35,10 @@
 import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.security.token.Token;
 import org.apache.ozone.test.TestClock;
@@ -238,6 +241,8 @@ public void testConstructValidateAndDecryptSTSTokenExpiredSecretKey() throws Exc
     // Create a mock secret key that is expired
     final ManagedSecretKey expiredSecretKey = mock(ManagedSecretKey.class);
     when(expiredSecretKey.isExpired()).thenReturn(true);
+    final Instant now = Instant.now();
+    when(expiredSecretKey.getExpiryTime()).thenReturn(now);
 
     final SecretKeyClient mockKeyClient = mock(SecretKeyClient.class);
     when(mockKeyClient.getSecretKey(any())).thenReturn(expiredSecretKey);
@@ -373,4 +378,76 @@ public void testEnsureEssentialFieldsArePresentInTokenMissingSecretAccessKey() {
         .isInstanceOf(SecretManager.InvalidToken.class)
         .hasMessage("Invalid STS token - secretAccessKey is null/empty");
   }
+
+  @Test
+  public void testEnsureResolvedStsFieldsInvariantsSuccess() throws Exception {
+    final String tokenString = tokenSecretManager.createSTSTokenString(
+        TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock);
+
+    final S3Authentication s3Auth = S3Authentication.newBuilder()
+        .setSessionToken(tokenString)
+        .setResolvedStsSessionPolicy(SESSION_POLICY)
+        .setResolvedStsRoleArn(ROLE_ARN)
+        .setResolvedStsOriginalAccessKeyId(ORIGINAL_ACCESS_KEY)
+        .setResolvedStsTempAccessKeyId(TEMP_ACCESS_KEY)
+        .setResolvedStsSecretKeyId(secretKeyId.toString())
+        .build();
+
+    final OMRequest request = OMRequest.newBuilder()
+        .setCmdType(Type.CreateBucket)
+        .setClientId("client-id")
+        .setS3Authentication(s3Auth)
+        .build();
+
+    STSSecurityUtil.ensureResolvedStsFieldsInvariants(request);
+  }
+
+  @Test
+  public void testEnsureResolvedStsFieldsInvariantsMissingSessionToken() {
+    final S3Authentication s3Auth = S3Authentication.newBuilder()
+        .setResolvedStsSessionPolicy(SESSION_POLICY)
+        .build();
+
+    final OMRequest request = OMRequest.newBuilder()
+        .setCmdType(Type.CreateBucket)
+        .setClientId("client-id")
+        .setS3Authentication(s3Auth)
+        .build();
+
+    assertThatThrownBy(() -> STSSecurityUtil.ensureResolvedStsFieldsInvariants(request))
+        .isInstanceOf(OMException.class)
+        .hasMessageContaining("Resolved STS fields must be empty when sessionToken is not present");
+  }
+
+  @Test
+  public void testEnsureResolvedStsFieldsInvariantsMissingResolvedFields() throws Exception {
+    final String tokenString = tokenSecretManager.createSTSTokenString(
+        TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS,
+        SECRET_ACCESS_KEY, SESSION_POLICY, clock);
+
+    final S3Authentication s3Auth = S3Authentication.newBuilder()
+        .setSessionToken(tokenString)
+        .build();
+
+    final OMRequest request = OMRequest.newBuilder()
+        .setCmdType(Type.CreateBucket)
+        .setClientId("client-id")
+        .setS3Authentication(s3Auth)
+        .build();
+
+    assertThatThrownBy(() -> STSSecurityUtil.ensureResolvedStsFieldsInvariants(request))
+        .isInstanceOf(OMException.class)
+        .hasMessageContaining("Resolved STS fields must be present when sessionToken is present");
+  }
+
+  @Test
+  public void testEnsureResolvedStsFieldsInvariantsNoS3Auth() throws Exception {
+    final OMRequest request = OMRequest.newBuilder()
+        .setCmdType(Type.CreateBucket)
+        .setClientId("client-id")
+        .build();
+
+    // Should not throw
+    STSSecurityUtil.ensureResolvedStsFieldsInvariants(request);
+  }
 }