Skip to content

Commit 9504881

Browse files
fmorg-gitfmorg-git
authored
HDDS-14809. [STS] Part 5 - IAM Session Policy and ListBucket improvements (#9900)
1 parent 9b3577e commit 9504881

2 files changed

Lines changed: 168 additions & 17 deletions

File tree

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/iam/IamSessionPolicyResolver.java‎

Lines changed: 43 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -154,11 +154,20 @@ public static Set<AssumeRoleRequest.OzoneGrant> resolve(String policyJson, Strin
154154
continue;
155155
}
156156

157+
// s3:prefix is only applicable to the ListBucket action because we don't support ListBucketVersions
158+
// (see https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazons3.html and search for
159+
// s3:prefix). If a statement carries a Condition, non-ListBucket actions (ex GetObject, PutObject,
160+
// ListBucketMultipartUploads, etc.) in that statement do not apply.
161+
final Set<S3Action> filteredS3Actions = filterActionsWhenConditionPresent(mappedS3Actions, condition);
162+
if (filteredS3Actions.isEmpty()) {
163+
continue;
164+
}
165+
157166
// Categorize resources according to bucket resource, object resource, etc
158167
final Set<ResourceSpec> resourceSpecs = validateAndCategorizeResources(authorizerType, resources);
159168

160169
// For each action, map to Ozone objects (paths) and acls based on resource specs and prefixes
161-
createPathsAndPermissions(volumeName, authorizerType, mappedS3Actions, resourceSpecs, condition, objToAclsMap);
170+
createPathsAndPermissions(volumeName, authorizerType, filteredS3Actions, resourceSpecs, condition, objToAclsMap);
162171
}
163172

164173
// Group accumulated objects by their ACL sets to create final result
@@ -359,6 +368,23 @@ static Set<S3Action> mapPolicyActionsToS3Actions(Set<String> actions) {
359368
return mappedActions;
360369
}
361370

371+
/**
372+
* Filters out actions when a Condition is present if the action is not ListBucket.
373+
*/
374+
private static Set<S3Action> filterActionsWhenConditionPresent(Set<S3Action> mappedS3Actions, Condition condition) {
375+
if (condition == null) {
376+
return mappedS3Actions;
377+
}
378+
379+
if (mappedS3Actions.contains(S3Action.LIST_BUCKET) || mappedS3Actions.contains(S3Action.ALL_S3)) {
380+
final Set<S3Action> filteredActions = new HashSet<>();
381+
filteredActions.add(S3Action.LIST_BUCKET);
382+
return filteredActions;
383+
}
384+
385+
return Collections.emptySet();
386+
}
387+
362388
/**
363389
* Validates that wildcard bucket patterns are not used with native authorizer.
364390
*/
@@ -474,7 +500,7 @@ private static void processResourceSpecWithActions(String volumeName, Authorizer
474500
Preconditions.checkArgument(
475501
authorizerType != AuthorizerType.NATIVE,
476502
"ResourceSpec type ANY not supported for OzoneNativeAuthorizer");
477-
processResourceTypeAny(volumeName, mappedS3Actions, objToAclsMap);
503+
processResourceTypeAny(volumeName, authorizerType, mappedS3Actions, condition, objToAclsMap);
478504
break;
479505
case BUCKET:
480506
processBucketResource(volumeName, mappedS3Actions, resourceSpec, condition, authorizerType, objToAclsMap);
@@ -509,12 +535,24 @@ private static void processResourceSpecWithActions(String volumeName, Authorizer
509535
* Handles ResourceType.ANY (*).
510536
* Example: "Resource": "*"
511537
*/
512-
private static void processResourceTypeAny(String volumeName, Set<S3Action> mappedS3Actions,
513-
Map<IOzoneObj, Set<ACLType>> objToAclsMap) {
538+
private static void processResourceTypeAny(String volumeName, AuthorizerType authorizerType,
539+
Set<S3Action> mappedS3Actions, Condition condition, Map<IOzoneObj, Set<ACLType>> objToAclsMap) {
514540
for (S3Action action : mappedS3Actions) {
515541
addAclsForObj(objToAclsMap, volumeObj(volumeName), action.volumePerms);
516542
addAclsForObj(objToAclsMap, bucketObj(volumeName, "*"), action.bucketPerms);
517-
addAclsForObj(objToAclsMap, keyObj(volumeName, "*", "*"), action.objectPerms);
543+
if (condition != null && condition.prefixes != null && !condition.prefixes.isEmpty() &&
544+
(action == S3Action.LIST_BUCKET || action == S3Action.ALL_S3)) {
545+
for (String prefix : condition.prefixes) {
546+
// If operator is StringEquals, ignore wildcard prefixes - this is AWS behavior
547+
if (STRING_EQUALS.equals(condition.operator) && hasWildcard(prefix)) {
548+
continue;
549+
}
550+
createObjectResourcesFromConditionPrefix(
551+
volumeName, authorizerType, ResourceSpec.any(), prefix, objToAclsMap, EnumSet.of(READ));
552+
}
553+
} else {
554+
addAclsForObj(objToAclsMap, keyObj(volumeName, "*", "*"), action.objectPerms);
555+
}
518556
}
519557
}
520558

‎hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java‎

Lines changed: 125 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1258,6 +1258,30 @@ public void testAllActionsForKey() throws OMException {
12581258
assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
12591259
}
12601260

1261+
@Test
1262+
public void testAllActionsForKeyWithPrefixCondition() throws OMException {
1263+
final String json = "{\n" +
1264+
" \"Statement\": [{\n" +
1265+
" \"Effect\": \"Allow\",\n" +
1266+
" \"Action\": \"s3:*\",\n" +
1267+
" \"Resource\": \"arn:aws:s3:::my-bucket/*\",\n" +
1268+
" \"Condition\": {\n" +
1269+
" \"StringLike\": {\n" +
1270+
" \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
1271+
" }\n" +
1272+
" }\n" +
1273+
" }]\n" +
1274+
"}";
1275+
1276+
final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
1277+
final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
1278+
1279+
// Ensure what we got is what we expected - only ListBucket supports s3:prefix and that is a bucket action,
1280+
// not object action
1281+
assertThat(resolvedFromNativeAuthorizer).isEmpty();
1282+
assertThat(resolvedFromRangerAuthorizer).isEmpty();
1283+
}
1284+
12611285
@Test
12621286
public void testAllActionsForBucket() throws OMException {
12631287
final String json = "{\n" +
@@ -1287,6 +1311,46 @@ public void testAllActionsForBucket() throws OMException {
12871311
assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
12881312
}
12891313

1314+
@Test
1315+
public void testAllActionsForBucketWithPrefixCondition() throws OMException {
1316+
final String json = "{\n" +
1317+
" \"Statement\": [{\n" +
1318+
" \"Effect\": \"Allow\",\n" +
1319+
" \"Action\": \"s3:*\",\n" +
1320+
" \"Resource\": \"arn:aws:s3:::my-bucket\",\n" +
1321+
" \"Condition\": {\n" +
1322+
" \"StringLike\": {\n" +
1323+
" \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
1324+
" }\n" +
1325+
" }\n" +
1326+
" }]\n" +
1327+
"}";
1328+
1329+
final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
1330+
final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
1331+
1332+
// Ensure what we got is what we expected
1333+
final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
1334+
// Expected for native: READ, LIST ACLs for bucket (only ListBucket supports s3:prefix); volume READ;
1335+
// prefix "team/folder", "team/folder/" READ
1336+
final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
1337+
final Set<ACLType> bucketAcls = acls(READ, LIST);
1338+
expectedResolvedNative.add(
1339+
new OzoneGrant(objSet(volume(), prefix("my-bucket", "team/folder"), prefix("my-bucket", "team/folder/")),
1340+
acls(READ)));
1341+
expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
1342+
assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
1343+
1344+
// Expected for Ranger: READ, LIST ACLs for bucket (only ListBucket supports s3:prefix); volume READ,
1345+
// key "team/folder", "team/folder/*" READ
1346+
final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
1347+
expectedResolvedRanger.add(
1348+
new OzoneGrant(objSet(volume(), key("my-bucket", "team/folder"), key("my-bucket", "team/folder/*")),
1349+
acls(READ)));
1350+
expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
1351+
assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
1352+
}
1353+
12901354
@Test
12911355
public void testMultipleResourcesInSeparateStatements() throws OMException {
12921356
final String json = "{\n" +
@@ -1517,11 +1581,11 @@ public void testIgnoresUnsupportedActionsWhenSupportedActionsAreIncluded() throw
15171581
" \"Effect\": \"Allow\",\n" +
15181582
" \"Action\": [\n" +
15191583
" \"s3:GetAccelerateConfiguration\",\n" + // unsupported action
1520-
" \"s3:GetBucketAcl\",\n" +
1584+
" \"s3:GetBucketAcl\",\n" + // ignored because it doesn't support s3:prefix condition
15211585
" \"s3:GetObject\",\n" + // object-level action not applied for bucket
15221586
" \"s3:GetObjectAcl\",\n" + // unsupported action
15231587
" \"s3:ListBucket\",\n" +
1524-
" \"s3:ListBucketMultipartUploads\"\n" +
1588+
" \"s3:ListBucketMultipartUploads\"\n" + // ignored because it doesn't support s3:prefix condition
15251589
" ],\n" +
15261590
" \"Resource\": \"arn:aws:s3:::bucket1\",\n" +
15271591
" \"Condition\": {\n" +
@@ -1539,16 +1603,16 @@ public void testIgnoresUnsupportedActionsWhenSupportedActionsAreIncluded() throw
15391603
// Ensure what we got is what we expected
15401604
final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
15411605

1542-
// Expected for native: READ, LIST, READ_ACL bucket acls; volume and prefixes "team/folder", "team/folder/" READ
1606+
// Expected for native: READ, LIST bucket acls; volume and prefixes "team/folder", "team/folder/" READ
15431607
final Set<IOzoneObj> bucketSet = objSet(bucket("bucket1"));
1544-
final Set<ACLType> bucketAcls = acls(READ, LIST, READ_ACL);
1608+
final Set<ACLType> bucketAcls = acls(READ, LIST);
15451609
expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
15461610
expectedResolvedNative.add(new OzoneGrant(
15471611
objSet(volume(), prefix("bucket1", "team/folder"), prefix("bucket1", "team/folder/")), acls(READ)));
15481612
assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
15491613

15501614
final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
1551-
// Expected for Ranger: READ, LIST, READ_ACL bucket acls; volume and keys "team/folder" and "team/folder/*" READ
1615+
// Expected for Ranger: READ, LIST bucket acls; volume and keys "team/folder" and "team/folder/*" READ
15521616
expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
15531617
expectedResolvedRanger.add(new OzoneGrant(
15541618
objSet(volume(), key("bucket1", "team/folder"), key("bucket1", "team/folder/*")), acls(READ)));
@@ -1569,17 +1633,37 @@ public void testMultiplePrefixesWithWildcards() throws OMException {
15691633
final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
15701634
final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
15711635

1572-
// Ensure what we got is what we expected
1636+
// s3:prefix conditions do not apply to object actions like s3:GetObject.
1637+
assertThat(resolvedFromNativeAuthorizer).isEmpty();
1638+
assertThat(resolvedFromRangerAuthorizer).isEmpty();
1639+
}
1640+
1641+
@Test
1642+
public void testListAndGetWithPrefixConditionSkipsObjectAction() throws OMException {
1643+
final String json = "{\n" +
1644+
" \"Statement\": [{\n" +
1645+
" \"Effect\": \"Allow\",\n" +
1646+
" \"Action\": [\"s3:ListBucket\", \"s3:GetObject\"],\n" +
1647+
" \"Resource\": [\"arn:aws:s3:::logs\", \"arn:aws:s3:::logs/*\"],\n" +
1648+
" \"Condition\": { \"StringLike\": { \"s3:prefix\": \"team/*\" } }\n" +
1649+
" }]\n" +
1650+
"}";
1651+
1652+
final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
1653+
final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
1654+
1655+
// Expected for native (GetObject is ignored because s3:prefix is present): READ, LIST bucket acls; volume READ;
1656+
// prefix "log/team" READ
15731657
final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
1574-
// Expected for native: READ acl on prefix "" (condition prefixes are ignored); bucket READ; volume READ;
1575-
final Set<IOzoneObj> readObjectsNative = objSet(prefix("logs", ""), bucket("logs"), volume());
1576-
expectedResolvedNative.add(new OzoneGrant(readObjectsNative, acls(READ)));
1658+
expectedResolvedNative.add(new OzoneGrant(objSet(bucket("logs")), acls(READ, LIST)));
1659+
expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("logs", "team/")), acls(READ)));
15771660
assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
15781661

1662+
// Expected for Ranger (GetObject is ignored because s3:prefix is present): READ, LIST bucket acls; volume READ;
1663+
// key "log/team/*" READ
15791664
final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
1580-
// Expected for Ranger: READ acl on key "*" (condition prefixes are ignored)
1581-
final Set<IOzoneObj> keySet = objSet(key("logs", "*"), bucket("logs"), volume());
1582-
expectedResolvedRanger.add(new OzoneGrant(keySet, acls(READ)));
1665+
expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("logs")), acls(READ, LIST)));
1666+
expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("logs", "team/*")), acls(READ)));
15831667
assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
15841668
}
15851669

@@ -1701,6 +1785,35 @@ public void testObjectActionOnAllResources() throws OMException {
17011785
assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
17021786
}
17031787

1788+
@Test
1789+
public void testAllActionsOnAllResourcesWithPrefixCondition() throws OMException {
1790+
final String json = "{\n" +
1791+
" \"Statement\": [{\n" +
1792+
" \"Effect\": \"Allow\",\n" +
1793+
" \"Action\": \"s3:*\",\n" +
1794+
" \"Resource\": \"*\",\n" +
1795+
" \"Condition\": {\n" +
1796+
" \"StringLike\": {\n" +
1797+
" \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
1798+
" }\n" +
1799+
" }\n" +
1800+
" }]\n" +
1801+
"}";
1802+
1803+
// Wildcards on bucket are not supported for Native authorizer
1804+
expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
1805+
1806+
final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
1807+
// Ensure what we got is what we expected
1808+
final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
1809+
// Expected for Ranger: (only ListBucket supports s3:prefix) READ volume; READ, LIST acl on bucket;
1810+
// READ on key "team/folder", "team/folder/*"
1811+
expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("*")), acls(READ, LIST)));
1812+
expectedResolvedRanger.add(
1813+
new OzoneGrant(objSet(volume(), key("*", "team/folder"), key("*", "team/folder/*")), acls(READ)));
1814+
assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
1815+
}
1816+
17041817
@Test
17051818
public void testAllActionsOnAllResources() throws OMException {
17061819
final String json = "{\n" +

0 commit comments

Comments
 (0)