Skip to content

Commit 8ee502a

Browse files
xichen01xichen01
authored
HDDS-12794. Improve Lifecycle rule validity checking (#9593)
1 parent cfa9cfe commit 8ee502a

9 files changed

Lines changed: 350 additions & 80 deletions

File tree

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,13 @@
1717

1818
package org.apache.hadoop.ozone.om.helpers;
1919

20-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
21-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
20+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
21+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
22+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTagUniqAndLength;
23+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
2224

2325
import jakarta.annotation.Nullable;
26+
import java.util.Collections;
2427
import net.jcip.annotations.Immutable;
2528
import org.apache.commons.lang3.tuple.Pair;
2629
import org.apache.hadoop.ozone.OzoneConsts;
@@ -59,12 +62,17 @@ private OmLCFilter(Builder builder) {
5962

6063
/**
6164
* Validates the OmLCFilter.
62-
* Ensures that only one of prefix, tag, or andOperator is set.
63-
* You can specify an empty filter, in which case the rule applies to all objects in the bucket.
64-
* Prefix can be "", in which case the rule applies to all objects in the bucket.
6565
* Ref: <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/intro-lifecycle-filters.html#filter-examples">...</a>
66-
* If the validation fails, an OMException is thrown.
66+
* - Only one of prefix, tag, or andOperator is set.
67+
* - You can specify an empty filter, in which case the rule applies to all objects in the bucket.
68+
* - Prefix can be "", in which case the rule applies to all objects in the bucket.
69+
* - Prefix length must be a length between 0 and 1024.
70+
* - Tag's key must be a length between 1 and 128.
71+
* - Tag's value must be a length between 0 and 256.
72+
* - For FSO bucket, the prefix must be normalized and valid path.
73+
* - Prefix cannot be the Trash directory or any of its subdirectories.
6774
*
75+
* @param layout The bucket layout for validation
6876
* @throws OMException if the filter is invalid.
6977
*/
7078
public void valid(BucketLayout layout) throws OMException {
@@ -78,17 +86,17 @@ public void valid(BucketLayout layout) throws OMException {
7886
OMException.ResultCodes.INVALID_REQUEST);
7987
}
8088

89+
if (hasPrefix) {
90+
validatePrefixLength(prefix);
91+
validateTrashPrefix(prefix);
92+
}
93+
94+
if (hasTag()) {
95+
validateTagUniqAndLength(Collections.singletonMap(tagKey, tagValue));
96+
}
97+
8198
if (hasPrefix && layout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
82-
String normalizedPrefix = normalizePrefix(prefix);
83-
if (!normalizedPrefix.equals(prefix)) {
84-
throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix +
85-
" instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST);
86-
}
87-
try {
88-
isValidKeyPath(normalizedPrefix);
89-
} catch (OMException e) {
90-
throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST);
91-
}
99+
validateAndNormalizePrefix(prefix);
92100
}
93101

94102
if (andOperator != null) {

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@
1717

1818
package org.apache.hadoop.ozone.om.helpers;
1919

20-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
21-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
20+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
21+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
22+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
2223

2324
import jakarta.annotation.Nullable;
2425
import java.util.ArrayList;
@@ -142,14 +143,17 @@ public boolean isTagEnable() {
142143

143144
/**
144145
* Validates the lifecycle rule.
145-
* - ID length should not exceed the allowed limit
146-
* - At least one action must be specified
147-
* - Filter and Prefix cannot be used together
148-
* - Filter and prefix cannot both be null
146+
* - ID length should not exceed the allowed limit.
147+
* - At least one action must be specified, and the expiration type Action can have at most one.
148+
* - Filter and Prefix cannot be used together.
149+
* - Filter and prefix cannot both be null.
149150
* - Prefix can be "", in which case the rule applies to all objects in the bucket.
150-
* - Actions must be valid
151-
* - Filter must be valid
152-
* - There must be at most one Expiration action per rule
151+
* - Prefix length must be a length between 0 and 1024.
152+
* - For FSO bucket, the prefix must be normalized and valid path.
153+
* - Prefix cannot be the Trash directory or any of its subdirectories.
154+
* - Actions must be valid.
155+
* - Filter must be valid.
156+
* - There must be at most one Expiration action per rule.
153157
*
154158
* @param bucketLayout The bucket layout for validation
155159
* @param creationTime The creation time of the lifecycle configuration in milliseconds since epoch
@@ -189,17 +193,13 @@ public void valid(BucketLayout bucketLayout, Long creationTime) throws OMExcepti
189193
OMException.ResultCodes.INVALID_REQUEST);
190194
}
191195

196+
if (prefix != null) {
197+
validatePrefixLength(prefix);
198+
validateTrashPrefix(prefix);
199+
}
200+
192201
if (prefix != null && bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
193-
String normalizedPrefix = normalizePrefix(prefix);
194-
if (!normalizedPrefix.equals(prefix)) {
195-
throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix +
196-
" instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST);
197-
}
198-
try {
199-
isValidKeyPath(normalizedPrefix);
200-
} catch (OMException e) {
201-
throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST);
202-
}
202+
validateAndNormalizePrefix(prefix);
203203
}
204204

205205
if (filter != null) {

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@
1717

1818
package org.apache.hadoop.ozone.om.helpers;
1919

20-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
21-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
20+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
21+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
22+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTagUniqAndLength;
23+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
2224

2325
import jakarta.annotation.Nonnull;
2426
import jakarta.annotation.Nullable;
@@ -77,7 +79,14 @@ public boolean isDirectoryStylePrefix() {
7779
* - If there are tags and no prefix, the tags should be more than one.
7880
* - Prefix can be "".
7981
* - Prefix alone is not allowed.
82+
* - Prefix length must be a length between 0 and 1024.
83+
* - The key of a tag must be unique.
84+
* - Tag's key must be a length between 1 and 128.
85+
* - Tag's value must be a length between 0 and 256.
86+
* - Prefix cannot be the Trash directory or any of its subdirectories.
87+
* - For FSO bucket, the prefix must be normalized and valid path
8088
*
89+
* @param layout The bucket layout for validation
8190
* @throws OMException if the validation fails.
8291
*/
8392
public void valid(BucketLayout layout) throws OMException {
@@ -102,17 +111,17 @@ public void valid(BucketLayout layout) throws OMException {
102111
OMException.ResultCodes.INVALID_REQUEST);
103112
}
104113

114+
if (hasTags) {
115+
validateTagUniqAndLength(tags);
116+
}
117+
118+
if (hasPrefix) {
119+
validatePrefixLength(prefix);
120+
validateTrashPrefix(prefix);
121+
}
122+
105123
if (hasPrefix && layout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
106-
String normalizedPrefix = normalizePrefix(prefix);
107-
if (!normalizedPrefix.equals(prefix)) {
108-
throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix +
109-
" instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST);
110-
}
111-
try {
112-
isValidKeyPath(normalizedPrefix);
113-
} catch (OMException e) {
114-
throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST);
115-
}
124+
validateAndNormalizePrefix(prefix);
116125
}
117126
}
118127

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleUtils.java‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hadoop.ozone.om.helpers;
19+
20+
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
21+
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
22+
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
23+
24+
import java.nio.charset.StandardCharsets;
25+
import java.util.HashSet;
26+
import java.util.Map;
27+
import java.util.Set;
28+
import org.apache.commons.lang3.StringUtils;
29+
import org.apache.hadoop.fs.FileSystem;
30+
import org.apache.hadoop.ozone.om.exceptions.OMException;
31+
32+
/**
33+
* Utility class for Ozone Lifecycle.
34+
*/
35+
public final class OmLifecycleUtils {
36+
37+
// Ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
38+
public static final int MAX_PREFIX_LENGTH = 1024;
39+
40+
// Ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html
41+
public static final int MAX_TAG_KEY_LENGTH = 128;
42+
public static final int MAX_TAG_VALUE_LENGTH = 256;
43+
44+
private OmLifecycleUtils() {
45+
}
46+
47+
/**
48+
* Check if the prefix is a Trash path.
49+
*
50+
* @param prefix the prefix to check
51+
* @throws OMException if the prefix is a trash path
52+
*/
53+
public static void validateTrashPrefix(String prefix) throws OMException {
54+
if (StringUtils.isEmpty(prefix)) {
55+
return;
56+
}
57+
// Remove leading slash if present for validation
58+
String p = prefix.startsWith(OZONE_URI_DELIMITER) ? prefix.substring(1) : prefix;
59+
60+
if (p.startsWith(FileSystem.TRASH_PREFIX + OZONE_URI_DELIMITER) ||
61+
p.equals(FileSystem.TRASH_PREFIX)) {
62+
throw new OMException("Lifecycle rule prefix cannot be trash root " +
63+
FileSystem.TRASH_PREFIX + OZONE_URI_DELIMITER, OMException.ResultCodes.INVALID_REQUEST);
64+
}
65+
}
66+
67+
/**
68+
* Normalize and validate the prefix for FILE_SYSTEM_OPTIMIZED layout.
69+
*
70+
* @param prefix the prefix to validate
71+
* @throws OMException if the prefix is invalid
72+
*/
73+
public static void validateAndNormalizePrefix(String prefix) throws OMException {
74+
String normalizedPrefix = normalizePrefix(prefix);
75+
if (!normalizedPrefix.equals(prefix)) {
76+
throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix +
77+
" instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST);
78+
}
79+
try {
80+
isValidKeyPath(normalizedPrefix);
81+
} catch (OMException e) {
82+
throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST);
83+
}
84+
}
85+
86+
/**
87+
* Validate prefix length.
88+
*
89+
* @param prefix the prefix to validate
90+
* @throws OMException if the prefix length exceeds 1024
91+
*/
92+
public static void validatePrefixLength(String prefix) throws OMException {
93+
if (prefix != null && prefix.getBytes(StandardCharsets.UTF_8).length > MAX_PREFIX_LENGTH) {
94+
throw new OMException("The maximum size of a prefix is " + MAX_PREFIX_LENGTH,
95+
OMException.ResultCodes.INVALID_REQUEST);
96+
}
97+
}
98+
99+
/**
100+
* Validate tag key, value length and the uniqueness of the key.
101+
*
102+
* @param tags the tags to validate
103+
* @throws OMException if the tag key or value is invalid
104+
*/
105+
public static void validateTagUniqAndLength(Map<String, String> tags) throws OMException {
106+
if (tags == null) {
107+
return;
108+
}
109+
Set<String> keys = new HashSet<>();
110+
for (Map.Entry<String, String> entry : tags.entrySet()) {
111+
String key = entry.getKey();
112+
String value = entry.getValue();
113+
114+
if (StringUtils.isEmpty(key) || key.getBytes(StandardCharsets.UTF_8).length > MAX_TAG_KEY_LENGTH) {
115+
throw new OMException("A Tag's Key must be a length between 1 and " +
116+
MAX_TAG_KEY_LENGTH, OMException.ResultCodes.INVALID_REQUEST);
117+
}
118+
119+
if (!StringUtils.isEmpty(value) && value.getBytes(StandardCharsets.UTF_8).length > MAX_TAG_VALUE_LENGTH) {
120+
throw new OMException("A Tag's Value must be a length between 0 and " +
121+
MAX_TAG_VALUE_LENGTH, OMException.ResultCodes.INVALID_REQUEST);
122+
}
123+
124+
if (!keys.add(key)) {
125+
throw new OMException("Duplicate Tag Keys are not allowed",
126+
OMException.ResultCodes.INVALID_REQUEST);
127+
}
128+
}
129+
}
130+
}
131+

‎hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@
3030
import static org.junit.jupiter.api.Assertions.assertNull;
3131

3232
import java.util.Collections;
33+
import org.apache.commons.lang3.RandomStringUtils;
3334
import org.apache.commons.lang3.tuple.Pair;
35+
import org.apache.hadoop.fs.FileSystem;
3436
import org.apache.hadoop.ozone.om.exceptions.OMException;
3537
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleFilter;
3638
import org.junit.jupiter.api.Test;
@@ -90,6 +92,32 @@ public void testInValidFilter() {
9092

9193
}
9294

95+
@Test
96+
public void testFilterValidation() {
97+
// 1. Prefix is Trash path
98+
OmLCFilter.Builder trashPrefixFilter = getOmLCFilterBuilder(FileSystem.TRASH_PREFIX, null, null);
99+
assertOMException(() -> trashPrefixFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
100+
"Lifecycle rule prefix cannot be trash root");
101+
102+
// 2. Prefix too long
103+
String longPrefix = RandomStringUtils.randomAlphanumeric(1025);
104+
OmLCFilter.Builder longPrefixFilter = getOmLCFilterBuilder(longPrefix, null, null);
105+
assertOMException(() -> longPrefixFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
106+
"The maximum size of a prefix is 1024");
107+
108+
// 3. Tag key too long
109+
String longKey = RandomStringUtils.randomAlphanumeric(129);
110+
OmLCFilter.Builder longKeyFilter = getOmLCFilterBuilder(null, Pair.of(longKey, "value"), null);
111+
assertOMException(() -> longKeyFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
112+
"A Tag's Key must be a length between 1 and 128");
113+
114+
// 4. Tag value too long
115+
String longValue = RandomStringUtils.randomAlphanumeric(257);
116+
OmLCFilter.Builder longValueFilter = getOmLCFilterBuilder(null, Pair.of("key", longValue), null);
117+
assertOMException(() -> longValueFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
118+
"A Tag's Value must be a length between 0 and 256");
119+
}
120+
93121
@Test
94122
public void testProtobufConversion() throws OMException {
95123
// Only prefix

0 commit comments

Comments
 (0)