Skip to content

Commit 70315aa

Browse files
xichen01xichen01
committed
HDDS-12794. Improve Lifecycle rule validity checking
1 parent dd04f4b commit 70315aa

9 files changed

Lines changed: 324 additions & 81 deletions

File tree

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,13 @@
1717

1818
package org.apache.hadoop.ozone.om.helpers;
1919

20-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
21-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
20+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
21+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
22+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTagUniqAndLength;
23+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
2224

2325
import jakarta.annotation.Nullable;
26+
import java.util.Collections;
2427
import net.jcip.annotations.Immutable;
2528
import org.apache.commons.lang3.tuple.Pair;
2629
import org.apache.hadoop.ozone.OzoneConsts;
@@ -78,17 +81,17 @@ public void valid(BucketLayout layout) throws OMException {
7881
OMException.ResultCodes.INVALID_REQUEST);
7982
}
8083

84+
if (hasPrefix) {
85+
validatePrefixLength(prefix);
86+
validateTrashPrefix(prefix);
87+
}
88+
89+
if (hasTag()) {
90+
validateTagUniqAndLength(Collections.singletonMap(tagKey, tagValue));
91+
}
92+
8193
if (hasPrefix && layout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
82-
String normalizedPrefix = normalizePrefix(prefix);
83-
if (!normalizedPrefix.equals(prefix)) {
84-
throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix +
85-
" instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST);
86-
}
87-
try {
88-
isValidKeyPath(normalizedPrefix);
89-
} catch (OMException e) {
90-
throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST);
91-
}
94+
validateAndNormalizePrefix(prefix);
9295
}
9396

9497
if (andOperator != null) {

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java‎

Lines changed: 9 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@
1717

1818
package org.apache.hadoop.ozone.om.helpers;
1919

20-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
21-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
20+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
21+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
22+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
2223

2324
import jakarta.annotation.Nullable;
2425
import java.util.ArrayList;
@@ -142,14 +143,6 @@ public boolean isTagEnable() {
142143

143144
/**
144145
* Validates the lifecycle rule.
145-
* - ID length should not exceed the allowed limit
146-
* - At least one action must be specified
147-
* - Filter and Prefix cannot be used together
148-
* - Filter and prefix cannot both be null
149-
* - Prefix can be "", in which case the rule applies to all objects in the bucket.
150-
* - Actions must be valid
151-
* - Filter must be valid
152-
* - There must be at most one Expiration action per rule
153146
*
154147
* @param bucketLayout The bucket layout for validation
155148
* @param creationTime The creation time of the lifecycle configuration in milliseconds since epoch
@@ -189,17 +182,13 @@ public void valid(BucketLayout bucketLayout, Long creationTime) throws OMExcepti
189182
OMException.ResultCodes.INVALID_REQUEST);
190183
}
191184

185+
if (prefix != null) {
186+
validatePrefixLength(prefix);
187+
validateTrashPrefix(prefix);
188+
}
189+
192190
if (prefix != null && bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
193-
String normalizedPrefix = normalizePrefix(prefix);
194-
if (!normalizedPrefix.equals(prefix)) {
195-
throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix +
196-
" instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST);
197-
}
198-
try {
199-
isValidKeyPath(normalizedPrefix);
200-
} catch (OMException e) {
201-
throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST);
202-
}
191+
validateAndNormalizePrefix(prefix);
203192
}
204193

205194
if (filter != null) {

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@
1717

1818
package org.apache.hadoop.ozone.om.helpers;
1919

20-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
21-
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
20+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
21+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
22+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTagUniqAndLength;
23+
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
2224

2325
import jakarta.annotation.Nonnull;
2426
import jakarta.annotation.Nullable;
@@ -73,10 +75,6 @@ public boolean isDirectoryStylePrefix() {
7375
/**
7476
* Validates the OmLifecycleRuleAndOperator.
7577
* Ensures the following:
76-
* - Either tags or prefix must be specified.
77-
* - If there are tags and no prefix, the tags should be more than one.
78-
* - Prefix can be "".
79-
* - Prefix alone is not allowed.
8078
*
8179
* @throws OMException if the validation fails.
8280
*/
@@ -102,17 +100,17 @@ public void valid(BucketLayout layout) throws OMException {
102100
OMException.ResultCodes.INVALID_REQUEST);
103101
}
104102

103+
if (hasTags) {
104+
validateTagUniqAndLength(tags);
105+
}
106+
107+
if (hasPrefix) {
108+
validatePrefixLength(prefix);
109+
validateTrashPrefix(prefix);
110+
}
111+
105112
if (hasPrefix && layout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
106-
String normalizedPrefix = normalizePrefix(prefix);
107-
if (!normalizedPrefix.equals(prefix)) {
108-
throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix +
109-
" instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST);
110-
}
111-
try {
112-
isValidKeyPath(normalizedPrefix);
113-
} catch (OMException e) {
114-
throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST);
115-
}
113+
validateAndNormalizePrefix(prefix);
116114
}
117115
}
118116

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleUtils.java‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hadoop.ozone.om.helpers;
19+
20+
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
21+
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
22+
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
23+
24+
import java.nio.charset.StandardCharsets;
25+
import java.util.HashSet;
26+
import java.util.Map;
27+
import java.util.Set;
28+
import org.apache.hadoop.fs.FileSystem;
29+
import org.apache.hadoop.ozone.om.exceptions.OMException;
30+
31+
/**
32+
* Utility class for Ozone Lifecycle.
33+
*/
34+
public final class OmLifecycleUtils {
35+
36+
// Ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
37+
public static final int MAX_PREFIX_LENGTH = 1024;
38+
39+
// Ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html
40+
public static final int MAX_TAG_KEY_LENGTH = 128;
41+
public static final int MAX_TAG_VALUE_LENGTH = 256;
42+
43+
private OmLifecycleUtils() {
44+
}
45+
46+
/**
47+
* Check if the prefix is a Trash path.
48+
*
49+
* @param prefix the prefix to check
50+
* @throws OMException if the prefix is a trash path
51+
*/
52+
public static void validateTrashPrefix(String prefix) throws OMException {
53+
if (org.apache.commons.lang3.StringUtils.isEmpty(prefix)) {
54+
return;
55+
}
56+
// Remove leading slash if present for validation
57+
String p = prefix.startsWith(OZONE_URI_DELIMITER) ? prefix.substring(1) : prefix;
58+
59+
if (p.startsWith(FileSystem.TRASH_PREFIX + OZONE_URI_DELIMITER) ||
60+
p.equals(FileSystem.TRASH_PREFIX)) {
61+
throw new OMException("Lifecycle rule prefix cannot be trash root " +
62+
FileSystem.TRASH_PREFIX + OZONE_URI_DELIMITER, OMException.ResultCodes.INVALID_REQUEST);
63+
}
64+
}
65+
66+
/**
67+
* Normalize and validate the prefix for FILE_SYSTEM_OPTIMIZED layout.
68+
*
69+
* @param prefix the prefix to validate
70+
* @throws OMException if the prefix is invalid
71+
*/
72+
public static void validateAndNormalizePrefix(String prefix) throws OMException {
73+
String normalizedPrefix = normalizePrefix(prefix);
74+
if (!normalizedPrefix.equals(prefix)) {
75+
throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix +
76+
" instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST);
77+
}
78+
try {
79+
isValidKeyPath(normalizedPrefix);
80+
} catch (OMException e) {
81+
throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST);
82+
}
83+
}
84+
85+
/**
86+
* Validate prefix length.
87+
*
88+
* @param prefix the prefix to validate
89+
* @throws OMException if the prefix length exceeds 1024
90+
*/
91+
public static void validatePrefixLength(String prefix) throws OMException {
92+
if (prefix != null && prefix.getBytes(StandardCharsets.UTF_8).length > MAX_PREFIX_LENGTH) {
93+
throw new OMException("The maximum size of a prefix is " + MAX_PREFIX_LENGTH,
94+
OMException.ResultCodes.INVALID_REQUEST);
95+
}
96+
}
97+
98+
/**
99+
* Validate tag key, value length and the uniqueness of the key.
100+
*
101+
* @param tags the tags to validate
102+
* @throws OMException if the tag key or value is invalid
103+
*/
104+
public static void validateTagUniqAndLength(Map<String, String> tags) throws OMException {
105+
if (tags == null) {
106+
return;
107+
}
108+
Set<String> keys = new HashSet<>();
109+
for (Map.Entry<String, String> entry : tags.entrySet()) {
110+
String key = entry.getKey();
111+
String value = entry.getValue();
112+
113+
if (key == null || key.isEmpty() ||
114+
key.getBytes(StandardCharsets.UTF_8).length > MAX_TAG_KEY_LENGTH) {
115+
throw new OMException("A Tag's Key must be a length between 1 and " +
116+
MAX_TAG_KEY_LENGTH, OMException.ResultCodes.INVALID_REQUEST);
117+
}
118+
119+
if (value != null && value.getBytes(StandardCharsets.UTF_8).length > MAX_TAG_VALUE_LENGTH) {
120+
throw new OMException("A Tag's Value must be a length between 0 and " +
121+
MAX_TAG_VALUE_LENGTH, OMException.ResultCodes.INVALID_REQUEST);
122+
}
123+
124+
if (!keys.add(key)) {
125+
throw new OMException("Duplicate Tag Keys are not allowed",
126+
OMException.ResultCodes.INVALID_REQUEST);
127+
}
128+
}
129+
}
130+
}
131+

‎hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@
3030
import static org.junit.jupiter.api.Assertions.assertNull;
3131

3232
import java.util.Collections;
33+
import org.apache.commons.lang3.RandomStringUtils;
3334
import org.apache.commons.lang3.tuple.Pair;
35+
import org.apache.hadoop.fs.FileSystem;
3436
import org.apache.hadoop.ozone.om.exceptions.OMException;
3537
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleFilter;
3638
import org.junit.jupiter.api.Test;
@@ -90,6 +92,32 @@ public void testInValidFilter() {
9092

9193
}
9294

95+
@Test
96+
public void testFilterValidation() {
97+
// 1. Prefix is Trash path
98+
OmLCFilter.Builder trashPrefixFilter = getOmLCFilterBuilder(FileSystem.TRASH_PREFIX, null, null);
99+
assertOMException(() -> trashPrefixFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
100+
"Lifecycle rule prefix cannot be trash root");
101+
102+
// 2. Prefix too long
103+
String longPrefix = RandomStringUtils.randomAlphanumeric(1025);
104+
OmLCFilter.Builder longPrefixFilter = getOmLCFilterBuilder(longPrefix, null, null);
105+
assertOMException(() -> longPrefixFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
106+
"The maximum size of a prefix is 1024");
107+
108+
// 3. Tag key too long
109+
String longKey = RandomStringUtils.randomAlphanumeric(129);
110+
OmLCFilter.Builder longKeyFilter = getOmLCFilterBuilder(null, Pair.of(longKey, "value"), null);
111+
assertOMException(() -> longKeyFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
112+
"A Tag's Key must be a length between 1 and 128");
113+
114+
// 4. Tag value too long
115+
String longValue = RandomStringUtils.randomAlphanumeric(257);
116+
OmLCFilter.Builder longValueFilter = getOmLCFilterBuilder(null, Pair.of("key", longValue), null);
117+
assertOMException(() -> longValueFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
118+
"A Tag's Value must be a length between 0 and 256");
119+
}
120+
93121
@Test
94122
public void testProtobufConversion() throws OMException {
95123
// Only prefix

0 commit comments

Comments
 (0)