diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java index 0ef39db6f288..387c4b24df17 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java @@ -17,10 +17,13 @@ package org.apache.hadoop.ozone.om.helpers; -import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath; -import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTagUniqAndLength; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix; import jakarta.annotation.Nullable; +import java.util.Collections; import net.jcip.annotations.Immutable; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.ozone.OzoneConsts; @@ -59,12 +62,17 @@ private OmLCFilter(Builder builder) { /** * Validates the OmLCFilter. - * Ensures that only one of prefix, tag, or andOperator is set. - * You can specify an empty filter, in which case the rule applies to all objects in the bucket. - * Prefix can be "", in which case the rule applies to all objects in the bucket. * Ref: ... - * If the validation fails, an OMException is thrown. + * - Only one of prefix, tag, or andOperator is set. + * - You can specify an empty filter, in which case the rule applies to all objects in the bucket. + * - Prefix can be "", in which case the rule applies to all objects in the bucket. + * - Prefix length must be a length between 0 and 1024. + * - Tag's key must be a length between 1 and 128. + * - Tag's value must be a length between 0 and 256. + * - For FSO bucket, the prefix must be normalized and valid path. + * - Prefix cannot be the Trash directory or any of its subdirectories. * + * @param layout The bucket layout for validation * @throws OMException if the filter is invalid. */ public void valid(BucketLayout layout) throws OMException { @@ -78,17 +86,17 @@ public void valid(BucketLayout layout) throws OMException { OMException.ResultCodes.INVALID_REQUEST); } + if (hasPrefix) { + validatePrefixLength(prefix); + validateTrashPrefix(prefix); + } + + if (hasTag()) { + validateTagUniqAndLength(Collections.singletonMap(tagKey, tagValue)); + } + if (hasPrefix && layout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { - String normalizedPrefix = normalizePrefix(prefix); - if (!normalizedPrefix.equals(prefix)) { - throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix + - " instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST); - } - try { - isValidKeyPath(normalizedPrefix); - } catch (OMException e) { - throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST); - } + validateAndNormalizePrefix(prefix); } if (andOperator != null) { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java index b539830006a6..00875842655c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java @@ -17,8 +17,9 @@ package org.apache.hadoop.ozone.om.helpers; -import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath; -import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix; import jakarta.annotation.Nullable; import java.util.ArrayList; @@ -142,14 +143,17 @@ public boolean isTagEnable() { /** * Validates the lifecycle rule. - * - ID length should not exceed the allowed limit - * - At least one action must be specified - * - Filter and Prefix cannot be used together - * - Filter and prefix cannot both be null + * - ID length should not exceed the allowed limit. + * - At least one action must be specified, and the expiration type Action can have at most one. + * - Filter and Prefix cannot be used together. + * - Filter and prefix cannot both be null. * - Prefix can be "", in which case the rule applies to all objects in the bucket. - * - Actions must be valid - * - Filter must be valid - * - There must be at most one Expiration action per rule + * - Prefix length must be a length between 0 and 1024. + * - For FSO bucket, the prefix must be normalized and valid path. + * - Prefix cannot be the Trash directory or any of its subdirectories. + * - Actions must be valid. + * - Filter must be valid. + * - There must be at most one Expiration action per rule. * * @param bucketLayout The bucket layout for validation * @param creationTime The creation time of the lifecycle configuration in milliseconds since epoch @@ -189,17 +193,13 @@ public void valid(BucketLayout bucketLayout, Long creationTime) throws OMExcepti OMException.ResultCodes.INVALID_REQUEST); } + if (prefix != null) { + validatePrefixLength(prefix); + validateTrashPrefix(prefix); + } + if (prefix != null && bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { - String normalizedPrefix = normalizePrefix(prefix); - if (!normalizedPrefix.equals(prefix)) { - throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix + - " instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST); - } - try { - isValidKeyPath(normalizedPrefix); - } catch (OMException e) { - throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST); - } + validateAndNormalizePrefix(prefix); } if (filter != null) { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java index 2abef21c8558..70228846c228 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java @@ -17,8 +17,10 @@ package org.apache.hadoop.ozone.om.helpers; -import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath; -import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTagUniqAndLength; +import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; @@ -77,7 +79,14 @@ public boolean isDirectoryStylePrefix() { * - If there are tags and no prefix, the tags should be more than one. * - Prefix can be "". * - Prefix alone is not allowed. + * - Prefix length must be a length between 0 and 1024. + * - The key of a tag must be unique. + * - Tag's key must be a length between 1 and 128. + * - Tag's value must be a length between 0 and 256. + * - Prefix cannot be the Trash directory or any of its subdirectories. + * - For FSO bucket, the prefix must be normalized and valid path * + * @param layout The bucket layout for validation * @throws OMException if the validation fails. */ public void valid(BucketLayout layout) throws OMException { @@ -102,17 +111,17 @@ public void valid(BucketLayout layout) throws OMException { OMException.ResultCodes.INVALID_REQUEST); } + if (hasTags) { + validateTagUniqAndLength(tags); + } + + if (hasPrefix) { + validatePrefixLength(prefix); + validateTrashPrefix(prefix); + } + if (hasPrefix && layout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { - String normalizedPrefix = normalizePrefix(prefix); - if (!normalizedPrefix.equals(prefix)) { - throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix + - " instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST); - } - try { - isValidKeyPath(normalizedPrefix); - } catch (OMException e) { - throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST); - } + validateAndNormalizePrefix(prefix); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleUtils.java new file mode 100644 index 000000000000..32781280919d --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleUtils.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; +import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath; +import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.ozone.om.exceptions.OMException; + +/** + * Utility class for Ozone Lifecycle. + */ +public final class OmLifecycleUtils { + + // Ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + public static final int MAX_PREFIX_LENGTH = 1024; + + // Ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html + public static final int MAX_TAG_KEY_LENGTH = 128; + public static final int MAX_TAG_VALUE_LENGTH = 256; + + private OmLifecycleUtils() { + } + + /** + * Check if the prefix is a Trash path. + * + * @param prefix the prefix to check + * @throws OMException if the prefix is a trash path + */ + public static void validateTrashPrefix(String prefix) throws OMException { + if (StringUtils.isEmpty(prefix)) { + return; + } + // Remove leading slash if present for validation + String p = prefix.startsWith(OZONE_URI_DELIMITER) ? prefix.substring(1) : prefix; + + if (p.startsWith(FileSystem.TRASH_PREFIX + OZONE_URI_DELIMITER) || + p.equals(FileSystem.TRASH_PREFIX)) { + throw new OMException("Lifecycle rule prefix cannot be trash root " + + FileSystem.TRASH_PREFIX + OZONE_URI_DELIMITER, OMException.ResultCodes.INVALID_REQUEST); + } + } + + /** + * Normalize and validate the prefix for FILE_SYSTEM_OPTIMIZED layout. + * + * @param prefix the prefix to validate + * @throws OMException if the prefix is invalid + */ + public static void validateAndNormalizePrefix(String prefix) throws OMException { + String normalizedPrefix = normalizePrefix(prefix); + if (!normalizedPrefix.equals(prefix)) { + throw new OMException("Prefix format is not supported. Please use " + normalizedPrefix + + " instead of " + prefix + ".", OMException.ResultCodes.INVALID_REQUEST); + } + try { + isValidKeyPath(normalizedPrefix); + } catch (OMException e) { + throw new OMException("Prefix is not a valid key path: " + prefix, OMException.ResultCodes.INVALID_REQUEST); + } + } + + /** + * Validate prefix length. + * + * @param prefix the prefix to validate + * @throws OMException if the prefix length exceeds 1024 + */ + public static void validatePrefixLength(String prefix) throws OMException { + if (prefix != null && prefix.getBytes(StandardCharsets.UTF_8).length > MAX_PREFIX_LENGTH) { + throw new OMException("The maximum size of a prefix is " + MAX_PREFIX_LENGTH, + OMException.ResultCodes.INVALID_REQUEST); + } + } + + /** + * Validate tag key, value length and the uniqueness of the key. + * + * @param tags the tags to validate + * @throws OMException if the tag key or value is invalid + */ + public static void validateTagUniqAndLength(Map tags) throws OMException { + if (tags == null) { + return; + } + Set keys = new HashSet<>(); + for (Map.Entry entry : tags.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + + if (StringUtils.isEmpty(key) || key.getBytes(StandardCharsets.UTF_8).length > MAX_TAG_KEY_LENGTH) { + throw new OMException("A Tag's Key must be a length between 1 and " + + MAX_TAG_KEY_LENGTH, OMException.ResultCodes.INVALID_REQUEST); + } + + if (!StringUtils.isEmpty(value) && value.getBytes(StandardCharsets.UTF_8).length > MAX_TAG_VALUE_LENGTH) { + throw new OMException("A Tag's Value must be a length between 0 and " + + MAX_TAG_VALUE_LENGTH, OMException.ResultCodes.INVALID_REQUEST); + } + + if (!keys.add(key)) { + throw new OMException("Duplicate Tag Keys are not allowed", + OMException.ResultCodes.INVALID_REQUEST); + } + } + } +} + diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java index bd3809ba10a6..75f693d8461d 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java @@ -30,7 +30,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; import java.util.Collections; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleFilter; import org.junit.jupiter.api.Test; @@ -90,6 +92,32 @@ public void testInValidFilter() { } + @Test + public void testFilterValidation() { + // 1. Prefix is Trash path + OmLCFilter.Builder trashPrefixFilter = getOmLCFilterBuilder(FileSystem.TRASH_PREFIX, null, null); + assertOMException(() -> trashPrefixFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST, + "Lifecycle rule prefix cannot be trash root"); + + // 2. Prefix too long + String longPrefix = RandomStringUtils.randomAlphanumeric(1025); + OmLCFilter.Builder longPrefixFilter = getOmLCFilterBuilder(longPrefix, null, null); + assertOMException(() -> longPrefixFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST, + "The maximum size of a prefix is 1024"); + + // 3. Tag key too long + String longKey = RandomStringUtils.randomAlphanumeric(129); + OmLCFilter.Builder longKeyFilter = getOmLCFilterBuilder(null, Pair.of(longKey, "value"), null); + assertOMException(() -> longKeyFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST, + "A Tag's Key must be a length between 1 and 128"); + + // 4. Tag value too long + String longValue = RandomStringUtils.randomAlphanumeric(257); + OmLCFilter.Builder longValueFilter = getOmLCFilterBuilder(null, Pair.of("key", longValue), null); + assertOMException(() -> longValueFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST, + "A Tag's Value must be a length between 0 and 256"); + } + @Test public void testProtobufConversion() throws OMException { // Only prefix diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java index b9993bce114d..4524e4adfa87 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java @@ -34,7 +34,9 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRule; import org.junit.jupiter.api.Test; @@ -186,6 +188,90 @@ public void testDuplicateRuleIDs() throws OMException { assertOMException(() -> config.buildAndValid(), INVALID_REQUEST, "Duplicate rule IDs found"); } + @Test + public void testTrashPrefixValidation() throws OMException { + long currentTime = System.currentTimeMillis(); + OmLCExpiration exp = new OmLCExpiration.Builder() + .setDays(30) + .build(); + + // Case 1: Prefix is .Trash + OmLCRule.Builder r1 = new OmLCRule.Builder() + .setId("trash-rule-1") + .setEnabled(true) + .setPrefix(FileSystem.TRASH_PREFIX) + .setAction(exp); + assertOMException(() -> r1.build().valid(BucketLayout.DEFAULT, currentTime), INVALID_REQUEST, + "Lifecycle rule prefix cannot be trash root"); + + // Case 4: Prefix is .Trash/subdir + OmLCRule.Builder r2 = new OmLCRule.Builder() + .setId("trash-rule-2") + .setEnabled(true) + .setPrefix(FileSystem.TRASH_PREFIX + "/user") + .setAction(exp); + assertOMException(() -> r2.build().valid(BucketLayout.DEFAULT, currentTime), INVALID_REQUEST, + "Lifecycle rule prefix cannot be trash root"); + } + + @Test + public void testTagValidation() throws OMException { + long currentTime = System.currentTimeMillis(); + OmLCExpiration exp = new OmLCExpiration.Builder() + .setDays(30) + .build(); + + // Case 1: Tag key too long + String longKey = RandomStringUtils.randomAlphanumeric(OmLifecycleUtils.MAX_TAG_KEY_LENGTH + 1); + OmLCFilter filterKeyTooLong = getOmLCFilterBuilder(null, Pair.of(longKey, "value"), null).build(); + OmLCRule.Builder r1 = new OmLCRule.Builder() + .setId("long-tag-key") + .setEnabled(true) + .setFilter(filterKeyTooLong) + .setAction(exp); + assertOMException(() -> r1.build().valid(BucketLayout.DEFAULT, currentTime), INVALID_REQUEST, + "A Tag's Key must be a length between 1 and 128"); + + // Case 2: Tag value too long + String longValue = RandomStringUtils.randomAlphanumeric(OmLifecycleUtils.MAX_TAG_VALUE_LENGTH + 1); + OmLCFilter filterValueTooLong = getOmLCFilterBuilder(null, Pair.of("key", longValue), null).build(); + OmLCRule.Builder r2 = new OmLCRule.Builder() + .setId("long-tag-value") + .setEnabled(true) + .setFilter(filterValueTooLong) + .setAction(exp); + assertOMException(() -> r2.build().valid(BucketLayout.DEFAULT, currentTime), INVALID_REQUEST, + "A Tag's Value must be a length between 0 and 256"); + } + + @Test + public void testPrefixLengthValidation() throws OMException { + long currentTime = System.currentTimeMillis(); + OmLCExpiration exp = new OmLCExpiration.Builder() + .setDays(30) + .build(); + + // Case 1: Prefix too long + String longPrefix = RandomStringUtils.randomAlphanumeric(OmLifecycleUtils.MAX_PREFIX_LENGTH + 1); + OmLCRule.Builder r1 = new OmLCRule.Builder() + .setId("long-prefix") + .setEnabled(true) + .setPrefix(longPrefix) + .setAction(exp); + assertOMException(() -> r1.build().valid(BucketLayout.DEFAULT, currentTime), INVALID_REQUEST, + "The maximum size of a prefix is 1024"); + + // Case 2: Filter Prefix too long + OmLCFilter filterPrefixTooLong = getOmLCFilterBuilder(longPrefix, null, null).build(); + OmLCRule.Builder r2 = new OmLCRule.Builder() + .setId("filter-long-prefix") + .setEnabled(true) + .setFilter(filterPrefixTooLong) + .setAction(exp); + assertOMException(() -> r2.build().valid(BucketLayout.DEFAULT, currentTime), INVALID_REQUEST, + "The maximum size of a prefix is 1024"); + } + @Test public void testProtobufConversion() throws OMException { // Only Filter diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleRuleAndOperator.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleRuleAndOperator.java index 43c7ffbc41b8..164846bba7eb 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleRuleAndOperator.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleRuleAndOperator.java @@ -28,6 +28,8 @@ import com.google.common.collect.ImmutableMap; import java.util.Collections; import java.util.Map; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRuleAndOperator; import org.junit.jupiter.api.Test; @@ -72,6 +74,36 @@ public void testInValidAndOperator() { "Either 'Tags' or 'Prefix' must be specified."); } + @Test + public void testValidation() { + // 1. Prefix is Trash path + OmLifecycleRuleAndOperator.Builder trashPrefixAndOp = getOmLCAndOperatorBuilder( + FileSystem.TRASH_PREFIX, Collections.singletonMap("tag1", "value1")); + assertOMException(() -> trashPrefixAndOp.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST, + "Lifecycle rule prefix cannot be trash root"); + + // 2. Prefix too long + String longPrefix = RandomStringUtils.randomAlphanumeric(1025); + OmLifecycleRuleAndOperator.Builder longPrefixAndOp = getOmLCAndOperatorBuilder( + longPrefix, Collections.singletonMap("tag1", "value1")); + assertOMException(() -> longPrefixAndOp.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST, + "The maximum size of a prefix is 1024"); + + // 3. Tag key too long + String longKey = RandomStringUtils.randomAlphanumeric(129); + OmLifecycleRuleAndOperator.Builder longKeyAndOp = getOmLCAndOperatorBuilder( + "prefix", Collections.singletonMap(longKey, "value")); + assertOMException(() -> longKeyAndOp.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST, + "A Tag's Key must be a length between 1 and 128"); + + // 4. Tag value too long + String longValue = RandomStringUtils.randomAlphanumeric(257); + OmLifecycleRuleAndOperator.Builder longValueAndOp = getOmLCAndOperatorBuilder( + "prefix", Collections.singletonMap("key", longValue)); + assertOMException(() -> longValueAndOp.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST, + "A Tag's Value must be a length between 0 and 256"); + } + @Test public void testProtobufConversion() throws OMException { // Prefix and tags diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java index f115ef181e21..e5779660f960 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java @@ -372,19 +372,12 @@ public BackgroundTaskResult call() { private void evaluateFSOBucket(OmVolumeArgs volume, OmBucketInfo bucket, String bucketKey, Table keyTable, List ruleList, LimitedExpiredObjectList expiredKeyList, LimitedExpiredObjectList expiredDirList) { - List prefixStartsWithTrashRuleList = - ruleList.stream().filter(r -> r.isPrefixEnable() && r.getEffectivePrefix().startsWith( - TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)).collect(Collectors.toList()); List prefixRuleList = ruleList.stream().filter(r -> r.isPrefixEnable()).collect(Collectors.toList()); // r.isPrefixEnable() == false means empty filter List noPrefixRuleList = ruleList.stream().filter(r -> !r.isPrefixEnable()).collect(Collectors.toList()); - prefixRuleList.removeAll(prefixStartsWithTrashRuleList); - prefixStartsWithTrashRuleList.stream().forEach( - r -> LOG.info("Skip rule {} as its prefix starts with {}", r, TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)); - for (OmLCRule rule : prefixRuleList) { // find KeyInfo of each directory for prefix List dirList; @@ -710,15 +703,6 @@ private void evaluateBucket(OmBucketInfo bucketInfo, String volumeName = bucketInfo.getVolumeName(); String bucketName = bucketInfo.getBucketName(); - if (bucketInfo.getBucketLayout() == BucketLayout.LEGACY) { - List prefixStartsWithTrashRuleList = - ruleList.stream().filter(r -> r.isPrefixEnable() && r.getEffectivePrefix().startsWith( - TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)).collect(Collectors.toList()); - ruleList.removeAll(prefixStartsWithTrashRuleList); - prefixStartsWithTrashRuleList.stream().forEach( - r -> LOG.info("Skip rule {} as its prefix starts with {}", r, TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)); - } - // use bucket name as key iterator prefix try (TableIterator> keyTblItr = keyTable.iterator(omMetadataManager.getBucketKey(volumeName, bucketName))) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java index 62b9dc0ba6aa..6842b7452b18 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java @@ -38,6 +38,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assumptions.assumeTrue; @@ -1404,24 +1405,14 @@ void testMoveToTrash(BucketLayout bucketLayout, String prefix) throws IOExceptio } // create new policy to test rule with prefix ".Trash/" is ignored during lifecycle evaluation now = ZonedDateTime.now(ZoneOffset.UTC); - date = now.plusSeconds(EXPIRE_SECONDS); - createLifecyclePolicy(volumeName, bucketName, bucketLayout, TRASH_PREFIX + OM_KEY_PREFIX, - null, date.toString(), true); - - GenericTestUtils.waitFor( - () -> log.getOutput().contains("Skip rule") && - log.getOutput().contains("as its prefix starts with " + TRASH_PREFIX + OM_KEY_PREFIX), - WAIT_CHECK_INTERVAL, 5000); - deleteLifecyclePolicy(volumeName, bucketName); + final String expiredDate = now.plusSeconds(EXPIRE_SECONDS).toString(); + assertThrowsExactly(OMException.class, () -> createLifecyclePolicy( + volumeName, bucketName, bucketLayout, TRASH_PREFIX + OM_KEY_PREFIX, null, expiredDate, true)); // create new policy to test rule with prefix ".Trash" is ignored during lifecycle evaluation - now = ZonedDateTime.now(ZoneOffset.UTC); - date = now.plusSeconds(EXPIRE_SECONDS); - createLifecyclePolicy(volumeName, bucketName, bucketLayout, TRASH_PREFIX, null, date.toString(), true); - - GenericTestUtils.waitFor( - () -> log.getOutput().contains("Skip evaluate trash directory " + TRASH_PREFIX), WAIT_CHECK_INTERVAL, 5000); - deleteLifecyclePolicy(volumeName, bucketName); + assertThrowsExactly(OMException.class, () -> createLifecyclePolicy( + volumeName, bucketName, bucketLayout, TRASH_PREFIX, + null, expiredDate, true)); // create new policy to test rule with prefix ".Tras" is ignored during lifecycle evaluation now = ZonedDateTime.now(ZoneOffset.UTC); @@ -1429,8 +1420,9 @@ void testMoveToTrash(BucketLayout bucketLayout, String prefix) throws IOExceptio createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, ".Tras", null, date.toString(), true); GenericTestUtils.waitFor( - () -> log.getOutput().contains("Skip evaluate trash directory " + TRASH_PREFIX), WAIT_CHECK_INTERVAL, 5000); + () -> log.getOutput().contains("No expired keys/dirs found/remained for bucket"), WAIT_CHECK_INTERVAL, 5000); deleteLifecyclePolicy(volumeName, bucketName); + log.clearOutput(); // create new policy to test trash directory is skipped during lifecycle evaluation now = ZonedDateTime.now(ZoneOffset.UTC);