From df9d920b0141a1176c35963ad028159153b5c4f0 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sat, 6 Dec 2025 15:52:19 +0800 Subject: [PATCH 1/8] HDDS-8511. Allow only FSO type of bucket to be created with bucket name including non-S3 compliant charaters when the om server side config 'ozone.om.namespace.s3.strict' is set false --- .../request/bucket/OMBucketCreateRequest.java | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 23cbac4704ac..34e9b8d57f7f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; @@ -93,9 +94,40 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { CreateBucketRequest createBucketRequest = getOmRequest().getCreateBucketRequest(); BucketInfo bucketInfo = createBucketRequest.getBucketInfo(); + + // Convert BucketLayoutProto -> BucketLayout enum. + BucketLayout bucketLayout = + BucketLayout.valueOf(bucketInfo.getBucketLayout().name()); + + // Determine if this bucket belongs to the S3 namespace. + String s3VolumeName = ozoneManager.getConfiguration().get( + OzoneConfigKeys.OZONE_S3_VOLUME_NAME, + OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT); + boolean isS3Bucket = + s3VolumeName != null && s3VolumeName.equals(bucketInfo.getVolumeName()); + + // Compute effectiveStrict based on global strict flag, S3 namespace, + // and bucket layout. + boolean globalStrict = ozoneManager.isStrictS3(); + boolean effectiveStrict = globalStrict; + + + // When strict=false, only S3 buckets with FSO layout should allow + // non-S3-compliant characters (e.g. underscore). + // All other S3 bucket layouts must still enforce strict naming rules. + if (!globalStrict && isS3Bucket) { + if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { + // S3 + FSO + strict=false → 放寬（允許 '_' 等非 S3 字元） + effectiveStrict = false; + } else { + // S3 + 非 FSO + strict=false → 仍然嚴格（不允許 '_'） + effectiveStrict = true; + } + } + // Verify resource name OmUtils.validateBucketName(bucketInfo.getBucketName(), - ozoneManager.isStrictS3()); + effectiveStrict); // ACL check during preExecute if (ozoneManager.getAclsEnabled()) { From 28ccd86e656f52df6f126029afcf70231505a9bc Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Mon, 8 Dec 2025 14:54:11 +0800 Subject: [PATCH 2/8] Add comments to clarify logic --- .../hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 34e9b8d57f7f..5d468a2e6f17 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -117,10 +117,10 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { // All other S3 bucket layouts must still enforce strict naming rules. if (!globalStrict && isS3Bucket) { if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { - // S3 + FSO + strict=false → 放寬（允許 '_' 等非 S3 字元） + // S3 + FSO + strict=false → bypass strict S3 validation (allow '_' and other non-S3 characters) effectiveStrict = false; } else { - // S3 + 非 FSO + strict=false → 仍然嚴格（不允許 '_'） + // S3 + non-FSO + strict=false → strict S3 validation still applies ( '_' and other non-S3 characters are not permitted ) effectiveStrict = true; } } From a7349c6074b2eea1025ace2afbd72f9b0c9af34b Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Mon, 8 Dec 2025 15:13:56 +0800 Subject: [PATCH 3/8] Fix checkStyle --- .../hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 5d468a2e6f17..673e75f0a869 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -120,7 +120,8 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { // S3 + FSO + strict=false → bypass strict S3 validation (allow '_' and other non-S3 characters) effectiveStrict = false; } else { - // S3 + non-FSO + strict=false → strict S3 validation still applies ( '_' and other non-S3 characters are not permitted ) + // S3 + non-FSO + strict=false → strict S3 validation still applies + // '_' and other non-S3 characters are not permitted effectiveStrict = true; } } From d3fcfc026ddf14bdcfe313a6ff9b544a3608e95a Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Wed, 17 Dec 2025 23:42:54 +0800 Subject: [PATCH 4/8] Add Unit test --- .../request/bucket/OMBucketCreateRequest.java | 38 ++----------- .../TestOMBucketCreateRequestWithFSO.java | 37 +++++++++++-- ...tOMBucketCreateRequestWithObjectStore.java | 55 +++++++++++++++++++ 3 files changed, 92 insertions(+), 38 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 673e75f0a869..96b943fa0d7c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -95,40 +95,10 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { getOmRequest().getCreateBucketRequest(); BucketInfo bucketInfo = createBucketRequest.getBucketInfo(); - // Convert BucketLayoutProto -> BucketLayout enum. - BucketLayout bucketLayout = - BucketLayout.valueOf(bucketInfo.getBucketLayout().name()); - - // Determine if this bucket belongs to the S3 namespace. - String s3VolumeName = ozoneManager.getConfiguration().get( - OzoneConfigKeys.OZONE_S3_VOLUME_NAME, - OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT); - boolean isS3Bucket = - s3VolumeName != null && s3VolumeName.equals(bucketInfo.getVolumeName()); - - // Compute effectiveStrict based on global strict flag, S3 namespace, - // and bucket layout. - boolean globalStrict = ozoneManager.isStrictS3(); - boolean effectiveStrict = globalStrict; - - - // When strict=false, only S3 buckets with FSO layout should allow - // non-S3-compliant characters (e.g. underscore). - // All other S3 bucket layouts must still enforce strict naming rules. - if (!globalStrict && isS3Bucket) { - if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { - // S3 + FSO + strict=false → bypass strict S3 validation (allow '_' and other non-S3 characters) - effectiveStrict = false; - } else { - // S3 + non-FSO + strict=false → strict S3 validation still applies - // '_' and other non-S3 characters are not permitted - effectiveStrict = true; - } - } - - // Verify resource name - OmUtils.validateBucketName(bucketInfo.getBucketName(), - effectiveStrict); + BucketLayout bucketLayout = BucketLayout.fromProto(bucketInfo.getBucketLayout()); + boolean strict = ozoneManager.isStrictS3() + || bucketLayout.isObjectStore(ozoneManager.getConfig().isFileSystemPathEnabled()); + OmUtils.validateBucketName(bucketInfo.getBucketName(), strict); // ACL check during preExecute if (ozoneManager.getAclsEnabled()) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java index b9bd4029a624..a861bb28c7f4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java @@ -20,10 +20,7 @@ import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newBucketInfoBuilder; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newCreateBucketRequest; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketLayoutProto.FILE_SYSTEM_OPTIMIZED; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.when; import java.util.UUID; @@ -182,4 +179,36 @@ protected void doValidateAndUpdateCache(String volumeName, String bucketName, verifySuccessCreateBucketResponse(omClientResponse.getOMResponse()); } + + @Test + public void testNonS3BucketNameAllowedForFSOWhenStrictDisabled() throws Exception { + // Arrange + ozoneManager.getConfiguration().setBoolean( + OMConfigKeys.OZONE_OM_NAMESPACE_STRICT_S3, false); // 如果常數名稱不同，改成實際的 key + + when(ozoneManager.getOMDefaultBucketLayout()).thenReturn( + BucketLayout.FILE_SYSTEM_OPTIMIZED); + + String volumeName = UUID.randomUUID().toString(); + String bucketName = "bucket_with_underscore"; // non-S3-compliant name + addCreateVolumeToTable(volumeName, omMetadataManager); + + OzoneManagerProtocolProtos.BucketInfo.Builder bucketInfo = + newBucketInfoBuilder(bucketName, volumeName) + .setBucketLayout(FILE_SYSTEM_OPTIMIZED) + .addMetadata(OMRequestTestUtils.fsoMetadata()); + + OMRequest originalRequest = newCreateBucketRequest(bucketInfo).build(); + OMBucketCreateRequest req = new OMBucketCreateRequest(originalRequest); + + // Act + OMRequest modifiedRequest = req.preExecute(ozoneManager); + + // Assert: validateAndUpdateCache should succeed + assertDoesNotThrow(() -> { + OMBucketCreateRequest omReq = new OMBucketCreateRequest(modifiedRequest); + omReq.setUGI(UserGroupInformation.getCurrentUser()); + omReq.validateAndUpdateCache(ozoneManager, 1); + }); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java new file mode 100644 index 000000000000..ee0a514f5fc2 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java @@ -0,0 +1,55 @@ +package org.apache.hadoop.ozone.om.request.bucket; + +import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newBucketInfoBuilder; +import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newCreateBucketRequest; +import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; + +import java.util.UUID; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestOMBucketCreateRequestWithObjectStore extends TestOMBucketCreateRequest { + + @BeforeEach + public void setupWithObjectStore() { + ozoneManager.getConfiguration().set( + OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, + OMConfigKeys.OZONE_BUCKET_LAYOUT_OBJECT_STORE); // 或 LEGACY + } + + @Test + public void testNonS3BucketNameRejectedForObjectStoreWhenStrictDisabled() + throws Exception { + + // strict mode disabled + ozoneManager.getConfiguration().setBoolean( + OMConfigKeys.OZONE_OM_NAMESPACE_STRICT_S3, false); + + String volumeName = UUID.randomUUID().toString(); + String bucketName = "bucket_with_underscore"; // non-S3-compliant + addCreateVolumeToTable(volumeName, omMetadataManager); + + // Explicitly set bucket layout to OBJECT_STORE so the test doesn't depend on + // defaults or mocked OM behavior. + OzoneManagerProtocolProtos.BucketInfo.Builder bucketInfo = + newBucketInfoBuilder(bucketName, volumeName) + .setBucketLayout( + OzoneManagerProtocolProtos.BucketLayoutProto.OBJECT_STORE); + + OMRequest originalRequest = newCreateBucketRequest(bucketInfo).build(); + OMBucketCreateRequest req = new OMBucketCreateRequest(originalRequest); + + OMException ex = assertThrows(OMException.class, + () -> req.preExecute(ozoneManager)); + + assertEquals(OMException.ResultCodes.INVALID_BUCKET_NAME, ex.getResult()); + } +} From c3aad96485e6f9839de1df4bebc6e4c6aa095d80 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sat, 20 Dec 2025 09:21:46 +0800 Subject: [PATCH 5/8] Add tests for strict bucket name validation by layout --- .../request/bucket/OMBucketCreateRequest.java | 1 - .../bucket/TestOMBucketCreateRequest.java | 26 +++++++++++++++++- .../TestOMBucketCreateRequestWithFSO.java | 6 ++++- ...tOMBucketCreateRequestWithObjectStore.java | 27 +++++++++++++++---- 4 files changed, 52 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 96b943fa0d7c..17bd3774cd73 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -36,7 +36,6 @@ import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java index 09f3e0b9d601..8aa79aba4f34 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java @@ -322,7 +322,7 @@ public void testAcceptNonS3CompliantBucketNameCreationWithStrictS3False() {"bucket_underscore", "_bucket___multi_underscore_", "bucket_"}; when(ozoneManager.isStrictS3()).thenReturn(false); for (String bucketName : nonS3CompliantBucketName) { - acceptBucketCreationHelper(volumeName, bucketName); + acceptFSOBucketCreationHelper(volumeName, bucketName); } } @@ -478,4 +478,28 @@ public static void addCreateVolumeToTable(String volumeName, .setOwnerName(UUID.randomUUID().toString()).build(); OMRequestTestUtils.addVolumeToOM(omMetadataManager, omVolumeArgs); } + + protected OMBucketCreateRequest doPreExecute(String volumeName, + String bucketName, + OzoneManagerProtocolProtos.BucketLayoutProto layout) throws Exception { + + OzoneManagerProtocolProtos.BucketInfo.Builder bucketInfo = + newBucketInfoBuilder(bucketName, volumeName) + .setBucketLayout(layout); + + if (layout == OzoneManagerProtocolProtos.BucketLayoutProto.FILE_SYSTEM_OPTIMIZED) { + bucketInfo.addMetadata(OMRequestTestUtils.fsoMetadata()); + } + + return doPreExecute(bucketInfo); + } + + private void acceptFSOBucketCreationHelper(String volumeName, String bucketName) + throws Exception { + OMBucketCreateRequest omBucketCreateRequest = + doPreExecute(volumeName, bucketName, + OzoneManagerProtocolProtos.BucketLayoutProto.FILE_SYSTEM_OPTIMIZED); + doValidateAndUpdateCache(volumeName, bucketName, + omBucketCreateRequest.getOmRequest()); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java index a861bb28c7f4..63622cf35c7d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java @@ -20,7 +20,11 @@ import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newBucketInfoBuilder; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newCreateBucketRequest; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketLayoutProto.FILE_SYSTEM_OPTIMIZED; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; import java.util.UUID; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java index ee0a514f5fc2..89b3920d25ae 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java @@ -1,28 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.ozone.om.request.bucket; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newBucketInfoBuilder; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newCreateBucketRequest; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.when; import java.util.UUID; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.security.UserGroupInformation; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +/** + * Tests bucket creation behavior for ObjectStore / Legacy bucket layouts. + */ public class TestOMBucketCreateRequestWithObjectStore extends TestOMBucketCreateRequest { @BeforeEach public void setupWithObjectStore() { ozoneManager.getConfiguration().set( OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, - OMConfigKeys.OZONE_BUCKET_LAYOUT_OBJECT_STORE); // 或 LEGACY + OMConfigKeys.OZONE_BUCKET_LAYOUT_OBJECT_STORE); } @Test From cad0e4a0c9fbe1ba39648586e19934b7447695a8 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sat, 20 Dec 2025 11:33:11 +0800 Subject: [PATCH 6/8] Prevent NPE Exception for TestOMClientRequestWithUserInfo --- .../ozone/om/request/bucket/OMBucketCreateRequest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 17bd3774cd73..fd072b13883c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OmConfig; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; @@ -95,8 +96,14 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { BucketInfo bucketInfo = createBucketRequest.getBucketInfo(); BucketLayout bucketLayout = BucketLayout.fromProto(bucketInfo.getBucketLayout()); + OmConfig omConfig = ozoneManager.getConfig(); + boolean fsPathEnabled = false; + if (omConfig != null) { + fsPathEnabled = omConfig.isFileSystemPathEnabled(); + } + boolean strict = ozoneManager.isStrictS3() - || bucketLayout.isObjectStore(ozoneManager.getConfig().isFileSystemPathEnabled()); + || bucketLayout.isObjectStore(fsPathEnabled); OmUtils.validateBucketName(bucketInfo.getBucketName(), strict); // ACL check during preExecute From 3a62c7d5cf607b689bf2fa3f6a31b09982a3cb87 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sun, 21 Dec 2025 08:30:30 +0800 Subject: [PATCH 7/8] Address review feedback --- .../request/bucket/OMBucketCreateRequest.java | 9 +-- .../TestOMClientRequestWithUserInfo.java | 6 ++ .../bucket/TestOMBucketCreateRequest.java | 35 +++++++-- ...tOMBucketCreateRequestWithObjectStore.java | 72 ------------------- 4 files changed, 37 insertions(+), 85 deletions(-) delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index fd072b13883c..17bd3774cd73 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -43,7 +43,6 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; -import org.apache.hadoop.ozone.om.OmConfig; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; @@ -96,14 +95,8 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { BucketInfo bucketInfo = createBucketRequest.getBucketInfo(); BucketLayout bucketLayout = BucketLayout.fromProto(bucketInfo.getBucketLayout()); - OmConfig omConfig = ozoneManager.getConfig(); - boolean fsPathEnabled = false; - if (omConfig != null) { - fsPathEnabled = omConfig.isFileSystemPathEnabled(); - } - boolean strict = ozoneManager.isStrictS3() - || bucketLayout.isObjectStore(fsPathEnabled); + || bucketLayout.isObjectStore(ozoneManager.getConfig().isFileSystemPathEnabled()); OmUtils.validateBucketName(bucketInfo.getBucketName(), strict); // ACL check during preExecute diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java index d8618b8ed8fd..145ffe9854fe 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java @@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OmConfig; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -79,6 +80,11 @@ public void setup() throws Exception { when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); + // mock OmConfig to avoid NPE in OMBucketCreateRequest.preExecute + OmConfig omConfig = mock(OmConfig.class); + when(omConfig.isFileSystemPathEnabled()).thenReturn(false); + when(ozoneManager.getConfig()).thenReturn(omConfig); + // Mock version manager to avoid NPE in preExecute OMLayoutVersionManager versionManager = mock(OMLayoutVersionManager.class); when(versionManager.getMetadataLayoutVersion()).thenReturn(0); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java index 8aa79aba4f34..f0a185131f9b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java @@ -326,6 +326,34 @@ public void testAcceptNonS3CompliantBucketNameCreationWithStrictS3False() } } + @Test + public void testNonS3BucketNameRejectedForObjectStoreWhenStrictDisabled() + throws Exception { + + // strict mode disabled + ozoneManager.getConfiguration().setBoolean( + OMConfigKeys.OZONE_OM_NAMESPACE_STRICT_S3, false); + + String volumeName = UUID.randomUUID().toString(); + String bucketName = "bucket_with_underscore"; // non-S3-compliant + addCreateVolumeToTable(volumeName, omMetadataManager); + + // Explicitly set bucket layout to OBJECT_STORE so the test doesn't depend on + // defaults or mocked OM behavior. + OzoneManagerProtocolProtos.BucketInfo.Builder bucketInfo = + newBucketInfoBuilder(bucketName, volumeName) + .setBucketLayout( + OzoneManagerProtocolProtos.BucketLayoutProto.OBJECT_STORE); + + OMRequest originalRequest = newCreateBucketRequest(bucketInfo).build(); + OMBucketCreateRequest req = new OMBucketCreateRequest(originalRequest); + + OMException ex = assertThrows(OMException.class, + () -> req.preExecute(ozoneManager)); + + assertEquals(OMException.ResultCodes.INVALID_BUCKET_NAME, ex.getResult()); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) public void testIgnoreClientACL(boolean ignoreClientACLs) throws Exception { @@ -480,17 +508,14 @@ public static void addCreateVolumeToTable(String volumeName, } protected OMBucketCreateRequest doPreExecute(String volumeName, - String bucketName, - OzoneManagerProtocolProtos.BucketLayoutProto layout) throws Exception { - + String bucketName, + OzoneManagerProtocolProtos.BucketLayoutProto layout) throws Exception { OzoneManagerProtocolProtos.BucketInfo.Builder bucketInfo = newBucketInfoBuilder(bucketName, volumeName) .setBucketLayout(layout); - if (layout == OzoneManagerProtocolProtos.BucketLayoutProto.FILE_SYSTEM_OPTIMIZED) { bucketInfo.addMetadata(OMRequestTestUtils.fsoMetadata()); } - return doPreExecute(bucketInfo); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java deleted file mode 100644 index 89b3920d25ae..000000000000 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithObjectStore.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.request.bucket; - -import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newBucketInfoBuilder; -import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newCreateBucketRequest; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.util.UUID; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * Tests bucket creation behavior for ObjectStore / Legacy bucket layouts. - */ -public class TestOMBucketCreateRequestWithObjectStore extends TestOMBucketCreateRequest { - - @BeforeEach - public void setupWithObjectStore() { - ozoneManager.getConfiguration().set( - OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, - OMConfigKeys.OZONE_BUCKET_LAYOUT_OBJECT_STORE); - } - - @Test - public void testNonS3BucketNameRejectedForObjectStoreWhenStrictDisabled() - throws Exception { - - // strict mode disabled - ozoneManager.getConfiguration().setBoolean( - OMConfigKeys.OZONE_OM_NAMESPACE_STRICT_S3, false); - - String volumeName = UUID.randomUUID().toString(); - String bucketName = "bucket_with_underscore"; // non-S3-compliant - addCreateVolumeToTable(volumeName, omMetadataManager); - - // Explicitly set bucket layout to OBJECT_STORE so the test doesn't depend on - // defaults or mocked OM behavior. - OzoneManagerProtocolProtos.BucketInfo.Builder bucketInfo = - newBucketInfoBuilder(bucketName, volumeName) - .setBucketLayout( - OzoneManagerProtocolProtos.BucketLayoutProto.OBJECT_STORE); - - OMRequest originalRequest = newCreateBucketRequest(bucketInfo).build(); - OMBucketCreateRequest req = new OMBucketCreateRequest(originalRequest); - - OMException ex = assertThrows(OMException.class, - () -> req.preExecute(ozoneManager)); - - assertEquals(OMException.ResultCodes.INVALID_BUCKET_NAME, ex.getResult()); - } -} From 5609201515b3d396fd70ce9c453fb40c4c68d9ae Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Wed, 31 Dec 2025 01:10:17 +0800 Subject: [PATCH 8/8] Refine strict bucket name validation and simplify tests --- .../request/bucket/OMBucketCreateRequest.java | 5 ++++- .../bucket/TestOMBucketCreateRequest.java | 19 ++++--------------- .../TestOMBucketCreateRequestWithFSO.java | 2 +- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 17bd3774cd73..ac308b800bc8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -96,7 +96,10 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { BucketLayout bucketLayout = BucketLayout.fromProto(bucketInfo.getBucketLayout()); boolean strict = ozoneManager.isStrictS3() - || bucketLayout.isObjectStore(ozoneManager.getConfig().isFileSystemPathEnabled()); + || bucketLayout == BucketLayout.OBJECT_STORE; + + // OBS (Object Store) buckets must follow strict S3 bucket naming rules. + // FSO and LEGACY buckets are not strictly bound to S3 naming semantics. OmUtils.validateBucketName(bucketInfo.getBucketName(), strict); // ACL check during preExecute diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java index f0a185131f9b..49bf00b1f8f2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java @@ -507,23 +507,12 @@ public static void addCreateVolumeToTable(String volumeName, OMRequestTestUtils.addVolumeToOM(omMetadataManager, omVolumeArgs); } - protected OMBucketCreateRequest doPreExecute(String volumeName, - String bucketName, - OzoneManagerProtocolProtos.BucketLayoutProto layout) throws Exception { - OzoneManagerProtocolProtos.BucketInfo.Builder bucketInfo = - newBucketInfoBuilder(bucketName, volumeName) - .setBucketLayout(layout); - if (layout == OzoneManagerProtocolProtos.BucketLayoutProto.FILE_SYSTEM_OPTIMIZED) { - bucketInfo.addMetadata(OMRequestTestUtils.fsoMetadata()); - } - return doPreExecute(bucketInfo); - } - private void acceptFSOBucketCreationHelper(String volumeName, String bucketName) throws Exception { - OMBucketCreateRequest omBucketCreateRequest = - doPreExecute(volumeName, bucketName, - OzoneManagerProtocolProtos.BucketLayoutProto.FILE_SYSTEM_OPTIMIZED); + OzoneManagerProtocolProtos.BucketInfo.Builder bucketInfo = + newBucketInfoBuilder(bucketName, volumeName) + .setBucketLayout(OzoneManagerProtocolProtos.BucketLayoutProto.FILE_SYSTEM_OPTIMIZED); + OMBucketCreateRequest omBucketCreateRequest = doPreExecute(bucketInfo); doValidateAndUpdateCache(volumeName, bucketName, omBucketCreateRequest.getOmRequest()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java index 63622cf35c7d..107e697971f8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java @@ -188,7 +188,7 @@ protected void doValidateAndUpdateCache(String volumeName, String bucketName, public void testNonS3BucketNameAllowedForFSOWhenStrictDisabled() throws Exception { // Arrange ozoneManager.getConfiguration().setBoolean( - OMConfigKeys.OZONE_OM_NAMESPACE_STRICT_S3, false); // 如果常數名稱不同，改成實際的 key + OMConfigKeys.OZONE_OM_NAMESPACE_STRICT_S3, false); when(ozoneManager.getOMDefaultBucketLayout()).thenReturn( BucketLayout.FILE_SYSTEM_OPTIMIZED);