diff --git a/dev-support/ci/xml_to_md.py b/dev-support/ci/xml_to_md.py index a059e00672c8..d299e3365e6e 100644 --- a/dev-support/ci/xml_to_md.py +++ b/dev-support/ci/xml_to_md.py @@ -64,12 +64,19 @@ def parse_xml_file(xml_content, properties): raise ValueError(f"Property '{name}' is missing a description.") tag = prop.findtext('tag', '') - properties[name] = Property( + p = Property( name=name.strip(), value=prop.findtext('value', '').strip(), tag=tag, description=' '.join(description.split()).strip() ) + if name in properties and p != properties[name]: + msg = f"Duplicate property '{name}'" + print(msg) + print(properties[name]) + print(p) + raise ValueError(msg) + properties[name] = p return properties def format_properties(properties): diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java index 0002284b7f90..dba10525d2bd 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java @@ -189,8 +189,11 @@ public class OzoneClientConfig { @Config(key = "ozone.client.max.ec.stripe.write.retries", defaultValue = "10", - description = "Ozone EC client to retry stripe to new block group on" + - " failures.", + description = "When EC stripe write failed, client will request to allocate new block group " + + "and write the failed stripe into new block group. If the same stripe failure " + + "continued in newly acquired block group also, then it will retry by requesting " + + "to allocate new block group again. This configuration is used to limit these " + + "number of retries. By default the number of retries are 10.", tags = ConfigTag.CLIENT) private int maxECStripeWriteRetries = 10; @@ -244,8 +247,12 @@ public class OzoneClientConfig { @Config(key = "ozone.client.fs.default.bucket.layout", defaultValue = "FILE_SYSTEM_OPTIMIZED", type = ConfigType.STRING, - description = "The bucket layout used by buckets created using OFS. " + - "Valid values include FILE_SYSTEM_OPTIMIZED and LEGACY", + description = + "Default bucket layout value used when buckets are created using OFS. " + + "Supported values are LEGACY and FILE_SYSTEM_OPTIMIZED. " + + "FILE_SYSTEM_OPTIMIZED: This layout allows the bucket to support atomic rename/delete operations and " + + "also allows interoperability between S3 and FS APIs. Keys written via S3 API with a '/' delimiter " + + "will create intermediate directories.", tags = ConfigTag.CLIENT) private String fsDefaultBucketLayout = "FILE_SYSTEM_OPTIMIZED"; @@ -650,4 +657,22 @@ public enum ChecksumCombineMode { MD5MD5CRC, // MD5 of block checksums, which are MD5 over chunk CRCs COMPOSITE_CRC // Block/chunk-independent composite CRC } + + /** + * String keys for tests and grep. + */ + public static final class Keys { + public static final String OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT = + "ozone.client.fs.default.bucket.layout"; + public static final String OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES = + "ozone.client.max.ec.stripe.write.retries"; + } + + /** + * Default values for tests. + */ + public static final class Defaults { + public static final String OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT = + OzoneConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED; + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java index 0876bd22ea4f..9d3d7ef85cbe 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java @@ -33,16 +33,16 @@ public class ScmConfig extends ReconfigurableConfig { @Config(key = "hdds.scm.kerberos.principal", type = ConfigType.STRING, - defaultValue = "", - tags = { ConfigTag.SECURITY, ConfigTag.OZONE }, - description = "This Kerberos principal is used by the SCM service." + defaultValue = "SCM/_HOST@REALM", + tags = { ConfigTag.SCM, ConfigTag.SECURITY, ConfigTag.OZONE }, + description = "The SCM service principal. e.g. scm/_HOST@REALM.COM" ) private String principal; @Config(key = "hdds.scm.kerberos.keytab.file", type = ConfigType.STRING, - defaultValue = "", - tags = { ConfigTag.SECURITY, ConfigTag.OZONE }, + defaultValue = "/etc/security/keytabs/SCM.keytab", + tags = { ConfigTag.SCM, ConfigTag.SECURITY, ConfigTag.OZONE }, description = "The keytab file used by SCM daemon to login as " + "its service principal." ) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 2c44fa881c41..a90bacc9fee2 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -627,6 +627,7 @@ public final class ScmConfigKeys { public static final String OZONE_SCM_HA_RATIS_SERVER_RPC_FIRST_ELECTION_TIMEOUT = "ozone.scm.ha.raft.server.rpc.first-election.timeout"; + public static final String HDDS_SCM_HTTP_AUTH_TYPE = "hdds.scm.http.auth.type"; /** * Never constructed. diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 654818a1aa5a..6edf71e80662 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -177,10 +177,6 @@ public final class OzoneConfigKeys { "ozone.scm.block.size"; public static final String OZONE_SCM_BLOCK_SIZE_DEFAULT = "256MB"; - public static final String OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES = - "ozone.client.max.ec.stripe.write.retries"; - public static final String OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES_DEFAULT = - "10"; public static final String OZONE_CLIENT_EC_GRPC_RETRIES_ENABLED = "ozone.client.ec.grpc.retries.enabled"; public static final boolean OZONE_CLIENT_EC_GRPC_RETRIES_ENABLED_DEFAULT @@ -628,11 +624,6 @@ public final class OzoneConfigKeys { public static final String OZONE_BUCKET_LAYOUT_OBJECT_STORE = "OBJECT_STORE"; - public static final String OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT = - "ozone.client.fs.default.bucket.layout"; - public static final String OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT = - OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED; - public static final String OZONE_S3G_DEFAULT_BUCKET_LAYOUT_KEY = "ozone.s3g.default.bucket.layout"; public static final String OZONE_S3G_DEFAULT_BUCKET_LAYOUT_DEFAULT = diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index e159eb6948b8..05047cdd9454 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -800,16 +800,6 @@ Then maximum 500000/(100/8) = 40000 blocks will be sent to each DN in every interval. - - ozone.scm.block.deletion.per.dn.distribution.factor - 8 - OZONE, SCM - - Factor with which number of delete blocks sent to each datanode in every interval. - If total number of DNs are 100 and hdds.scm.block.deletion.per-interval.max is 500000 - Then maximum 500000/(100/8) = 40000 blocks will be sent to each DN in every interval. - - ozone.scm.block.size 256MB @@ -1822,37 +1812,6 @@ - - hdds.scm.kerberos.keytab.file - /etc/security/keytabs/SCM.keytab - SCM, SECURITY, KERBEROS - The keytab file used by SCM daemon to login as its service principal. - - - - hdds.scm.kerberos.principal - SCM/_HOST@REALM - SCM, SECURITY, KERBEROS - The SCM service principal. e.g. scm/_HOST@REALM.COM - - - hdds.scm.http.auth.kerberos.principal - HTTP/_HOST@REALM - SCM, SECURITY, KERBEROS - - SCM http server service principal if SPNEGO is enabled for SCM http server. - - - - hdds.scm.http.auth.kerberos.keytab - /etc/security/keytabs/HTTP.keytab - SCM, SECURITY, KERBEROS - - The keytab file used by SCM http server to login as its service - principal if SPNEGO is enabled for SCM http server. - - - ozone.s3.administrators @@ -4226,7 +4185,7 @@ 60000 OZONE, PERFORMANCE, S3GATEWAY - OM/SCM/DN/S3GATEWAY Server connection timeout in milliseconds. + Ozone HTTP server connection timeout in milliseconds. @@ -4290,18 +4249,6 @@ - - ozone.client.max.ec.stripe.write.retries - 10 - CLIENT - - When EC stripe write failed, client will request to allocate new block group and write the failed stripe into new - block group. If the same stripe failure continued in newly acquired block group also, then it will retry by - requesting to allocate new block group again. This configuration is used to limit these number of retries. By - default the number of retries are 10. - - - ozone.client.ec.grpc.retries.enabled true @@ -4382,18 +4329,6 @@ - ozone.client.fs.default.bucket.layout - FILE_SYSTEM_OPTIMIZED - OZONE, CLIENT - - Default bucket layout value used when buckets are created using OFS. - Supported values are LEGACY and FILE_SYSTEM_OPTIMIZED. - FILE_SYSTEM_OPTIMIZED: This layout allows the bucket to support atomic rename/delete operations and - also allows interoperability between S3 and FS APIs. Keys written via S3 API with a "/" delimiter - will create intermediate directories. - - - ozone.om.namespace.s3.strict true OZONE, OM diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMHTTPServerConfig.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMHTTPServerConfig.java index 079de6629962..f402178562a8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMHTTPServerConfig.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMHTTPServerConfig.java @@ -30,21 +30,20 @@ public class SCMHTTPServerConfig { @Config(key = "hdds.scm.http.auth.kerberos.principal", type = ConfigType.STRING, - defaultValue = "", - tags = {ConfigTag.SECURITY}, - description = "This Kerberos principal is used when communicating to " + - "the HTTP server of SCM.The protocol used is SPNEGO." + defaultValue = "HTTP/_HOST@REALM", + tags = { ConfigTag.KERBEROS, ConfigTag.SCM, ConfigTag.SECURITY }, + description = "SCM http server service principal if SPNEGO is enabled for SCM http server." ) - private String principal = ""; + private String principal; @Config(key = "hdds.scm.http.auth.kerberos.keytab", type = ConfigType.STRING, - defaultValue = "", - tags = {ConfigTag.SECURITY}, + defaultValue = "/etc/security/keytabs/HTTP.keytab", + tags = { ConfigTag.KERBEROS, ConfigTag.SCM, ConfigTag.SECURITY }, description = "The keytab file used by SCM http server to login" + - " as its service principal." + " as its service principal if SPNEGO is enabled for SCM http server." ) - private String keytab = ""; + private String keytab; public void setKerberosPrincipal(String kerberosPrincipal) { this.principal = kerberosPrincipal; @@ -76,9 +75,6 @@ public static class ConfigStrings { SCMHTTPServerConfig.class.getAnnotation(ConfigGroup.class).prefix() + "."; - public static final String HDDS_SCM_HTTP_AUTH_TYPE = - HDDS_SCM_HTTP_AUTH_CONFIG_PREFIX + "type"; - public static final String HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY = HDDS_SCM_HTTP_AUTH_CONFIG_PREFIX + "kerberos.principal"; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java index 57b4a5768bef..674eaf624c09 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java @@ -83,7 +83,7 @@ public StorageContainerManagerHttpServer(MutableConfigurationSource conf, @Override protected String getHttpAuthType() { - return SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_AUTH_TYPE; + return ScmConfigKeys.HDDS_SCM_HTTP_AUTH_TYPE; } @Override diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/HttpAuthProbe.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/HttpAuthProbe.java index 90f9000d4117..466f97ef6ee0 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/HttpAuthProbe.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/HttpAuthProbe.java @@ -19,7 +19,7 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.recon.ReconServerConfigKeys; @@ -41,7 +41,7 @@ public String name() { public ProbeResult test(OzoneConfiguration conf) { print(conf, OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE); - print(conf, SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_AUTH_TYPE); + print(conf, ScmConfigKeys.HDDS_SCM_HTTP_AUTH_TYPE); print(conf, HddsConfigKeys.HDDS_DATANODE_HTTP_AUTH_TYPE); print(conf, ReconServerConfigKeys.OZONE_RECON_HTTP_AUTH_TYPE); //Used key directly to avoid cyclic dependency diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java index f89b2c1d563b..af4521387615 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientFactory; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; @@ -694,7 +695,7 @@ public void testStripeWriteRetriesOnAllNodeFailures() throws Exception { public void testStripeWriteRetriesOn4FailuresWith3RetriesAllowed() throws Exception { OzoneConfiguration con = createConfiguration(); - con.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3); + con.setInt(OzoneClientConfig.Keys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3); int[] nodesIndexesToMarkFailure = new int[4]; nodesIndexesToMarkFailure[0] = 0; @@ -895,7 +896,7 @@ public void testLargeWriteOfMultipleStripesWithStripeFailure() OzoneConfiguration con = createConfiguration(); // block size of 3KB could hold 3 full stripes con.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 3, StorageUnit.KB); - con.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3); + con.setInt(OzoneClientConfig.Keys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3); MultiNodePipelineBlockAllocator blkAllocator = new MultiNodePipelineBlockAllocator(con, dataBlocks + parityBlocks, 15); @@ -968,7 +969,7 @@ public void testPartialStripeWithPartialChunkRetry() OzoneConfiguration con = createConfiguration(); // block size of 3KB could hold 3 full stripes con.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 3, StorageUnit.KB); - con.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3); + con.setInt(OzoneClientConfig.Keys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3); MultiNodePipelineBlockAllocator blkAllocator = new MultiNodePipelineBlockAllocator(con, dataBlocks + parityBlocks, 15); createNewClient(con, blkAllocator); @@ -1029,7 +1030,7 @@ void testDiscardPreAllocatedBlocksPreventRetryExceeds() close(); OzoneConfiguration con = createConfiguration(); int maxRetries = 3; - con.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, + con.setInt(OzoneClientConfig.Keys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, maxRetries); MultiNodePipelineBlockAllocator blkAllocator = new MultiNodePipelineBlockAllocator(con, dataBlocks + parityBlocks, diff --git a/hadoop-ozone/httpfsgateway/src/main/resources/httpfs-default.xml b/hadoop-ozone/httpfsgateway/src/main/resources/httpfs-default.xml index b695ea791059..8dbe4b46b4ba 100644 --- a/hadoop-ozone/httpfsgateway/src/main/resources/httpfs-default.xml +++ b/hadoop-ozone/httpfsgateway/src/main/resources/httpfs-default.xml @@ -54,14 +54,6 @@ - - hadoop.http.idle_timeout.ms - 60000 - - Httpfs Server connection timeout in milliseconds. - - - hadoop.http.max.threads 1000 diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index 16eea0519c0f..dadb3dc5baa0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -95,6 +95,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.mapreduce.Job; @@ -233,7 +234,7 @@ void initClusterAndEnv() throws IOException, InterruptedException, TimeoutExcept conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, bucketLayout.name()); } else { - conf.set(OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT, + conf.set(OzoneClientConfig.Keys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT, OzoneConfigKeys.OZONE_BUCKET_LAYOUT_LEGACY); conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, bucketLayout.name()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java index 62428079e83b..033020310c6d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; @@ -70,7 +69,7 @@ public abstract class TestOzoneFSBucketLayout implements NonHATests.TestCase { "Buckets created with OBJECT_STORE layout do not support file " + "system semantics."); ERROR_MAP.put(UNKNOWN_LAYOUT, "Unsupported value provided for " + - OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT); + OzoneClientConfig.Keys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT); } static Collection validDefaultBucketLayouts() { @@ -140,7 +139,7 @@ void fileSystemWithValidBucketLayout(String layout) throws IOException { objectStore.getClientProxy().getBucketDetails(volumeName, bucketName); String expectedLayout = layout.isEmpty() - ? OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT + ? OzoneClientConfig.Defaults.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT : layout; assertEquals(expectedLayout, bucketInfo.getBucketLayout().name()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index be9f1ac00e96..d4a340b311a1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@ -21,9 +21,7 @@ import org.apache.hadoop.conf.TestConfigurationFieldsBase; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.recon.ReconConfigKeys; -import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; import org.apache.hadoop.hdds.server.http.HttpServer2; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -45,9 +43,6 @@ public void initializeMemberVariables() { ReconConfigKeys.class, ReconServerConfigKeys.class, S3GatewayConfigKeys.class, S3SecretConfigKeys.class, - SCMHTTPServerConfig.class, - SCMHTTPServerConfig.ConfigStrings.class, - ScmConfig.ConfigStrings.class }; errorIfMissingConfigProps = true; errorIfMissingXmlProps = true; diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index ab1f5c27625c..c502f9096c5f 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -130,7 +130,8 @@ public class BasicRootedOzoneClientAdapterImpl private boolean securityEnabled; private int configuredDnPort; private BucketLayout defaultOFSBucketLayout; - private OzoneConfiguration config; + private final OzoneConfiguration config; + private final OzoneClientConfig clientConfig; /** * Create new OzoneClientAdapter implementation. @@ -218,6 +219,7 @@ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort, OzoneConfigKeys.HDDS_CONTAINER_IPC_PORT, OzoneConfigKeys.HDDS_CONTAINER_IPC_PORT_DEFAULT); + clientConfig = conf.getObject(OzoneClientConfig.class); // Fetches the bucket layout to be used by OFS. try { initDefaultFsBucketLayout(conf); @@ -245,11 +247,10 @@ private void initDefaultFsBucketLayout(OzoneConfiguration conf) throws OMException { try { this.defaultOFSBucketLayout = BucketLayout.fromString( - conf.get(OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT, - OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT)); + clientConfig.getFsDefaultBucketLayout()); } catch (IllegalArgumentException iae) { throw new OMException("Unsupported value provided for " + - OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT + + OzoneClientConfig.Keys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT + ". Supported values are " + BucketLayout.FILE_SYSTEM_OPTIMIZED + " and " + BucketLayout.LEGACY + ".", OMException.ResultCodes.INVALID_REQUEST); @@ -260,7 +261,7 @@ private void initDefaultFsBucketLayout(OzoneConfiguration conf) throw new OMException( "Buckets created with OBJECT_STORE layout do not support file " + "system semantics. Supported values for config " + - OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT + + OzoneClientConfig.Keys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT + " include " + BucketLayout.FILE_SYSTEM_OPTIMIZED + " and " + BucketLayout.LEGACY, OMException.ResultCodes.INVALID_REQUEST); } @@ -1298,7 +1299,7 @@ public boolean isFSOptimizedBucket() { public FileChecksum getFileChecksum(String keyName, long length) throws IOException { OzoneClientConfig.ChecksumCombineMode combineMode = - config.getObject(OzoneClientConfig.class).getChecksumCombineMode(); + clientConfig.getChecksumCombineMode(); if (combineMode == null) { return null; }