HDDS-15108. Remove duplicate keys from ozone-default.xml

dev-support/ci/xml_to_md.py

@@ -64,12 +64,19 @@ def parse_xml_file(xml_content, properties):
       raise ValueError(f"Property '{name}' is missing a description.")
     tag = prop.findtext('tag', '')
 
-    properties[name] = Property(
+    p = Property(
       name=name.strip(),
       value=prop.findtext('value', '').strip(),
       tag=tag,
       description=' '.join(description.split()).strip()
     )
+    if name in properties and p != properties[name]:
+      msg = f"Duplicate property '{name}'"
+      print(msg)
+      print(properties[name])
+      print(p)
+      raise ValueError(msg)
+    properties[name] = p
   return properties
 
 def format_properties(properties):

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

@@ -189,8 +189,11 @@ public class OzoneClientConfig {
 
   @Config(key = "ozone.client.max.ec.stripe.write.retries",
       defaultValue = "10",
-      description = "Ozone EC client to retry stripe to new block group on" +
-          " failures.",
+      description = "When EC stripe write failed, client will request to allocate new block group "
+          + "and write the failed stripe into new block group. If the same stripe failure "
+          + "continued in newly acquired block group also, then it will retry by requesting "
+          + "to allocate new block group again. This configuration is used to limit these "
+          + "number of retries. By default the number of retries are 10.",
       tags = ConfigTag.CLIENT)
   private int maxECStripeWriteRetries = 10;
 
@@ -244,8 +247,12 @@ public class OzoneClientConfig {
   @Config(key = "ozone.client.fs.default.bucket.layout",
       defaultValue = "FILE_SYSTEM_OPTIMIZED",
       type = ConfigType.STRING,
-      description = "The bucket layout used by buckets created using OFS. " +
-          "Valid values include FILE_SYSTEM_OPTIMIZED and LEGACY",
+      description =
+          "Default bucket layout value used when buckets are created using OFS. "
+          + "Supported values are LEGACY and FILE_SYSTEM_OPTIMIZED. "
+          + "FILE_SYSTEM_OPTIMIZED: This layout allows the bucket to support atomic rename/delete operations and "
+          + "also allows interoperability between S3 and FS APIs. Keys written via S3 API with a '/' delimiter "
+          + "will create intermediate directories.",
       tags = ConfigTag.CLIENT)
   private String fsDefaultBucketLayout = "FILE_SYSTEM_OPTIMIZED";
 
@@ -650,4 +657,22 @@ public enum ChecksumCombineMode {
     MD5MD5CRC,  // MD5 of block checksums, which are MD5 over chunk CRCs
     COMPOSITE_CRC  // Block/chunk-independent composite CRC
   }
+
+  /**
+   * String keys for tests and grep.
+   */
+  public static final class Keys {
+    public static final String OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT =
+        "ozone.client.fs.default.bucket.layout";
+    public static final String OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES =
+        "ozone.client.max.ec.stripe.write.retries";
+  }
+
+  /**
+   * Default values for tests.
+   */
+  public static final class Defaults {
+    public static final String OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT =
+        OzoneConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED;
+  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java

@@ -33,16 +33,16 @@ public class ScmConfig extends ReconfigurableConfig {
 
   @Config(key = "hdds.scm.kerberos.principal",
       type = ConfigType.STRING,
-      defaultValue = "",
-      tags = { ConfigTag.SECURITY, ConfigTag.OZONE },
-      description = "This Kerberos principal is used by the SCM service."
+      defaultValue = "SCM/_HOST@REALM",
+      tags = { ConfigTag.SCM, ConfigTag.SECURITY, ConfigTag.OZONE },
+      description = "The SCM service principal. e.g. scm/_HOST@REALM.COM"
   )
   private String principal;
 
   @Config(key = "hdds.scm.kerberos.keytab.file",
       type = ConfigType.STRING,
-      defaultValue = "",
-      tags = { ConfigTag.SECURITY, ConfigTag.OZONE },
+      defaultValue = "/etc/security/keytabs/SCM.keytab",
+      tags = { ConfigTag.SCM, ConfigTag.SECURITY, ConfigTag.OZONE },
       description = "The keytab file used by SCM daemon to login as " +
           "its service principal."
   )

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -627,6 +627,7 @@ public final class ScmConfigKeys {
 
   public static final String OZONE_SCM_HA_RATIS_SERVER_RPC_FIRST_ELECTION_TIMEOUT
       = "ozone.scm.ha.raft.server.rpc.first-election.timeout";
+  public static final String HDDS_SCM_HTTP_AUTH_TYPE = "hdds.scm.http.auth.type";
 
   /**
    * Never constructed.

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java

@@ -177,10 +177,6 @@ public final class OzoneConfigKeys {
       "ozone.scm.block.size";
   public static final String OZONE_SCM_BLOCK_SIZE_DEFAULT = "256MB";
 
-  public static final String OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES =
-      "ozone.client.max.ec.stripe.write.retries";
-  public static final String OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES_DEFAULT =
-      "10";
   public static final String OZONE_CLIENT_EC_GRPC_RETRIES_ENABLED =
       "ozone.client.ec.grpc.retries.enabled";
   public static final boolean OZONE_CLIENT_EC_GRPC_RETRIES_ENABLED_DEFAULT
@@ -628,11 +624,6 @@ public final class OzoneConfigKeys {
   public static final String OZONE_BUCKET_LAYOUT_OBJECT_STORE =
       "OBJECT_STORE";
 
-  public static final String OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT =
-      "ozone.client.fs.default.bucket.layout";
-  public static final String OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT =
-      OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED;
-
   public static final String OZONE_S3G_DEFAULT_BUCKET_LAYOUT_KEY =
       "ozone.s3g.default.bucket.layout";
   public static final String OZONE_S3G_DEFAULT_BUCKET_LAYOUT_DEFAULT =

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -800,16 +800,6 @@
       Then maximum 500000/(100/8) = 40000 blocks will be sent to each DN in every interval.
     </description>
   </property>
-  <property>
-    <name>ozone.scm.block.deletion.per.dn.distribution.factor</name>
-    <value>8</value>
-    <tag>OZONE, SCM</tag>
-    <description>
-      Factor with which number of delete blocks sent to each datanode in every interval.
-      If total number of DNs are 100 and hdds.scm.block.deletion.per-interval.max is 500000
-      Then maximum 500000/(100/8) = 40000 blocks will be sent to each DN in every interval.
-    </description>
-  </property>
   <property>
     <name>ozone.scm.block.size</name>
     <value>256MB</value>
@@ -1822,37 +1812,6 @@
     </description>
   </property>
 
-  <property>
-    <name>hdds.scm.kerberos.keytab.file</name>
-    <value>/etc/security/keytabs/SCM.keytab</value>
-    <tag>SCM, SECURITY, KERBEROS</tag>
-    <description> The keytab file used by SCM daemon to login as its service principal.
-    </description>
-  </property>
-  <property>
-    <name>hdds.scm.kerberos.principal</name>
-    <value>SCM/_HOST@REALM</value>
-    <tag>SCM, SECURITY, KERBEROS</tag>
-    <description>The SCM service principal. e.g. scm/_HOST@REALM.COM</description>
-  </property>
-  <property>
-    <name>hdds.scm.http.auth.kerberos.principal</name>
-    <value>HTTP/_HOST@REALM</value>
-    <tag>SCM, SECURITY, KERBEROS</tag>
-    <description>
-      SCM http server service principal if SPNEGO is enabled for SCM http server.
-    </description>
-  </property>
-  <property>
-    <name>hdds.scm.http.auth.kerberos.keytab</name>
-    <value>/etc/security/keytabs/HTTP.keytab</value>
-    <tag>SCM, SECURITY, KERBEROS</tag>
-    <description>
-      The keytab file used by SCM http server to login as its service
-      principal if SPNEGO is enabled for SCM http server.
-    </description>
-  </property>
-
   <property>
     <name>ozone.s3.administrators</name>
     <value/>
@@ -4226,7 +4185,7 @@
     <value>60000</value>
     <tag>OZONE, PERFORMANCE, S3GATEWAY</tag>
     <description>
-      OM/SCM/DN/S3GATEWAY Server connection timeout in milliseconds.
+      Ozone HTTP server connection timeout in milliseconds.
     </description>
   </property>
 
@@ -4290,18 +4249,6 @@
     </description>
   </property>
 
-  <property>
-    <name>ozone.client.max.ec.stripe.write.retries</name>
-    <value>10</value>
-    <tag>CLIENT</tag>
-    <description>
-      When EC stripe write failed, client will request to allocate new block group and write the failed stripe into new
-      block group. If the same stripe failure continued in newly acquired block group also, then it will retry by
-      requesting to allocate new block group again. This configuration is used to limit these number of retries. By
-      default the number of retries are 10.
-    </description>
-  </property>
-
   <property>
     <name>ozone.client.ec.grpc.retries.enabled</name>
     <value>true</value>
@@ -4382,18 +4329,6 @@
   </property>
 
   <property>
-    <name>ozone.client.fs.default.bucket.layout</name>
-    <value>FILE_SYSTEM_OPTIMIZED</value>
-    <tag>OZONE, CLIENT</tag>
-    <description>
-      Default bucket layout value used when buckets are created using OFS.
-      Supported values are LEGACY and FILE_SYSTEM_OPTIMIZED.
-      FILE_SYSTEM_OPTIMIZED: This layout allows the bucket to support atomic rename/delete operations and
-      also allows interoperability between S3 and FS APIs. Keys written via S3 API with a "/" delimiter
-      will create intermediate directories.
-    </description>
-  </property>
-    <property>
     <name>ozone.om.namespace.s3.strict</name>
     <value>true</value>
     <tag>OZONE, OM</tag>

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMHTTPServerConfig.java

@@ -30,21 +30,20 @@ public class SCMHTTPServerConfig {
 
   @Config(key = "hdds.scm.http.auth.kerberos.principal",
       type = ConfigType.STRING,
-      defaultValue = "",
-      tags = {ConfigTag.SECURITY},
-      description = "This Kerberos principal is used when communicating to " +
-          "the HTTP server of SCM.The protocol used is SPNEGO."
+      defaultValue = "HTTP/_HOST@REALM",
+      tags = { ConfigTag.KERBEROS, ConfigTag.SCM, ConfigTag.SECURITY },
+      description = "SCM http server service principal if SPNEGO is enabled for SCM http server."
   )
-  private String principal = "";
+  private String principal;
 
   @Config(key = "hdds.scm.http.auth.kerberos.keytab",
       type = ConfigType.STRING,
-      defaultValue = "",
-      tags = {ConfigTag.SECURITY},
+      defaultValue = "/etc/security/keytabs/HTTP.keytab",
+      tags = { ConfigTag.KERBEROS, ConfigTag.SCM, ConfigTag.SECURITY },
       description = "The keytab file used by SCM http server to login" +
-          " as its service principal."
+          " as its service principal if SPNEGO is enabled for SCM http server."
   )
-  private String keytab = "";
+  private String keytab;
 
   public void setKerberosPrincipal(String kerberosPrincipal) {
     this.principal = kerberosPrincipal;
@@ -76,9 +75,6 @@ public static class ConfigStrings {
         SCMHTTPServerConfig.class.getAnnotation(ConfigGroup.class).prefix() +
             ".";
 
-    public static final String HDDS_SCM_HTTP_AUTH_TYPE =
-        HDDS_SCM_HTTP_AUTH_CONFIG_PREFIX + "type";
-
     public static final String HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY =
         HDDS_SCM_HTTP_AUTH_CONFIG_PREFIX + "kerberos.principal";
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java

@@ -83,7 +83,7 @@ public StorageContainerManagerHttpServer(MutableConfigurationSource conf,
 
   @Override
   protected String getHttpAuthType() {
-    return SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_AUTH_TYPE;
+    return ScmConfigKeys.HDDS_SCM_HTTP_AUTH_TYPE;
   }
 
   @Override

hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/HttpAuthProbe.java

@@ -19,7 +19,7 @@
 
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.recon.ReconServerConfigKeys;
 
@@ -41,7 +41,7 @@ public String name() {
   public ProbeResult test(OzoneConfiguration conf) {
 
     print(conf, OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE);
-    print(conf, SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_AUTH_TYPE);
+    print(conf, ScmConfigKeys.HDDS_SCM_HTTP_AUTH_TYPE);
     print(conf, HddsConfigKeys.HDDS_DATANODE_HTTP_AUTH_TYPE);
     print(conf, ReconServerConfigKeys.OZONE_RECON_HTTP_AUTH_TYPE);
     //Used key directly to avoid cyclic dependency

hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java

@@ -44,6 +44,7 @@
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
 import org.apache.hadoop.hdds.scm.XceiverClientFactory;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
 import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -694,7 +695,7 @@ public void testStripeWriteRetriesOnAllNodeFailures() throws Exception {
   public void testStripeWriteRetriesOn4FailuresWith3RetriesAllowed()
       throws Exception {
     OzoneConfiguration con = createConfiguration();
-    con.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3);
+    con.setInt(OzoneClientConfig.Keys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3);
 
     int[] nodesIndexesToMarkFailure = new int[4];
     nodesIndexesToMarkFailure[0] = 0;
@@ -895,7 +896,7 @@ public void testLargeWriteOfMultipleStripesWithStripeFailure()
     OzoneConfiguration con = createConfiguration();
     // block size of 3KB could hold 3 full stripes
     con.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 3, StorageUnit.KB);
-    con.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3);
+    con.setInt(OzoneClientConfig.Keys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3);
     MultiNodePipelineBlockAllocator blkAllocator =
         new MultiNodePipelineBlockAllocator(con, dataBlocks + parityBlocks,
             15);
@@ -968,7 +969,7 @@ public void testPartialStripeWithPartialChunkRetry()
     OzoneConfiguration con = createConfiguration();
     // block size of 3KB could hold 3 full stripes
     con.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 3, StorageUnit.KB);
-    con.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3);
+    con.setInt(OzoneClientConfig.Keys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3);
     MultiNodePipelineBlockAllocator blkAllocator =
         new MultiNodePipelineBlockAllocator(con, dataBlocks + parityBlocks, 15);
     createNewClient(con, blkAllocator);
@@ -1029,7 +1030,7 @@ void testDiscardPreAllocatedBlocksPreventRetryExceeds()
     close();
     OzoneConfiguration con = createConfiguration();
     int maxRetries = 3;
-    con.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES,
+    con.setInt(OzoneClientConfig.Keys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES,
         maxRetries);
     MultiNodePipelineBlockAllocator blkAllocator =
         new MultiNodePipelineBlockAllocator(con, dataBlocks + parityBlocks,

hadoop-ozone/httpfsgateway/src/main/resources/httpfs-default.xml

@@ -54,14 +54,6 @@
   </property>
 
   <!-- HTTP properties -->
-  <property>
-    <name>hadoop.http.idle_timeout.ms</name>
-    <value>60000</value>
-    <description>
-      Httpfs Server connection timeout in milliseconds.
-    </description>
-  </property>
-
   <property>
     <name>hadoop.http.max.threads</name>
     <value>1000</value>

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java

@@ -95,6 +95,7 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
 import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.mapreduce.Job;
@@ -233,7 +234,7 @@ void initClusterAndEnv() throws IOException, InterruptedException, TimeoutExcept
       conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
           bucketLayout.name());
     } else {
-      conf.set(OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT,
+      conf.set(OzoneClientConfig.Keys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT,
           OzoneConfigKeys.OZONE_BUCKET_LAYOUT_LEGACY);
       conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
           bucketLayout.name());

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java

@@ -34,7 +34,6 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.OzoneClientConfig;
 import org.apache.hadoop.hdds.utils.IOUtils;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.TestDataUtil;
 import org.apache.hadoop.ozone.client.ObjectStore;
@@ -70,7 +69,7 @@ public abstract class TestOzoneFSBucketLayout implements NonHATests.TestCase {
         "Buckets created with OBJECT_STORE layout do not support file " +
             "system semantics.");
     ERROR_MAP.put(UNKNOWN_LAYOUT, "Unsupported value provided for " +
-        OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT);
+        OzoneClientConfig.Keys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT);
   }
 
   static Collection<String> validDefaultBucketLayouts() {
@@ -140,7 +139,7 @@ void fileSystemWithValidBucketLayout(String layout) throws IOException {
           objectStore.getClientProxy().getBucketDetails(volumeName, bucketName);
 
       String expectedLayout = layout.isEmpty()
-          ? OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT
+          ? OzoneClientConfig.Defaults.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT
           : layout;
       assertEquals(expectedLayout, bucketInfo.getBucketLayout().name());
     }