HDDS-14188. NodesOutOfSpace renamed to NonWritableNodes, include DNs not accepting writes (#9518)

Author: sarvekshayr

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java

@@ -48,9 +48,11 @@
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import javax.management.ObjectName;
 import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.conf.StorageUnit;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -1279,8 +1281,8 @@ public Map<String, String> getNodeStatistics() {
     nodeStateStatistics(nodeStatistics);
     // Statistics node space
     nodeSpaceStatistics(nodeStatistics);
-    // Statistics node readOnly
-    nodeOutOfSpaceStatistics(nodeStatistics);
+    // Statistics node non-writable
+    nodeNonWritableStatistics(nodeStatistics);
     // todo: Statistics of other instances
     return nodeStatistics;
   }
@@ -1368,43 +1370,59 @@ private void nodeSpaceStatistics(Map<String, String> nodeStatics) {
     nodeStatics.put(SpaceStatistics.NON_SCM_USED.getLabel(), nonScmUsed);
   }
 
-  private void nodeOutOfSpaceStatistics(Map<String, String> nodeStatics) {
-    List<DatanodeInfo> allNodes = getAllNodes();
-    long blockSize = (long) conf.getStorageSize(
-        OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
-        OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT,
-        StorageUnit.BYTES);
-    long minRatisVolumeSizeBytes = (long) conf.getStorageSize(
-        ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
-        ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT,
-        StorageUnit.BYTES);
-    long containerSize = (long) conf.getStorageSize(
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
-        StorageUnit.BYTES);
-
-    int nodeOutOfSpaceCount = (int) allNodes.parallelStream()
-        .filter(dn -> !hasEnoughSpace(dn, minRatisVolumeSizeBytes, containerSize, conf)
-            && !hasEnoughCommittedVolumeSpace(dn, blockSize))
+  private void nodeNonWritableStatistics(Map<String, String> nodeStatics) {
+    int nonWritableNodesCount = (int) getAllNodes().parallelStream()
+        .filter(new NonWritableNodeFilter(conf))
         .count();
 
-    nodeStatics.put("NodesOutOfSpace", String.valueOf(nodeOutOfSpaceCount));
-  }
-  
-  /**
-   * Check if any volume in the datanode has committed space >= blockSize.
-   *
-   * @return true if any volume has committed space >= blockSize, false otherwise
-   */
-  private boolean hasEnoughCommittedVolumeSpace(DatanodeInfo dnInfo, long blockSize) {
-    for (StorageReportProto reportProto : dnInfo.getStorageReports()) {
-      if (reportProto.getCommitted() >= blockSize) {
-        return true;
+    nodeStatics.put("NonWritableNodes", String.valueOf(nonWritableNodesCount));
+  }
+
+  static class NonWritableNodeFilter implements Predicate<DatanodeInfo> {
+
+    private final long blockSize;
+    private final long minRatisVolumeSizeBytes;
+    private final long containerSize;
+    private final ConfigurationSource conf;
+
+    NonWritableNodeFilter(ConfigurationSource conf) {
+      blockSize = (long) conf.getStorageSize(
+          OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+          OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT,
+          StorageUnit.BYTES);
+      minRatisVolumeSizeBytes = (long) conf.getStorageSize(
+          ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
+          ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT,
+          StorageUnit.BYTES);
+      containerSize = (long) conf.getStorageSize(
+          ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+          ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
+          StorageUnit.BYTES);
+      this.conf = conf;
+    }
+
+    @Override
+    public boolean test(DatanodeInfo dn) {
+      return !dn.getNodeStatus().isNodeWritable()
+          || (!hasEnoughSpace(dn, minRatisVolumeSizeBytes, containerSize, conf)
+          && !hasEnoughCommittedVolumeSpace(dn));
+    }
+
+    /**
+     * Check if any volume in the datanode has committed space >= blockSize.
+     *
+     * @return true if any volume has committed space >= blockSize, false otherwise
+     */
+    private boolean hasEnoughCommittedVolumeSpace(DatanodeInfo dnInfo) {
+      for (StorageReportProto reportProto : dnInfo.getStorageReports()) {
+        if (reportProto.getCommitted() >= blockSize) {
+          return true;
+        }
       }
+      LOG.debug("Datanode {} has no volumes with committed space >= {} bytes",
+          dnInfo.getID(), blockSize);
+      return false;
     }
-    LOG.debug("Datanode {} has no volumes with committed space >= {} bytes",
-        dnInfo.getID(), blockSize);
-    return false;
   }
 
   /**

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java

@@ -157,12 +157,13 @@ public void getMetrics(MetricsCollector collector, boolean all) {
     metrics.addGauge(
         Interns.info("AllNodes", "Number of datanodes"), totalNodeCount);
 
-    String nodesOutOfSpace = nodeStatistics.get("NodesOutOfSpace");
-    if (nodesOutOfSpace != null) {
+    String nonWritableNodes = nodeStatistics.get("NonWritableNodes");
+    if (nonWritableNodes != null) {
       metrics.addGauge(
-          Interns.info("NodesOutOfSpace", "Number of datanodes that are out of space because " +
-              "they cannot allocate new containers or write to existing ones."),
-          Integer.parseInt(nodesOutOfSpace));
+          Interns.info("NonWritableNodes", "Number of datanodes that cannot accept new writes because " +
+              "they are either not in IN_SERVICE and HEALTHY state, cannot allocate new containers or " +
+              "cannot write to existing containers."),
+          Integer.parseInt(nonWritableNodes));
     }
 
     for (Map.Entry<String, Long> e : nodeInfo.entrySet()) {

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java

@@ -227,7 +227,7 @@ public void testNodeCountAndInfoMetricsReported() throws Exception {
     assertGauge("AllNodes", 1,
         getMetrics(SCMNodeMetrics.class.getSimpleName()));
     // The DN has no metadata volumes, so hasEnoughSpace() returns false indicating the DN is out of space.
-    assertGauge("NodesOutOfSpace", 1,
+    assertGauge("NonWritableNodes", 1,
         getMetrics(SCMNodeMetrics.class.getSimpleName()));
     assertGauge("TotalCapacity", 100L,
         getMetrics(SCMNodeMetrics.class.getSimpleName()));