HDDS-13969. Add SCM metric for number of datanodes out of space (#9339)

Author: sarvekshayr

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java

@@ -22,6 +22,7 @@
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY_READONLY;
+import static org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy.hasEnoughSpace;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
@@ -51,8 +52,10 @@
 import javax.management.ObjectName;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageUnit;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.DatanodeID;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto;
@@ -67,6 +70,7 @@
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.VersionInfo;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -84,6 +88,7 @@
 import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.protocol.VersionResponse;
 import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
@@ -122,6 +127,7 @@ public class SCMNodeManager implements NodeManager {
   private final SCMNodeMetrics metrics;
   // Node manager MXBean
   private ObjectName nmInfoBean;
+  private final OzoneConfiguration conf;
   private final SCMStorageConfig scmStorageConfig;
   private final NetworkTopology clusterMap;
   private final Function<String, String> nodeResolver;
@@ -170,6 +176,7 @@ public SCMNodeManager(
       SCMContext scmContext,
       HDDSLayoutVersionManager layoutVersionManager,
       Function<String, String> nodeResolver) {
+    this.conf = conf;
     this.scmNodeEventPublisher = eventPublisher;
     this.nodeStateManager = new NodeStateManager(conf, eventPublisher,
         layoutVersionManager, scmContext);
@@ -1274,6 +1281,8 @@ public Map<String, String> getNodeStatistics() {
     nodeStateStatistics(nodeStatistics);
     // Statistics node space
     nodeSpaceStatistics(nodeStatistics);
+    // Statistics node readOnly
+    nodeOutOfSpaceStatistics(nodeStatistics);
     // todo: Statistics of other instances
     return nodeStatistics;
   }
@@ -1361,6 +1370,49 @@ private void nodeSpaceStatistics(Map<String, String> nodeStatics) {
     nodeStatics.put(SpaceStatistics.NON_SCM_USED.getLabel(), nonScmUsed);
   }
 
+  private void nodeOutOfSpaceStatistics(Map<String, String> nodeStatics) {
+    List<DatanodeInfo> allNodes = getAllNodes();
+    long blockSize = (long) conf.getStorageSize(
+        OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+        OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT,
+        StorageUnit.BYTES);
+    long minRatisVolumeSizeBytes = (long) conf.getStorageSize(
+        ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
+        ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT,
+        StorageUnit.BYTES);
+    long containerSize = (long) conf.getStorageSize(
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
+        StorageUnit.BYTES);
+
+    int nodeOutOfSpaceCount = (int) allNodes.parallelStream()
+        .filter(dn -> !hasEnoughSpace(dn, minRatisVolumeSizeBytes, containerSize, conf)
+            && !hasContainerWithSpace(dn, blockSize, containerSize))
+        .count();
+
+    nodeStatics.put("NodesOutOfSpace", String.valueOf(nodeOutOfSpaceCount));
+  }
+  
+  /**
+   * Check if a datanode has any OPEN container with enough space to accept new blocks.
+   */
+  private boolean hasContainerWithSpace(DatanodeInfo dnInfo, long blockSize, long containerSize) {
+    try {
+      Set<ContainerID> containers = getContainers(dnInfo);
+      for (ContainerID containerID : containers) {
+        ContainerInfo containerInfo = scmContext.getScm().getContainerManager().getContainer(containerID);
+        
+        if (containerInfo.getState() == HddsProtos.LifeCycleState.OPEN &&
+            containerInfo.getUsedBytes() + blockSize <= containerSize) {
+          return true;
+        }
+      }
+    } catch (Exception e) {
+      LOG.debug("Error checking containers for datanode {}: {}", dnInfo.getID(), e.getMessage());
+    }
+    return false;
+  }
+
   /**
    * Based on the current time and the last heartbeat, calculate the time difference
    * and get a string of the relative value. E.g. "2s ago", "1m 2s ago", etc.

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java

@@ -132,6 +132,7 @@ void incNumNodeCommandQueueReportProcessingFailed() {
   public void getMetrics(MetricsCollector collector, boolean all) {
     Map<String, Map<String, Integer>> nodeCount = managerMXBean.getNodeCount();
     Map<String, Long> nodeInfo = managerMXBean.getNodeInfo();
+    Map<String, String> nodeStatistics = managerMXBean.getNodeStatistics();
     int totalNodeCount = 0;
     /**
      * Loop over the Node map and create a metric for the cross product of all
@@ -156,6 +157,15 @@ public void getMetrics(MetricsCollector collector, boolean all) {
     metrics.addGauge(
         Interns.info("AllNodes", "Number of datanodes"), totalNodeCount);
 
+    String nodesOutOfSpace = nodeStatistics.get("NodesOutOfSpace");
+    if (nodesOutOfSpace != null) {
+      metrics.addGauge(
+          Interns.info("NodesOutOfSpace", "Number of datanodes that cannot accept new writes because " +
+              "they lack either sufficient metadata space, data volume space for creating new containers " +
+              "or free space in existing open containers."),
+          Integer.parseInt(nodesOutOfSpace));
+    }
+
     for (Map.Entry<String, Long> e : nodeInfo.entrySet()) {
       metrics.addGauge(
           Interns.info(e.getKey(), diskMetricDescription(e.getKey())),

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java

@@ -226,6 +226,9 @@ public void testNodeCountAndInfoMetricsReported() throws Exception {
         getMetrics(SCMNodeMetrics.class.getSimpleName()));
     assertGauge("AllNodes", 1,
         getMetrics(SCMNodeMetrics.class.getSimpleName()));
+    // The DN has no metadata volumes, so hasEnoughSpace() returns false indicating the DN is out of space.
+    assertGauge("NodesOutOfSpace", 1,
+        getMetrics(SCMNodeMetrics.class.getSimpleName()));
     assertGauge("TotalCapacity", 100L,
         getMetrics(SCMNodeMetrics.class.getSimpleName()));
     assertGauge("TotalUsed", 10L,