HDDS-13945. Show datanode reserved space in StorageDistributionEndpoint

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java

@@ -1063,7 +1063,7 @@ public static double calculateAvgUtilization(List<DatanodeUsageInfo> nodes) {
       return 0;
     }
     SCMNodeStat aggregatedStats = new SCMNodeStat(
-        0, 0, 0, 0, 0);
+        0, 0, 0, 0, 0, 0);
     for (DatanodeUsageInfo node : nodes) {
       aggregatedStats.add(node.getScmNodeStat());
     }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/NodeStat.java

@@ -53,6 +53,12 @@ interface NodeStat {
    */
   LongMetric getFreeSpaceToSpare();
 
+  /**
+   * Get the reserved space on the node.
+   * @return the reserved space on the node
+   */
+  LongMetric getReserved();
+
   /**
    * Set the total/used/remaining space.
    * @param capacity - total space.
@@ -61,7 +67,7 @@ interface NodeStat {
    */
   @VisibleForTesting
   void set(long capacity, long used, long remain, long committed,
-           long freeSpaceToSpare);
+           long freeSpaceToSpare, long reserved);
 
   /**
    * Adding of the stat.

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeMetric.java

@@ -46,9 +46,9 @@ public SCMNodeMetric(SCMNodeStat stat) {
    */
   @VisibleForTesting
   public SCMNodeMetric(long capacity, long used, long remaining,
-                       long committed, long freeSpaceToSpare) {
+                       long committed, long freeSpaceToSpare, long reserved) {
     this.stat = new SCMNodeStat();
-    this.stat.set(capacity, used, remaining, committed, freeSpaceToSpare);
+    this.stat.set(capacity, used, remaining, committed, freeSpaceToSpare, reserved);
   }
 
   /**
@@ -159,7 +159,7 @@ public SCMNodeStat get() {
   public void set(SCMNodeStat value) {
     stat.set(value.getCapacity().get(), value.getScmUsed().get(),
         value.getRemaining().get(), value.getCommitted().get(),
-        value.getFreeSpaceToSpare().get());
+        value.getFreeSpaceToSpare().get(), value.getReserved().get());
   }
 
   /**

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeStat.java

@@ -29,18 +29,19 @@ public class SCMNodeStat implements NodeStat {
   private LongMetric remaining;
   private LongMetric committed;
   private LongMetric freeSpaceToSpare;
+  private LongMetric reserved;
 
   public SCMNodeStat() {
-    this(0L, 0L, 0L, 0L, 0L);
+    this(0L, 0L, 0L, 0L, 0L, 0L);
   }
 
   public SCMNodeStat(SCMNodeStat other) {
     this(other.capacity.get(), other.scmUsed.get(), other.remaining.get(),
-        other.committed.get(), other.freeSpaceToSpare.get());
+        other.committed.get(), other.freeSpaceToSpare.get(), other.reserved.get());
   }
 
   public SCMNodeStat(long capacity, long used, long remaining, long committed,
-                     long freeSpaceToSpare) {
+                     long freeSpaceToSpare, long reserved) {
     Preconditions.checkArgument(capacity >= 0, "Capacity cannot be " +
         "negative.");
     Preconditions.checkArgument(used >= 0, "used space cannot be " +
@@ -52,6 +53,7 @@ public SCMNodeStat(long capacity, long used, long remaining, long committed,
     this.remaining = new LongMetric(remaining);
     this.committed = new LongMetric(committed);
     this.freeSpaceToSpare = new LongMetric(freeSpaceToSpare);
+    this.reserved = new LongMetric(reserved);
   }
 
   /**
@@ -96,6 +98,15 @@ public LongMetric getFreeSpaceToSpare() {
     return freeSpaceToSpare;
   }
 
+  /**
+   * Get the reserved space on the node.
+   * @return the reserved space on the node
+   */
+  @Override
+  public LongMetric getReserved() {
+    return reserved;
+  }
+
   /**
    * Set the capacity, used and remaining space on a datanode.
    *
@@ -106,7 +117,7 @@ public LongMetric getFreeSpaceToSpare() {
   @Override
   @VisibleForTesting
   public void set(long newCapacity, long newUsed, long newRemaining,
-                  long newCommitted, long newFreeSpaceToSpare) {
+                  long newCommitted, long newFreeSpaceToSpare, long newReserved) {
     Preconditions.checkArgument(newCapacity >= 0, "Capacity cannot be " +
         "negative.");
     Preconditions.checkArgument(newUsed >= 0, "used space cannot be " +
@@ -119,6 +130,7 @@ public void set(long newCapacity, long newUsed, long newRemaining,
     this.remaining = new LongMetric(newRemaining);
     this.committed = new LongMetric(newCommitted);
     this.freeSpaceToSpare = new LongMetric(newFreeSpaceToSpare);
+    this.reserved = new LongMetric(newReserved);
   }
 
   /**
@@ -133,8 +145,8 @@ public SCMNodeStat add(NodeStat stat) {
     this.scmUsed.set(this.getScmUsed().get() + stat.getScmUsed().get());
     this.remaining.set(this.getRemaining().get() + stat.getRemaining().get());
     this.committed.set(this.getCommitted().get() + stat.getCommitted().get());
-    this.freeSpaceToSpare.set(this.freeSpaceToSpare.get() +
-        stat.getFreeSpaceToSpare().get());
+    this.freeSpaceToSpare.set(this.freeSpaceToSpare.get() + stat.getFreeSpaceToSpare().get());
+    this.reserved.set(this.reserved.get() + stat.getReserved().get());
     return this;
   }
 
@@ -150,8 +162,8 @@ public SCMNodeStat subtract(NodeStat stat) {
     this.scmUsed.set(this.getScmUsed().get() - stat.getScmUsed().get());
     this.remaining.set(this.getRemaining().get() - stat.getRemaining().get());
     this.committed.set(this.getCommitted().get() - stat.getCommitted().get());
-    this.freeSpaceToSpare.set(freeSpaceToSpare.get() -
-        stat.getFreeSpaceToSpare().get());
+    this.freeSpaceToSpare.set(freeSpaceToSpare.get() - stat.getFreeSpaceToSpare().get());
+    this.reserved.set(reserved.get() - stat.getReserved().get());
     return this;
   }
 
@@ -163,15 +175,16 @@ public boolean equals(Object to) {
           scmUsed.isEqual(tempStat.getScmUsed().get()) &&
           remaining.isEqual(tempStat.getRemaining().get()) &&
           committed.isEqual(tempStat.getCommitted().get()) &&
-          freeSpaceToSpare.isEqual(tempStat.freeSpaceToSpare.get());
+          freeSpaceToSpare.isEqual(tempStat.freeSpaceToSpare.get()) &&
+          reserved.isEqual(tempStat.reserved.get());
     }
     return false;
   }
 
   @Override
   public int hashCode() {
     return Long.hashCode(capacity.get() ^ scmUsed.get() ^ remaining.get() ^
-        committed.get() ^ freeSpaceToSpare.get());
+        committed.get() ^ freeSpaceToSpare.get() ^ reserved.get());
   }
 
   @Override
@@ -180,6 +193,9 @@ public String toString() {
         "capacity=" + capacity.get() +
         ", scmUsed=" + scmUsed.get() +
         ", remaining=" + remaining.get() +
+        ", committed=" + committed.get() +
+        ", freeSpaceToSpare=" + freeSpaceToSpare.get() +
+        ", reserved=" + reserved.get() +
         '}';
   }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java

@@ -938,16 +938,18 @@ public SCMNodeStat getStats() {
     long remaining = 0L;
     long committed = 0L;
     long freeSpaceToSpare = 0L;
+    long reserved = 0L;
 
     for (SCMNodeStat stat : getNodeStats().values()) {
       capacity += stat.getCapacity().get();
       used += stat.getScmUsed().get();
       remaining += stat.getRemaining().get();
       committed += stat.getCommitted().get();
       freeSpaceToSpare += stat.getFreeSpaceToSpare().get();
+      reserved += stat.getReserved().get();
     }
     return new SCMNodeStat(capacity, used, remaining, committed,
-        freeSpaceToSpare);
+        freeSpaceToSpare, reserved);
   }
 
   /**
@@ -1055,6 +1057,7 @@ private SCMNodeStat getNodeStatInternal(DatanodeDetails datanodeDetails) {
       long remaining = 0L;
       long committed = 0L;
       long freeSpaceToSpare = 0L;
+      long reserved = 0L;
 
       final DatanodeInfo datanodeInfo = nodeStateManager
           .getNode(datanodeDetails);
@@ -1066,9 +1069,10 @@ private SCMNodeStat getNodeStatInternal(DatanodeDetails datanodeDetails) {
         remaining += reportProto.getRemaining();
         committed += reportProto.getCommitted();
         freeSpaceToSpare += reportProto.getFreeSpaceToSpare();
+        reserved += reportProto.getReserved();
       }
       return new SCMNodeStat(capacity, used, remaining, committed,
-          freeSpaceToSpare);
+          freeSpaceToSpare, reserved);
     } catch (NodeNotFoundException e) {
       LOG.warn("Cannot generate NodeStat, datanode {} not found.", datanodeDetails);
       return null;

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java

@@ -218,7 +218,7 @@ private void populateNodeMetric(DatanodeDetails datanodeDetails, int x) {
         NODES[x % NODES.length].capacity - NODES[x % NODES.length].used;
     newStat.set(
         (NODES[x % NODES.length].capacity),
-        (NODES[x % NODES.length].used), remaining, 0, 100000);
+        (NODES[x % NODES.length].used), remaining, 0, 100000, 0);
     this.nodeMetricMap.put(datanodeDetails, newStat);
     aggregateStat.add(newStat);
 

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java

@@ -460,7 +460,7 @@ private void createCluster(int[] sizeArray) {
       }
       SCMNodeStat stat = new SCMNodeStat(datanodeCapacity, datanodeUsedSpace,
           datanodeCapacity - datanodeUsedSpace, 0,
-          datanodeCapacity - datanodeUsedSpace - 1);
+          datanodeCapacity - datanodeUsedSpace - 1, 0);
       nodesInCluster.get(i).setScmNodeStat(stat);
     }
   }

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestFindTargetStrategy.java

@@ -53,11 +53,11 @@ public void testFindTargetGreedyByUsage() {
 
     //create three datanodes with different usageinfo
     DatanodeUsageInfo dui1 = new DatanodeUsageInfo(MockDatanodeDetails
-        .randomDatanodeDetails(), new SCMNodeStat(100, 0, 40, 0, 30));
+        .randomDatanodeDetails(), new SCMNodeStat(100, 0, 40, 0, 30, 0));
     DatanodeUsageInfo dui2 = new DatanodeUsageInfo(MockDatanodeDetails
-        .randomDatanodeDetails(), new SCMNodeStat(100, 0, 60, 0, 30));
+        .randomDatanodeDetails(), new SCMNodeStat(100, 0, 60, 0, 30, 0));
     DatanodeUsageInfo dui3 = new DatanodeUsageInfo(MockDatanodeDetails
-        .randomDatanodeDetails(), new SCMNodeStat(100, 0, 80, 0, 30));
+        .randomDatanodeDetails(), new SCMNodeStat(100, 0, 80, 0, 30, 0));
 
     //insert in ascending order
     overUtilizedDatanodes.add(dui1);
@@ -92,11 +92,11 @@ public void testFindTargetGreedyByUsage() {
   public void testResetPotentialTargets() {
     // create three datanodes with different usage infos
     DatanodeUsageInfo dui1 = new DatanodeUsageInfo(MockDatanodeDetails
-        .randomDatanodeDetails(), new SCMNodeStat(100, 30, 70, 0, 50));
+        .randomDatanodeDetails(), new SCMNodeStat(100, 30, 70, 0, 50, 0));
     DatanodeUsageInfo dui2 = new DatanodeUsageInfo(MockDatanodeDetails
-        .randomDatanodeDetails(), new SCMNodeStat(100, 20, 80, 0, 60));
+        .randomDatanodeDetails(), new SCMNodeStat(100, 20, 80, 0, 60, 0));
     DatanodeUsageInfo dui3 = new DatanodeUsageInfo(MockDatanodeDetails
-        .randomDatanodeDetails(), new SCMNodeStat(100, 10, 90, 0, 70));
+        .randomDatanodeDetails(), new SCMNodeStat(100, 10, 90, 0, 70, 0));
 
     List<DatanodeUsageInfo> potentialTargets = new ArrayList<>();
     potentialTargets.add(dui1);
@@ -171,18 +171,18 @@ public void testFindTargetGreedyByNetworkTopology() {
     List<DatanodeUsageInfo> overUtilizedDatanodes = new ArrayList<>();
     //set the farthest target with the lowest usage info
     overUtilizedDatanodes.add(
-        new DatanodeUsageInfo(target5, new SCMNodeStat(100, 0, 90, 0, 80)));
+        new DatanodeUsageInfo(target5, new SCMNodeStat(100, 0, 90, 0, 80, 0)));
     //set the tree targets, which have the same network topology distance
     //to source , with different usage info
     overUtilizedDatanodes.add(
-        new DatanodeUsageInfo(target2, new SCMNodeStat(100, 0, 20, 0, 10)));
+        new DatanodeUsageInfo(target2, new SCMNodeStat(100, 0, 20, 0, 10, 0)));
     overUtilizedDatanodes.add(
-        new DatanodeUsageInfo(target3, new SCMNodeStat(100, 0, 40, 0, 30)));
+        new DatanodeUsageInfo(target3, new SCMNodeStat(100, 0, 40, 0, 30, 0)));
     overUtilizedDatanodes.add(
-        new DatanodeUsageInfo(target4, new SCMNodeStat(100, 0, 60, 0, 50)));
+        new DatanodeUsageInfo(target4, new SCMNodeStat(100, 0, 60, 0, 50, 0)));
     //set the nearest target with the highest usage info
     overUtilizedDatanodes.add(
-        new DatanodeUsageInfo(target1, new SCMNodeStat(100, 0, 10, 0, 5)));
+        new DatanodeUsageInfo(target1, new SCMNodeStat(100, 0, 10, 0, 5, 0)));
 
 
     FindTargetGreedyByNetworkTopology findTargetGreedyByNetworkTopology =

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestableCluster.java

@@ -80,7 +80,7 @@ public final class TestableCluster {
 
       SCMNodeStat stat = new SCMNodeStat(datanodeCapacity, datanodeUsedSpace,
           datanodeCapacity - datanodeUsedSpace, 0,
-          datanodeCapacity - datanodeUsedSpace - 1);
+          datanodeCapacity - datanodeUsedSpace - 1, 0);
       nodesInCluster[i].setScmNodeStat(stat);
       clusterUsedSpace += datanodeUsedSpace;
       clusterCapacity += datanodeCapacity;

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java

@@ -105,13 +105,13 @@ public void chooseDatanodes() throws SCMException {
         .thenReturn(new ArrayList<>(datanodes));
 
     when(mockNodeManager.getNodeStat(any()))
-        .thenReturn(new SCMNodeMetric(100L, 0L, 100L, 0, 90));
+        .thenReturn(new SCMNodeMetric(100L, 0L, 100L, 0, 90, 0));
     when(mockNodeManager.getNodeStat(datanodes.get(2)))
-        .thenReturn(new SCMNodeMetric(100L, 90L, 10L, 0, 9));
+        .thenReturn(new SCMNodeMetric(100L, 90L, 10L, 0, 9, 0));
     when(mockNodeManager.getNodeStat(datanodes.get(3)))
-        .thenReturn(new SCMNodeMetric(100L, 80L, 20L, 0, 19));
+        .thenReturn(new SCMNodeMetric(100L, 80L, 20L, 0, 19, 0));
     when(mockNodeManager.getNodeStat(datanodes.get(4)))
-        .thenReturn(new SCMNodeMetric(100L, 70L, 30L, 0, 20));
+        .thenReturn(new SCMNodeMetric(100L, 70L, 30L, 0, 20, 0));
     when(mockNodeManager.getNode(any(DatanodeID.class))).thenAnswer(
             invocation -> datanodes.stream()
                 .filter(dn -> dn.getID().equals(invocation.getArgument(0)))

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerUtil.java

@@ -317,13 +317,13 @@ public void testDatanodesWithInSufficientDiskSpaceAreExcluded() throws NodeNotFo
     when(replicationManager.getNodeManager()).thenReturn(nodeManagerMock);
     doReturn(fullDn).when(nodeManagerMock).getNode(fullDn.getID());
     doReturn(new SCMNodeMetric(50 * oneGb, 20 * oneGb, 30 * oneGb, 5 * oneGb,
-        20 * oneGb)).when(nodeManagerMock).getNodeStat(fullDn);
+        20 * oneGb, 0)).when(nodeManagerMock).getNodeStat(fullDn);
     doReturn(spaceAvailableDn).when(nodeManagerMock).getNode(spaceAvailableDn.getID());
     doReturn(new SCMNodeMetric(50 * oneGb, 10 * oneGb, 40 * oneGb, 5 * oneGb,
-        20 * oneGb)).when(nodeManagerMock).getNodeStat(spaceAvailableDn);
+        20 * oneGb, 0)).when(nodeManagerMock).getNodeStat(spaceAvailableDn);
     doReturn(expiredOpDn).when(nodeManagerMock).getNode(expiredOpDn.getID());
     doReturn(new SCMNodeMetric(50 * oneGb, 20 * oneGb, 30 * oneGb, 5 * oneGb,
-        20 * oneGb)).when(nodeManagerMock).getNodeStat(expiredOpDn);
+        20 * oneGb, 0)).when(nodeManagerMock).getNodeStat(expiredOpDn);
 
     when(replicationManager.getNodeStatus(any())).thenAnswer(
         invocation -> {

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/choose/algorithms/TestCapacityPipelineChoosePolicy.java

@@ -53,13 +53,13 @@ public void testChoosePipeline() throws Exception {
     // used       0   10    20    30
     NodeManager mockNodeManager = mock(NodeManager.class);
     when(mockNodeManager.getNodeStat(datanodes.get(0)))
-        .thenReturn(new SCMNodeMetric(100L, 0, 100L, 0, 0));
+        .thenReturn(new SCMNodeMetric(100L, 0, 100L, 0, 0, 0));
     when(mockNodeManager.getNodeStat(datanodes.get(1)))
-        .thenReturn(new SCMNodeMetric(100L, 10L, 90L, 0, 0));
+        .thenReturn(new SCMNodeMetric(100L, 10L, 90L, 0, 0, 0));
     when(mockNodeManager.getNodeStat(datanodes.get(2)))
-        .thenReturn(new SCMNodeMetric(100L, 20L, 80L, 0, 0));
+        .thenReturn(new SCMNodeMetric(100L, 20L, 80L, 0, 0, 0));
     when(mockNodeManager.getNodeStat(datanodes.get(3)))
-        .thenReturn(new SCMNodeMetric(100L, 30L, 70L, 0, 0));
+        .thenReturn(new SCMNodeMetric(100L, 30L, 70L, 0, 0, 0));
 
     PipelineChoosePolicy policy = new CapacityPipelineChoosePolicy().init(mockNodeManager);
 

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestDatanodeMetrics.java

@@ -30,13 +30,13 @@
 public class TestDatanodeMetrics {
   @Test
   public void testSCMNodeMetric() {
-    SCMNodeStat stat = new SCMNodeStat(100L, 10L, 90L, 0, 80);
+    SCMNodeStat stat = new SCMNodeStat(100L, 10L, 90L, 0, 80, 0);
     assertEquals((long) stat.getCapacity().get(), 100L);
     assertEquals(10L, (long) stat.getScmUsed().get());
     assertEquals(90L, (long) stat.getRemaining().get());
     SCMNodeMetric metric = new SCMNodeMetric(stat);
 
-    SCMNodeStat newStat = new SCMNodeStat(100L, 10L, 90L, 0, 80);
+    SCMNodeStat newStat = new SCMNodeStat(100L, 10L, 90L, 0, 80, 0);
     assertEquals(100L, (long) stat.getCapacity().get());
     assertEquals(10L, (long) stat.getScmUsed().get());
     assertEquals(90L, (long) stat.getRemaining().get());
@@ -52,8 +52,8 @@ public void testSCMNodeMetric() {
     assertTrue(metric.isGreater(zeroMetric.get()));
 
     // Another case when nodes have similar weight
-    SCMNodeStat stat1 = new SCMNodeStat(10000000L, 50L, 9999950L, 0, 100000);
-    SCMNodeStat stat2 = new SCMNodeStat(10000000L, 51L, 9999949L, 0, 100000);
+    SCMNodeStat stat1 = new SCMNodeStat(10000000L, 50L, 9999950L, 0, 100000, 0);
+    SCMNodeStat stat2 = new SCMNodeStat(10000000L, 51L, 9999949L, 0, 100000, 0);
     assertTrue(new SCMNodeMetric(stat2).isGreater(stat1));
   }
 }