Skip to content

Commit 72f47df

Browse files
priyeshkarathapriyeshkaratha
committed
adding reserved in nodestat
1 parent 661e13a commit 72f47df

17 files changed

Lines changed: 76 additions & 50 deletions

File tree

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1063,7 +1063,7 @@ public static double calculateAvgUtilization(List<DatanodeUsageInfo> nodes) {
10631063
return 0;
10641064
}
10651065
SCMNodeStat aggregatedStats = new SCMNodeStat(
1066-
0, 0, 0, 0, 0);
1066+
0, 0, 0, 0, 0, 0);
10671067
for (DatanodeUsageInfo node : nodes) {
10681068
aggregatedStats.add(node.getScmNodeStat());
10691069
}

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/NodeStat.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ interface NodeStat {
5353
*/
5454
LongMetric getFreeSpaceToSpare();
5555

56+
/**
57+
* Get the reserved space on the node.
58+
* @return the reserved space on the node
59+
*/
60+
LongMetric getReserved();
61+
5662
/**
5763
* Set the total/used/remaining space.
5864
* @param capacity - total space.
@@ -61,7 +67,7 @@ interface NodeStat {
6167
*/
6268
@VisibleForTesting
6369
void set(long capacity, long used, long remain, long committed,
64-
long freeSpaceToSpare);
70+
long freeSpaceToSpare, long reserved);
6571

6672
/**
6773
* Adding of the stat.

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeMetric.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ public SCMNodeMetric(SCMNodeStat stat) {
4646
*/
4747
@VisibleForTesting
4848
public SCMNodeMetric(long capacity, long used, long remaining,
49-
long committed, long freeSpaceToSpare) {
49+
long committed, long freeSpaceToSpare, long reserved) {
5050
this.stat = new SCMNodeStat();
51-
this.stat.set(capacity, used, remaining, committed, freeSpaceToSpare);
51+
this.stat.set(capacity, used, remaining, committed, freeSpaceToSpare, reserved);
5252
}
5353

5454
/**
@@ -159,7 +159,7 @@ public SCMNodeStat get() {
159159
public void set(SCMNodeStat value) {
160160
stat.set(value.getCapacity().get(), value.getScmUsed().get(),
161161
value.getRemaining().get(), value.getCommitted().get(),
162-
value.getFreeSpaceToSpare().get());
162+
value.getFreeSpaceToSpare().get(), value.getReserved().get());
163163
}
164164

165165
/**

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeStat.java‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,18 +29,19 @@ public class SCMNodeStat implements NodeStat {
2929
private LongMetric remaining;
3030
private LongMetric committed;
3131
private LongMetric freeSpaceToSpare;
32+
private LongMetric reserved;
3233

3334
public SCMNodeStat() {
34-
this(0L, 0L, 0L, 0L, 0L);
35+
this(0L, 0L, 0L, 0L, 0L, 0L);
3536
}
3637

3738
public SCMNodeStat(SCMNodeStat other) {
3839
this(other.capacity.get(), other.scmUsed.get(), other.remaining.get(),
39-
other.committed.get(), other.freeSpaceToSpare.get());
40+
other.committed.get(), other.freeSpaceToSpare.get(), other.reserved.get());
4041
}
4142

4243
public SCMNodeStat(long capacity, long used, long remaining, long committed,
43-
long freeSpaceToSpare) {
44+
long freeSpaceToSpare, long reserved) {
4445
Preconditions.checkArgument(capacity >= 0, "Capacity cannot be " +
4546
"negative.");
4647
Preconditions.checkArgument(used >= 0, "used space cannot be " +
@@ -52,6 +53,7 @@ public SCMNodeStat(long capacity, long used, long remaining, long committed,
5253
this.remaining = new LongMetric(remaining);
5354
this.committed = new LongMetric(committed);
5455
this.freeSpaceToSpare = new LongMetric(freeSpaceToSpare);
56+
this.reserved = new LongMetric(reserved);
5557
}
5658

5759
/**
@@ -96,6 +98,15 @@ public LongMetric getFreeSpaceToSpare() {
9698
return freeSpaceToSpare;
9799
}
98100

101+
/**
102+
* Get the reserved spade on the node.
103+
* @return the reserved spade on the node
104+
*/
105+
@Override
106+
public LongMetric getReserved() {
107+
return reserved;
108+
}
109+
99110
/**
100111
* Set the capacity, used and remaining space on a datanode.
101112
*
@@ -106,7 +117,7 @@ public LongMetric getFreeSpaceToSpare() {
106117
@Override
107118
@VisibleForTesting
108119
public void set(long newCapacity, long newUsed, long newRemaining,
109-
long newCommitted, long newFreeSpaceToSpare) {
120+
long newCommitted, long newFreeSpaceToSpare, long newReserved) {
110121
Preconditions.checkArgument(newCapacity >= 0, "Capacity cannot be " +
111122
"negative.");
112123
Preconditions.checkArgument(newUsed >= 0, "used space cannot be " +
@@ -119,6 +130,7 @@ public void set(long newCapacity, long newUsed, long newRemaining,
119130
this.remaining = new LongMetric(newRemaining);
120131
this.committed = new LongMetric(newCommitted);
121132
this.freeSpaceToSpare = new LongMetric(newFreeSpaceToSpare);
133+
this.reserved = new LongMetric(newReserved);
122134
}
123135

124136
/**
@@ -133,8 +145,8 @@ public SCMNodeStat add(NodeStat stat) {
133145
this.scmUsed.set(this.getScmUsed().get() + stat.getScmUsed().get());
134146
this.remaining.set(this.getRemaining().get() + stat.getRemaining().get());
135147
this.committed.set(this.getCommitted().get() + stat.getCommitted().get());
136-
this.freeSpaceToSpare.set(this.freeSpaceToSpare.get() +
137-
stat.getFreeSpaceToSpare().get());
148+
this.freeSpaceToSpare.set(this.freeSpaceToSpare.get() + stat.getFreeSpaceToSpare().get());
149+
this.reserved.set(this.reserved.get() + stat.getReserved().get());
138150
return this;
139151
}
140152

@@ -150,8 +162,8 @@ public SCMNodeStat subtract(NodeStat stat) {
150162
this.scmUsed.set(this.getScmUsed().get() - stat.getScmUsed().get());
151163
this.remaining.set(this.getRemaining().get() - stat.getRemaining().get());
152164
this.committed.set(this.getCommitted().get() - stat.getCommitted().get());
153-
this.freeSpaceToSpare.set(freeSpaceToSpare.get() -
154-
stat.getFreeSpaceToSpare().get());
165+
this.freeSpaceToSpare.set(freeSpaceToSpare.get() - stat.getFreeSpaceToSpare().get());
166+
this.reserved.set(reserved.get() - stat.getReserved().get());
155167
return this;
156168
}
157169

@@ -163,15 +175,16 @@ public boolean equals(Object to) {
163175
scmUsed.isEqual(tempStat.getScmUsed().get()) &&
164176
remaining.isEqual(tempStat.getRemaining().get()) &&
165177
committed.isEqual(tempStat.getCommitted().get()) &&
166-
freeSpaceToSpare.isEqual(tempStat.freeSpaceToSpare.get());
178+
freeSpaceToSpare.isEqual(tempStat.freeSpaceToSpare.get()) &&
179+
reserved.isEqual(tempStat.reserved.get());
167180
}
168181
return false;
169182
}
170183

171184
@Override
172185
public int hashCode() {
173186
return Long.hashCode(capacity.get() ^ scmUsed.get() ^ remaining.get() ^
174-
committed.get() ^ freeSpaceToSpare.get());
187+
committed.get() ^ freeSpaceToSpare.get() ^ reserved.get());
175188
}
176189

177190
@Override
@@ -180,6 +193,9 @@ public String toString() {
180193
"capacity=" + capacity.get() +
181194
", scmUsed=" + scmUsed.get() +
182195
", remaining=" + remaining.get() +
196+
", committed=" + committed.get() +
197+
", freeSpaceToSpare=" + freeSpaceToSpare.get() +
198+
", reserved=" + reserved.get() +
183199
'}';
184200
}
185201
}

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -938,16 +938,18 @@ public SCMNodeStat getStats() {
938938
long remaining = 0L;
939939
long committed = 0L;
940940
long freeSpaceToSpare = 0L;
941+
long reserved = 0L;
941942

942943
for (SCMNodeStat stat : getNodeStats().values()) {
943944
capacity += stat.getCapacity().get();
944945
used += stat.getScmUsed().get();
945946
remaining += stat.getRemaining().get();
946947
committed += stat.getCommitted().get();
947948
freeSpaceToSpare += stat.getFreeSpaceToSpare().get();
949+
reserved += stat.getReserved().get();
948950
}
949951
return new SCMNodeStat(capacity, used, remaining, committed,
950-
freeSpaceToSpare);
952+
freeSpaceToSpare, reserved);
951953
}
952954

953955
/**
@@ -1055,6 +1057,7 @@ private SCMNodeStat getNodeStatInternal(DatanodeDetails datanodeDetails) {
10551057
long remaining = 0L;
10561058
long committed = 0L;
10571059
long freeSpaceToSpare = 0L;
1060+
long reserved = 0L;
10581061

10591062
final DatanodeInfo datanodeInfo = nodeStateManager
10601063
.getNode(datanodeDetails);
@@ -1066,9 +1069,10 @@ private SCMNodeStat getNodeStatInternal(DatanodeDetails datanodeDetails) {
10661069
remaining += reportProto.getRemaining();
10671070
committed += reportProto.getCommitted();
10681071
freeSpaceToSpare += reportProto.getFreeSpaceToSpare();
1072+
reserved += reportProto.getReserved();
10691073
}
10701074
return new SCMNodeStat(capacity, used, remaining, committed,
1071-
freeSpaceToSpare);
1075+
freeSpaceToSpare, reserved);
10721076
} catch (NodeNotFoundException e) {
10731077
LOG.warn("Cannot generate NodeStat, datanode {} not found.", datanodeDetails);
10741078
return null;

‎hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ private void populateNodeMetric(DatanodeDetails datanodeDetails, int x) {
218218
NODES[x % NODES.length].capacity - NODES[x % NODES.length].used;
219219
newStat.set(
220220
(NODES[x % NODES.length].capacity),
221-
(NODES[x % NODES.length].used), remaining, 0, 100000);
221+
(NODES[x % NODES.length].used), remaining, 0, 100000, 0);
222222
this.nodeMetricMap.put(datanodeDetails, newStat);
223223
aggregateStat.add(newStat);
224224

‎hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,7 @@ private void createCluster(int[] sizeArray) {
460460
}
461461
SCMNodeStat stat = new SCMNodeStat(datanodeCapacity, datanodeUsedSpace,
462462
datanodeCapacity - datanodeUsedSpace, 0,
463-
datanodeCapacity - datanodeUsedSpace - 1);
463+
datanodeCapacity - datanodeUsedSpace - 1, 0);
464464
nodesInCluster.get(i).setScmNodeStat(stat);
465465
}
466466
}

‎hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestFindTargetStrategy.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,11 @@ public void testFindTargetGreedyByUsage() {
5353

5454
//create three datanodes with different usageinfo
5555
DatanodeUsageInfo dui1 = new DatanodeUsageInfo(MockDatanodeDetails
56-
.randomDatanodeDetails(), new SCMNodeStat(100, 0, 40, 0, 30));
56+
.randomDatanodeDetails(), new SCMNodeStat(100, 0, 40, 0, 30, 0));
5757
DatanodeUsageInfo dui2 = new DatanodeUsageInfo(MockDatanodeDetails
58-
.randomDatanodeDetails(), new SCMNodeStat(100, 0, 60, 0, 30));
58+
.randomDatanodeDetails(), new SCMNodeStat(100, 0, 60, 0, 30, 0));
5959
DatanodeUsageInfo dui3 = new DatanodeUsageInfo(MockDatanodeDetails
60-
.randomDatanodeDetails(), new SCMNodeStat(100, 0, 80, 0, 30));
60+
.randomDatanodeDetails(), new SCMNodeStat(100, 0, 80, 0, 30, 0));
6161

6262
//insert in ascending order
6363
overUtilizedDatanodes.add(dui1);
@@ -92,11 +92,11 @@ public void testFindTargetGreedyByUsage() {
9292
public void testResetPotentialTargets() {
9393
// create three datanodes with different usage infos
9494
DatanodeUsageInfo dui1 = new DatanodeUsageInfo(MockDatanodeDetails
95-
.randomDatanodeDetails(), new SCMNodeStat(100, 30, 70, 0, 50));
95+
.randomDatanodeDetails(), new SCMNodeStat(100, 30, 70, 0, 50, 0));
9696
DatanodeUsageInfo dui2 = new DatanodeUsageInfo(MockDatanodeDetails
97-
.randomDatanodeDetails(), new SCMNodeStat(100, 20, 80, 0, 60));
97+
.randomDatanodeDetails(), new SCMNodeStat(100, 20, 80, 0, 60, 0));
9898
DatanodeUsageInfo dui3 = new DatanodeUsageInfo(MockDatanodeDetails
99-
.randomDatanodeDetails(), new SCMNodeStat(100, 10, 90, 0, 70));
99+
.randomDatanodeDetails(), new SCMNodeStat(100, 10, 90, 0, 70, 0));
100100

101101
List<DatanodeUsageInfo> potentialTargets = new ArrayList<>();
102102
potentialTargets.add(dui1);
@@ -171,18 +171,18 @@ public void testFindTargetGreedyByNetworkTopology() {
171171
List<DatanodeUsageInfo> overUtilizedDatanodes = new ArrayList<>();
172172
//set the farthest target with the lowest usage info
173173
overUtilizedDatanodes.add(
174-
new DatanodeUsageInfo(target5, new SCMNodeStat(100, 0, 90, 0, 80)));
174+
new DatanodeUsageInfo(target5, new SCMNodeStat(100, 0, 90, 0, 80, 0)));
175175
//set the tree targets, which have the same network topology distance
176176
//to source , with different usage info
177177
overUtilizedDatanodes.add(
178-
new DatanodeUsageInfo(target2, new SCMNodeStat(100, 0, 20, 0, 10)));
178+
new DatanodeUsageInfo(target2, new SCMNodeStat(100, 0, 20, 0, 10, 0)));
179179
overUtilizedDatanodes.add(
180-
new DatanodeUsageInfo(target3, new SCMNodeStat(100, 0, 40, 0, 30)));
180+
new DatanodeUsageInfo(target3, new SCMNodeStat(100, 0, 40, 0, 30, 0)));
181181
overUtilizedDatanodes.add(
182-
new DatanodeUsageInfo(target4, new SCMNodeStat(100, 0, 60, 0, 50)));
182+
new DatanodeUsageInfo(target4, new SCMNodeStat(100, 0, 60, 0, 50, 0)));
183183
//set the nearest target with the highest usage info
184184
overUtilizedDatanodes.add(
185-
new DatanodeUsageInfo(target1, new SCMNodeStat(100, 0, 10, 0, 5)));
185+
new DatanodeUsageInfo(target1, new SCMNodeStat(100, 0, 10, 0, 5, 0)));
186186

187187

188188
FindTargetGreedyByNetworkTopology findTargetGreedyByNetworkTopology =

‎hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestableCluster.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public final class TestableCluster {
8080

8181
SCMNodeStat stat = new SCMNodeStat(datanodeCapacity, datanodeUsedSpace,
8282
datanodeCapacity - datanodeUsedSpace, 0,
83-
datanodeCapacity - datanodeUsedSpace - 1);
83+
datanodeCapacity - datanodeUsedSpace - 1, 0);
8484
nodesInCluster[i].setScmNodeStat(stat);
8585
clusterUsedSpace += datanodeUsedSpace;
8686
clusterCapacity += datanodeCapacity;

‎hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -105,13 +105,13 @@ public void chooseDatanodes() throws SCMException {
105105
.thenReturn(new ArrayList<>(datanodes));
106106

107107
when(mockNodeManager.getNodeStat(any()))
108-
.thenReturn(new SCMNodeMetric(100L, 0L, 100L, 0, 90));
108+
.thenReturn(new SCMNodeMetric(100L, 0L, 100L, 0, 90, 0));
109109
when(mockNodeManager.getNodeStat(datanodes.get(2)))
110-
.thenReturn(new SCMNodeMetric(100L, 90L, 10L, 0, 9));
110+
.thenReturn(new SCMNodeMetric(100L, 90L, 10L, 0, 9, 0));
111111
when(mockNodeManager.getNodeStat(datanodes.get(3)))
112-
.thenReturn(new SCMNodeMetric(100L, 80L, 20L, 0, 19));
112+
.thenReturn(new SCMNodeMetric(100L, 80L, 20L, 0, 19, 0));
113113
when(mockNodeManager.getNodeStat(datanodes.get(4)))
114-
.thenReturn(new SCMNodeMetric(100L, 70L, 30L, 0, 20));
114+
.thenReturn(new SCMNodeMetric(100L, 70L, 30L, 0, 20, 0));
115115
when(mockNodeManager.getNode(any(DatanodeID.class))).thenAnswer(
116116
invocation -> datanodes.stream()
117117
.filter(dn -> dn.getID().equals(invocation.getArgument(0)))

0 commit comments

Comments
 (0)