Skip to content

Commit e0b804e

Browse files
sreejasahithisreejasahithi
authored
HDDS-13095. Support sorting by most/least used nodes in ozone admin datanode list (#8520)
1 parent f7eb010 commit e0b804e

2 files changed

Lines changed: 193 additions & 0 deletions

File tree

‎hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@
1717

1818
package org.apache.hadoop.hdds.scm.cli.datanode;
1919

20+
import com.fasterxml.jackson.annotation.JsonInclude;
2021
import com.google.common.base.Strings;
2122
import java.io.IOException;
2223
import java.util.Collections;
2324
import java.util.List;
25+
import java.util.Objects;
2426
import java.util.UUID;
2527
import java.util.stream.Collectors;
2628
import java.util.stream.Stream;
@@ -63,6 +65,9 @@ public class ListInfoSubcommand extends ScmSubcommand {
6365
defaultValue = "false")
6466
private boolean json;
6567

68+
@CommandLine.ArgGroup(exclusive = true, multiplicity = "0..1")
69+
private UsageSortingOptions usageSortingOptions;
70+
6671
@CommandLine.Mixin
6772
private ListLimitOptions listLimitOptions;
6873

@@ -71,6 +76,16 @@ public class ListInfoSubcommand extends ScmSubcommand {
7176

7277
private List<Pipeline> pipelines;
7378

79+
static class UsageSortingOptions {
80+
@CommandLine.Option(names = {"--most-used"},
81+
description = "Show datanodes sorted by Utilization (most to least).")
82+
private boolean mostUsed;
83+
84+
@CommandLine.Option(names = {"--least-used"},
85+
description = "Show datanodes sorted by Utilization (least to most).")
86+
private boolean leastUsed;
87+
}
88+
7489
@Override
7590
public void execute(ScmClient scmClient) throws IOException {
7691
pipelines = scmClient.listPipelines();
@@ -123,6 +138,38 @@ public void execute(ScmClient scmClient) throws IOException {
123138

124139
private List<DatanodeWithAttributes> getAllNodes(ScmClient scmClient)
125140
throws IOException {
141+
142+
// If sorting is requested
143+
if (usageSortingOptions != null && (usageSortingOptions.mostUsed || usageSortingOptions.leastUsed)) {
144+
boolean sortByMostUsed = usageSortingOptions.mostUsed;
145+
List<HddsProtos.DatanodeUsageInfoProto> usageInfos = scmClient.getDatanodeUsageInfo(sortByMostUsed,
146+
Integer.MAX_VALUE);
147+
148+
return usageInfos.stream()
149+
.map(p -> {
150+
String uuidStr = p.getNode().getUuid();
151+
UUID parsedUuid = UUID.fromString(uuidStr);
152+
153+
try {
154+
HddsProtos.Node node = scmClient.queryNode(parsedUuid);
155+
long capacity = p.getCapacity();
156+
long used = capacity - p.getRemaining();
157+
double percentUsed = (capacity > 0) ? (used * 100.0) / capacity : 0.0;
158+
return new DatanodeWithAttributes(
159+
DatanodeDetails.getFromProtoBuf(node.getNodeID()),
160+
node.getNodeOperationalStates(0),
161+
node.getNodeStates(0),
162+
used,
163+
capacity,
164+
percentUsed);
165+
} catch (IOException e) {
166+
return null;
167+
}
168+
})
169+
.filter(Objects::nonNull)
170+
.collect(Collectors.toList());
171+
}
172+
126173
List<HddsProtos.Node> nodes = scmClient.queryNode(null,
127174
null, HddsProtos.QueryScope.CLUSTER, "");
128175

@@ -165,12 +212,24 @@ private void printDatanodeInfo(DatanodeWithAttributes dna) {
165212
System.out.println("Operational State: " + dna.getOpState());
166213
System.out.println("Health State: " + dna.getHealthState());
167214
System.out.println("Related pipelines:\n" + pipelineListInfo);
215+
216+
if (dna.getUsed() != null && dna.getCapacity() != null && dna.getUsed() >= 0 && dna.getCapacity() > 0) {
217+
System.out.println("Capacity: " + dna.getCapacity());
218+
System.out.println("Used: " + dna.getUsed());
219+
System.out.printf("Percentage Used : %.2f%%%n%n", dna.getPercentUsed());
220+
}
168221
}
169222

170223
private static class DatanodeWithAttributes {
171224
private DatanodeDetails datanodeDetails;
172225
private HddsProtos.NodeOperationalState operationalState;
173226
private HddsProtos.NodeState healthState;
227+
@JsonInclude(JsonInclude.Include.NON_NULL)
228+
private Long used = null;
229+
@JsonInclude(JsonInclude.Include.NON_NULL)
230+
private Long capacity = null;
231+
@JsonInclude(JsonInclude.Include.NON_NULL)
232+
private Double percentUsed = null;
174233

175234
DatanodeWithAttributes(DatanodeDetails dn,
176235
HddsProtos.NodeOperationalState opState,
@@ -180,6 +239,20 @@ private static class DatanodeWithAttributes {
180239
this.healthState = healthState;
181240
}
182241

242+
DatanodeWithAttributes(DatanodeDetails dn,
243+
HddsProtos.NodeOperationalState opState,
244+
HddsProtos.NodeState healthState,
245+
long used,
246+
long capacity,
247+
double percentUsed) {
248+
this.datanodeDetails = dn;
249+
this.operationalState = opState;
250+
this.healthState = healthState;
251+
this.used = used;
252+
this.capacity = capacity;
253+
this.percentUsed = percentUsed;
254+
}
255+
183256
public DatanodeDetails getDatanodeDetails() {
184257
return datanodeDetails;
185258
}
@@ -191,5 +264,17 @@ public HddsProtos.NodeOperationalState getOpState() {
191264
public HddsProtos.NodeState getHealthState() {
192265
return healthState;
193266
}
267+
268+
public Long getUsed() {
269+
return used;
270+
}
271+
272+
public Long getCapacity() {
273+
return capacity;
274+
}
275+
276+
public Double getPercentUsed() {
277+
return percentUsed;
278+
}
194279
}
195280
}

‎hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import java.nio.charset.StandardCharsets;
3434
import java.util.ArrayList;
3535
import java.util.Arrays;
36+
import java.util.Collections;
3637
import java.util.List;
3738
import java.util.UUID;
3839
import java.util.regex.Matcher;
@@ -42,6 +43,8 @@
4243
import org.junit.jupiter.api.AfterEach;
4344
import org.junit.jupiter.api.BeforeEach;
4445
import org.junit.jupiter.api.Test;
46+
import org.junit.jupiter.params.ParameterizedTest;
47+
import org.junit.jupiter.params.provider.CsvSource;
4548
import picocli.CommandLine;
4649

4750
/**
@@ -212,6 +215,111 @@ public void testDataNodeByUuidOutput()
212215
"Expected UUID " + nodes.get(0).getNodeID().getUuid() + " but got: " + uuid);
213216
}
214217

218+
@ParameterizedTest
219+
@CsvSource({
220+
"true, --most-used, descending",
221+
"false, --least-used, ascending"
222+
})
223+
public void testUsedOrderingAndOutput(
224+
boolean mostUsed,
225+
String cliFlag,
226+
String orderDirection) throws Exception {
227+
228+
ScmClient scmClient = mock(ScmClient.class);
229+
List<HddsProtos.DatanodeUsageInfoProto> usageList = new ArrayList<>();
230+
List<HddsProtos.Node> nodeList = getNodeDetails();
231+
232+
for (int i = 0; i < 4; i++) {
233+
long remaining = 1000 - (100 * i);
234+
usageList.add(HddsProtos.DatanodeUsageInfoProto.newBuilder()
235+
.setNode(nodeList.get(i).getNodeID())
236+
.setRemaining(remaining)
237+
.setCapacity(1000)
238+
.build());
239+
240+
when(scmClient.queryNode(UUID.fromString(nodeList.get(i).getNodeID().getUuid())))
241+
.thenReturn(nodeList.get(i));
242+
}
243+
244+
if (mostUsed) {
245+
Collections.reverse(usageList); // For most-used only
246+
}
247+
248+
when(scmClient.getDatanodeUsageInfo(mostUsed, Integer.MAX_VALUE)).thenReturn(usageList);
249+
when(scmClient.listPipelines()).thenReturn(new ArrayList<>());
250+
251+
// ----- JSON output test -----
252+
CommandLine c = new CommandLine(cmd);
253+
c.parseArgs(cliFlag, "--json");
254+
cmd.execute(scmClient);
255+
256+
String jsonOutput = outContent.toString(DEFAULT_ENCODING);
257+
JsonNode root = new ObjectMapper().readTree(jsonOutput);
258+
259+
assertTrue(root.isArray(), "JSON output should be an array");
260+
assertEquals(4, root.size(), "Expected 4 nodes in JSON output");
261+
262+
for (JsonNode node : root) {
263+
assertTrue(node.has("used"), "Missing 'used'");
264+
assertTrue(node.has("capacity"), "Missing 'capacity'");
265+
assertTrue(node.has("percentUsed"), "Missing 'percentUsed'");
266+
}
267+
268+
validateOrdering(root, orderDirection);
269+
270+
outContent.reset();
271+
272+
// ----- Text output test -----
273+
c = new CommandLine(cmd);
274+
c.parseArgs(cliFlag);
275+
cmd.execute(scmClient);
276+
277+
String textOutput = outContent.toString(DEFAULT_ENCODING);
278+
validateOrderingFromTextOutput(textOutput, orderDirection);
279+
}
280+
281+
private void validateOrdering(JsonNode root, String orderDirection) {
282+
for (int i = 0; i < root.size() - 1; i++) {
283+
long usedCurrent = root.get(i).get("used").asLong();
284+
long capacityCurrent = root.get(i).get("capacity").asLong();
285+
long usedNext = root.get(i + 1).get("used").asLong();
286+
long capacityNext = root.get(i + 1).get("capacity").asLong();
287+
double ratio1 = (capacityCurrent == 0) ? 0.0 : (double) usedCurrent / capacityCurrent;
288+
double ratio2 = (capacityNext == 0) ? 0.0 : (double) usedNext / capacityNext;
289+
290+
if ("ascending".equals(orderDirection)) {
291+
assertTrue(ratio1 <= ratio2, "Expected ascending order, got: " + ratio1 + " > " + ratio2);
292+
} else {
293+
assertTrue(ratio1 >= ratio2, "Expected descending order, got: " + ratio1 + " < " + ratio2);
294+
}
295+
}
296+
}
297+
298+
private void validateOrderingFromTextOutput(String output, String orderDirection) {
299+
Pattern usedPattern = Pattern.compile("Used: (\\d+)");
300+
Pattern capacityPattern = Pattern.compile("Capacity: (\\d+)");
301+
Matcher usedMatcher = usedPattern.matcher(output);
302+
Matcher capacityMatcher = capacityPattern.matcher(output);
303+
304+
List<Double> usageRatios = new ArrayList<>();
305+
while (usedMatcher.find() && capacityMatcher.find()) {
306+
long used = Long.parseLong(usedMatcher.group(1));
307+
long capacity = Long.parseLong(capacityMatcher.group(1));
308+
double usage = (capacity == 0) ? 0.0 : (double) used / capacity;
309+
usageRatios.add(usage);
310+
}
311+
312+
for (int i = 0; i < usageRatios.size() - 1; i++) {
313+
double ratio1 = usageRatios.get(i);
314+
double ratio2 = usageRatios.get(i + 1);
315+
if ("ascending".equals(orderDirection)) {
316+
assertTrue(ratio1 <= ratio2, "Expected ascending order, got: " + ratio1 + " > " + ratio2);
317+
} else {
318+
assertTrue(ratio1 >= ratio2, "Expected descending order, got: " + ratio1 + " < " + ratio2);
319+
}
320+
}
321+
}
322+
215323
private List<HddsProtos.Node> getNodeDetails() {
216324
List<HddsProtos.Node> nodes = new ArrayList<>();
217325

0 commit comments

Comments
 (0)