Skip to content

Commit c874233

Browse files
committed
Rebase and update fix
1 parent 79d76b1 commit c874233

15 files changed

Lines changed: 53 additions & 69 deletions

File tree

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ private ContainerInfo createContainer(Pipeline pipeline, String owner)
252252
private ContainerInfo allocateContainer(final Pipeline pipeline,
253253
final String owner)
254254
throws IOException {
255-
if (!pipelineManager.hasEnoughSpace(pipeline, maxContainerSize)) {
255+
if (!pipelineManager.hasEnoughSpace(pipeline)) {
256256
LOG.debug("Cannot allocate a new container because pipeline {} does not have the required space {}.",
257257
pipeline, maxContainerSize);
258258
return null;
@@ -287,8 +287,6 @@ private ContainerInfo allocateContainer(final Pipeline pipeline,
287287
scmContainerManagerMetrics.incNumSuccessfulCreateContainers();
288288
// Record pending allocation - tracks containers scheduled but not yet written
289289
pendingContainerTracker.recordPendingAllocation(pipeline, containerID);
290-
LOG.debug("Allocated container {} on pipeline {}. Recorded as pending on {} DataNodes",
291-
containerID, pipeline.getId(), pipeline.getNodes().size());
292290

293291
return containerStateManager.getContainer(containerID);
294292
}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,6 @@ public void onMessage(final DatanodeDetails datanodeDetails,
9797
* action.
9898
*/
9999
LOG.info("A dead datanode is detected. {}", datanodeDetails);
100-
PendingContainerTracker pending = nodeManager.getPendingContainerTracker();
101-
if (pending != null) {
102-
pending.clearPendingForDatanode(datanodeDetails);
103-
}
104100
closeContainers(datanodeDetails, publisher);
105101
destroyPipelines(datanodeDetails);
106102

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -432,5 +432,5 @@ default void removeNode(DatanodeDetails datanodeDetails) throws NodeNotFoundExce
432432
* True if the node can accept another container of the given size, accounting for
433433
* {@link #getPendingContainerTracker()}.
434434
*/
435-
boolean hasSpaceForNewContainerAllocation(DatanodeDetails node, long containerSize);
435+
boolean hasSpaceForNewContainerAllocation(DatanodeDetails node);
436436
}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -251,19 +251,16 @@ public PendingContainerTracker getPendingContainerTracker() {
251251
* SCM pending allocations.
252252
*/
253253
@Override
254-
public boolean hasSpaceForNewContainerAllocation(DatanodeDetails node,
255-
long containerSize) {
256-
if (node == null) {
257-
return false;
258-
}
254+
public boolean hasSpaceForNewContainerAllocation(DatanodeDetails node) {
255+
Objects.requireNonNull(node, "node==null");
259256
try {
260257
DatanodeInfo datanodeInfo = getDatanodeInfo(node);
261258
if (datanodeInfo == null) {
262259
LOG.warn("DatanodeInfo not found for node {}", node.getUuidString());
263260
return false;
264261
}
265262
return pendingContainerTracker.hasEffectiveAllocatableSpaceForNewContainer(
266-
node, datanodeInfo, containerSize);
263+
node, datanodeInfo);
267264
} catch (Exception e) {
268265
LOG.warn("Error checking allocatable space for node {}", node.getUuidString(), e);
269266
return false;
@@ -1145,8 +1142,6 @@ private SCMNodeStat getNodeStatInternal(DatanodeDetails datanodeDetails) {
11451142
freeSpaceToSpare += reportProto.getFreeSpaceToSpare();
11461143
reserved += reportProto.getReserved();
11471144
}
1148-
// SCM-side pending container allocations (not yet in DN reports) count toward committed.
1149-
committed += pendingContainerTracker.getPendingAllocationSize(datanodeDetails);
11501145
return new SCMNodeStat(capacity, used, remaining, committed,
11511146
freeSpaceToSpare, reserved);
11521147
} catch (NodeNotFoundException e) {

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,14 @@ void incNumPendingContainersRemoved() {
137137
numPendingContainersRemoved.incr();
138138
}
139139

140+
public long getNumPendingContainersAdded() {
141+
return numPendingContainersAdded.value();
142+
}
143+
144+
public long getNumPendingContainersRemoved() {
145+
return numPendingContainersRemoved.value();
146+
}
147+
140148
void incNumSkippedFullNodeContainerAllocation() {
141149
numSkippedFullNodeContainerAllocation.incr();
142150
}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,6 @@ public StaleNodeHandler(NodeManager nodeManager,
4646
@Override
4747
public void onMessage(DatanodeDetails datanodeDetails,
4848
EventPublisher publisher) {
49-
PendingContainerTracker pending = nodeManager.getPendingContainerTracker();
50-
if (pending != null) {
51-
pending.clearPendingForDatanode(datanodeDetails);
52-
}
5349
Set<PipelineID> pipelineIds =
5450
nodeManager.getPipelines(datanodeDetails);
5551
LOG.info("Datanode {} moved to stale state. Finalizing its pipelines {}",

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -213,13 +213,12 @@ void reinitialize(Table<PipelineID, Pipeline> pipelineStore)
213213
void releaseWriteLock();
214214

215215
/**
216-
* Checks whether all Datanodes in the specified pipeline have greater than the specified space, containerSize.
216+
* Checks whether all Datanodes in the specified pipeline have greater than the container size.
217217
* @param pipeline pipeline to check
218-
* @param containerSize the required amount of space
219-
* @return false if all the volumes on any Datanode in the pipeline have space less than equal to the specified
218+
* @return false if all the volumes on any Datanode in the pipeline have space less than equal to the
220219
* containerSize, otherwise true
221220
*/
222-
boolean hasEnoughSpace(Pipeline pipeline, long containerSize);
221+
boolean hasEnoughSpace(Pipeline pipeline);
223222

224223
int openContainerLimit(List<DatanodeDetails> datanodes);
225224

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -634,9 +634,9 @@ private boolean isOpenWithUnregisteredNodes(Pipeline pipeline) {
634634
}
635635

636636
@Override
637-
public boolean hasEnoughSpace(Pipeline pipeline, long containerSize) {
637+
public boolean hasEnoughSpace(Pipeline pipeline) {
638638
for (DatanodeDetails node : pipeline.getNodes()) {
639-
if (!nodeManager.hasSpaceForNewContainerAllocation(node, containerSize)) {
639+
if (!nodeManager.hasSpaceForNewContainerAllocation(node)) {
640640
return false;
641641
}
642642
}

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -945,20 +945,21 @@ public void setNumHealthyVolumes(int value) {
945945

946946
@Override
947947
public PendingContainerTracker getPendingContainerTracker() {
948+
int rollIntervalMs = 5 * 60 * 1000;
948949
if (pendingContainerTracker == null) {
949-
pendingContainerTracker = new PendingContainerTracker(5L * 1024 * 1024 * 1024);
950+
pendingContainerTracker = new PendingContainerTracker(5L * 1024 * 1024 * 1024, rollIntervalMs, null);
950951
}
951952
return pendingContainerTracker;
952953
}
953954

954955
@Override
955-
public boolean hasSpaceForNewContainerAllocation(DatanodeDetails node, long containerSize) {
956+
public boolean hasSpaceForNewContainerAllocation(DatanodeDetails node) {
956957
DatanodeInfo info = getDatanodeInfo(node);
957958
if (info == null) {
958959
return false;
959960
}
960961
return getPendingContainerTracker()
961-
.hasEffectiveAllocatableSpaceForNewContainer(node, info, containerSize);
962+
.hasEffectiveAllocatableSpaceForNewContainer(node, info);
962963
}
963964

964965
/**

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -439,14 +439,15 @@ public Boolean isNodeRegistered(DatanodeDetails datanodeDetails) {
439439

440440
@Override
441441
public PendingContainerTracker getPendingContainerTracker() {
442+
int rollIntervalMs = 5 * 60 * 1000;
442443
if (pendingContainerTracker == null) {
443-
pendingContainerTracker = new PendingContainerTracker(5L * 1024 * 1024 * 1024);
444+
pendingContainerTracker = new PendingContainerTracker(5L * 1024 * 1024 * 1024, rollIntervalMs, null);
444445
}
445446
return pendingContainerTracker;
446447
}
447448

448449
@Override
449-
public boolean hasSpaceForNewContainerAllocation(DatanodeDetails node, long containerSize) {
450+
public boolean hasSpaceForNewContainerAllocation(DatanodeDetails node) {
450451
return true;
451452
}
452453

0 commit comments

Comments
 (0)