Skip to content

Commit 3c31e24

Browse files
devmadhuudevmadhuu
authored
HDDS-14758. Mismatch in Recon container count between Cluster State and Container Summary (#10074)
1 parent fff83a1 commit 3c31e24

26 files changed

Lines changed: 3810 additions & 224 deletions

File tree

‎hadoop-hdds/common/src/main/resources/ozone-default.xml‎

Lines changed: 25 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3515,24 +3515,6 @@
35153515
OM snapshot.
35163516
</description>
35173517
</property>
3518-
<property>
3519-
<name>ozone.recon.scm.connection.request.timeout</name>
3520-
<value>5s</value>
3521-
<tag>OZONE, RECON, SCM</tag>
3522-
<description>
3523-
Connection request timeout in milliseconds for HTTP call made by Recon to
3524-
request SCM DB snapshot.
3525-
</description>
3526-
</property>
3527-
<property>
3528-
<name>ozone.recon.scm.connection.timeout</name>
3529-
<value>5s</value>
3530-
<tag>OZONE, RECON, SCM</tag>
3531-
<description>
3532-
Connection timeout for HTTP call in milliseconds made by Recon to request
3533-
SCM snapshot.
3534-
</description>
3535-
</property>
35363518
<property>
35373519
<name>ozone.recon.scmclient.rpc.timeout</name>
35383520
<value>1m</value>
@@ -3625,11 +3607,11 @@
36253607
</property>
36263608
<property>
36273609
<name>ozone.recon.scm.container.threshold</name>
3628-
<value>100</value>
3610+
<value>1000000</value>
36293611
<tag>OZONE, RECON, SCM</tag>
36303612
<description>
3631-
Threshold value for the difference in number of containers
3632-
in SCM and RECON.
3613+
Container-count drift threshold used during initial SCM DB setup to decide
3614+
whether Recon should refresh from an SCM snapshot before serving requests.
36333615
</description>
36343616
</property>
36353617
<property>
@@ -4608,20 +4590,34 @@
46084590
</property>
46094591

46104592
<property>
4611-
<name>ozone.recon.scm.snapshot.task.initial.delay</name>
4593+
<name>ozone.recon.scm.container.sync.task.initial.delay</name>
46124594
<value>1m</value>
4613-
<tag>OZONE, MANAGEMENT, RECON</tag>
4595+
<tag>OZONE, MANAGEMENT, RECON, SCM</tag>
46144596
<description>
4615-
Initial delay in MINUTES by Recon to request SCM DB Snapshot.
4597+
Initial delay before Recon starts the incremental SCM container sync task.
4598+
This gives Recon startup enough time to initialize the SCM DB before the
4599+
first incremental sync runs.
46164600
</description>
46174601
</property>
4618-
46194602
<property>
4620-
<name>ozone.recon.scm.snapshot.task.interval.delay</name>
4621-
<value>24h</value>
4622-
<tag>OZONE, MANAGEMENT, RECON</tag>
4603+
<name>ozone.recon.scm.container.sync.task.interval.delay</name>
4604+
<value>6h</value>
4605+
<tag>OZONE, MANAGEMENT, RECON, SCM</tag>
4606+
<description>
4607+
Interval between incremental SCM container sync runs in Recon. Each cycle
4608+
evaluates drift between SCM and Recon and either runs the targeted
4609+
multi-pass sync or takes no action.
4610+
</description>
4611+
</property>
4612+
<property>
4613+
<name>ozone.recon.scm.deleted.container.check.batch.size</name>
4614+
<value>1000000</value>
4615+
<tag>OZONE, RECON, SCM, PERFORMANCE</tag>
46234616
<description>
4624-
Interval in MINUTES by Recon to request SCM DB Snapshot.
4617+
Maximum number of SCM DELETED containers fetched per page during targeted
4618+
Recon container sync. DELETED sync reads SCM's DELETED list and reconciles
4619+
Recon forward to DELETED; the configured value is capped by the Hadoop IPC
4620+
message-size limit.
46254621
</description>
46264622
</property>
46274623
<property>

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1248,10 +1248,12 @@ public long getContainerCount() throws IOException {
12481248
public long getContainerCount(HddsProtos.LifeCycleState state)
12491249
throws IOException {
12501250
GetContainerCountRequestProto request =
1251-
GetContainerCountRequestProto.newBuilder().build();
1251+
GetContainerCountRequestProto.newBuilder()
1252+
.setState(state)
1253+
.build();
12521254

12531255
GetContainerCountResponseProto response =
1254-
submitRequest(Type.GetClosedContainerCount,
1256+
submitRequest(Type.GetContainerCount,
12551257
builder -> builder.setGetContainerCountRequest(request))
12561258
.getGetContainerCountResponse();
12571259
return response.getContainerCount();

‎hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,7 @@ message GetPipelineResponseProto {
470470
}
471471

472472
message GetContainerCountRequestProto {
473+
optional LifeCycleState state = 1;
473474
}
474475

475476
message GetContainerCountResponseProto {

‎hadoop-hdds/interface-admin/src/main/resources/proto.lock‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1544,7 +1544,15 @@
15441544
]
15451545
},
15461546
{
1547-
"name": "GetContainerCountRequestProto"
1547+
"name": "GetContainerCountRequestProto",
1548+
"fields": [
1549+
{
1550+
"id": 1,
1551+
"name": "state",
1552+
"type": "LifeCycleState",
1553+
"optional": true
1554+
}
1555+
]
15481556
},
15491557
{
15501558
"name": "GetContainerCountResponseProto",
@@ -2358,4 +2366,4 @@
23582366
}
23592367
}
23602368
]
2361-
}
2369+
}

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,24 @@ List<ContainerInfo> getContainers(ContainerID startID,
124124
*/
125125
int getContainerStateCount(LifeCycleState state);
126126

127+
/**
128+
* Returns the total number of containers across all lifecycle states.
129+
*
130+
* <p>Default implementation sums {@link #getContainerStateCount(LifeCycleState)}
131+
* for every {@link LifeCycleState} value — each call is O(1), so the total
132+
* is O(number of states) rather than O(total containers). Automatically
133+
* includes any new states added to the enum in the future.
134+
*
135+
* @return total container count
136+
*/
137+
default long getTotalContainerCount() {
138+
long total = 0;
139+
for (LifeCycleState state : LifeCycleState.values()) {
140+
total += getContainerStateCount(state);
141+
}
142+
return total;
143+
}
144+
127145
/**
128146
* Returns true if the container exist, false otherwise.
129147
* @param id Container ID

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,16 @@ private void initialize() throws IOException {
241241
Objects.requireNonNull(container, "container == null");
242242
containers.addContainer(container);
243243
if (container.getState() == LifeCycleState.OPEN) {
244+
if (container.getPipelineID() == null) {
245+
// This can happen in Recon when SCM returns an OPEN container after
246+
// its pipeline metadata has already been cleaned up. Keep the
247+
// container record, but skip pipeline registration because there is
248+
// no pipeline ID to look up.
249+
LOG.warn("Found container {} which is in OPEN state without a "
250+
+ "pipeline ID. Skipping pipeline registration during SCM "
251+
+ "start.", container);
252+
continue;
253+
}
244254
try {
245255
pipelineManager.addContainerToPipelineSCMStart(
246256
container.getPipelineID(), container.containerID());
@@ -261,8 +271,12 @@ private void initialize() throws IOException {
261271
getContainerStateChangeActions() {
262272
final Map<LifeCycleEvent, CheckedConsumer<ContainerInfo, IOException>>
263273
actions = new EnumMap<>(LifeCycleEvent.class);
264-
actions.put(FINALIZE, info -> pipelineManager
265-
.removeContainerFromPipeline(info.getPipelineID(), info.containerID()));
274+
actions.put(FINALIZE, info -> {
275+
if (info.getPipelineID() != null) {
276+
pipelineManager.removeContainerFromPipeline(
277+
info.getPipelineID(), info.containerID());
278+
}
279+
});
266280
return actions;
267281
}
268282

@@ -335,12 +349,23 @@ public void addContainer(final ContainerInfoProto containerInfo)
335349
transactionBuffer.addToBuffer(containerStore,
336350
containerID, container);
337351
containers.addContainer(container);
338-
if (pipelineManager.containsPipeline(pipelineID)) {
352+
if (pipelineID != null && pipelineManager.containsPipeline(pipelineID)) {
339353
pipelineManager.addContainerToPipeline(pipelineID, containerID);
340354
} else if (containerInfo.getState().
341355
equals(LifeCycleState.OPEN)) {
342-
// Pipeline should exist, but not
343-
throw new PipelineNotFoundException();
356+
if (pipelineID != null) {
357+
// The container names a pipeline, but that pipeline is not in
358+
// the pipeline manager. Preserve the existing failure path for
359+
// this inconsistent OPEN container state.
360+
throw new PipelineNotFoundException();
361+
}
362+
// There is no pipeline ID to look up or register. This can happen
363+
// on Recon sync paths when SCM returns an OPEN container after its
364+
// pipeline metadata has already been cleaned up. Keep the
365+
// container record so Recon does not miss it permanently, but skip
366+
// pipeline tracking until later reports/syncs advance the state.
367+
LOG.warn("Adding OPEN container {} without pipeline tracking "
368+
+ "because its pipeline ID is null.", containerID);
344369
}
345370
//recon may receive report of closed container,
346371
// no corresponding Pipeline can be synced for scm.

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1350,9 +1350,12 @@ public DatanodeUsageInfoResponseProto getDatanodeUsageInfo(
13501350
public GetContainerCountResponseProto getContainerCount(
13511351
StorageContainerLocationProtocolProtos.GetContainerCountRequestProto
13521352
request) throws IOException {
1353+
long containerCount = request.hasState()
1354+
? impl.getContainerCount(request.getState())
1355+
: impl.getContainerCount();
13531356

13541357
return GetContainerCountResponseProto.newBuilder()
1355-
.setContainerCount(impl.getContainerCount())
1358+
.setContainerCount(containerCount)
13561359
.build();
13571360
}
13581361

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -417,8 +417,13 @@ public List<ContainerWithPipeline> getExistContainerWithPipelinesInBatch(
417417
ContainerWithPipeline cp = getContainerWithPipelineCommon(containerID);
418418
cpList.add(cp);
419419
} catch (IOException ex) {
420-
//not found , just go ahead
421-
LOG.error("Container with common pipeline not found: {}", ex);
420+
// ContainerWithPipeline.pipeline is required in the protobuf response,
421+
// so this RPC cannot return container metadata with a null pipeline.
422+
// Keep the "exist" semantics by excluding only this container from the
423+
// batch result instead of failing the entire request.
424+
LOG.warn("Container {} exists but its pipeline could not be resolved; "
425+
+ "excluding it from getExistContainerWithPipelinesInBatch result. "
426+
+ "Cause: {}", containerID, ex.getMessage());
422427
}
423428
}
424429
return cpList;
@@ -1530,7 +1535,7 @@ public Token<?> getContainerToken(ContainerID containerID)
15301535
@Override
15311536
public long getContainerCount() throws IOException {
15321537
try {
1533-
long count = scm.getContainerManager().getContainers().size();
1538+
long count = scm.getContainerManager().getTotalContainerCount();
15341539
AUDIT.logReadSuccess(buildAuditMessageForSuccess(
15351540
SCMAction.GET_CONTAINER_COUNT, null));
15361541
return count;

‎hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,12 @@
2020
import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
2121
import static org.apache.hadoop.hdds.scm.HddsTestUtils.getContainer;
2222
import static org.apache.hadoop.hdds.scm.HddsTestUtils.getECContainer;
23+
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
2324
import static org.junit.jupiter.api.Assertions.assertEquals;
2425
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
2526
import static org.junit.jupiter.api.Assertions.fail;
2627
import static org.mockito.ArgumentMatchers.eq;
28+
import static org.mockito.ArgumentMatchers.isNull;
2729
import static org.mockito.Mockito.any;
2830
import static org.mockito.Mockito.doAnswer;
2931
import static org.mockito.Mockito.mock;
@@ -85,6 +87,7 @@ public class TestContainerStateManager {
8587
private File testDir;
8688
private DBStore dbStore;
8789
private Pipeline pipeline;
90+
private PipelineManager pipelineManager;
8891
private MockNodeManager nodeManager;
8992
private ContainerManager containerManager;
9093
private SCMContext scmContext;
@@ -96,7 +99,7 @@ public void init() throws IOException, TimeoutException, InvalidStateTransitionE
9699
SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true);
97100
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
98101
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
99-
PipelineManager pipelineManager = mock(PipelineManager.class);
102+
pipelineManager = mock(PipelineManager.class);
100103
pipeline = Pipeline.newBuilder().setState(Pipeline.PipelineState.CLOSED)
101104
.setId(PipelineID.randomId())
102105
.setReplicationConfig(StandaloneReplicationConfig.getInstance(
@@ -402,6 +405,31 @@ public void testGetContainerIDs() throws IOException {
402405
HddsProtos.LifeCycleState.CLOSED, ContainerID.MIN, 10).size());
403406
}
404407

408+
@Test
409+
public void testReinitializeWithOpenContainerWithoutPipelineID()
410+
throws Exception {
411+
ContainerID containerID = ContainerID.valueOf(3L);
412+
ContainerInfo openContainerInfo = new ContainerInfo.Builder()
413+
.setContainerID(containerID.getId())
414+
.setState(HddsProtos.LifeCycleState.OPEN)
415+
.setSequenceId(100L)
416+
.setOwner("scm")
417+
.setReplicationConfig(
418+
RatisReplicationConfig
419+
.getInstance(ReplicationFactor.THREE))
420+
.build();
421+
422+
SCMDBDefinition.CONTAINERS.getTable(dbStore)
423+
.put(containerID, openContainerInfo);
424+
425+
assertDoesNotThrow(() -> containerStateManager.reinitialize(
426+
SCMDBDefinition.CONTAINERS.getTable(dbStore)));
427+
assertEquals(HddsProtos.LifeCycleState.OPEN,
428+
containerStateManager.getContainer(containerID).getState());
429+
verify(pipelineManager, times(0))
430+
.addContainerToPipelineSCMStart(isNull(), eq(containerID));
431+
}
432+
405433
@Test
406434
public void testSequenceIdOnStateUpdate() throws Exception {
407435
ContainerID containerID = ContainerID.valueOf(3L);

‎hadoop-ozone/dist/src/main/compose/ozone/docker-config‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*
2323
OZONE-SITE.XML_ozone.om.address=om
2424
OZONE-SITE.XML_ozone.om.http-address=om:9874
2525
OZONE-SITE.XML_ozone.scm.http-address=scm:9876
26-
OZONE-SITE.XML_ozone.scm.container.size=1GB
26+
OZONE-SITE.XML_ozone.scm.container.size=100MB
2727
OZONE-SITE.XML_ozone.scm.block.size=1MB
2828
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
2929
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
@@ -43,6 +43,15 @@ OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
4343
OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
4444
OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
4545
OZONE-SITE.XML_ozone.recon.om.snapshot.task.initial.delay=20s
46+
OZONE-SITE.XML_ozone.recon.scm.container.sync.task.initial.delay=30s
47+
OZONE-SITE.XML_ozone.recon.scm.container.sync.task.interval.delay=2m
48+
OZONE-SITE.XML_ozone.recon.scm.snapshot.task.interval.delay=30m
49+
OZONE-SITE.XML_ozone.recon.scm.container.threshold=20
50+
OZONE-SITE.XML_ozone.recon.scm.per.state.drift.threshold=1
51+
OZONE-SITE.XML_ozone.recon.scm.deleted.container.check.batch.size=50
52+
OZONE-SITE.XML_hdds.heartbeat.recon.interval=5m
53+
OZONE-SITE.XML_hdds.container.report.interval=1h
54+
OZONE-SITE.XML_hdds.pipeline.report.interval=5m
4655
OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
4756
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
4857
OZONE-SITE.XML_hdds.container.report.interval=60s
@@ -51,8 +60,8 @@ OZONE-SITE.XML_ozone.scm.dead.node.interval=45s
5160
OZONE-SITE.XML_hdds.heartbeat.interval=5s
5261
OZONE-SITE.XML_ozone.scm.close.container.wait.duration=5s
5362
OZONE-SITE.XML_hdds.scm.replication.thread.interval=15s
54-
OZONE-SITE.XML_hdds.scm.replication.under.replicated.interval=5s
55-
OZONE-SITE.XML_hdds.scm.replication.over.replicated.interval=5s
63+
OZONE-SITE.XML_hdds.scm.replication.under.replicated.interval=10s
64+
OZONE-SITE.XML_hdds.scm.replication.over.replicated.interval=2m
5665
OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=30s
5766
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
5867

0 commit comments

Comments
 (0)