Skip to content

Commit c524015

Browse files
CRZbulabulaCRZbulabula
authored
Clarify RemoveDataNode single-replica error and add diagnostics for the no-available-RegionGroup race (#17878)
1 parent 86987e4 commit c524015

7 files changed

Lines changed: 188 additions & 20 deletions

File tree

‎integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveLastDataNodeIT.java‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.confignode.it.removedatanode;
21+
22+
import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
23+
import org.apache.iotdb.consensus.ConsensusFactory;
24+
import org.apache.iotdb.it.env.EnvFactory;
25+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
26+
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
27+
import org.apache.iotdb.jdbc.IoTDBSQLException;
28+
29+
import org.junit.After;
30+
import org.junit.Assert;
31+
import org.junit.Before;
32+
import org.junit.Test;
33+
import org.junit.experimental.categories.Category;
34+
import org.junit.runner.RunWith;
35+
36+
import java.sql.Connection;
37+
import java.sql.ResultSet;
38+
import java.sql.Statement;
39+
import java.util.HashSet;
40+
import java.util.Set;
41+
42+
import static org.apache.iotdb.confignode.it.removedatanode.IoTDBRemoveDataNodeUtils.generateRemoveString;
43+
import static org.apache.iotdb.confignode.it.removedatanode.IoTDBRemoveDataNodeUtils.selectRemoveDataNodes;
44+
import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly;
45+
46+
/**
47+
* Removing the last DataNode of a single-replica cluster must be rejected. This only needs a 1C1D
48+
* cluster, so it lives in the 1C1D (LocalStandaloneIT) suite, separate from the multi-DataNode
49+
* removal tests in {@link IoTDBRemoveDataNodeNormalIT}.
50+
*/
51+
@Category({LocalStandaloneIT.class})
52+
@RunWith(IoTDBTestRunner.class)
53+
public class IoTDBRemoveLastDataNodeIT {
54+
55+
private static final String SHOW_DATANODES = "show datanodes";
56+
57+
@Before
58+
public void setUp() throws Exception {
59+
EnvFactory.getEnv()
60+
.getConfig()
61+
.getCommonConfig()
62+
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
63+
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
64+
}
65+
66+
@After
67+
public void tearDown() throws InterruptedException {
68+
EnvFactory.getEnv().cleanClusterEnvironment();
69+
}
70+
71+
@Test
72+
public void failWhenRemovingLastSingleReplicaDataNodeUseSQL() throws Exception {
73+
// With a single replica (schema_replication_factor and data_replication_factor are both 1),
74+
// removing DataNodes is still supported as long as more than one DataNode remains, but the last
75+
// remaining DataNode cannot be removed because there is nowhere to migrate its regions to.
76+
// Here we set up 1C1D with single replica and try to remove the only DataNode, which must fail
77+
// because removing it would leave the cluster with no DataNode.
78+
EnvFactory.getEnv()
79+
.getConfig()
80+
.getCommonConfig()
81+
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
82+
.setSchemaReplicationFactor(1)
83+
.setDataReplicationFactor(1)
84+
.setDefaultDataRegionGroupNumPerDatabase(1);
85+
EnvFactory.getEnv().initClusterEnvironment(1, 1);
86+
87+
try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection());
88+
final Statement statement = makeItCloseQuietly(connection.createStatement());
89+
final ResultSet resultSet = statement.executeQuery(SHOW_DATANODES)) {
90+
final Set<Integer> allDataNodeId = new HashSet<>();
91+
while (resultSet.next()) {
92+
allDataNodeId.add(resultSet.getInt(ColumnHeaderConstant.NODE_ID));
93+
}
94+
95+
final String removeDataNodeSQL =
96+
generateRemoveString(selectRemoveDataNodes(allDataNodeId, 1));
97+
try {
98+
statement.execute(removeDataNodeSQL);
99+
Assert.fail(
100+
"Remove DataNode should fail when it would leave no DataNode under single replica");
101+
} catch (final IoTDBSQLException e) {
102+
// The unified rejection message reports the gap and, for a single replica, appends the
103+
// "at least one DataNode must always remain" hint.
104+
Assert.assertTrue(e.getMessage(), e.getMessage().contains("Cannot remove"));
105+
Assert.assertTrue(e.getMessage(), e.getMessage().contains("single replica"));
106+
Assert.assertFalse(
107+
e.getMessage(), e.getMessage().contains("Failed to remove all requested data nodes"));
108+
}
109+
}
110+
}
111+
}

‎iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,11 @@ public final class ProcedureMessages {
445445
"Failed to push topic meta to dataNodes, details: %s";
446446
public static final String FAILED_TO_REMOVE_DATA_NODE_BECAUSE_IT_IS_NOT_IN =
447447
"Failed to remove data node {} because it is not in running and the configuration of cluster is one replication";
448+
449+
public static final String FAILED_TO_REMOVE_DATA_NODE_WOULD_LEAVE_TOO_FEW =
450+
"Cannot remove %d DataNode(s): the cluster has %d available DataNode(s) and must retain at least %d of them (max(schema_replication_factor=%d, data_replication_factor=%d)) so that every region keeps enough replicas, but this request would leave only %d.";
451+
public static final String FAILED_TO_REMOVE_DATA_NODE_SINGLE_REPLICA_HINT =
452+
" With a single replica there is nowhere to migrate regions to, so at least one DataNode must always remain.";
448453
public static final String FAILED_TO_ROLLBACK_ALTER_PIPE_DETAILS_METADATA_WILL_BE_SYNCHRONIZED =
449454
"Failed to rollback alter pipe {}, details: {}, metadata will be synchronized later.";
450455
public static final String FAILED_TO_ROLLBACK_COMMIT_SET_TEMPLATE_ON_PATH_DUE_TO =

‎iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,11 @@ public final class ProcedureMessages {
445445
"Failed to push topic meta to dataNodes, details: %s";
446446
public static final String FAILED_TO_REMOVE_DATA_NODE_BECAUSE_IT_IS_NOT_IN =
447447
"Failed to remove data node {} because it is not in running and the configuration of cluster is one replication";
448+
449+
public static final String FAILED_TO_REMOVE_DATA_NODE_WOULD_LEAVE_TOO_FEW =
450+
"无法移除 %d 个 DataNode：集群当前有 %d 个可用 DataNode，且至少需保留 %d 个（max(schema_replication_factor=%d, data_replication_factor=%d)），以保证每个 Region 仍有足够的副本；但本次请求执行后将只剩 %d 个。";
451+
public static final String FAILED_TO_REMOVE_DATA_NODE_SINGLE_REPLICA_HINT =
452+
" 单副本下没有其它节点可供迁移 Region，因此必须始终保留至少一个 DataNode。";
448453
public static final String FAILED_TO_ROLLBACK_ALTER_PIPE_DETAILS_METADATA_WILL_BE_SYNCHRONIZED =
449454
"Failed to rollback alter pipe {}, details: {}, metadata will be synchronized later.";
450455
public static final String FAILED_TO_ROLLBACK_COMMIT_SET_TEMPLATE_ON_PATH_DUE_TO =

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@
108108
import java.util.Collections;
109109
import java.util.HashMap;
110110
import java.util.HashSet;
111+
import java.util.LinkedHashMap;
111112
import java.util.LinkedList;
112113
import java.util.List;
113114
import java.util.Map;
@@ -993,6 +994,23 @@ public List<Pair<Long, TConsensusGroupId>> getSortedRegionGroupSlotsCounter(
993994
}
994995

995996
if (result.isEmpty()) {
997+
// Diagnostic for the intermittent "no available RegionGroup" CI failures: dump every
998+
// RegionGroup visible in PartitionInfo for this Database together with its LoadCache status.
999+
// This pinpoints whether PartitionInfo simply has no RegionGroup yet (newly created
1000+
// RegionGroup not exposed) or it has some but all of them are currently Disabled.
1001+
// Only logged on the failure path right before throwing, so it never floods the log.
1002+
final Map<TConsensusGroupId, RegionGroupStatus> visibleRegionGroupStatusMap =
1003+
new LinkedHashMap<>();
1004+
regionGroupSlotsCounter.forEach(
1005+
slotsCounter ->
1006+
visibleRegionGroupStatusMap.put(
1007+
slotsCounter.getRight(),
1008+
getLoadManager().getRegionGroupStatus(slotsCounter.getRight())));
1009+
LOGGER.warn(
1010+
"No available {} RegionGroup for Database: {}. RegionGroups visible in PartitionInfo and their LoadCache status: {}",
1011+
type,
1012+
database,
1013+
visibleRegionGroupStatusMap);
9961014
throw new NoAvailableRegionGroupException(type, Collections.singletonList(database));
9971015
}
9981016

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -511,12 +511,15 @@ public List<TSStatus> notifyRegionMigrationToAllDataNodes(
511511
return clientHandler.getResponseList();
512512
}
513513

514-
public void persistRegionGroup(CreateRegionGroupsPlan createRegionGroupsPlan) {
514+
public TSStatus persistRegionGroup(CreateRegionGroupsPlan createRegionGroupsPlan) {
515515
// Persist the allocation result
516516
try {
517-
getConsensusManager().write(createRegionGroupsPlan);
517+
return getConsensusManager().write(createRegionGroupsPlan);
518518
} catch (ConsensusException e) {
519519
LOG.warn("Failed in the write API executing the consensus layer due to: ", e);
520+
return new TSStatus(TSStatusCode.CREATE_REGION_ERROR.getStatusCode())
521+
.setMessage(
522+
"Failed to persist RegionGroup allocation in the consensus layer: " + e.getMessage());
520523
}
521524
}
522525

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java‎

Lines changed: 35 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -567,19 +567,25 @@ public TSStatus checkRegionReplication(RemoveDataNodePlan removeDataNodePlan) {
567567
// when the configuration is one replication, it will be failed if the data node is not in
568568
// running state.
569569
if (CONF.getSchemaReplicationFactor() == 1 || CONF.getDataReplicationFactor() == 1) {
570-
for (TDataNodeLocation dataNodeLocation : removedDataNodes) {
571-
// check whether removed data node is in running state
572-
if (!NodeStatus.Running.equals(
573-
configManager.getLoadManager().getNodeStatus(dataNodeLocation.getDataNodeId()))) {
574-
removedDataNodes.remove(dataNodeLocation);
575-
LOGGER.error(
576-
ProcedureMessages.FAILED_TO_REMOVE_DATA_NODE_BECAUSE_IT_IS_NOT_IN, dataNodeLocation);
577-
}
578-
if (removedDataNodes.isEmpty()) {
579-
status.setCode(TSStatusCode.NO_ENOUGH_DATANODE.getStatusCode());
580-
status.setMessage(ProcedureMessages.FAILED_TO_REMOVE_ALL_REQUESTED_DATA_NODES);
581-
return status;
582-
}
570+
final List<TDataNodeLocation> notRunningDataNodes =
571+
removedDataNodes.stream()
572+
.filter(
573+
dataNodeLocation ->
574+
!NodeStatus.Running.equals(
575+
configManager
576+
.getLoadManager()
577+
.getNodeStatus(dataNodeLocation.getDataNodeId())))
578+
.collect(Collectors.toList());
579+
notRunningDataNodes.forEach(
580+
dataNodeLocation ->
581+
LOGGER.error(
582+
ProcedureMessages.FAILED_TO_REMOVE_DATA_NODE_BECAUSE_IT_IS_NOT_IN,
583+
dataNodeLocation));
584+
removedDataNodes.removeAll(notRunningDataNodes);
585+
if (removedDataNodes.isEmpty()) {
586+
status.setCode(TSStatusCode.NO_ENOUGH_DATANODE.getStatusCode());
587+
status.setMessage(ProcedureMessages.FAILED_TO_REMOVE_ALL_REQUESTED_DATA_NODES);
588+
return status;
583589
}
584590
}
585591

@@ -593,13 +599,25 @@ public TSStatus checkRegionReplication(RemoveDataNodePlan removeDataNodePlan) {
593599
.count();
594600
if (availableDatanodeSize - removedDataNodeSize < NodeInfo.getMinimumDataNode()) {
595601
status.setCode(TSStatusCode.NO_ENOUGH_DATANODE.getStatusCode());
596-
status.setMessage(
602+
// Report the concrete numbers so operators can see the gap: how many DataNodes are being
603+
// removed, how many are available, the minimum that must remain (the larger of the schema and
604+
// data replication factors) and how many would be left.
605+
String message =
597606
String.format(
598-
"Can't remove datanode due to the limit of replication factor, "
599-
+ "availableDataNodeSize: %s, maxReplicaFactor: %s, max allowed removed Data Node size is: %s",
607+
ProcedureMessages.FAILED_TO_REMOVE_DATA_NODE_WOULD_LEAVE_TOO_FEW,
608+
removedDataNodeSize,
600609
availableDatanodeSize,
601610
NodeInfo.getMinimumDataNode(),
602-
(availableDatanodeSize - NodeInfo.getMinimumDataNode())));
611+
CONF.getSchemaReplicationFactor(),
612+
CONF.getDataReplicationFactor(),
613+
availableDatanodeSize - removedDataNodeSize);
614+
if (NodeInfo.getMinimumDataNode() == 1) {
615+
// With a single replica (schema_replication_factor and data_replication_factor are both 1)
616+
// the only copy of each region lives on one DataNode, so at least one DataNode must always
617+
// remain: there is nowhere to migrate its regions to.
618+
message += ProcedureMessages.FAILED_TO_REMOVE_DATA_NODE_SINGLE_REPLICA_HINT;
619+
}
620+
status.setMessage(message);
603621
}
604622
return status;
605623
}

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@
2323
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
2424
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
2525
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
26+
import org.apache.iotdb.common.rpc.thrift.TSStatus;
2627
import org.apache.iotdb.commons.cluster.RegionStatus;
28+
import org.apache.iotdb.commons.exception.IoTDBException;
2729
import org.apache.iotdb.commons.utils.TestOnly;
2830
import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils;
2931
import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
@@ -36,10 +38,12 @@
3638
import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionCreateTask;
3739
import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionDeleteTask;
3840
import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
41+
import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
3942
import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure;
4043
import org.apache.iotdb.confignode.procedure.state.CreateRegionGroupsState;
4144
import org.apache.iotdb.confignode.procedure.store.ProcedureType;
4245
import org.apache.iotdb.consensus.exception.ConsensusException;
46+
import org.apache.iotdb.rpc.TSStatusCode;
4347

4448
import org.slf4j.Logger;
4549
import org.slf4j.LoggerFactory;
@@ -173,7 +177,11 @@ protected Flow executeFromState(
173177
}
174178
}));
175179

176-
env.persistRegionGroup(persistPlan);
180+
final TSStatus persistStatus = env.persistRegionGroup(persistPlan);
181+
if (persistStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
182+
setFailure(new ProcedureException(new IoTDBException(persistStatus)));
183+
return Flow.NO_MORE_STATE;
184+
}
177185
try {
178186
env.getConfigManager().getConsensusManager().write(offerPlan);
179187
} catch (final ConsensusException e) {

0 commit comments

Comments
 (0)