Merge branch 'master' of https://github.com/apache/iotdb into time-partition-boundary-overflow-fix

# Conflicts:
#	iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java

Author: Caideyipi

integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseRegionControlIT.java

@@ -39,6 +39,7 @@
 
 import java.io.IOException;
 import java.sql.Connection;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Collections;
@@ -164,6 +165,53 @@ public void testRegionGroupNumControlThroughCreation()
     }
   }
 
+  @Test
+  public void testRecreateWithPartialMaxRegionGroupNumUsesDefaultCounterpart() throws SQLException {
+    try (final Connection connection = EnvFactory.getEnv().getConnection();
+        final Statement statement = connection.createStatement()) {
+      final String database = "root.rg_tree_create_schema";
+      statement.execute(String.format("CREATE DATABASE %s;", database));
+
+      try (final ResultSet resultSet =
+          statement.executeQuery("SHOW DATABASES DETAILS " + database)) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(
+            testDefaultSchemaRegionGroupNumPerDatabase,
+            resultSet.getInt("MaxSchemaRegionGroupNum"));
+        Assert.assertEquals(
+            testDefaultDataRegionGroupNumPerDatabase, resultSet.getInt("MaxDataRegionGroupNum"));
+        Assert.assertFalse(resultSet.next());
+      }
+
+      statement.execute(String.format("DROP DATABASE %s;", database));
+      statement.execute(
+          String.format("CREATE DATABASE %s WITH MAX_SCHEMA_REGION_GROUP_NUM=3;", database));
+
+      try (final ResultSet resultSet =
+          statement.executeQuery("SHOW DATABASES DETAILS " + database)) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(3, resultSet.getInt("MaxSchemaRegionGroupNum"));
+        Assert.assertEquals(
+            testDefaultDataRegionGroupNumPerDatabase, resultSet.getInt("MaxDataRegionGroupNum"));
+        Assert.assertFalse(resultSet.next());
+      }
+
+      final String dataDatabase = "root.rg_tree_create_data";
+      statement.execute(
+          String.format("CREATE DATABASE %s WITH MAX_DATA_REGION_GROUP_NUM=4;", dataDatabase));
+
+      try (final ResultSet resultSet =
+          statement.executeQuery("SHOW DATABASES DETAILS " + dataDatabase)) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(
+            testDefaultSchemaRegionGroupNumPerDatabase,
+            resultSet.getInt("MaxSchemaRegionGroupNum"));
+        Assert.assertEquals(4, resultSet.getInt("MaxDataRegionGroupNum"));
+        Assert.assertFalse(resultSet.next());
+      }
+    }
+  }
+
   @Test
   public void testRegionGroupNumControlThroughAlter()
       throws SQLException, ClientManagerException, IOException, InterruptedException, TException {
@@ -236,6 +284,69 @@ public void testRegionGroupNumControlThroughAlter()
     }
   }
 
+  @Test
+  public void testAlterMaxDataRegionGroupNumCannotDecrease() throws SQLException {
+    try (final Connection connection = EnvFactory.getEnv().getConnection();
+        final Statement statement = connection.createStatement()) {
+      final String database = "root.rg_tree_decrease";
+      statement.execute(
+          String.format("CREATE DATABASE %s WITH MAX_DATA_REGION_GROUP_NUM=8;", database));
+      statement.execute(
+          String.format("ALTER DATABASE %s WITH MAX_DATA_REGION_GROUP_NUM=16;", database));
+
+      final SQLException exception =
+          Assert.assertThrows(
+              SQLException.class,
+              () ->
+                  statement.execute(
+                      String.format(
+                          "ALTER DATABASE %s WITH MAX_DATA_REGION_GROUP_NUM=12;", database)));
+      Assert.assertTrue(
+          exception.getMessage(),
+          exception
+              .getMessage()
+              .contains(
+                  "MaxDataRegionGroupNum should be greater than or equal to current max "
+                      + "DataRegionGroupNum: 16."));
+
+      try (final ResultSet resultSet =
+          statement.executeQuery("SHOW DATABASES DETAILS " + database)) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(16, resultSet.getInt("MaxDataRegionGroupNum"));
+        Assert.assertFalse(resultSet.next());
+      }
+
+      final String schemaDatabase = "root.rg_tree_decrease_schema";
+      statement.execute(
+          String.format("CREATE DATABASE %s WITH MAX_SCHEMA_REGION_GROUP_NUM=4;", schemaDatabase));
+      statement.execute(
+          String.format("ALTER DATABASE %s WITH MAX_SCHEMA_REGION_GROUP_NUM=5;", schemaDatabase));
+
+      final SQLException schemaException =
+          Assert.assertThrows(
+              SQLException.class,
+              () ->
+                  statement.execute(
+                      String.format(
+                          "ALTER DATABASE %s WITH MAX_SCHEMA_REGION_GROUP_NUM=3;",
+                          schemaDatabase)));
+      Assert.assertTrue(
+          schemaException.getMessage(),
+          schemaException
+              .getMessage()
+              .contains(
+                  "MaxSchemaRegionGroupNum should be greater than or equal to current max "
+                      + "SchemaRegionGroupNum: 5."));
+
+      try (final ResultSet resultSet =
+          statement.executeQuery("SHOW DATABASES DETAILS " + schemaDatabase)) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(5, resultSet.getInt("MaxSchemaRegionGroupNum"));
+        Assert.assertFalse(resultSet.next());
+      }
+    }
+  }
+
   @Test
   public void testDeprecatedRegionGroupNumSqlRejected() throws SQLException {
     try (final Connection connection = EnvFactory.getEnv().getConnection();

integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBMigrateMultiRegionForIoTV1IT.java

@@ -37,6 +37,7 @@
 
 import java.sql.Connection;
 import java.sql.Statement;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -51,6 +52,7 @@
 @RunWith(IoTDBTestRunner.class)
 public class IoTDBMigrateMultiRegionForIoTV1IT extends IoTDBRegionOperationReliabilityITFramework {
   private static final String MULTI_REGION_MIGRATE_FORMAT = "migrate region %s from %d to %d";
+  private static final String EXPAND_FORMAT = "extend region %d to %d";
 
   private static final Logger LOGGER =
       LoggerFactory.getLogger(IoTDBMigrateMultiRegionForIoTV1IT.class);
@@ -90,6 +92,10 @@ public void multiRegionMigrateTest() throws Exception {
       Map<Integer, Set<Integer>> regionMap = getAllRegionMap(statement);
       Set<Integer> allDataNodeId = getAllDataNodes(statement);
 
+      // INSERTION1 only creates one schema region and one data region; with replication factor 1
+      // they are often placed on different DataNodes. Colocate them before multi-region migrate.
+      regionMap = ensureDataNodeHostsMultipleRegions(statement, client, regionMap);
+
       // With replication factor 1 every region lives on exactly one DataNode. Pick a source
       // DataNode that hosts at least two regions, then migrate all its regions to a fresh
       // destination DataNode.
@@ -177,6 +183,66 @@ public void multiRegionMigrateTest() throws Exception {
     }
   }
 
+  private Map<Integer, Set<Integer>> ensureDataNodeHostsMultipleRegions(
+      Statement statement,
+      SyncConfigNodeIServiceClient client,
+      Map<Integer, Set<Integer>> regionMap)
+      throws Exception {
+    if (hasDataNodeHostingMultipleRegions(regionMap)) {
+      return regionMap;
+    }
+    List<Integer> regionIds = new ArrayList<>(regionMap.keySet());
+    Assert.assertTrue("Need at least two regions to colocate", regionIds.size() >= 2);
+
+    int firstRegion = regionIds.get(0);
+    int secondRegion = regionIds.get(1);
+    int targetDataNode = regionMap.get(secondRegion).iterator().next();
+    regionGroupExpand(statement, client, firstRegion, targetDataNode);
+    return getAllRegionMap(statement);
+  }
+
+  private boolean hasDataNodeHostingMultipleRegions(Map<Integer, Set<Integer>> regionMap) {
+    return regionMap.values().stream()
+        .flatMap(Set::stream)
+        .collect(Collectors.groupingBy(dataNodeId -> dataNodeId, Collectors.counting()))
+        .values()
+        .stream()
+        .anyMatch(count -> count >= 2);
+  }
+
+  private void regionGroupExpand(
+      Statement statement,
+      SyncConfigNodeIServiceClient client,
+      int selectedRegion,
+      int targetDataNode)
+      throws Exception {
+    Awaitility.await()
+        .atMost(10, TimeUnit.SECONDS)
+        .pollInterval(1, TimeUnit.SECONDS)
+        .until(
+            () -> {
+              statement.execute(String.format(EXPAND_FORMAT, selectedRegion, targetDataNode));
+              return true;
+            });
+
+    Predicate<TShowRegionResp> expandRegionPredicate =
+        tShowRegionResp -> {
+          Map<Integer, Set<Integer>> newRegionMap =
+              getRunningRegionMap(tShowRegionResp.getRegionInfoList());
+          Set<Integer> dataNodes = newRegionMap.get(selectedRegion);
+          return dataNodes != null && dataNodes.contains(targetDataNode);
+        };
+
+    awaitUntilSuccess(
+        client,
+        selectedRegion,
+        expandRegionPredicate,
+        Optional.of(targetDataNode),
+        Optional.empty());
+
+    LOGGER.info("Region {} has expanded to DataNode {}", selectedRegion, targetDataNode);
+  }
+
   private int selectDataNodeHostingMultipleRegions(Map<Integer, Set<Integer>> regionMap) {
     Map<Integer, Long> regionCountPerDataNode =
         regionMap.values().stream()

…onMigrateWithDeletionMultiDataDirIT.java …onMigrateWithDeletionMultiDataDirIT.javaintegration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateWithDeletionMultiDataDirIT.java renamed to integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionMigrateWithDeletionMultiDataDirIT.java integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateWithDeletionMultiDataDirIT.java renamed to integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionMigrateWithDeletionMultiDataDirIT.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1;
+package org.apache.iotdb.confignode.it.regionmigration.pass.commit;
 
 import org.apache.iotdb.consensus.ConsensusFactory;
 import org.apache.iotdb.it.env.EnvFactory;
@@ -43,12 +43,12 @@
 
 import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getAllDataNodes;
 import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getDataRegionMapWithLeader;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.MULTI_DATA_DIRS;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitModsVisibleOnReplicas;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitRegionReplicas;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitTsFileResourceVisibleOnReplicas;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitTsFileVisibleOnReplicas;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.getReplicaDataNodeIds;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.MULTI_DATA_DIRS;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.awaitModsVisibleOnReplicas;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.awaitRegionReplicas;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.awaitTsFileResourceVisibleOnReplicas;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.awaitTsFileVisibleOnReplicas;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.getReplicaDataNodeIds;
 
 /**
  * Tree-model coverage for IoTConsensus region migration over multiple data dirs: a deletion (mods)

…rateWithDeletionMultiDataDirTableIT.java …rateWithDeletionMultiDataDirTableIT.javaintegration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateWithDeletionMultiDataDirTableIT.java renamed to integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionMigrateWithDeletionMultiDataDirTableIT.java integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateWithDeletionMultiDataDirTableIT.java renamed to integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionMigrateWithDeletionMultiDataDirTableIT.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1;
+package org.apache.iotdb.confignode.it.regionmigration.pass.commit;
 
 import org.apache.iotdb.consensus.ConsensusFactory;
 import org.apache.iotdb.isession.SessionConfig;
@@ -45,12 +45,12 @@
 
 import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getAllDataNodes;
 import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getDataRegionMapWithLeader;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.MULTI_DATA_DIRS;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitModsVisibleOnReplicas;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitRegionReplicas;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitTsFileResourceVisibleOnReplicas;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitTsFileVisibleOnReplicas;
-import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.getReplicaDataNodeIds;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.MULTI_DATA_DIRS;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.awaitModsVisibleOnReplicas;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.awaitRegionReplicas;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.awaitTsFileResourceVisibleOnReplicas;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.awaitTsFileVisibleOnReplicas;
+import static org.apache.iotdb.confignode.it.regionmigration.pass.commit.RegionMigrateFileAssertions.getReplicaDataNodeIds;
 
 /**
  * Table-model twin of {@link IoTDBRegionMigrateWithDeletionMultiDataDirIT}: a deletion (mods) must

…y/iotv1/RegionMigrateFileAssertions.java …/commit/RegionMigrateFileAssertions.javaintegration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/RegionMigrateFileAssertions.java renamed to integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/RegionMigrateFileAssertions.java integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/RegionMigrateFileAssertions.java renamed to integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/RegionMigrateFileAssertions.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1;
+package org.apache.iotdb.confignode.it.regionmigration.pass.commit;
 
 import org.apache.iotdb.it.env.EnvFactory;
 import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;

integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateNormalITForIoTV2BatchIT.java

@@ -21,13 +21,21 @@
 
 import org.apache.iotdb.commons.utils.KillPoint.KillNode;
 import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework;
+import org.apache.iotdb.it.env.EnvFactory;
 import org.apache.iotdb.it.framework.IoTDBTestRunner;
 import org.apache.iotdb.itbase.category.ClusterIT;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly;
+
 @Category({ClusterIT.class})
 @RunWith(IoTDBTestRunner.class)
 public class IoTDBRegionMigrateNormalITForIoTV2BatchIT
@@ -41,4 +49,30 @@ public void normal1C2DTest() throws Exception {
   public void normal3C3DTest() throws Exception {
     successTest(2, 3, 3, 3, noKillPoints(), noKillPoints(), KillNode.ALL_NODES);
   }
+
+  @Test
+  public void migrateRegionWithDegradedTimeIndexTest() throws Exception {
+    // set TsFileResource memory to 0 to trigger degrading
+    EnvFactory.getEnv().getConfig().getCommonConfig().setQueryMemoryProportion("1:1:1:1:1:1:0");
+
+    successTest(1, 1, 1, 2, noKillPoints(), noKillPoints(), KillNode.ALL_NODES);
+
+    try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection());
+        final Statement statement = makeItCloseQuietly(connection.createStatement())) {
+      assertCounts(statement, 1, 1);
+      statement.execute("INSERT INTO root.sg.d1(timestamp,speed,temperature) values(101, 3, 4)");
+      assertCounts(statement, 2, 2);
+    }
+  }
+
+  private static void assertCounts(
+      final Statement statement, final long expectedSpeedCount, final long expectedTemperatureCount)
+      throws Exception {
+    try (final ResultSet resultSet =
+        statement.executeQuery("select count(speed), count(temperature) from root.sg.d1")) {
+      Assert.assertTrue(resultSet.next());
+      Assert.assertEquals(expectedSpeedCount, resultSet.getLong(1));
+      Assert.assertEquals(expectedTemperatureCount, resultSet.getLong(2));
+    }
+  }
 }

integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java

@@ -71,7 +71,7 @@ public void testDoubleLivingInvalidForwardingParameter() throws Exception {
               "create pipe %s"
                   + " with source ("
                   + "'capture.tree'='false',"
-                  + "'mode.double-living'='true')"
+                  + "'double-living'='true')"
                   + " with sink ("
                   + "'node-urls'='%s')",
               "p1", receiverDataNode.getIpAndPortString()));
@@ -84,7 +84,7 @@ public void testDoubleLivingInvalidForwardingParameter() throws Exception {
               "create pipe %s"
                   + " with source ("
                   + "'capture.table'='false',"
-                  + "'mode.double-living'='true')"
+                  + "'source.mode.double-living'='true')"
                   + " with sink ("
                   + "'node-urls'='%s')",
               "p2", receiverDataNode.getIpAndPortString()));
@@ -286,7 +286,7 @@ public void testDoubleLivingIsolation() throws Exception {
           String.format(
               "create pipe %s"
                   + " with source ("
-                  + "'mode.double-living'='true')"
+                  + "'double-living'='true')"
                   + " with sink ("
                   + "'node-urls'='%s')",
               treePipeName, receiverDataNode.getIpAndPortString()));
@@ -308,7 +308,7 @@ public void testDoubleLivingIsolation() throws Exception {
           String.format(
               "create pipe %s"
                   + " with source ("
-                  + "'mode.double-living'='true')"
+                  + "'source.mode.double-living'='true')"
                   + " with sink ("
                   + "'node-urls'='%s')",
               tablePipeName, receiverDataNode.getIpAndPortString()));
@@ -350,7 +350,7 @@ public void testDoubleLivingCreatesSameNamePipesWithSeparateDialects() throws Ex
       statement.execute(
           String.format(
               "create pipe %s"
-                  + " with source ('mode.double-living'='true')"
+                  + " with source ('extractor.double-living'='true')"
                   + " with sink ('node-urls'='%s')",
               pipeName, receiverDataNode.getIpAndPortString()));
     }