From 6c934ea176b8faf9ed8d93ec4f4eea8013929039 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 1 Aug 2025 14:11:10 +0800 Subject: [PATCH 1/4] Pipe: Fixed the default param of single entry of disruptor queue & Banned memory checks from some missing pipe ITs & Do not check non-user pipes (#16069) * Update CommonConfig.java * refactor * try-fix * test * revert-pom * fix * fix --- .../env/cluster/config/MppCommonConfig.java | 6 + .../cluster/config/MppSharedCommonConfig.java | 7 + .../env/remote/config/RemoteCommonConfig.java | 5 + .../apache/iotdb/itbase/env/CommonConfig.java | 2 + .../it/autocreate/AbstractPipeDualAutoIT.java | 2 + .../autocreate/IoTDBPipeAutoConflictIT.java | 2 + .../it/autocreate/IoTDBPipeClusterIT.java | 2 + .../it/autocreate/IoTDBPipeIdempotentIT.java | 2 + .../it/autocreate/IoTDBPipeProcessorIT.java | 2 + .../it/autocreate/IoTDBPipeProtocolIT.java | 2 + .../IoTDBPipeSinkCompressionIT.java | 2 + .../pipe/it/autocreate/IoTDBPipeSourceIT.java | 2 + .../it/autocreate/IoTDBPipeWithLoadIT.java | 2 + .../manual/basic/IoTDBPipePermissionIT.java | 371 ++++++ .../manual/basic/IoTDBPipeProtocolIT.java | 498 ++++++++ .../manual/basic/IoTDBPipeSourceIT.java | 1005 ++++++++++++++++ .../manual/enhanced/IoTDBPipeClusterIT.java | 1054 +++++++++++++++++ .../it/manual/AbstractPipeDualManualIT.java | 2 + .../cluster/IoTDBSubscriptionRestartIT.java | 1 + .../it/local/AbstractSubscriptionLocalIT.java | 1 + .../triple/AbstractSubscriptionTripleIT.java | 1 + .../iotdb/tools/it/ExportTsFileTestIT.java | 1 + .../agent/task/PipeDataNodeTaskAgent.java | 135 ++- ...icalDataRegionTsFileAndDeletionSource.java | 947 +++++++++++++++ .../PipeInsertionDataNodeListener.java | 4 + .../iotdb/commons/conf/CommonConfig.java | 4 +- .../pipe/agent/task/PipeTaskAgent.java | 2 + pom.xml | 1 + 28 files changed, 4003 insertions(+), 62 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java index b77d8916b205b..2c1059272da87 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java @@ -429,6 +429,12 @@ public CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode) { return this; } + @Override + public CommonConfig setPipeMemoryManagementEnabled(boolean pipeMemoryManagementEnabled) { + setProperty("pipe_memory_management_enabled", String.valueOf(pipeMemoryManagementEnabled)); + return this; + } + @Override public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) { setProperty("pipe_enable_memory_checked", String.valueOf(isPipeEnableMemoryCheck)); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java index 2461c1e6ba2d4..d131bf862c099 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java @@ -438,6 +438,13 @@ public CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode) { return this; } + @Override + public CommonConfig setPipeMemoryManagementEnabled(boolean pipeMemoryManagementEnabled) { + dnConfig.setPipeMemoryManagementEnabled(pipeMemoryManagementEnabled); + cnConfig.setPipeMemoryManagementEnabled(pipeMemoryManagementEnabled); + return this; + } + @Override public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) { dnConfig.setIsPipeEnableMemoryCheck(isPipeEnableMemoryCheck); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java index e1de42382b6a6..32061709b271f 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java @@ -308,6 +308,11 @@ public CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode) { return this; } + @Override + public CommonConfig setPipeMemoryManagementEnabled(boolean pipeMemoryManagementEnabled) { + return this; + } + @Override public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index 09a4adce9440b..24e23a31e5f9a 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -138,6 +138,8 @@ CommonConfig setEnableAutoLeaderBalanceForIoTConsensus( CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode); + CommonConfig setPipeMemoryManagementEnabled(boolean pipeMemoryManagementEnabled); + CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck); CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java index 59478fc36e67f..6d3f2e85d8bd5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java @@ -49,6 +49,7 @@ protected void setupConfig() { .setAutoCreateSchemaEnabled(true) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); receiverEnv .getConfig() @@ -56,6 +57,7 @@ protected void setupConfig() { .setAutoCreateSchemaEnabled(true) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); // 10 min, assert that the operations will not time out diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java index 9da5591bb2f76..b89483840cb58 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java @@ -60,6 +60,7 @@ public void setUp() { .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); receiverEnv .getConfig() @@ -68,6 +69,7 @@ public void setUp() { .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); // 10 min, assert that the operations will not time out diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java index c8d5bc0b47174..3a4ca6fcada14 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java @@ -75,6 +75,7 @@ public void setUp() { .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); receiverEnv @@ -86,6 +87,7 @@ public void setUp() { .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); // 10 min, assert that the operations will not time out diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java index 771d50c97c68a..4e1270577d814 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java @@ -66,6 +66,7 @@ public void setUp() { .setDefaultSchemaRegionGroupNumPerDatabase(1) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); receiverEnv .getConfig() @@ -73,6 +74,7 @@ public void setUp() { .setAutoCreateSchemaEnabled(true) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); // 10 min, assert that the operations will not time out diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java index 13a63a585a8dc..bbf4f206b59f5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java @@ -60,6 +60,7 @@ public void setUp() { .setTimestampPrecision("ms") .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); receiverEnv .getConfig() @@ -67,6 +68,7 @@ public void setUp() { .setAutoCreateSchemaEnabled(true) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); // 10 min, assert that the operations will not time out diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java index 1f64676852579..ed68cc23ebada 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java @@ -75,6 +75,7 @@ private void innerSetUp( .setDataRegionConsensusProtocolClass(dataRegionConsensus) .setSchemaReplicationFactor(schemaRegionReplicationFactor) .setDataReplicationFactor(dataRegionReplicationFactor) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); receiverEnv .getConfig() @@ -85,6 +86,7 @@ private void innerSetUp( .setDataRegionConsensusProtocolClass(dataRegionConsensus) .setSchemaReplicationFactor(schemaRegionReplicationFactor) .setDataReplicationFactor(dataRegionReplicationFactor) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); // 10 min, assert that the operations will not time out diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSinkCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSinkCompressionIT.java index 9a39b0b2f4ada..59cfa4321e4b0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSinkCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSinkCompressionIT.java @@ -67,6 +67,7 @@ public void setUp() { .setAutoCreateSchemaEnabled(true) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); receiverEnv @@ -76,6 +77,7 @@ public void setUp() { .setPipeAirGapReceiverEnabled(true) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); // 10 min, assert that the operations will not time out diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSourceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSourceIT.java index 117d7396ec174..fe3c550334436 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSourceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSourceIT.java @@ -72,6 +72,7 @@ public void setUp() { .setEnableSeqSpaceCompaction(false) .setEnableUnseqSpaceCompaction(false) .setEnableCrossSpaceCompaction(false) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); receiverEnv .getConfig() @@ -79,6 +80,7 @@ public void setUp() { .setAutoCreateSchemaEnabled(true) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); // 10 min, assert that the operations will not time out diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java index c71e212b629a3..d87aa3b5fae1c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java @@ -62,6 +62,7 @@ public void setUp() { .setEnableSeqSpaceCompaction(false) .setEnableUnseqSpaceCompaction(false) .setEnableCrossSpaceCompaction(false) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); receiverEnv .getConfig() @@ -69,6 +70,7 @@ public void setUp() { .setAutoCreateSchemaEnabled(true) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); // 10 min, assert that the operations will not time out diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java new file mode 100644 index 0000000000000..83616911639bf --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipePermissionIT extends AbstractPipeTableModelDualManualIT { + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(false) + .setDefaultSchemaRegionGroupNumPerDatabase(1) + .setTimestampPrecision("ms") + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(false) + .setTimestampPrecision("ms") + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false) + .setSchemaReplicationFactor(3) + .setDataReplicationFactor(2); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(3, 3); + } + + @Test + public void testSourcePermission() { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, "create user `thulab` 'passwD@123456'", null)) { + return; + } + + // Shall fail if username is specified without password + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe a2b" + + " with source (" + + "'user'='thulab'" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + fail("When the 'user' or 'username' is specified, password must be specified too."); + } catch (final SQLException ignore) { + // Expected + } + + // Shall fail if password is wrong + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe a2b" + + " with source (" + + "'user'='thulab'" + + "'password'='hack'" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + fail("Shall fail if password is wrong."); + } catch (final SQLException ignore) { + // Expected + } + + // Use current session, user is root + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe a2b" + + " with source (" + + "'inclusion'='all'," + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail("Create pipe without user shall succeed if use the current session"); + } + + // Alter to another user, shall fail because of lack of password + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ('username'='thulab')"); + fail("Alter pipe shall fail if only user is specified"); + } catch (final SQLException ignore) { + // Expected + } + + // Successfully alter + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + "alter pipe a2b modify source ('username'='thulab', 'password'='passwD@123456')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail("Alter pipe shall not fail if user and password are specified"); + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + + // Shall not be transferred + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "count databases", + "count,", + Collections.singleton("1,"), + "information_schema"); + + // Grant some privilege + if (!TestUtils.tryExecuteNonQueryWithRetry( + "test", BaseEnv.TABLE_SQL_DIALECT, senderEnv, "grant INSERT on any to user thulab", null)) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + + // Shall not be transferred + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show tables from test1", + "TableName,TTL(ms),", + Collections.singleton("test1,INF,"), + "information_schema"); + + // Alter pipe, throw exception if no privileges + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ('skipif'='')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Write some data + if (!TableModelUtils.insertData("test", "test", 0, 100, senderEnv)) { + return; + } + + try { + TableModelUtils.createDataBaseAndTable(receiverEnv, "test", "test"); + } catch (final Exception | Error ignore) { + // Ignore because the db/table may be transferred because sender user may see these + } + + // Exception, block here + TableModelUtils.assertCountDataAlwaysOnEnv("test", "test", 0, receiverEnv); + + // Grant SELECT privilege + if (!TestUtils.tryExecuteNonQueriesWithRetry( + "test", + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + Arrays.asList("grant SELECT on any to user thulab", "start pipe a2b"), + null)) { + return; + } + + // Will finally pass + TableModelUtils.assertCountData( + "test", + "test", + 100, + receiverEnv, + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }); + } + + @Test + public void testReceiverPermission() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, "create user testUser 'passwD@123456'", null)) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + final String dbName = "test"; + final String tbName = "test"; + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.capture.tree", "false"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.user", "testUser"); + connectorAttributes.put("connector.password", "passwD@123456"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.startPipeExtended(new TStartPipeReq("testPipe").setIsTableModel(true)).getCode()); + + TableModelUtils.createDataBaseAndTable(senderEnv, tbName, dbName); + + // Write some data + if (!TableModelUtils.insertData(dbName, tbName, 0, 100, senderEnv)) { + return; + } + + // Shall not be transferred + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "show databases", + "Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", + Collections.singleton("information_schema,INF,null,null,null,"), + (String) null); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + "information_schema", + BaseEnv.TABLE_SQL_DIALECT, + receiverEnv, + "grant insert,create on database test to user testUser", + null)) { + return; + } + + // Will finally pass + TableModelUtils.assertCountData( + dbName, + tbName, + 100, + receiverEnv, + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }); + + // Alter pipe, skip if no privileges + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe testPipe modify sink ('skipif'='no-privileges')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + final String dbName2 = "test2"; + + // Write some data + if (!TableModelUtils.insertData(dbName2, tbName, 0, 100, senderEnv)) { + return; + } + + // Shall not be transferred + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "count databases", "count,", Collections.singleton("2,"), (String) null); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + "information_schema", + BaseEnv.TABLE_SQL_DIALECT, + receiverEnv, + "grant insert,create on database test2 to user testUser", + null)) { + return; + } + + if (!TableModelUtils.insertData(dbName2, tbName, 100, 200, senderEnv)) { + return; + } + + // Will finally pass + TableModelUtils.assertCountData( + dbName2, + tbName, + 100, + receiverEnv, + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java new file mode 100644 index 0000000000000..939e84b9f4961 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java @@ -0,0 +1,498 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +/** Test pipe's basic functionalities under multiple cluster and consensus protocol settings. */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeProtocolIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + } + + private void innerSetUp( + final String configNodeConsensus, + final String schemaRegionConsensus, + final String dataRegionConsensus, + final int configNodesNum, + final int dataNodesNum, + int schemaRegionReplicationFactor, + int dataRegionReplicationFactor) { + schemaRegionReplicationFactor = Math.min(schemaRegionReplicationFactor, dataNodesNum); + dataRegionReplicationFactor = Math.min(dataRegionReplicationFactor, dataNodesNum); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(dataRegionConsensus) + .setSchemaReplicationFactor(schemaRegionReplicationFactor) + .setDataReplicationFactor(dataRegionReplicationFactor) + .setDnConnectionTimeoutMs(600000) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(configNodeConsensus) + .setSchemaRegionConsensusProtocolClass(schemaRegionConsensus) + .setDataRegionConsensusProtocolClass(dataRegionConsensus) + .setSchemaReplicationFactor(schemaRegionReplicationFactor) + .setDataReplicationFactor(dataRegionReplicationFactor) + .setDnConnectionTimeoutMs(600000) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + + senderEnv.initClusterEnvironment(configNodesNum, dataNodesNum); + receiverEnv.initClusterEnvironment(configNodesNum, dataNodesNum); + } + + @Test + public void test1C1DWithRatisRatisIot() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 1, + 1, + 1, + 1); + doTest(); + } + + @Test + public void test1C1DWithSimpleSimpleIot() throws Exception { + innerSetUp( + ConsensusFactory.SIMPLE_CONSENSUS, + ConsensusFactory.SIMPLE_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 1, + 1, + 1, + 1); + doTest(); + } + + @Test + public void test1C1DWithRatisRatisSimple() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.SIMPLE_CONSENSUS, + 1, + 1, + 1, + 1); + doTest(); + } + + @Test + public void test3C3DWith3SchemaRegionFactor3DataRegionFactor() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 3, + 3, + 3, + 3); + doTest(); + } + + @Test + public void test3C3DWith3SchemaRegionFactor2DataRegionFactor() throws Exception { + innerSetUp( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.RATIS_CONSENSUS, + ConsensusFactory.IOT_CONSENSUS, + 3, + 3, + 3, + 2); + doTest(); + } + + @Test + public void testPipeOnBothSenderAndReceiver() throws Exception { + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(3) + .setDataReplicationFactor(2) + .setDnConnectionTimeoutMs(600000) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(1) + .setDnConnectionTimeoutMs(600000) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + + senderEnv.initClusterEnvironment(3, 3); + receiverEnv.initClusterEnvironment(1, 1); + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.streaming", "true"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + } + + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final String senderIp = senderDataNode.getIp(); + final int senderPort = senderDataNode.getPort(); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(receiverEnv, "test1", "test1"); + insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, receiverEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test.*"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.streaming", "true"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", senderIp); + connectorAttributes.put("connector.port", Integer.toString(senderPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.assertCountData("test1", "test1", 100, senderEnv, handleFailure); + } + } + + private void doTest() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test.*"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.streaming", "true"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.assertData("test", "test", 0, 100, receiverEnv, handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); + } + } + + @Test + public void testSyncConnectorUseNodeUrls() throws Exception { + doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName()); + } + + @Test + public void testAsyncConnectorUseNodeUrls() throws Exception { + doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName()); + } + + @Test + public void testAirGapConnectorUseNodeUrls() throws Exception { + doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); + } + + private void doTestUseNodeUrls(String connectorName) throws Exception { + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setPipeAirGapReceiverEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(1) + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setDnConnectionTimeoutMs(600000) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setPipeAirGapReceiverEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(3) + .setDataReplicationFactor(2) + .setDnConnectionTimeoutMs(600000) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + + senderEnv.initClusterEnvironment(1, 1); + receiverEnv.initClusterEnvironment(1, 3); + + final StringBuilder nodeUrlsBuilder = new StringBuilder(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) { + if (connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) { + // Use default port for convenience + nodeUrlsBuilder + .append(wrapper.getIp()) + .append(":") + .append(wrapper.getPipeAirGapReceiverPort()) + .append(","); + } else { + nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(","); + } + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", connectorName); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.batch.max-delay-seconds", "1"); + connectorAttributes.put("connector.batch.size-bytes", "2048"); + connectorAttributes.put("connector.node-urls", nodeUrlsBuilder.toString()); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test.*"); + extractorAttributes.put("table-name", "test.*"); + extractorAttributes.put("inclusion", "data.insert"); + extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("mode.strict", "true"); + extractorAttributes.put("user", "root"); + + // Test forced-log mode, in open releases this might be "file" + extractorAttributes.put("realtime.mode", "forced-log"); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); + + extractorAttributes.replace("realtime.mode", "file"); + + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 300, 400, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java new file mode 100644 index 0000000000000..5dcf2d0e8e4af --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java @@ -0,0 +1,1005 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeSourceIT extends AbstractPipeTableModelDualManualIT { + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + // Disable sender compaction for tsfile determination in loose range test + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setDnConnectionTimeoutMs(600000) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH"); + + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDnConnectionTimeoutMs(600000) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + + @Test + public void testMatchingMultipleDatabases() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + boolean insertResult = true; + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.table-name", "test"); + extractorAttributes.put("extractor.pattern", "root.db1"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + assertTimeseriesCountOnReceiver(receiverEnv, 0); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db1.d1 (time, at1) values (1, 10)", + "insert into root.db2.d1 (time, at1) values (1, 20)", + "flush"), + null)) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + extractorAttributes.replace("extractor.pattern", "root.db2"); + extractorAttributes.replace("extractor.table-name", "test1"); + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + assertTimeseriesCountOnReceiver(receiverEnv, 2); + + Thread.sleep(10000); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p2").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db1.d1 (time, at1) values (2, 11)", + "insert into root.db2.d1 (time, at1) values (2, 21)", + "flush"), + null)) { + return; + } + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + extractorAttributes.remove("extractor.pattern"); // no pattern, will match all databases + extractorAttributes.remove("extractor.table-name"); + extractorAttributes.remove("extractor.database-name"); + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db*.**", + "count(root.db1.d1.at1),count(root.db2.d1.at1),", + Collections.singleton("2,2,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); + } + } + + @Test + public void testHistoryAndRealtime() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1) values (1, 10)", + "insert into root.db.d2 (time, at1) values (1, 20)", + "insert into root.db.d3 (time, at1) values (1, 30)", + "insert into root.db.d4 (time, at1) values (1, 40)", + "flush"), + null)) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + insertResult = TableModelUtils.insertData("test", "test2", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test3"); + insertResult = TableModelUtils.insertData("test", "test3", 0, 100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test4", "test"); + insertResult = TableModelUtils.insertData("test", "test4", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("extractor.database", "test"); + extractorAttributes.put("extractor.table", "test2"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.pattern", "root.db.d2"); + extractorAttributes.put("extractor.history.enable", "false"); + extractorAttributes.put("extractor.realtime.enable", "true"); + extractorAttributes.put("user", "root"); + TSStatus status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + extractorAttributes.replace("extractor.table-name", "test3"); + extractorAttributes.replace("extractor.pattern", "root.db.d3"); + extractorAttributes.replace("extractor.history.enable", "true"); + extractorAttributes.replace("extractor.realtime.enable", "false"); + status = + client.createPipe( + new TCreatePipeReq("p3", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); + + extractorAttributes.replace("extractor.table-name", "test4"); + extractorAttributes.replace("extractor.pattern", "root.db.d4"); + extractorAttributes.replace("extractor.history.enable", "true"); + extractorAttributes.replace("extractor.realtime.enable", "true"); + status = + client.createPipe( + new TCreatePipeReq("p4", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p4").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1) values (2, 11)", + "insert into root.db.d2 (time, at1) values (2, 21)", + "insert into root.db.d3 (time, at1) values (2, 31)", + "insert into root.db.d4 (time, at1) values (2, 41), (3, 51)"), + null)) { + return; + } + + insertResult = TableModelUtils.insertData("test", "test2", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test3", 0, 100, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test4", 0, 200, senderEnv); + if (!insertResult) { + return; + } + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.** where time <= 1", + "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", + Collections.singleton("1,0,1,"), + handleFailure); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.** where time >= 2", + "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", + Collections.singleton("2,1,0,"), + handleFailure); + + TableModelUtils.assertData("test", "test2", 100, 200, receiverEnv, handleFailure); + + TableModelUtils.assertData("test", "test3", 100, 200, receiverEnv, handleFailure); + + TableModelUtils.assertData("test", "test4", 100, 200, receiverEnv, handleFailure); + } + } + + @Ignore + @Test + public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "insert into root.db.d2 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "flush"), + null)) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test1", 0, 10, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + insertResult = TableModelUtils.insertData("test", "test2", 0, 10, senderEnv); + if (!insertResult) { + return; + } + + // wait for flush to complete + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Collections.singletonList("flush"), null)) { + return; + } + Thread.sleep(10000); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.table-name", "test1"); + extractorAttributes.put("extractor.pattern", "root.db.d1"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + extractorAttributes.put("extractor.history.enable", "true"); + // 1970-01-01T08:00:02+08:00 + extractorAttributes.put("extractor.history.start-time", "2"); + extractorAttributes.put("extractor.history.end-time", "3"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.**", + "count(root.db.d1.at1),", + Collections.singleton("2,"), + handleFailure); + + extractorAttributes.remove("extractor.table-name"); + extractorAttributes.remove("extractor.pattern"); + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.**", + "count(root.db.d1.at1),count(root.db.d2.at1),", + Collections.singleton("2,2,"), + handleFailure); + + TableModelUtils.assertData("test", "test1", 2, 4, receiverEnv, handleFailure); + TableModelUtils.assertData("test", "test2", 2, 4, receiverEnv, handleFailure); + } + } + + @Test + public void testExtractorTimeRangeMatch() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // insert history data + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "insert into root.db.d2 (time, at1)" + + " values (6, 6), (7, 7), (8, 8), (9, 9), (10, 10)", + "flush"), + null)) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test1", 0, 10, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + insertResult = TableModelUtils.insertData("test", "test2", 0, 10, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.start-time", "2"); + extractorAttributes.put("source.end-time", "4"); + extractorAttributes.put("user", "root"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.**", + "count(root.db.d1.at1),", + Collections.singleton("3,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + + // Insert realtime data that overlapped with time range + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d3 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "flush"), + null)) { + return; + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test"); + insertResult = TableModelUtils.insertData("test", "test3", 0, 5, senderEnv); + if (!insertResult) { + return; + } + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.**", + "count(root.db.d1.at1),count(root.db.d3.at1),", + Collections.singleton("3,3,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test3", 3, receiverEnv, handleFailure); + + // Insert realtime data that does not overlap with time range + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d4 (time, at1)" + + " values (6, 6), (7, 7), (8, 8), (9, 9), (10, 10)", + "flush"), + null)) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test4", "test"); + insertResult = TableModelUtils.insertData("test", "test4", 6, 10, senderEnv); + if (!insertResult) { + return; + } + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "select count(*) from root.db.**", + "count(root.db.d1.at1),count(root.db.d3.at1),", + Collections.singleton("3,3,"), + 600, + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test3", 3, receiverEnv, handleFailure); + } + } + + @Test + public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + boolean insertResult = true; + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "insert into root.db.d2 (time, at1)" + + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", + "flush"), + null)) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test1", 0, 5, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + insertResult = TableModelUtils.insertData("test", "test2", 0, 5, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.capture.table", "true"); + extractorAttributes.put("source.capture.tree", "true"); + extractorAttributes.put("source.pattern", "root.db.d1"); + extractorAttributes.put("source.table-name", "test1"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.start-time", "2"); + // 1970-01-01T08:00:04+08:00 + extractorAttributes.put("source.end-time", "4"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.**", + "count(root.db.d1.at1),", + Collections.singleton("3,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + + extractorAttributes.remove("source.pattern"); + extractorAttributes.remove("source.table-name"); + status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.**", + "count(root.db.d1.at1),count(root.db.d2.at1),", + Collections.singleton("3,3,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test2", 3, receiverEnv, handleFailure); + } + } + + @Ignore + @Test + public void testHistoryLooseRange() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + boolean insertResult = true; + + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); + insertResult = TableModelUtils.insertData("test", "test1", 0, 2, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); + insertResult = TableModelUtils.insertData("test", "test2", 0, 2, senderEnv); + if (!insertResult) { + return; + } + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + // TsFile 1, extracted without parse + "insert into root.db.d1 (time, at1, at2)" + " values (1, 1, 2), (2, 3, 4)", + // TsFile 2, not extracted because pattern not overlapped + "insert into root.db1.d1 (time, at1, at2)" + " values (1, 1, 2), (2, 3, 4)", + "flush"), + null)) { + return; + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + // TsFile 3, not extracted because time range not overlapped + "insert into root.db.d1 (time, at1, at2)" + " values (3, 1, 2), (4, 3, 4)", "flush"), + null)) { + return; + } + + insertResult = TableModelUtils.insertData("test", "test1", 2, 5, senderEnv); + if (!insertResult) { + return; + } + + // wait for flush to complete + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, Collections.singletonList("flush"), null)) { + return; + } + Thread.sleep(10000); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.capture.table", "true"); + extractorAttributes.put("source.capture.tree", "true"); + extractorAttributes.put("source.table-name", "test1"); + extractorAttributes.put("source.path", "root.db.d1.at1"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.history.start-time", "1"); + extractorAttributes.put("source.history.end-time", "2"); + extractorAttributes.put("source.history.loose-range", "time, path"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.** group by level=0", + "count(root.*.*.*),", + Collections.singleton("4,"), + handleFailure); + + TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + } + } + + @Ignore + @Test + public void testRealtimeLooseRange() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + insertResult = TableModelUtils.insertData("test", "test2", 0, 20, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.capture.table", "true"); + extractorAttributes.put("source.capture.tree", "true"); + extractorAttributes.put("source.table-name", "test1"); + extractorAttributes.put("source.path", "root.db.d1.at1"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.realtime.loose-range", "time, path"); + extractorAttributes.put("source.start-time", "2"); + extractorAttributes.put("source.end-time", "10"); + extractorAttributes.put("source.realtime.mode", "batch"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1, at2)" + " values (1, 1, 2), (3, 3, 4)", "flush"), + null)) { + return; + } + + if (!insertResult) { + return; + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1)" + " values (5, 1), (16, 3)", + "insert into root.db.d1 (time, at1, at2)" + " values (5, 1, 2), (6, 3, 4)", + "flush"), + null)) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(at1) from root.db.d1 where time >= 2 and time <= 10", + new HashMap() { + { + put("count(root.db.d1.at1)", "3"); + } + }); + + insertResult = TableModelUtils.insertData("test", "test1", 10, 20, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test2", 10, 20, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.assertCountData("test", "test1", 20, receiverEnv, handleFailure); + } + } + + @Test + public void testTableModeSQLSupportNowFunc() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final String p1 = + String.format( + "create pipe p1" + + " with extractor (" + + "'capture.table'='true'," + + "'extractor.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(p1); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + final String p2 = + String.format( + "create pipe p2" + + " with extractor (" + + "'capture.table'='true'," + + "'extractor.history.enable'='true'," + + "'start-time'='now'," + + "'end-time'='now'," + + "'history.start-time'='now'," + + "'history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(p2); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + final String p3 = + String.format( + "create pipe p3" + + " with extractor (" + + "'capture.table'='true'," + + "'extractor.history.enable'='true'," + + "'extractor.start-time'='now'," + + "'extractor.end-time'='now'," + + "'extractor.history.start-time'='now'," + + "'extractor.history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(p3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + String alterP3 = + "alter pipe p3" + + " modify extractor (" + + "'history.enable'='true'," + + "'start-time'='now'," + + "'end-time'='now'," + + "'history.start-time'='now'," + + "'history.end-time'='now')"; + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(alterP3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + alterP3 = + "alter pipe p3" + + " modify extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.start-time'='now'," + + "'extractor.end-time'='now'," + + "'extractor.history.start-time'='now'," + + "'extractor.history.end-time'='now')"; + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(alterP3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + alterP3 = + "alter pipe p3" + + " modify source (" + + "'extractor.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')"; + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(alterP3); + } catch (final SQLException e) { + fail(e.getMessage()); + } + } + + private void assertTimeseriesCountOnReceiver(BaseEnv receiverEnv, int count) { + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "count timeseries root.db.**", + "count(timeseries),", + Collections.singleton(count + ",")); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java new file mode 100644 index 0000000000000..12f2857c7c18a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java @@ -0,0 +1,1054 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.RegionRoleType; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.env.cluster.env.AbstractEnv; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeClusterIT extends AbstractPipeTableModelDualManualIT { + + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setDnConnectionTimeoutMs(600000) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setDataReplicationFactor(2) + .setSchemaReplicationFactor(3) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setDnConnectionTimeoutMs(600000) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + + senderEnv.initClusterEnvironment(3, 3, 180); + receiverEnv.initClusterEnvironment(3, 3, 180); + } + + @Test + public void testMachineDowntimeAsync() { + testMachineDowntime("iotdb-thrift-connector"); + } + + @Test + public void testMachineDowntimeSync() { + testMachineDowntime("iotdb-thrift-sync-connector"); + } + + private void testMachineDowntime(String sink) { + StringBuilder a = new StringBuilder(); + for (DataNodeWrapper nodeWrapper : receiverEnv.getDataNodeWrapperList()) { + a.append(nodeWrapper.getIp()).append(":").append(nodeWrapper.getPort()); + a.append(","); + } + a.deleteCharAt(a.length() - 1); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.insertData("test", "test", 0, 1, senderEnv); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("user", "root"); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", sink); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.node-urls", a.toString()); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + TableModelUtils.assertCountData("test", "test", 1, receiverEnv); + receiverEnv.getDataNodeWrapper(0).stop(); + + // Ensure that the kill -9 operation is completed + Thread.sleep(5000); + TableModelUtils.insertData("test", "test", 1, 2, senderEnv); + } catch (Exception e) { + fail(e.getMessage()); + } + + for (DataNodeWrapper nodeWrapper : receiverEnv.getDataNodeWrapperList()) { + if (!nodeWrapper.isAlive()) { + continue; + } + TableModelUtils.assertCountData("test", "test", 2, receiverEnv, nodeWrapper); + return; + } + } + + @Test + public void testWithAllParametersInStreamingMode() throws Exception { + testWithAllParameters("true"); + } + + @Test + public void testWithAllParametersInNotStreamingMode() throws Exception { + testWithAllParameters("false"); + } + + private void testWithAllParameters(final String realtimeMode) throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + boolean insertResult = true; + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("start-time", "0"); + extractorAttributes.put("end-time", "199"); + extractorAttributes.put("mode.streaming", realtimeMode); + extractorAttributes.put("user", "root"); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.user", "root"); + connectorAttributes.put("connector.password", "root"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQuerySql("test"), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(0, 100), + "test", + handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", 100, 300, senderEnv); + if (!insertResult) { + return; + } + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQuerySql("test"), + TableModelUtils.generateHeaderResults(), + TableModelUtils.generateExpectedResults(0, 200), + "test", + handleFailure); + } + } + + // This function has a certain probability of triggering replica asynchrony. To ensure the success + // of the test, it will be retried 5 times. The exception will be thrown after five retries. + @Test + public void testPipeAfterDataRegionLeaderStop() throws Exception { + for (int retry = 0; retry < 5; retry++) { + try { + if (retry != 0) { + this.setUp(); + } + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("start-time", "0"); + extractorAttributes.put("end-time", "300"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + + final AtomicInteger leaderPort = new AtomicInteger(-1); + final TShowRegionResp showRegionResp = + client.showRegion(new TShowRegionReq().setIsTableModel(true)); + showRegionResp + .getRegionInfoList() + .forEach( + regionInfo -> { + if (RegionRoleType.Leader.getRoleType().equals(regionInfo.getRoleType())) { + leaderPort.set(regionInfo.getClientRpcPort()); + } + }); + + int leaderIndex = -1; + for (int i = 0; i < 3; ++i) { + if (senderEnv.getDataNodeWrapper(i).getPort() == leaderPort.get()) { + leaderIndex = i; + try { + senderEnv.shutdownDataNode(i); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + try { + TimeUnit.SECONDS.sleep(1); + } catch (final InterruptedException ignored) { + } + try { + senderEnv.startDataNode(i); + ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + } + } + if (leaderIndex == -1) { // ensure the leader is stopped + fail(); + } + + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 200, 300, senderEnv); + if (!insertResult) { + return; + } + + TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); + } + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Create a new pipe and write new data + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test1"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test1"); + extractorAttributes.put("start-time", "0"); + extractorAttributes.put("end-time", "300"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 300, 400, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 300, 400, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", 0, 301, receiverEnv, handleFailure); + TableModelUtils.assertData("test1", "test1", 0, 301, receiverEnv, handleFailure); + } + return; + } catch (Exception | Error e) { + if (retry < 4) { + this.tearDown(); + } else { + throw e; + } + } + } + } + + @Test + public void testPipeAfterRegisterNewDataNode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + if (!insertResult) { + return; + } + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + if (!insertResult) { + return; + } + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + final DataNodeWrapper newDataNode = + senderEnv.getDataNodeWrapper(senderEnv.getDataNodeWrapperList().size() - 1); + insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv, newDataNode); + insertResult = + insertResult + && TableModelUtils.insertData("test1", "test1", 200, 300, senderEnv, newDataNode); + if (!insertResult) { + return; + } + + TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); + } + + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // create a new pipe and write new data + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test1"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test1"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p2", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + + insertResult = TableModelUtils.insertData("test", "test", 300, 400, senderEnv); + insertResult = + insertResult && TableModelUtils.insertData("test1", "test1", 300, 400, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test1", "test1", 0, 400, receiverEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); + } + } + + @Test + public void testCreatePipeWhenRegisteringNewDataNode() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test1"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test1"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final Thread t = + new Thread( + () -> { + for (int i = 0; i < 30; ++i) { + try { + client.createPipe( + new TCreatePipeReq("p" + i, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + } catch (final TException e) { + // Not sure if the "createPipe" has succeeded + e.printStackTrace(); + return; + } + try { + Thread.sleep(100); + } catch (final Exception ignored) { + } + } + }); + t.start(); + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + t.join(); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(30, showPipeResult.size()); + } + } + + @Test + public void testRegisteringNewDataNodeWhenTransferringData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final AtomicInteger succeedNum = new AtomicInteger(0); + final Thread t = + new Thread( + () -> { + try { + for (int i = 100; i < 200; ++i) { + if (TableModelUtils.insertDataNotThrowError( + "test", "test", i, i + 1, senderEnv)) { + succeedNum.incrementAndGet(); + Thread.sleep(100); + } + } + } catch (final InterruptedException ignored) { + } + }); + t.start(); + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + t.join(); + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush", null)) { + return; + } + + TableModelUtils.assertCountData( + "test", "test", succeedNum.get() + 100, receiverEnv, handleFailure); + + try { + senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); + senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); + } catch (final Throwable e) { + e.printStackTrace(); + } + } + } + + @Test + public void testRegisteringNewDataNodeAfterTransferringData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + boolean insertResult = true; + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + int succeedNum = 0; + for (int i = 100; i < 200; ++i) { + if (TableModelUtils.insertDataNotThrowError("test", "test", i, i + 1, senderEnv)) { + succeedNum++; + } + } + + try { + senderEnv.registerNewDataNode(true); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + TableModelUtils.assertCountData("test", "test", succeedNum + 100, receiverEnv, handleFailure); + + try { + senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); + senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); + } catch (final Throwable e) { + e.printStackTrace(); + } + } + } + + @Test + public void testSenderRestartWhenTransferring() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + boolean insertResult = true; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + } + + int succeedNum = 0; + for (int i = 100; i < 200; ++i) { + if (TableModelUtils.insertDataNotThrowError("test", "test", i, i + 1, senderEnv)) { + succeedNum++; + } + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush", null)) { + return; + } + + try { + TestUtils.restartCluster(senderEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + TableModelUtils.assertCountData("test", "test", succeedNum + 100, receiverEnv, handleFailure); + } + + @Test + public void testConcurrentlyCreatePipeOfSameName() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final AtomicInteger successCount = new AtomicInteger(0); + final List threads = new ArrayList<>(); + for (int i = 0; i < 10; ++i) { + final Thread t = + new Thread( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + successCount.incrementAndGet(); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (final TException | ClientManagerException | IOException e) { + e.printStackTrace(); + } catch (final Exception e) { + // Fail iff pipe exception occurs + e.printStackTrace(); + fail(e.getMessage()); + } + }); + t.start(); + threads.add(t); + } + + for (Thread t : threads) { + t.join(); + } + Assert.assertEquals(1, successCount.get()); + + successCount.set(0); + for (int i = 0; i < 10; ++i) { + final Thread t = + new Thread( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TSStatus status = client.dropPipe("p1"); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + successCount.incrementAndGet(); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (final TException | ClientManagerException | IOException e) { + e.printStackTrace(); + } catch (final Exception e) { + // Fail iff pipe exception occurs + e.printStackTrace(); + fail(e.getMessage()); + } + }); + t.start(); + threads.add(t); + } + for (final Thread t : threads) { + t.join(); + } + + // Assert at least 1 drop operation succeeds + Assert.assertTrue(successCount.get() >= 1); + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(0, showPipeResult.size()); + } + } + + @Test + public void testCreate10PipesWithSameConnector() throws Exception { + testCreatePipesWithSameConnector(10); + } + + @Test + public void testCreate50PipesWithSameConnector() throws Exception { + testCreatePipesWithSameConnector(50); + } + + @Test + public void testCreate100PipesWithSameConnector() throws Exception { + testCreatePipesWithSameConnector(100); + } + + private void testCreatePipesWithSameConnector(final int pipeCount) throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final AtomicInteger successCount = new AtomicInteger(0); + final List threads = new ArrayList<>(); + for (int i = 0; i < pipeCount; ++i) { + final int finalI = i; + final Thread t = + new Thread( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p" + finalI, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + successCount.incrementAndGet(); + } catch (final InterruptedException e) { + e.printStackTrace(); + Thread.currentThread().interrupt(); + } catch (final TException | ClientManagerException | IOException e) { + e.printStackTrace(); + } catch (final Exception e) { + // Fail iff pipe exception occurs + e.printStackTrace(); + fail(e.getMessage()); + } + }); + t.start(); + threads.add(t); + } + for (final Thread t : threads) { + t.join(); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(successCount.get(), showPipeResult.size()); + showPipeResult = + client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(successCount.get(), showPipeResult.size()); + } + } + + @Test + public void testNegativeTimestamp() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + boolean insertResult = true; + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", -100, 100, senderEnv); + if (!insertResult) { + return; + } + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor", "iotdb-extractor"); + extractorAttributes.put("database-name", "test"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("table-name", "test"); + extractorAttributes.put("user", "root"); + + processorAttributes.put("processor", "do-nothing-processor"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TableModelUtils.assertData("test", "test", -100, 100, receiverEnv, handleFailure); + + insertResult = TableModelUtils.insertData("test", "test", -200, -100, senderEnv); + if (!insertResult) { + return; + } + TableModelUtils.assertData("test", "test", -200, 100, receiverEnv, handleFailure); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java index b7091a1db3299..a13e8dc152dbe 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java @@ -49,6 +49,7 @@ protected void setupConfig() { .setAutoCreateSchemaEnabled(false) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); receiverEnv .getConfig() @@ -56,6 +57,7 @@ protected void setupConfig() { .setAutoCreateSchemaEnabled(false) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); // 10 min, assert that the operations will not time out diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java index 8156139b84ceb..21490c2687668 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java @@ -83,6 +83,7 @@ public void setUp() throws Exception { .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) .setSchemaReplicationFactor(3) .setDataReplicationFactor(2) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); EnvFactory.getEnv().initClusterEnvironment(3, 3); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java index 7b5640234980c..fe667480c86bf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java @@ -37,6 +37,7 @@ public void setUp() throws Exception { .getConfig() .getCommonConfig() .setSubscriptionEnabled(true) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); EnvFactory.getEnv().initClusterEnvironment(); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java index e758bcb5a1719..6b125222e6135 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java @@ -74,6 +74,7 @@ protected void setUpConfig() { sender .getConfig() .getCommonConfig() + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false) .setSubscriptionPrefetchTsFileBatchMaxDelayInMs(500) .setSubscriptionPrefetchTsFileBatchMaxSizeInBytes(32 * 1024); diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java index 115dafbc5dc22..88b426d9a4af9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java @@ -57,6 +57,7 @@ public static void setUp() throws Exception { .getConfig() .getCommonConfig() .setSubscriptionEnabled(true) + .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); EnvFactory.getEnv().initClusterEnvironment(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 2fc0e6b02a8a2..43b6d4e3b5f91 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.MetaProgressIndex; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.task.PipeTask; import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; @@ -159,18 +160,17 @@ protected void createPipeTask( final PipeTaskMeta pipeTaskMeta) throws IllegalPathException { if (pipeTaskMeta.getLeaderNodeId() == CONFIG.getDataNodeId()) { - final PipeParameters extractorParameters = pipeStaticMeta.getExtractorParameters(); + final PipeParameters sourceParameters = pipeStaticMeta.getExtractorParameters(); final DataRegionId dataRegionId = new DataRegionId(consensusGroupId); final boolean needConstructDataRegionTask = StorageEngine.getInstance().getAllDataRegionIds().contains(dataRegionId) && DataRegionListeningFilter.shouldDataRegionBeListened( - extractorParameters, dataRegionId); + sourceParameters, dataRegionId); final boolean needConstructSchemaRegionTask = SchemaEngine.getInstance() .getAllSchemaRegionIds() .contains(new SchemaRegionId(consensusGroupId)) - && !SchemaRegionListeningFilter.parseListeningPlanTypeSet(extractorParameters) - .isEmpty(); + && !SchemaRegionListeningFilter.parseListeningPlanTypeSet(sourceParameters).isEmpty(); // Advance the extractor parameters parsing logic to avoid creating un-relevant pipeTasks if (needConstructDataRegionTask || needConstructSchemaRegionTask) { @@ -410,7 +410,7 @@ private void collectPipeMetaListInternal(final TDataNodeHeartbeatResp resp) thro || pipeTaskMap.entrySet().stream() .filter(entry -> dataRegionIds.contains(entry.getKey())) .allMatch(entry -> ((PipeDataNodeTask) entry.getValue()).isCompleted()); - final String extractorModeValue = + final String sourceModeValue = pipeMeta .getStaticMeta() .getExtractorParameters() @@ -422,9 +422,8 @@ private void collectPipeMetaListInternal(final TDataNodeHeartbeatResp resp) thro DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair( pipeMeta.getStaticMeta().getExtractorParameters()) .getLeft() - && (extractorModeValue.equalsIgnoreCase( - PipeSourceConstant.EXTRACTOR_MODE_QUERY_VALUE) - || extractorModeValue.equalsIgnoreCase( + && (sourceModeValue.equalsIgnoreCase(PipeSourceConstant.EXTRACTOR_MODE_QUERY_VALUE) + || sourceModeValue.equalsIgnoreCase( PipeSourceConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE)); final boolean isCompleted = isAllDataRegionCompleted && includeDataAndNeedDrop; @@ -665,24 +664,24 @@ public Map getAllConsensusPipe() { @Override protected void calculateMemoryUsage( - final PipeParameters extractorParameters, + final PipeStaticMeta staticMeta, + final PipeParameters sourceParameters, final PipeParameters processorParameters, - final PipeParameters connectorParameters) { - if (!PipeConfig.getInstance().isPipeEnableMemoryCheck()) { + final PipeParameters sinkParameters) { + if (!PipeConfig.getInstance().isPipeEnableMemoryCheck() + || !isInnerSource(sourceParameters) + || !PipeType.USER.equals(staticMeta.getPipeType())) { return; } - calculateInsertNodeQueueMemory(extractorParameters, processorParameters, connectorParameters); + calculateInsertNodeQueueMemory(sourceParameters); long needMemory = 0; - needMemory += - calculateTsFileParserMemory(extractorParameters, processorParameters, connectorParameters); - needMemory += - calculateSinkBatchMemory(extractorParameters, processorParameters, connectorParameters); - needMemory += - calculateSendTsFileReadBufferMemory( - extractorParameters, processorParameters, connectorParameters); + needMemory += calculateTsFileParserMemory(sourceParameters, sinkParameters); + needMemory += calculateSinkBatchMemory(sinkParameters); + needMemory += calculateSendTsFileReadBufferMemory(sourceParameters, sinkParameters); + needMemory += calculateAssignerMemory(sourceParameters); PipeMemoryManager pipeMemoryManager = PipeDataNodeResourceManager.memory(); final long freeMemorySizeInBytes = pipeMemoryManager.getFreeMemorySizeInBytes(); @@ -703,13 +702,22 @@ protected void calculateMemoryUsage( } } - private void calculateInsertNodeQueueMemory( - final PipeParameters extractorParameters, - final PipeParameters processorParameters, - final PipeParameters connectorParameters) { + private boolean isInnerSource(final PipeParameters sourceParameters) { + final String pluginName = + sourceParameters + .getStringOrDefault( + Arrays.asList(PipeSourceConstant.EXTRACTOR_KEY, PipeSourceConstant.SOURCE_KEY), + BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName()) + .toLowerCase(); - // Realtime extractor is enabled by default, so we only need to check the source realtime - if (!extractorParameters.getBooleanOrDefault( + return pluginName.equals(BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName()) + || pluginName.equals(BuiltinPipePlugin.IOTDB_SOURCE.getPipePluginName()); + } + + private void calculateInsertNodeQueueMemory(final PipeParameters sourceParameters) { + + // Realtime source is enabled by default, so we only need to check the source realtime + if (!sourceParameters.getBooleanOrDefault( Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY), EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE)) { return; @@ -717,7 +725,7 @@ private void calculateInsertNodeQueueMemory( // If the realtime mode is batch or file, we do not need to allocate memory final String realtimeMode = - extractorParameters.getStringByKeys( + sourceParameters.getStringByKeys( PipeSourceConstant.EXTRACTOR_REALTIME_MODE_KEY, PipeSourceConstant.SOURCE_REALTIME_MODE_KEY); if (PipeSourceConstant.EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE.equals(realtimeMode) @@ -739,53 +747,50 @@ private void calculateInsertNodeQueueMemory( } private long calculateTsFileParserMemory( - final PipeParameters extractorParameters, - final PipeParameters processorParameters, - final PipeParameters connectorParameters) { + final PipeParameters sourceParameters, final PipeParameters sinkParameters) { - // If the extractor is not history, we do not need to allocate memory + // If the source is not history, we do not need to allocate memory boolean isExtractorHistory = - extractorParameters.getBooleanOrDefault( + sourceParameters.getBooleanOrDefault( SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE) - || extractorParameters.getBooleanOrDefault( + || sourceParameters.getBooleanOrDefault( Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY), EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE); - // If the extractor is history, and has start/end time, we need to allocate memory + // If the source is history, and has start/end time, we need to allocate memory boolean isTSFileParser = isExtractorHistory - && extractorParameters.hasAnyAttributes( + && sourceParameters.hasAnyAttributes( EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY); isTSFileParser = isTSFileParser || (isExtractorHistory - && extractorParameters.hasAnyAttributes( + && sourceParameters.hasAnyAttributes( EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY)); - // if the extractor has start/end time, we need to allocate memory + // if the source has start/end time, we need to allocate memory isTSFileParser = isTSFileParser - || extractorParameters.hasAnyAttributes( - SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY); + || sourceParameters.hasAnyAttributes(SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY); isTSFileParser = isTSFileParser - || extractorParameters.hasAnyAttributes(SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY); + || sourceParameters.hasAnyAttributes(SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY); - // If the extractor has pattern or path, we need to allocate memory + // If the source has pattern or path, we need to allocate memory isTSFileParser = isTSFileParser - || extractorParameters.hasAnyAttributes(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY); + || sourceParameters.hasAnyAttributes(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY); isTSFileParser = - isTSFileParser || extractorParameters.hasAnyAttributes(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY); + isTSFileParser || sourceParameters.hasAnyAttributes(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY); - // If the extractor is not hybrid, we do need to allocate memory + // If the source is not hybrid, we do need to allocate memory isTSFileParser = isTSFileParser || !PipeSinkConstant.CONNECTOR_FORMAT_HYBRID_VALUE.equals( - connectorParameters.getStringOrDefault( + sinkParameters.getStringOrDefault( Arrays.asList( PipeSinkConstant.CONNECTOR_FORMAT_KEY, PipeSinkConstant.SINK_FORMAT_KEY), PipeSinkConstant.CONNECTOR_FORMAT_HYBRID_VALUE)); @@ -797,15 +802,12 @@ private long calculateTsFileParserMemory( return PipeConfig.getInstance().getTsFileParserMemory(); } - private long calculateSinkBatchMemory( - final PipeParameters extractorParameters, - final PipeParameters processorParameters, - final PipeParameters connectorParameters) { + private long calculateSinkBatchMemory(final PipeParameters sinkParameters) { - // If the connector format is tsfile , we need to use batch + // If the sink format is tsfile , we need to use batch boolean needUseBatch = PipeSinkConstant.CONNECTOR_FORMAT_TS_FILE_VALUE.equals( - connectorParameters.getStringOrDefault( + sinkParameters.getStringOrDefault( Arrays.asList( PipeSinkConstant.CONNECTOR_FORMAT_KEY, PipeSinkConstant.SINK_FORMAT_KEY), PipeSinkConstant.CONNECTOR_FORMAT_HYBRID_VALUE)); @@ -814,9 +816,9 @@ private long calculateSinkBatchMemory( return PipeConfig.getInstance().getSinkBatchMemoryTsFile(); } - // If the connector is batch mode, we need to use batch + // If the sink is batch mode, we need to use batch needUseBatch = - connectorParameters.getBooleanOrDefault( + sinkParameters.getBooleanOrDefault( Arrays.asList( PipeSinkConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY, PipeSinkConstant.SINK_IOTDB_BATCH_MODE_ENABLE_KEY), @@ -830,23 +832,21 @@ private long calculateSinkBatchMemory( } private long calculateSendTsFileReadBufferMemory( - final PipeParameters extractorParameters, - final PipeParameters processorParameters, - final PipeParameters connectorParameters) { - // If the extractor is history enable, we need to transfer tsfile + final PipeParameters sourceParameters, final PipeParameters sinkParameters) { + // If the source is history enable, we need to transfer tsfile boolean needTransferTsFile = - extractorParameters.getBooleanOrDefault( + sourceParameters.getBooleanOrDefault( SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE) - || extractorParameters.getBooleanOrDefault( + || sourceParameters.getBooleanOrDefault( Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY), EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE); String format = - connectorParameters.getStringOrDefault( + sinkParameters.getStringOrDefault( Arrays.asList(PipeSinkConstant.CONNECTOR_FORMAT_KEY, PipeSinkConstant.SINK_FORMAT_KEY), PipeSinkConstant.CONNECTOR_FORMAT_HYBRID_VALUE); - // If the connector format is tsfile and hybrid, we need to transfer tsfile + // If the sink format is tsfile and hybrid, we need to transfer tsfile needTransferTsFile = needTransferTsFile || PipeSinkConstant.CONNECTOR_FORMAT_HYBRID_VALUE.equals(format) @@ -858,4 +858,19 @@ private long calculateSendTsFileReadBufferMemory( return PipeConfig.getInstance().getSendTsFileReadBuffer(); } + + private long calculateAssignerMemory(final PipeParameters sourceParameters) { + try { + if (!PipeInsertionDataNodeListener.getInstance().isEmpty() + || !DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(sourceParameters) + .getLeft()) { + return 0; + } + return PipeConfig.getInstance().getPipeExtractorAssignerDisruptorRingBufferSize() + * PipeConfig.getInstance().getPipeExtractorAssignerDisruptorRingBufferEntrySizeInBytes() + * Math.min(StorageEngine.getInstance().getDataRegionNumber(), 10); + } catch (final IllegalPathException e) { + return 0; + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java new file mode 100644 index 0000000000000..af38d626f4e31 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java @@ -0,0 +1,947 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.historical; + +import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.commons.consensus.index.ProgressIndex; +import org.apache.iotdb.commons.consensus.index.ProgressIndexType; +import org.apache.iotdb.commons.consensus.index.impl.HybridProgressIndex; +import org.apache.iotdb.commons.consensus.index.impl.RecoverProgressIndex; +import org.apache.iotdb.commons.consensus.index.impl.StateProgressIndex; +import org.apache.iotdb.commons.consensus.index.impl.TimeWindowStateProgressIndex; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; +import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskExtractorRuntimeEnvironment; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.resource.PersistentResource; +import org.apache.iotdb.commons.pipe.event.ProgressReportEvent; +import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.consensus.pipe.PipeConsensus; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; +import org.apache.iotdb.db.pipe.consensus.ReplicateProgressDataNodeManager; +import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResource; +import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager; +import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; +import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.apache.iotdb.db.pipe.processor.pipeconsensus.PipeConsensusProcessor; +import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; +import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter; +import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.DateTimeUtils; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; + +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.PlainDeviceID; +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_END_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_ENABLE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_END_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_ALL_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_PATH_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_TIME_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_START_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODE_STRICT_DEFAULT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODE_STRICT_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODS_DEFAULT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODS_ENABLE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODS_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_START_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_END_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_HISTORY_ENABLE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_HISTORY_END_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_HISTORY_LOOSE_RANGE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_HISTORY_START_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_MODE_STRICT_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_MODS_ENABLE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_MODS_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_START_TIME_KEY; +import static org.apache.iotdb.commons.pipe.source.IoTDBSource.getSkipIfNoPrivileges; +import static org.apache.tsfile.common.constant.TsFileConstant.PATH_ROOT; +import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; + +public class PipeHistoricalDataRegionTsFileAndDeletionSource + implements PipeHistoricalDataRegionSource { + + private static final Logger LOGGER = + LoggerFactory.getLogger(PipeHistoricalDataRegionTsFileAndDeletionSource.class); + + private static final Map DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP = new HashMap<>(); + + private static final String TREE_MODEL_EVENT_TABLE_NAME_PREFIX = PATH_ROOT + PATH_SEPARATOR; + + private String pipeName; + private long creationTime; + + private PipeTaskMeta pipeTaskMeta; + private ProgressIndex startIndex; + + private int dataRegionId; + + private TreePattern treePattern; + private TablePattern tablePattern; + + private boolean isModelDetected = false; + private boolean isTableModel; + private boolean isDbNameCoveredByPattern = false; + + private boolean isHistoricalExtractorEnabled = false; + private long historicalDataExtractionStartTime = Long.MIN_VALUE; // Event time + private long historicalDataExtractionEndTime = Long.MAX_VALUE; // Event time + + private boolean sloppyTimeRange; // true to disable time range filter after extraction + private boolean sloppyPattern; // true to disable pattern filter after extraction + + private Pair listeningOptionPair; + private boolean shouldExtractInsertion; + private boolean shouldExtractDeletion; + private boolean shouldTransferModFile; // Whether to transfer mods + protected String userName; + protected boolean skipIfNoPrivileges = true; + private boolean isTerminateSignalSent = false; + + private boolean isForwardingPipeRequests; + + private volatile boolean hasBeenStarted = false; + + private Queue pendingQueue; + private final Set filteredTsFileResources = new HashSet<>(); + + @Override + public void validate(final PipeParameterValidator validator) { + final PipeParameters parameters = validator.getParameters(); + + try { + listeningOptionPair = + DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters); + } catch (final Exception e) { + // compatible with the current validation framework + throw new PipeParameterNotValidException(e.getMessage()); + } + + if (parameters.hasAnyAttributes(EXTRACTOR_MODE_STRICT_KEY, SOURCE_MODE_STRICT_KEY)) { + final boolean isStrictMode = + parameters.getBooleanOrDefault( + Arrays.asList(EXTRACTOR_MODE_STRICT_KEY, SOURCE_MODE_STRICT_KEY), + EXTRACTOR_MODE_STRICT_DEFAULT_VALUE); + sloppyTimeRange = !isStrictMode; + sloppyPattern = !isStrictMode; + } else { + final String extractorHistoryLooseRangeValue = + parameters + .getStringOrDefault( + Arrays.asList(EXTRACTOR_HISTORY_LOOSE_RANGE_KEY, SOURCE_HISTORY_LOOSE_RANGE_KEY), + EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE) + .trim(); + if (EXTRACTOR_HISTORY_LOOSE_RANGE_ALL_VALUE.equalsIgnoreCase( + extractorHistoryLooseRangeValue)) { + sloppyTimeRange = true; + sloppyPattern = true; + } else { + final Set sloppyOptionSet = + Arrays.stream(extractorHistoryLooseRangeValue.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .map(String::toLowerCase) + .collect(Collectors.toSet()); + sloppyTimeRange = sloppyOptionSet.remove(EXTRACTOR_HISTORY_LOOSE_RANGE_TIME_VALUE); + sloppyPattern = sloppyOptionSet.remove(EXTRACTOR_HISTORY_LOOSE_RANGE_PATH_VALUE); + if (!sloppyOptionSet.isEmpty()) { + throw new PipeParameterNotValidException( + String.format( + "Parameters in set %s are not allowed in 'history.loose-range'", + sloppyOptionSet)); + } + } + } + + if (parameters.hasAnyAttributes( + SOURCE_START_TIME_KEY, + EXTRACTOR_START_TIME_KEY, + SOURCE_END_TIME_KEY, + EXTRACTOR_END_TIME_KEY)) { + isHistoricalExtractorEnabled = true; + + try { + historicalDataExtractionStartTime = + parameters.hasAnyAttributes(SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY) + ? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( + parameters.getStringByKeys(SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY)) + : Long.MIN_VALUE; + historicalDataExtractionEndTime = + parameters.hasAnyAttributes(SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY) + ? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( + parameters.getStringByKeys(SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY)) + : Long.MAX_VALUE; + if (historicalDataExtractionStartTime > historicalDataExtractionEndTime) { + throw new PipeParameterNotValidException( + String.format( + "%s (%s) [%s] should be less than or equal to %s (%s) [%s].", + SOURCE_START_TIME_KEY, + EXTRACTOR_START_TIME_KEY, + historicalDataExtractionStartTime, + SOURCE_END_TIME_KEY, + EXTRACTOR_END_TIME_KEY, + historicalDataExtractionEndTime)); + } + } catch (final PipeParameterNotValidException e) { + throw e; + } catch (final Exception e) { + // compatible with the current validation framework + throw new PipeParameterNotValidException(e.getMessage()); + } + + // return here + return; + } + + // Historical data extraction is enabled in the following cases: + // 1. System restarts the pipe. If the pipe is restarted but historical data extraction is not + // enabled, the pipe will lose some historical data. + // 2. User may set the EXTRACTOR_HISTORY_START_TIME and EXTRACTOR_HISTORY_END_TIME without + // enabling the historical data extraction, which may affect the realtime data extraction. + isHistoricalExtractorEnabled = + parameters.getBooleanOrDefault( + SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE) + || parameters.getBooleanOrDefault( + Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY), + EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE); + + try { + historicalDataExtractionStartTime = + parameters.hasAnyAttributes( + EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY) + ? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( + parameters.getStringByKeys( + EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY)) + : Long.MIN_VALUE; + historicalDataExtractionEndTime = + parameters.hasAnyAttributes(EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY) + ? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( + parameters.getStringByKeys( + EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY)) + : Long.MAX_VALUE; + if (historicalDataExtractionStartTime > historicalDataExtractionEndTime) { + throw new PipeParameterNotValidException( + String.format( + "%s (%s) [%s] should be less than or equal to %s (%s) [%s].", + EXTRACTOR_HISTORY_START_TIME_KEY, + SOURCE_HISTORY_START_TIME_KEY, + historicalDataExtractionStartTime, + EXTRACTOR_HISTORY_END_TIME_KEY, + SOURCE_HISTORY_END_TIME_KEY, + historicalDataExtractionEndTime)); + } + } catch (final Exception e) { + // Compatible with the current validation framework + throw new PipeParameterNotValidException(e.getMessage()); + } + } + + @Override + public void customize( + final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration) + throws IllegalPathException { + shouldExtractInsertion = listeningOptionPair.getLeft(); + shouldExtractDeletion = listeningOptionPair.getRight(); + // Do nothing if extract deletion + if (!shouldExtractInsertion) { + return; + } + + final PipeTaskExtractorRuntimeEnvironment environment = + (PipeTaskExtractorRuntimeEnvironment) configuration.getRuntimeEnvironment(); + + pipeName = environment.getPipeName(); + creationTime = environment.getCreationTime(); + pipeTaskMeta = environment.getPipeTaskMeta(); + if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + startIndex = + tryToExtractLocalProgressIndexForIoTV2(environment.getPipeTaskMeta().getProgressIndex()); + } else { + startIndex = environment.getPipeTaskMeta().getProgressIndex(); + } + + dataRegionId = environment.getRegionId(); + synchronized (DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP) { + DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.putIfAbsent(dataRegionId, 0L); + } + + treePattern = TreePattern.parsePipePatternFromSourceParameters(parameters); + tablePattern = TablePattern.parsePipePatternFromSourceParameters(parameters); + + final DataRegion dataRegion = + StorageEngine.getInstance().getDataRegion(new DataRegionId(environment.getRegionId())); + if (Objects.nonNull(dataRegion)) { + final String databaseName = dataRegion.getDatabaseName(); + if (Objects.nonNull(databaseName)) { + isTableModel = PathUtils.isTableModelDatabase(databaseName); + isModelDetected = true; + if (isTableModel) { + isDbNameCoveredByPattern = tablePattern.coversDb(databaseName); + } else { + isDbNameCoveredByPattern = treePattern.coversDb(databaseName); + } + } + } + + if (parameters.hasAnyAttributes(EXTRACTOR_MODS_KEY, SOURCE_MODS_KEY)) { + shouldTransferModFile = + parameters.getBooleanOrDefault( + Arrays.asList(EXTRACTOR_MODS_KEY, SOURCE_MODS_KEY), + EXTRACTOR_MODS_DEFAULT_VALUE + || // Should extract deletion + listeningOptionPair.getRight()); + } else { + shouldTransferModFile = + parameters.getBooleanOrDefault( + Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY), + EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE + || // Should extract deletion + listeningOptionPair.getRight()); + } + + userName = + parameters.getStringByKeys( + PipeSourceConstant.EXTRACTOR_IOTDB_USER_KEY, + PipeSourceConstant.SOURCE_IOTDB_USER_KEY, + PipeSourceConstant.EXTRACTOR_IOTDB_USERNAME_KEY, + PipeSourceConstant.SOURCE_IOTDB_USERNAME_KEY); + + skipIfNoPrivileges = getSkipIfNoPrivileges(parameters); + + isForwardingPipeRequests = + parameters.getBooleanOrDefault( + Arrays.asList( + PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, + PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), + PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); + + if (LOGGER.isInfoEnabled()) { + LOGGER.info( + "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy pattern {}, sloppy time range {}, should transfer mod file {}, username: {}, skip if no privileges: {}, is forwarding pipe requests: {}", + pipeName, + dataRegionId, + DateTimeUtils.convertLongToDate(historicalDataExtractionStartTime), + historicalDataExtractionStartTime, + DateTimeUtils.convertLongToDate(historicalDataExtractionEndTime), + historicalDataExtractionEndTime, + sloppyPattern, + sloppyTimeRange, + shouldTransferModFile, + userName, + skipIfNoPrivileges, + isForwardingPipeRequests); + } + } + + /** + * IoTV2 will only resend event that contains un-replicated local write data. So we only extract + * ProgressIndex containing local writes for comparison to prevent misjudgment on whether + * high-level tsFiles with mixed progressIndexes need to be retransmitted + * + * @return recoverProgressIndex dedicated in local DataNodeId or origin for fallback. + */ + private ProgressIndex tryToExtractLocalProgressIndexForIoTV2(ProgressIndex origin) { + // There are only 2 cases: + // 1. origin is RecoverProgressIndex + if (origin instanceof RecoverProgressIndex) { + RecoverProgressIndex toBeTransformed = (RecoverProgressIndex) origin; + return extractRecoverProgressIndex(toBeTransformed); + } + // 2. origin is HybridProgressIndex + else if (origin instanceof HybridProgressIndex) { + HybridProgressIndex toBeTransformed = (HybridProgressIndex) origin; + // if hybridProgressIndex contains recoverProgressIndex, which is what we expected. + if (toBeTransformed + .getType2Index() + .containsKey(ProgressIndexType.RECOVER_PROGRESS_INDEX.getType())) { + // 2.1. transform recoverProgressIndex + RecoverProgressIndex specificToBeTransformed = + (RecoverProgressIndex) + toBeTransformed + .getType2Index() + .get(ProgressIndexType.RECOVER_PROGRESS_INDEX.getType()); + return extractRecoverProgressIndex(specificToBeTransformed); + } + // if hybridProgressIndex doesn't contain recoverProgressIndex, which is not what we expected, + // fallback. + return origin; + } else { + // fallback + LOGGER.warn( + "Pipe {}@{}: unexpected ProgressIndex type {}, fallback to origin {}.", + pipeName, + dataRegionId, + origin.getType(), + origin); + return origin; + } + } + + private ProgressIndex extractRecoverProgressIndex(RecoverProgressIndex toBeTransformed) { + return new RecoverProgressIndex( + toBeTransformed.getDataNodeId2LocalIndex().entrySet().stream() + .filter( + entry -> + entry + .getKey() + .equals(IoTDBDescriptor.getInstance().getConfig().getDataNodeId())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + + @Override + public synchronized void start() { + if (!shouldExtractInsertion) { + hasBeenStarted = true; + return; + } + if (!StorageEngine.getInstance().isReadyForNonReadWriteFunctions()) { + LOGGER.info( + "Pipe {}@{}: failed to start to extract historical TsFile, storage engine is not ready. Will retry later.", + pipeName, + dataRegionId); + return; + } + hasBeenStarted = true; + + final DataRegion dataRegion = + StorageEngine.getInstance().getDataRegion(new DataRegionId(dataRegionId)); + if (Objects.isNull(dataRegion)) { + pendingQueue = new ArrayDeque<>(); + return; + } + + final long startHistoricalExtractionTime = System.currentTimeMillis(); + dataRegion.writeLock( + "Pipe: start to extract historical TsFile and Deletion(if uses pipeConsensus)"); + try { + List originalResourceList = new ArrayList<>(); + + if (shouldExtractInsertion) { + flushTsFilesForExtraction(dataRegion); + extractTsFiles(dataRegion, startHistoricalExtractionTime, originalResourceList); + } + if (shouldExtractDeletion) { + Optional.ofNullable(DeletionResourceManager.getInstance(String.valueOf(dataRegionId))) + .ifPresent(manager -> extractDeletions(manager, originalResourceList)); + } + + // Sort tsFileResource and deletionResource + long startTime = System.currentTimeMillis(); + LOGGER.info("Pipe {}@{}: start to sort all extracted resources", pipeName, dataRegionId); + originalResourceList.sort( + (o1, o2) -> + startIndex instanceof TimeWindowStateProgressIndex + ? Long.compare(o1.getFileStartTime(), o2.getFileStartTime()) + : o1.getProgressIndex().topologicalCompareTo(o2.getProgressIndex())); + pendingQueue = new ArrayDeque<>(originalResourceList); + + LOGGER.info( + "Pipe {}@{}: finish to sort all extracted resources, took {} ms", + pipeName, + dataRegionId, + System.currentTimeMillis() - startTime); + } finally { + dataRegion.writeUnlock(); + } + } + + private void flushTsFilesForExtraction(DataRegion dataRegion) { + LOGGER.info("Pipe {}@{}: start to flush data region", pipeName, dataRegionId); + + // Consider the scenario: a consensus pipe comes to the same region, followed by another pipe + // **immediately**, the latter pipe will skip the flush operation. + // Since a large number of consensus pipes are not created at the same time, resulting in no + // serious waiting for locks. Therefore, the flush operation is always performed for the + // consensus pipe, and the lastFlushed timestamp is not updated here. + if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + dataRegion.syncCloseAllWorkingTsFileProcessors(); + } else { + dataRegion.asyncCloseAllWorkingTsFileProcessors(); + } + } + + private void extractTsFiles( + final DataRegion dataRegion, + final long startHistoricalExtractionTime, + final List originalResourceList) { + final TsFileManager tsFileManager = dataRegion.getTsFileManager(); + tsFileManager.readLock(); + try { + final int originalSequenceTsFileCount = tsFileManager.size(true); + final int originalUnsequenceTsFileCount = tsFileManager.size(false); + LOGGER.info( + "Pipe {}@{}: start to extract historical TsFile, original sequence file count {}, " + + "original unsequence file count {}, start progress index {}", + pipeName, + dataRegionId, + originalSequenceTsFileCount, + originalUnsequenceTsFileCount, + startIndex); + + final Collection sequenceTsFileResources = + tsFileManager.getTsFileList(true).stream() + .peek(originalResourceList::add) + .filter( + resource -> + isHistoricalExtractorEnabled + && + // Some resource is marked as deleted but not removed from the list. + !resource.isDeleted() + // Some resource is generated by pipe. We ignore them if the pipe should + // not transfer pipe requests. + && (!resource.isGeneratedByPipe() || isForwardingPipeRequests) + && ( + // Some resource may not be closed due to the control of + // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them. + !resource.isClosed() + && Optional.ofNullable(resource.getProcessor()) + .map(TsFileProcessor::alreadyMarkedClosing) + .orElse(true) + || mayTsFileContainUnprocessedData(resource) + && isTsFileResourceOverlappedWithTimeRange(resource) + && mayTsFileResourceOverlappedWithPattern(resource))) + .collect(Collectors.toList()); + filteredTsFileResources.addAll(sequenceTsFileResources); + + final Collection unsequenceTsFileResources = + tsFileManager.getTsFileList(false).stream() + .peek(originalResourceList::add) + .filter( + resource -> + isHistoricalExtractorEnabled + && + // Some resource is marked as deleted but not removed from the list. + !resource.isDeleted() + // Some resource is generated by pipe. We ignore them if the pipe should + // not transfer pipe requests. + && (!resource.isGeneratedByPipe() || isForwardingPipeRequests) + && ( + // Some resource may not be closed due to the control of + // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them. + !resource.isClosed() + && Optional.ofNullable(resource.getProcessor()) + .map(TsFileProcessor::alreadyMarkedClosing) + .orElse(true) + || mayTsFileContainUnprocessedData(resource) + && isTsFileResourceOverlappedWithTimeRange(resource) + && mayTsFileResourceOverlappedWithPattern(resource))) + .collect(Collectors.toList()); + filteredTsFileResources.addAll(unsequenceTsFileResources); + + filteredTsFileResources.removeIf( + resource -> { + // Pin the resource, in case the file is removed by compaction or anything. + // Will unpin it after the PipeTsFileInsertionEvent is created and pinned. + try { + PipeDataNodeResourceManager.tsfile() + .pinTsFileResource(resource, shouldTransferModFile, pipeName); + return false; + } catch (final IOException e) { + LOGGER.warn("Pipe: failed to pin TsFileResource {}", resource.getTsFilePath(), e); + return true; + } + }); + + LOGGER.info( + "Pipe {}@{}: finish to extract historical TsFile, extracted sequence file count {}/{}, " + + "extracted unsequence file count {}/{}, extracted file count {}/{}, took {} ms", + pipeName, + dataRegionId, + sequenceTsFileResources.size(), + originalSequenceTsFileCount, + unsequenceTsFileResources.size(), + originalUnsequenceTsFileCount, + filteredTsFileResources.size(), + originalSequenceTsFileCount + originalUnsequenceTsFileCount, + System.currentTimeMillis() - startHistoricalExtractionTime); + } finally { + tsFileManager.readUnlock(); + } + } + + private boolean mayTsFileContainUnprocessedData(final TsFileResource resource) { + if (startIndex instanceof TimeWindowStateProgressIndex) { + // The resource is closed thus the TsFileResource#getFileEndTime() is safe to use + return ((TimeWindowStateProgressIndex) startIndex).getMinTime() <= resource.getFileEndTime(); + } + + if (startIndex instanceof StateProgressIndex) { + startIndex = ((StateProgressIndex) startIndex).getInnerProgressIndex(); + } + + if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + // For consensus pipe, we only focus on the progressIndex that is generated from local write + // instead of replication or something else. + ProgressIndex dedicatedProgressIndex = + tryToExtractLocalProgressIndexForIoTV2(resource.getMaxProgressIndexAfterClose()); + return greaterThanStartIndex(resource, dedicatedProgressIndex); + } + return greaterThanStartIndex(resource, resource.getMaxProgressIndexAfterClose()); + } + + private boolean greaterThanStartIndex(PersistentResource resource, ProgressIndex progressIndex) { + if (!startIndex.isAfter(progressIndex) && !startIndex.equals(progressIndex)) { + LOGGER.info( + "Pipe {}@{}: resource {} meets mayTsFileContainUnprocessedData condition, extractor progressIndex: {}, resource ProgressIndex: {}", + pipeName, + dataRegionId, + resource, + startIndex, + progressIndex); + return true; + } + return false; + } + + private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource resource) { + final Set deviceSet; + try { + final Map deviceIsAlignedMap = + PipeDataNodeResourceManager.tsfile() + .getDeviceIsAlignedMapFromCache(resource.getTsFile(), false); + deviceSet = + Objects.nonNull(deviceIsAlignedMap) ? deviceIsAlignedMap.keySet() : resource.getDevices(); + } catch (final IOException e) { + LOGGER.warn( + "Pipe {}@{}: failed to get devices from TsFile {}, extract it anyway", + pipeName, + dataRegionId, + resource.getTsFilePath(), + e); + return true; + } + + return deviceSet.stream() + .anyMatch( + deviceID -> { + if (!isModelDetected) { + detectModel(resource, deviceID); + isModelDetected = true; + } + + return isTableModel + ? (tablePattern.isTableModelDataAllowedToBeCaptured() + && tablePattern.matchesDatabase(resource.getDatabaseName()) + && tablePattern.matchesTable(deviceID.getTableName())) + : (treePattern.isTreeModelDataAllowedToBeCaptured() + && treePattern.mayOverlapWithDevice(deviceID)); + }); + } + + private void detectModel(final TsFileResource resource, final IDeviceID deviceID) { + this.isTableModel = + !(deviceID instanceof PlainDeviceID + || deviceID.getTableName().startsWith(TREE_MODEL_EVENT_TABLE_NAME_PREFIX) + || deviceID.getTableName().equals(PATH_ROOT)); + + final String databaseName = resource.getDatabaseName(); + isDbNameCoveredByPattern = + isTableModel + ? tablePattern.isTableModelDataAllowedToBeCaptured() + && tablePattern.coversDb(databaseName) + : treePattern.isTreeModelDataAllowedToBeCaptured() + && treePattern.coversDb(databaseName); + } + + private boolean isTsFileResourceOverlappedWithTimeRange(final TsFileResource resource) { + return !(resource.getFileEndTime() < historicalDataExtractionStartTime + || historicalDataExtractionEndTime < resource.getFileStartTime()); + } + + private boolean isTsFileResourceCoveredByTimeRange(final TsFileResource resource) { + return historicalDataExtractionStartTime <= resource.getFileStartTime() + && historicalDataExtractionEndTime >= resource.getFileEndTime(); + } + + private void extractDeletions( + final DeletionResourceManager deletionResourceManager, + final List resourceList) { + LOGGER.info("Pipe {}@{}: start to extract deletions", pipeName, dataRegionId); + long startTime = System.currentTimeMillis(); + List allDeletionResources = deletionResourceManager.getAllDeletionResources(); + final int originalDeletionCount = allDeletionResources.size(); + // For deletions that are filtered and will not be sent, we should manually decrease its + // reference count. Because the initial value of referenceCount is `ReplicaNum - 1` + allDeletionResources.stream() + .filter( + resource -> { + ProgressIndex toBeCompared = resource.getProgressIndex(); + if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + toBeCompared = tryToExtractLocalProgressIndexForIoTV2(toBeCompared); + } + return !greaterThanStartIndex(resource, toBeCompared); + }) + .forEach(DeletionResource::decreaseReference); + // Get deletions that should be sent. + allDeletionResources = + allDeletionResources.stream() + .filter( + resource -> { + ProgressIndex toBeCompared = resource.getProgressIndex(); + if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + toBeCompared = tryToExtractLocalProgressIndexForIoTV2(toBeCompared); + } + return greaterThanStartIndex(resource, toBeCompared); + }) + .collect(Collectors.toList()); + resourceList.addAll(allDeletionResources); + LOGGER.info( + "Pipe {}@{}: finish to extract deletions, extract deletions count {}/{}, took {} ms", + pipeName, + dataRegionId, + allDeletionResources.size(), + originalDeletionCount, + System.currentTimeMillis() - startTime); + } + + @Override + public synchronized Event supply() { + if (!hasBeenStarted && StorageEngine.getInstance().isReadyForNonReadWriteFunctions()) { + start(); + } + + if (Objects.isNull(pendingQueue)) { + return null; + } + + final PersistentResource resource = pendingQueue.poll(); + if (resource == null) { + return supplyTerminateEvent(); + } else if (resource instanceof TsFileResource) { + return supplyTsFileEvent((TsFileResource) resource); + } else { + return supplyDeletionEvent((DeletionResource) resource); + } + } + + private Event supplyTerminateEvent() { + final PipeTerminateEvent terminateEvent = + new PipeTerminateEvent(pipeName, creationTime, pipeTaskMeta, dataRegionId); + if (!terminateEvent.increaseReferenceCount( + PipeHistoricalDataRegionTsFileAndDeletionSource.class.getName())) { + LOGGER.warn( + "Pipe {}@{}: failed to increase reference count for terminate event, will resend it", + pipeName, + dataRegionId); + return null; + } + isTerminateSignalSent = true; + return terminateEvent; + } + + private Event supplyTsFileEvent(final TsFileResource resource) { + if (!filteredTsFileResources.contains(resource)) { + final ProgressReportEvent progressReportEvent = + new ProgressReportEvent( + pipeName, + creationTime, + pipeTaskMeta, + treePattern, + tablePattern, + userName, + skipIfNoPrivileges, + historicalDataExtractionStartTime, + historicalDataExtractionEndTime); + progressReportEvent.bindProgressIndex(resource.getMaxProgressIndex()); + final boolean isReferenceCountIncreased = + progressReportEvent.increaseReferenceCount( + PipeHistoricalDataRegionTsFileAndDeletionSource.class.getName()); + if (!isReferenceCountIncreased) { + LOGGER.warn( + "The reference count of the event {} cannot be increased, skipping it.", + progressReportEvent); + } + return isReferenceCountIncreased ? progressReportEvent : null; + } + + filteredTsFileResources.remove(resource); + + final PipeTsFileInsertionEvent event = + new PipeTsFileInsertionEvent( + isModelDetected ? isTableModel : null, + resource.getDatabaseName(), + resource, + null, + shouldTransferModFile, + false, + true, + pipeName, + creationTime, + pipeTaskMeta, + treePattern, + tablePattern, + userName, + skipIfNoPrivileges, + historicalDataExtractionStartTime, + historicalDataExtractionEndTime); + + // if using IoTV2, assign a replicateIndex for this event + if (DataRegionConsensusImpl.getInstance() instanceof PipeConsensus + && PipeConsensusProcessor.isShouldReplicate(event)) { + event.setReplicateIndexForIoTV2( + ReplicateProgressDataNodeManager.assignReplicateIndexForIoTV2(pipeName)); + LOGGER.info( + "[{}]Set {} for historical event {}", pipeName, event.getReplicateIndexForIoTV2(), event); + } + + if (sloppyPattern || isDbNameCoveredByPattern) { + event.skipParsingPattern(); + } + if (sloppyTimeRange || isTsFileResourceCoveredByTimeRange(resource)) { + event.skipParsingTime(); + } + + try { + final boolean isReferenceCountIncreased = + event.increaseReferenceCount( + PipeHistoricalDataRegionTsFileAndDeletionSource.class.getName()); + if (!isReferenceCountIncreased) { + LOGGER.warn( + "Pipe {}@{}: failed to increase reference count for historical tsfile event {}, will discard it", + pipeName, + dataRegionId, + event); + } + return isReferenceCountIncreased ? event : null; + } finally { + try { + PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, pipeName); + } catch (final IOException e) { + LOGGER.warn( + "Pipe {}@{}: failed to unpin TsFileResource after creating event, original path: {}", + pipeName, + dataRegionId, + resource.getTsFilePath()); + } + } + } + + private Event supplyDeletionEvent(final DeletionResource deletionResource) { + final PipeDeleteDataNodeEvent event = + new PipeDeleteDataNodeEvent( + deletionResource.getDeleteDataNode(), + pipeName, + creationTime, + pipeTaskMeta, + treePattern, + tablePattern, + userName, + skipIfNoPrivileges, + false); + + if (sloppyPattern || isDbNameCoveredByPattern) { + event.skipParsingPattern(); + } + if (sloppyTimeRange) { + event.skipParsingTime(); + } + + final boolean isReferenceCountIncreased = + event.increaseReferenceCount( + PipeHistoricalDataRegionTsFileAndDeletionSource.class.getName()); + if (!isReferenceCountIncreased) { + LOGGER.warn( + "Pipe {}@{}: failed to increase reference count for historical deletion event {}, will discard it", + pipeName, + dataRegionId, + event); + } else { + Optional.ofNullable(DeletionResourceManager.getInstance(String.valueOf(dataRegionId))) + .ifPresent( + manager -> + event.setDeletionResource( + manager.getDeletionResource(event.getDeleteDataNode()))); + } + return isReferenceCountIncreased ? event : null; + } + + @Override + public synchronized boolean hasConsumedAll() { + // If the pendingQueue is null when the function is called, it implies that the extractor only + // extracts deletion thus the historical event has nothing to consume. + return hasBeenStarted + && (Objects.isNull(pendingQueue) || pendingQueue.isEmpty() && isTerminateSignalSent); + } + + @Override + public int getPendingQueueSize() { + return Objects.nonNull(pendingQueue) ? pendingQueue.size() : 0; + } + + @Override + public synchronized void close() { + if (Objects.nonNull(pendingQueue)) { + pendingQueue.forEach( + resource -> { + if (resource instanceof TsFileResource) { + try { + PipeDataNodeResourceManager.tsfile() + .unpinTsFileResource((TsFileResource) resource, pipeName); + } catch (final IOException e) { + LOGGER.warn( + "Pipe {}@{}: failed to unpin TsFileResource after dropping pipe, original path: {}", + pipeName, + dataRegionId, + ((TsFileResource) resource).getTsFilePath()); + } + } + }); + pendingQueue.clear(); + pendingQueue = null; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java index aa32a6bf7f3fa..d6cfa6f6abc6f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java @@ -139,6 +139,10 @@ public void listenToDeleteData(DeleteDataNode node) { (key, value) -> value.publishToAssign(PipeRealtimeEventFactory.createRealtimeEvent(node))); } + public boolean isEmpty() { + return dataRegionId2Assigner.isEmpty(); + } + /////////////////////////////// singleton /////////////////////////////// private PipeInsertionDataNodeListener() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 1dd3bea411c2e..f0e0bb52e46e7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -255,8 +255,8 @@ public class CommonConfig { private long pipeMaxWaitFinishTime = 10 * 1000; - private int pipeExtractorAssignerDisruptorRingBufferSize = 65536; - private long pipeExtractorAssignerDisruptorRingBufferEntrySizeInBytes = 50; // 50B + private int pipeExtractorAssignerDisruptorRingBufferSize = 128; + private long pipeExtractorAssignerDisruptorRingBufferEntrySizeInBytes = 72 * KB; private long pipeExtractorMatcherCacheSize = 1024; private int pipeConnectorHandshakeTimeoutMs = 10 * 1000; // 10 seconds diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index 6eca59a3865f1..c7758526028aa 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -476,6 +476,7 @@ private boolean createPipe(final PipeMeta pipeMetaFromCoordinator) throws Illega final long creationTime = pipeMetaFromCoordinator.getStaticMeta().getCreationTime(); calculateMemoryUsage( + pipeMetaFromCoordinator.getStaticMeta(), pipeMetaFromCoordinator.getStaticMeta().getExtractorParameters(), pipeMetaFromCoordinator.getStaticMeta().getProcessorParameters(), pipeMetaFromCoordinator.getStaticMeta().getConnectorParameters()); @@ -521,6 +522,7 @@ private boolean createPipe(final PipeMeta pipeMetaFromCoordinator) throws Illega } protected void calculateMemoryUsage( + final PipeStaticMeta staticMeta, final PipeParameters extractorParameters, final PipeParameters processorParameters, final PipeParameters connectorParameters) { diff --git a/pom.xml b/pom.xml index 2ae4f3a04e44b..7b9cbbe367387 100644 --- a/pom.xml +++ b/pom.xml @@ -47,6 +47,7 @@ distribution example library-udf + integration-test From ffce094a26f00692d513ed0a68481e28a134f1c7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 1 Aug 2025 14:33:10 +0800 Subject: [PATCH 2/4] revert --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7b9cbbe367387..2ae4f3a04e44b 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,6 @@ distribution example library-udf - integration-test From fad055046ab870162a078ac98c91c79848cef252 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 1 Aug 2025 14:35:38 +0800 Subject: [PATCH 3/4] delete-table --- .../manual/basic/IoTDBPipePermissionIT.java | 371 ------ .../manual/basic/IoTDBPipeProtocolIT.java | 498 -------- .../manual/basic/IoTDBPipeSourceIT.java | 1005 ---------------- .../manual/enhanced/IoTDBPipeClusterIT.java | 1054 ----------------- 4 files changed, 2928 deletions(-) delete mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java delete mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java delete mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java delete mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java deleted file mode 100644 index 83616911639bf..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java +++ /dev/null @@ -1,371 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; -import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipePermissionIT extends AbstractPipeTableModelDualManualIT { - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - // TODO: delete ratis configurations - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(false) - .setDefaultSchemaRegionGroupNumPerDatabase(1) - .setTimestampPrecision("ms") - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(false) - .setTimestampPrecision("ms") - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false) - .setSchemaReplicationFactor(3) - .setDataReplicationFactor(2); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(3, 3); - } - - @Test - public void testSourcePermission() { - if (!TestUtils.tryExecuteNonQueryWithRetry( - senderEnv, "create user `thulab` 'passwD@123456'", null)) { - return; - } - - // Shall fail if username is specified without password - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe a2b" - + " with source (" - + "'user'='thulab'" - + "'capture.tree'='true'," - + "'capture.table'='true')" - + " with sink (" - + "'node-urls'='%s')", - receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); - fail("When the 'user' or 'username' is specified, password must be specified too."); - } catch (final SQLException ignore) { - // Expected - } - - // Shall fail if password is wrong - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe a2b" - + " with source (" - + "'user'='thulab'" - + "'password'='hack'" - + "'capture.tree'='true'," - + "'capture.table'='true')" - + " with sink (" - + "'node-urls'='%s')", - receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); - fail("Shall fail if password is wrong."); - } catch (final SQLException ignore) { - // Expected - } - - // Use current session, user is root - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe a2b" - + " with source (" - + "'inclusion'='all'," - + "'capture.tree'='true'," - + "'capture.table'='true')" - + " with sink (" - + "'node-urls'='%s')", - receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); - } catch (final SQLException e) { - e.printStackTrace(); - fail("Create pipe without user shall succeed if use the current session"); - } - - // Alter to another user, shall fail because of lack of password - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute("alter pipe a2b modify source ('username'='thulab')"); - fail("Alter pipe shall fail if only user is specified"); - } catch (final SQLException ignore) { - // Expected - } - - // Successfully alter - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute( - "alter pipe a2b modify source ('username'='thulab', 'password'='passwD@123456')"); - } catch (final SQLException e) { - e.printStackTrace(); - fail("Alter pipe shall not fail if user and password are specified"); - } - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - - // Shall not be transferred - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "count databases", - "count,", - Collections.singleton("1,"), - "information_schema"); - - // Grant some privilege - if (!TestUtils.tryExecuteNonQueryWithRetry( - "test", BaseEnv.TABLE_SQL_DIALECT, senderEnv, "grant INSERT on any to user thulab", null)) { - return; - } - - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - - // Shall not be transferred - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "show tables from test1", - "TableName,TTL(ms),", - Collections.singleton("test1,INF,"), - "information_schema"); - - // Alter pipe, throw exception if no privileges - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute("alter pipe a2b modify source ('skipif'='')"); - } catch (final SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Write some data - if (!TableModelUtils.insertData("test", "test", 0, 100, senderEnv)) { - return; - } - - try { - TableModelUtils.createDataBaseAndTable(receiverEnv, "test", "test"); - } catch (final Exception | Error ignore) { - // Ignore because the db/table may be transferred because sender user may see these - } - - // Exception, block here - TableModelUtils.assertCountDataAlwaysOnEnv("test", "test", 0, receiverEnv); - - // Grant SELECT privilege - if (!TestUtils.tryExecuteNonQueriesWithRetry( - "test", - BaseEnv.TABLE_SQL_DIALECT, - senderEnv, - Arrays.asList("grant SELECT on any to user thulab", "start pipe a2b"), - null)) { - return; - } - - // Will finally pass - TableModelUtils.assertCountData( - "test", - "test", - 100, - receiverEnv, - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }); - } - - @Test - public void testReceiverPermission() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueryWithRetry( - receiverEnv, "create user testUser 'passwD@123456'", null)) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - final String dbName = "test"; - final String tbName = "test"; - - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.capture.tree", "false"); - extractorAttributes.put("extractor.capture.table", "true"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.user", "testUser"); - connectorAttributes.put("connector.password", "passwD@123456"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client.startPipeExtended(new TStartPipeReq("testPipe").setIsTableModel(true)).getCode()); - - TableModelUtils.createDataBaseAndTable(senderEnv, tbName, dbName); - - // Write some data - if (!TableModelUtils.insertData(dbName, tbName, 0, 100, senderEnv)) { - return; - } - - // Shall not be transferred - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "show databases", - "Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", - Collections.singleton("information_schema,INF,null,null,null,"), - (String) null); - - if (!TestUtils.tryExecuteNonQueryWithRetry( - "information_schema", - BaseEnv.TABLE_SQL_DIALECT, - receiverEnv, - "grant insert,create on database test to user testUser", - null)) { - return; - } - - // Will finally pass - TableModelUtils.assertCountData( - dbName, - tbName, - 100, - receiverEnv, - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }); - - // Alter pipe, skip if no privileges - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute("alter pipe testPipe modify sink ('skipif'='no-privileges')"); - } catch (final SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - final String dbName2 = "test2"; - - // Write some data - if (!TableModelUtils.insertData(dbName2, tbName, 0, 100, senderEnv)) { - return; - } - - // Shall not be transferred - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "count databases", "count,", Collections.singleton("2,"), (String) null); - - if (!TestUtils.tryExecuteNonQueryWithRetry( - "information_schema", - BaseEnv.TABLE_SQL_DIALECT, - receiverEnv, - "grant insert,create on database test2 to user testUser", - null)) { - return; - } - - if (!TableModelUtils.insertData(dbName2, tbName, 100, 200, senderEnv)) { - return; - } - - // Will finally pass - TableModelUtils.assertCountData( - dbName2, - tbName, - 100, - receiverEnv, - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java deleted file mode 100644 index 939e84b9f4961..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java +++ /dev/null @@ -1,498 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Consumer; - -/** Test pipe's basic functionalities under multiple cluster and consensus protocol settings. */ -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeProtocolIT extends AbstractPipeTableModelDualManualIT { - - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - } - - private void innerSetUp( - final String configNodeConsensus, - final String schemaRegionConsensus, - final String dataRegionConsensus, - final int configNodesNum, - final int dataNodesNum, - int schemaRegionReplicationFactor, - int dataRegionReplicationFactor) { - schemaRegionReplicationFactor = Math.min(schemaRegionReplicationFactor, dataNodesNum); - dataRegionReplicationFactor = Math.min(dataRegionReplicationFactor, dataNodesNum); - - // TODO: delete ratis configurations - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(dataRegionConsensus) - .setSchemaReplicationFactor(schemaRegionReplicationFactor) - .setDataReplicationFactor(dataRegionReplicationFactor) - .setDnConnectionTimeoutMs(600000) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(configNodeConsensus) - .setSchemaRegionConsensusProtocolClass(schemaRegionConsensus) - .setDataRegionConsensusProtocolClass(dataRegionConsensus) - .setSchemaReplicationFactor(schemaRegionReplicationFactor) - .setDataReplicationFactor(dataRegionReplicationFactor) - .setDnConnectionTimeoutMs(600000) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - - senderEnv.initClusterEnvironment(configNodesNum, dataNodesNum); - receiverEnv.initClusterEnvironment(configNodesNum, dataNodesNum); - } - - @Test - public void test1C1DWithRatisRatisIot() throws Exception { - innerSetUp( - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.IOT_CONSENSUS, - 1, - 1, - 1, - 1); - doTest(); - } - - @Test - public void test1C1DWithSimpleSimpleIot() throws Exception { - innerSetUp( - ConsensusFactory.SIMPLE_CONSENSUS, - ConsensusFactory.SIMPLE_CONSENSUS, - ConsensusFactory.IOT_CONSENSUS, - 1, - 1, - 1, - 1); - doTest(); - } - - @Test - public void test1C1DWithRatisRatisSimple() throws Exception { - innerSetUp( - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.SIMPLE_CONSENSUS, - 1, - 1, - 1, - 1); - doTest(); - } - - @Test - public void test3C3DWith3SchemaRegionFactor3DataRegionFactor() throws Exception { - innerSetUp( - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.IOT_CONSENSUS, - 3, - 3, - 3, - 3); - doTest(); - } - - @Test - public void test3C3DWith3SchemaRegionFactor2DataRegionFactor() throws Exception { - innerSetUp( - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.RATIS_CONSENSUS, - ConsensusFactory.IOT_CONSENSUS, - 3, - 3, - 3, - 2); - doTest(); - } - - @Test - public void testPipeOnBothSenderAndReceiver() throws Exception { - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setSchemaReplicationFactor(3) - .setDataReplicationFactor(2) - .setDnConnectionTimeoutMs(600000) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setSchemaReplicationFactor(1) - .setDataReplicationFactor(1) - .setDnConnectionTimeoutMs(600000) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - - senderEnv.initClusterEnvironment(3, 3); - receiverEnv.initClusterEnvironment(1, 1); - - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult = true; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("table-name", "test.*"); - extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.streaming", "true"); - extractorAttributes.put("mode.snapshot", "false"); - extractorAttributes.put("mode.strict", "true"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - } - - final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); - final String senderIp = senderDataNode.getIp(); - final int senderPort = senderDataNode.getPort(); - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(receiverEnv, "test1", "test1"); - insertResult = TableModelUtils.insertData("test1", "test1", 0, 100, receiverEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("database-name", "test.*"); - extractorAttributes.put("table-name", "test.*"); - extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.streaming", "true"); - extractorAttributes.put("mode.snapshot", "false"); - extractorAttributes.put("mode.strict", "true"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderIp); - connectorAttributes.put("connector.port", Integer.toString(senderPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TableModelUtils.assertCountData("test1", "test1", 100, senderEnv, handleFailure); - } - } - - private void doTest() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult = true; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("database-name", "test.*"); - extractorAttributes.put("table-name", "test.*"); - extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.streaming", "true"); - extractorAttributes.put("mode.snapshot", "false"); - extractorAttributes.put("mode.strict", "true"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TableModelUtils.assertData("test", "test", 0, 100, receiverEnv, handleFailure); - - insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - - insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); - } - } - - @Test - public void testSyncConnectorUseNodeUrls() throws Exception { - doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName()); - } - - @Test - public void testAsyncConnectorUseNodeUrls() throws Exception { - doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName()); - } - - @Test - public void testAirGapConnectorUseNodeUrls() throws Exception { - doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); - } - - private void doTestUseNodeUrls(String connectorName) throws Exception { - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setPipeAirGapReceiverEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setSchemaReplicationFactor(1) - .setDataReplicationFactor(1) - .setEnableSeqSpaceCompaction(false) - .setEnableUnseqSpaceCompaction(false) - .setEnableCrossSpaceCompaction(false) - .setDnConnectionTimeoutMs(600000) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setPipeAirGapReceiverEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setSchemaReplicationFactor(3) - .setDataReplicationFactor(2) - .setDnConnectionTimeoutMs(600000) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - - senderEnv.initClusterEnvironment(1, 1); - receiverEnv.initClusterEnvironment(1, 3); - - final StringBuilder nodeUrlsBuilder = new StringBuilder(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult = true; - for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) { - if (connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) { - // Use default port for convenience - nodeUrlsBuilder - .append(wrapper.getIp()) - .append(":") - .append(wrapper.getPipeAirGapReceiverPort()) - .append(","); - } else { - nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(","); - } - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", connectorName); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.batch.max-delay-seconds", "1"); - connectorAttributes.put("connector.batch.size-bytes", "2048"); - connectorAttributes.put("connector.node-urls", nodeUrlsBuilder.toString()); - - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("database-name", "test.*"); - extractorAttributes.put("table-name", "test.*"); - extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.snapshot", "false"); - extractorAttributes.put("mode.strict", "true"); - extractorAttributes.put("user", "root"); - - // Test forced-log mode, in open releases this might be "file" - extractorAttributes.put("realtime.mode", "forced-log"); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); - - insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); - - extractorAttributes.replace("realtime.mode", "file"); - - status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - insertResult = TableModelUtils.insertData("test", "test", 300, 400, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java deleted file mode 100644 index 5dcf2d0e8e4af..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java +++ /dev/null @@ -1,1005 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; -import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Consumer; - -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeSourceIT extends AbstractPipeTableModelDualManualIT { - - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - // TODO: delete ratis configurations - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - // Disable sender compaction for tsfile determination in loose range test - .setEnableSeqSpaceCompaction(false) - .setEnableUnseqSpaceCompaction(false) - .setEnableCrossSpaceCompaction(false) - .setDnConnectionTimeoutMs(600000) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH"); - - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDnConnectionTimeoutMs(600000) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); - } - - @Test - public void testMatchingMultipleDatabases() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - boolean insertResult = true; - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.capture.table", "true"); - extractorAttributes.put("extractor.capture.tree", "true"); - extractorAttributes.put("extractor.database-name", "test"); - extractorAttributes.put("extractor.table-name", "test"); - extractorAttributes.put("extractor.pattern", "root.db1"); - extractorAttributes.put("extractor.inclusion", "data.insert"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - assertTimeseriesCountOnReceiver(receiverEnv, 0); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db1.d1 (time, at1) values (1, 10)", - "insert into root.db2.d1 (time, at1) values (1, 20)", - "flush"), - null)) { - return; - } - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - insertResult = - insertResult && TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); - if (!insertResult) { - return; - } - extractorAttributes.replace("extractor.pattern", "root.db2"); - extractorAttributes.replace("extractor.table-name", "test1"); - status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - assertTimeseriesCountOnReceiver(receiverEnv, 2); - - Thread.sleep(10000); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p2").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db1.d1 (time, at1) values (2, 11)", - "insert into root.db2.d1 (time, at1) values (2, 21)", - "flush"), - null)) { - return; - } - - insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); - insertResult = - insertResult && TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); - if (!insertResult) { - return; - } - extractorAttributes.remove("extractor.pattern"); // no pattern, will match all databases - extractorAttributes.remove("extractor.table-name"); - extractorAttributes.remove("extractor.database-name"); - status = - client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db*.**", - "count(root.db1.d1.at1),count(root.db2.d1.at1),", - Collections.singleton("2,2,"), - handleFailure); - - TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); - } - } - - @Test - public void testHistoryAndRealtime() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult = true; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1) values (1, 10)", - "insert into root.db.d2 (time, at1) values (1, 20)", - "insert into root.db.d3 (time, at1) values (1, 30)", - "insert into root.db.d4 (time, at1) values (1, 40)", - "flush"), - null)) { - return; - } - - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - insertResult = TableModelUtils.insertData("test", "test1", 0, 100, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - insertResult = TableModelUtils.insertData("test", "test2", 0, 100, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test3"); - insertResult = TableModelUtils.insertData("test", "test3", 0, 100, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.createDataBaseAndTable(senderEnv, "test4", "test"); - insertResult = TableModelUtils.insertData("test", "test4", 0, 100, senderEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - extractorAttributes.put("extractor.capture.table", "true"); - extractorAttributes.put("extractor.capture.tree", "true"); - extractorAttributes.put("extractor.database", "test"); - extractorAttributes.put("extractor.table", "test2"); - extractorAttributes.put("extractor.inclusion", "data.insert"); - extractorAttributes.put("extractor.pattern", "root.db.d2"); - extractorAttributes.put("extractor.history.enable", "false"); - extractorAttributes.put("extractor.realtime.enable", "true"); - extractorAttributes.put("user", "root"); - TSStatus status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - extractorAttributes.replace("extractor.table-name", "test3"); - extractorAttributes.replace("extractor.pattern", "root.db.d3"); - extractorAttributes.replace("extractor.history.enable", "true"); - extractorAttributes.replace("extractor.realtime.enable", "false"); - status = - client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); - - extractorAttributes.replace("extractor.table-name", "test4"); - extractorAttributes.replace("extractor.pattern", "root.db.d4"); - extractorAttributes.replace("extractor.history.enable", "true"); - extractorAttributes.replace("extractor.realtime.enable", "true"); - status = - client.createPipe( - new TCreatePipeReq("p4", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p4").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1) values (2, 11)", - "insert into root.db.d2 (time, at1) values (2, 21)", - "insert into root.db.d3 (time, at1) values (2, 31)", - "insert into root.db.d4 (time, at1) values (2, 41), (3, 51)"), - null)) { - return; - } - - insertResult = TableModelUtils.insertData("test", "test2", 100, 200, senderEnv); - if (!insertResult) { - return; - } - insertResult = TableModelUtils.insertData("test", "test3", 0, 100, senderEnv); - if (!insertResult) { - return; - } - insertResult = TableModelUtils.insertData("test", "test4", 0, 200, senderEnv); - if (!insertResult) { - return; - } - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.** where time <= 1", - "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", - Collections.singleton("1,0,1,"), - handleFailure); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.** where time >= 2", - "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", - Collections.singleton("2,1,0,"), - handleFailure); - - TableModelUtils.assertData("test", "test2", 100, 200, receiverEnv, handleFailure); - - TableModelUtils.assertData("test", "test3", 100, 200, receiverEnv, handleFailure); - - TableModelUtils.assertData("test", "test4", 100, 200, receiverEnv, handleFailure); - } - } - - @Ignore - @Test - public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult = true; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1)" - + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", - "insert into root.db.d2 (time, at1)" - + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", - "flush"), - null)) { - return; - } - - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - insertResult = TableModelUtils.insertData("test", "test1", 0, 10, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - insertResult = TableModelUtils.insertData("test", "test2", 0, 10, senderEnv); - if (!insertResult) { - return; - } - - // wait for flush to complete - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Collections.singletonList("flush"), null)) { - return; - } - Thread.sleep(10000); - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.capture.table", "true"); - extractorAttributes.put("extractor.capture.tree", "true"); - extractorAttributes.put("extractor.database-name", "test"); - extractorAttributes.put("extractor.table-name", "test1"); - extractorAttributes.put("extractor.pattern", "root.db.d1"); - extractorAttributes.put("extractor.inclusion", "data.insert"); - extractorAttributes.put("extractor.history.enable", "true"); - // 1970-01-01T08:00:02+08:00 - extractorAttributes.put("extractor.history.start-time", "2"); - extractorAttributes.put("extractor.history.end-time", "3"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),", - Collections.singleton("2,"), - handleFailure); - - extractorAttributes.remove("extractor.table-name"); - extractorAttributes.remove("extractor.pattern"); - status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),count(root.db.d2.at1),", - Collections.singleton("2,2,"), - handleFailure); - - TableModelUtils.assertData("test", "test1", 2, 4, receiverEnv, handleFailure); - TableModelUtils.assertData("test", "test2", 2, 4, receiverEnv, handleFailure); - } - } - - @Test - public void testExtractorTimeRangeMatch() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // insert history data - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1)" - + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", - "insert into root.db.d2 (time, at1)" - + " values (6, 6), (7, 7), (8, 8), (9, 9), (10, 10)", - "flush"), - null)) { - return; - } - - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - insertResult = TableModelUtils.insertData("test", "test1", 0, 10, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - insertResult = TableModelUtils.insertData("test", "test2", 0, 10, senderEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - extractorAttributes.put("extractor.capture.table", "true"); - extractorAttributes.put("extractor.capture.tree", "true"); - extractorAttributes.put("source.inclusion", "data.insert"); - extractorAttributes.put("source.start-time", "2"); - extractorAttributes.put("source.end-time", "4"); - extractorAttributes.put("user", "root"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),", - Collections.singleton("3,"), - handleFailure); - - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); - - // Insert realtime data that overlapped with time range - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d3 (time, at1)" - + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", - "flush"), - null)) { - return; - } - TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test"); - insertResult = TableModelUtils.insertData("test", "test3", 0, 5, senderEnv); - if (!insertResult) { - return; - } - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),count(root.db.d3.at1),", - Collections.singleton("3,3,"), - handleFailure); - - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); - TableModelUtils.assertCountData("test", "test3", 3, receiverEnv, handleFailure); - - // Insert realtime data that does not overlap with time range - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d4 (time, at1)" - + " values (6, 6), (7, 7), (8, 8), (9, 9), (10, 10)", - "flush"), - null)) { - return; - } - - TableModelUtils.createDataBaseAndTable(senderEnv, "test4", "test"); - insertResult = TableModelUtils.insertData("test", "test4", 6, 10, senderEnv); - if (!insertResult) { - return; - } - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),count(root.db.d3.at1),", - Collections.singleton("3,3,"), - 600, - handleFailure); - - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); - TableModelUtils.assertCountData("test", "test3", 3, receiverEnv, handleFailure); - } - } - - @Test - public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - boolean insertResult = true; - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1)" - + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", - "insert into root.db.d2 (time, at1)" - + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", - "flush"), - null)) { - return; - } - - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - insertResult = TableModelUtils.insertData("test", "test1", 0, 5, senderEnv); - if (!insertResult) { - return; - } - - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - insertResult = TableModelUtils.insertData("test", "test2", 0, 5, senderEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.capture.table", "true"); - extractorAttributes.put("source.capture.tree", "true"); - extractorAttributes.put("source.pattern", "root.db.d1"); - extractorAttributes.put("source.table-name", "test1"); - extractorAttributes.put("source.inclusion", "data.insert"); - extractorAttributes.put("source.start-time", "2"); - // 1970-01-01T08:00:04+08:00 - extractorAttributes.put("source.end-time", "4"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),", - Collections.singleton("3,"), - handleFailure); - - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); - - extractorAttributes.remove("source.pattern"); - extractorAttributes.remove("source.table-name"); - status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),count(root.db.d2.at1),", - Collections.singleton("3,3,"), - handleFailure); - - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); - TableModelUtils.assertCountData("test", "test2", 3, receiverEnv, handleFailure); - } - } - - @Ignore - @Test - public void testHistoryLooseRange() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - boolean insertResult = true; - - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - insertResult = TableModelUtils.insertData("test", "test1", 0, 2, senderEnv); - if (!insertResult) { - return; - } - - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - insertResult = TableModelUtils.insertData("test", "test2", 0, 2, senderEnv); - if (!insertResult) { - return; - } - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - // TsFile 1, extracted without parse - "insert into root.db.d1 (time, at1, at2)" + " values (1, 1, 2), (2, 3, 4)", - // TsFile 2, not extracted because pattern not overlapped - "insert into root.db1.d1 (time, at1, at2)" + " values (1, 1, 2), (2, 3, 4)", - "flush"), - null)) { - return; - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - // TsFile 3, not extracted because time range not overlapped - "insert into root.db.d1 (time, at1, at2)" + " values (3, 1, 2), (4, 3, 4)", "flush"), - null)) { - return; - } - - insertResult = TableModelUtils.insertData("test", "test1", 2, 5, senderEnv); - if (!insertResult) { - return; - } - - // wait for flush to complete - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, Collections.singletonList("flush"), null)) { - return; - } - Thread.sleep(10000); - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.capture.table", "true"); - extractorAttributes.put("source.capture.tree", "true"); - extractorAttributes.put("source.table-name", "test1"); - extractorAttributes.put("source.path", "root.db.d1.at1"); - extractorAttributes.put("source.inclusion", "data.insert"); - extractorAttributes.put("source.history.start-time", "1"); - extractorAttributes.put("source.history.end-time", "2"); - extractorAttributes.put("source.history.loose-range", "time, path"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.db.** group by level=0", - "count(root.*.*.*),", - Collections.singleton("4,"), - handleFailure); - - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); - } - } - - @Ignore - @Test - public void testRealtimeLooseRange() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult = true; - insertResult = TableModelUtils.insertData("test", "test2", 0, 20, senderEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("source.capture.table", "true"); - extractorAttributes.put("source.capture.tree", "true"); - extractorAttributes.put("source.table-name", "test1"); - extractorAttributes.put("source.path", "root.db.d1.at1"); - extractorAttributes.put("source.inclusion", "data.insert"); - extractorAttributes.put("source.realtime.loose-range", "time, path"); - extractorAttributes.put("source.start-time", "2"); - extractorAttributes.put("source.end-time", "10"); - extractorAttributes.put("source.realtime.mode", "batch"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1, at2)" + " values (1, 1, 2), (3, 3, 4)", "flush"), - null)) { - return; - } - - if (!insertResult) { - return; - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1)" + " values (5, 1), (16, 3)", - "insert into root.db.d1 (time, at1, at2)" + " values (5, 1, 2), (6, 3, 4)", - "flush"), - null)) { - return; - } - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(at1) from root.db.d1 where time >= 2 and time <= 10", - new HashMap() { - { - put("count(root.db.d1.at1)", "3"); - } - }); - - insertResult = TableModelUtils.insertData("test", "test1", 10, 20, senderEnv); - if (!insertResult) { - return; - } - insertResult = TableModelUtils.insertData("test", "test2", 10, 20, senderEnv); - if (!insertResult) { - return; - } - - TableModelUtils.assertCountData("test", "test1", 20, receiverEnv, handleFailure); - } - } - - @Test - public void testTableModeSQLSupportNowFunc() { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - final String p1 = - String.format( - "create pipe p1" - + " with extractor (" - + "'capture.table'='true'," - + "'extractor.history.enable'='true'," - + "'source.start-time'='now'," - + "'source.end-time'='now'," - + "'source.history.start-time'='now'," - + "'source.history.end-time'='now')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute(p1); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - final String p2 = - String.format( - "create pipe p2" - + " with extractor (" - + "'capture.table'='true'," - + "'extractor.history.enable'='true'," - + "'start-time'='now'," - + "'end-time'='now'," - + "'history.start-time'='now'," - + "'history.end-time'='now')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute(p2); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - final String p3 = - String.format( - "create pipe p3" - + " with extractor (" - + "'capture.table'='true'," - + "'extractor.history.enable'='true'," - + "'extractor.start-time'='now'," - + "'extractor.end-time'='now'," - + "'extractor.history.start-time'='now'," - + "'extractor.history.end-time'='now')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute(p3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - String alterP3 = - "alter pipe p3" - + " modify extractor (" - + "'history.enable'='true'," - + "'start-time'='now'," - + "'end-time'='now'," - + "'history.start-time'='now'," - + "'history.end-time'='now')"; - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute(alterP3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - alterP3 = - "alter pipe p3" - + " modify extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.start-time'='now'," - + "'extractor.end-time'='now'," - + "'extractor.history.start-time'='now'," - + "'extractor.history.end-time'='now')"; - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute(alterP3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - alterP3 = - "alter pipe p3" - + " modify source (" - + "'extractor.history.enable'='true'," - + "'source.start-time'='now'," - + "'source.end-time'='now'," - + "'source.history.start-time'='now'," - + "'source.history.end-time'='now')"; - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute(alterP3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - } - - private void assertTimeseriesCountOnReceiver(BaseEnv receiverEnv, int count) { - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "count timeseries root.db.**", - "count(timeseries),", - Collections.singleton(count + ",")); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java deleted file mode 100644 index 12f2857c7c18a..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java +++ /dev/null @@ -1,1054 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.exception.ClientManagerException; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.commons.cluster.RegionRoleType; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.env.AbstractEnv; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.apache.thrift.TException; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; - -import static org.junit.Assert.fail; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeClusterIT extends AbstractPipeTableModelDualManualIT { - - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setDnConnectionTimeoutMs(600000) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setDataReplicationFactor(2) - .setSchemaReplicationFactor(3) - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) - .setDnConnectionTimeoutMs(600000) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - - senderEnv.initClusterEnvironment(3, 3, 180); - receiverEnv.initClusterEnvironment(3, 3, 180); - } - - @Test - public void testMachineDowntimeAsync() { - testMachineDowntime("iotdb-thrift-connector"); - } - - @Test - public void testMachineDowntimeSync() { - testMachineDowntime("iotdb-thrift-sync-connector"); - } - - private void testMachineDowntime(String sink) { - StringBuilder a = new StringBuilder(); - for (DataNodeWrapper nodeWrapper : receiverEnv.getDataNodeWrapperList()) { - a.append(nodeWrapper.getIp()).append(":").append(nodeWrapper.getPort()); - a.append(","); - } - a.deleteCharAt(a.length() - 1); - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 1, senderEnv); - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor", "iotdb-extractor"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("user", "root"); - - processorAttributes.put("processor", "do-nothing-processor"); - - connectorAttributes.put("connector", sink); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.node-urls", a.toString()); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - TableModelUtils.assertCountData("test", "test", 1, receiverEnv); - receiverEnv.getDataNodeWrapper(0).stop(); - - // Ensure that the kill -9 operation is completed - Thread.sleep(5000); - TableModelUtils.insertData("test", "test", 1, 2, senderEnv); - } catch (Exception e) { - fail(e.getMessage()); - } - - for (DataNodeWrapper nodeWrapper : receiverEnv.getDataNodeWrapperList()) { - if (!nodeWrapper.isAlive()) { - continue; - } - TableModelUtils.assertCountData("test", "test", 2, receiverEnv, nodeWrapper); - return; - } - } - - @Test - public void testWithAllParametersInStreamingMode() throws Exception { - testWithAllParameters("true"); - } - - @Test - public void testWithAllParametersInNotStreamingMode() throws Exception { - testWithAllParameters("false"); - } - - private void testWithAllParameters(final String realtimeMode) throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - boolean insertResult = true; - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - if (!insertResult) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor", "iotdb-extractor"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("table-name", "test"); - extractorAttributes.put("start-time", "0"); - extractorAttributes.put("end-time", "199"); - extractorAttributes.put("mode.streaming", realtimeMode); - extractorAttributes.put("user", "root"); - - processorAttributes.put("processor", "do-nothing-processor"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.user", "root"); - connectorAttributes.put("connector.password", "root"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - TableModelUtils.getQuerySql("test"), - TableModelUtils.generateHeaderResults(), - TableModelUtils.generateExpectedResults(0, 100), - "test", - handleFailure); - - insertResult = TableModelUtils.insertData("test", "test", 100, 300, senderEnv); - if (!insertResult) { - return; - } - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - TableModelUtils.getQuerySql("test"), - TableModelUtils.generateHeaderResults(), - TableModelUtils.generateExpectedResults(0, 200), - "test", - handleFailure); - } - } - - // This function has a certain probability of triggering replica asynchrony. To ensure the success - // of the test, it will be retried 5 times. The exception will be thrown after five retries. - @Test - public void testPipeAfterDataRegionLeaderStop() throws Exception { - for (int retry = 0; retry < 5; retry++) { - try { - if (retry != 0) { - this.setUp(); - } - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult = true; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - insertResult = - insertResult && TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); - if (!insertResult) { - return; - } - - extractorAttributes.put("extractor", "iotdb-extractor"); - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test"); - extractorAttributes.put("start-time", "0"); - extractorAttributes.put("end-time", "300"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); - insertResult = - insertResult && TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); - if (!insertResult) { - return; - } - - final AtomicInteger leaderPort = new AtomicInteger(-1); - final TShowRegionResp showRegionResp = - client.showRegion(new TShowRegionReq().setIsTableModel(true)); - showRegionResp - .getRegionInfoList() - .forEach( - regionInfo -> { - if (RegionRoleType.Leader.getRoleType().equals(regionInfo.getRoleType())) { - leaderPort.set(regionInfo.getClientRpcPort()); - } - }); - - int leaderIndex = -1; - for (int i = 0; i < 3; ++i) { - if (senderEnv.getDataNodeWrapper(i).getPort() == leaderPort.get()) { - leaderIndex = i; - try { - senderEnv.shutdownDataNode(i); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - try { - TimeUnit.SECONDS.sleep(1); - } catch (final InterruptedException ignored) { - } - try { - senderEnv.startDataNode(i); - ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - } - } - if (leaderIndex == -1) { // ensure the leader is stopped - fail(); - } - - insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv); - insertResult = - insertResult && TableModelUtils.insertData("test1", "test1", 200, 300, senderEnv); - if (!insertResult) { - return; - } - - TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); - } - - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // Create a new pipe and write new data - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("database-name", "test1"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test1"); - extractorAttributes.put("start-time", "0"); - extractorAttributes.put("end-time", "300"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - insertResult = TableModelUtils.insertData("test", "test", 300, 400, senderEnv); - insertResult = - insertResult && TableModelUtils.insertData("test1", "test1", 300, 400, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.assertData("test", "test", 0, 301, receiverEnv, handleFailure); - TableModelUtils.assertData("test1", "test1", 0, 301, receiverEnv, handleFailure); - } - return; - } catch (Exception | Error e) { - if (retry < 4) { - this.tearDown(); - } else { - throw e; - } - } - } - } - - @Test - public void testPipeAfterRegisterNewDataNode() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult = true; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - insertResult = - insertResult && TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); - if (!insertResult) { - return; - } - - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); - insertResult = - insertResult && TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); - if (!insertResult) { - return; - } - try { - senderEnv.registerNewDataNode(true); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - final DataNodeWrapper newDataNode = - senderEnv.getDataNodeWrapper(senderEnv.getDataNodeWrapperList().size() - 1); - insertResult = TableModelUtils.insertData("test", "test", 200, 300, senderEnv, newDataNode); - insertResult = - insertResult - && TableModelUtils.insertData("test1", "test1", 200, 300, senderEnv, newDataNode); - if (!insertResult) { - return; - } - - TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); - } - - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // create a new pipe and write new data - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("database-name", "test1"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test1"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - - insertResult = TableModelUtils.insertData("test", "test", 300, 400, senderEnv); - insertResult = - insertResult && TableModelUtils.insertData("test1", "test1", 300, 400, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.assertData("test1", "test1", 0, 400, receiverEnv, handleFailure); - TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); - } - } - - @Test - public void testCreatePipeWhenRegisteringNewDataNode() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("database-name", "test1"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test1"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final Thread t = - new Thread( - () -> { - for (int i = 0; i < 30; ++i) { - try { - client.createPipe( - new TCreatePipeReq("p" + i, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - } catch (final TException e) { - // Not sure if the "createPipe" has succeeded - e.printStackTrace(); - return; - } - try { - Thread.sleep(100); - } catch (final Exception ignored) { - } - } - }); - t.start(); - try { - senderEnv.registerNewDataNode(true); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - t.join(); - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(30, showPipeResult.size()); - } - } - - @Test - public void testRegisteringNewDataNodeWhenTransferringData() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult = true; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final AtomicInteger succeedNum = new AtomicInteger(0); - final Thread t = - new Thread( - () -> { - try { - for (int i = 100; i < 200; ++i) { - if (TableModelUtils.insertDataNotThrowError( - "test", "test", i, i + 1, senderEnv)) { - succeedNum.incrementAndGet(); - Thread.sleep(100); - } - } - } catch (final InterruptedException ignored) { - } - }); - t.start(); - try { - senderEnv.registerNewDataNode(true); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - t.join(); - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush", null)) { - return; - } - - TableModelUtils.assertCountData( - "test", "test", succeedNum.get() + 100, receiverEnv, handleFailure); - - try { - senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); - senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); - } catch (final Throwable e) { - e.printStackTrace(); - } - } - } - - @Test - public void testRegisteringNewDataNodeAfterTransferringData() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - boolean insertResult = true; - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - if (!insertResult) { - return; - } - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - int succeedNum = 0; - for (int i = 100; i < 200; ++i) { - if (TableModelUtils.insertDataNotThrowError("test", "test", i, i + 1, senderEnv)) { - succeedNum++; - } - } - - try { - senderEnv.registerNewDataNode(true); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - TableModelUtils.assertCountData("test", "test", succeedNum + 100, receiverEnv, handleFailure); - - try { - senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); - senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); - } catch (final Throwable e) { - e.printStackTrace(); - } - } - } - - @Test - public void testSenderRestartWhenTransferring() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - boolean insertResult = true; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - } - - int succeedNum = 0; - for (int i = 100; i < 200; ++i) { - if (TableModelUtils.insertDataNotThrowError("test", "test", i, i + 1, senderEnv)) { - succeedNum++; - } - } - if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush", null)) { - return; - } - - try { - TestUtils.restartCluster(senderEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - - TableModelUtils.assertCountData("test", "test", succeedNum + 100, receiverEnv, handleFailure); - } - - @Test - public void testConcurrentlyCreatePipeOfSameName() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final AtomicInteger successCount = new AtomicInteger(0); - final List threads = new ArrayList<>(); - for (int i = 0; i < 10; ++i) { - final Thread t = - new Thread( - () -> { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - successCount.incrementAndGet(); - } - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (final TException | ClientManagerException | IOException e) { - e.printStackTrace(); - } catch (final Exception e) { - // Fail iff pipe exception occurs - e.printStackTrace(); - fail(e.getMessage()); - } - }); - t.start(); - threads.add(t); - } - - for (Thread t : threads) { - t.join(); - } - Assert.assertEquals(1, successCount.get()); - - successCount.set(0); - for (int i = 0; i < 10; ++i) { - final Thread t = - new Thread( - () -> { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final TSStatus status = client.dropPipe("p1"); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - successCount.incrementAndGet(); - } - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (final TException | ClientManagerException | IOException e) { - e.printStackTrace(); - } catch (final Exception e) { - // Fail iff pipe exception occurs - e.printStackTrace(); - fail(e.getMessage()); - } - }); - t.start(); - threads.add(t); - } - for (final Thread t : threads) { - t.join(); - } - - // Assert at least 1 drop operation succeeds - Assert.assertTrue(successCount.get() >= 1); - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(0, showPipeResult.size()); - } - } - - @Test - public void testCreate10PipesWithSameConnector() throws Exception { - testCreatePipesWithSameConnector(10); - } - - @Test - public void testCreate50PipesWithSameConnector() throws Exception { - testCreatePipesWithSameConnector(50); - } - - @Test - public void testCreate100PipesWithSameConnector() throws Exception { - testCreatePipesWithSameConnector(100); - } - - private void testCreatePipesWithSameConnector(final int pipeCount) throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final AtomicInteger successCount = new AtomicInteger(0); - final List threads = new ArrayList<>(); - for (int i = 0; i < pipeCount; ++i) { - final int finalI = i; - final Thread t = - new Thread( - () -> { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p" + finalI, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - successCount.incrementAndGet(); - } catch (final InterruptedException e) { - e.printStackTrace(); - Thread.currentThread().interrupt(); - } catch (final TException | ClientManagerException | IOException e) { - e.printStackTrace(); - } catch (final Exception e) { - // Fail iff pipe exception occurs - e.printStackTrace(); - fail(e.getMessage()); - } - }); - t.start(); - threads.add(t); - } - for (final Thread t : threads) { - t.join(); - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(successCount.get(), showPipeResult.size()); - showPipeResult = - client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(successCount.get(), showPipeResult.size()); - } - } - - @Test - public void testNegativeTimestamp() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - boolean insertResult = true; - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - insertResult = TableModelUtils.insertData("test", "test", -100, 100, senderEnv); - if (!insertResult) { - return; - } - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor", "iotdb-extractor"); - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("table-name", "test"); - extractorAttributes.put("user", "root"); - - processorAttributes.put("processor", "do-nothing-processor"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TableModelUtils.assertData("test", "test", -100, 100, receiverEnv, handleFailure); - - insertResult = TableModelUtils.insertData("test", "test", -200, -100, senderEnv); - if (!insertResult) { - return; - } - TableModelUtils.assertData("test", "test", -200, 100, receiverEnv, handleFailure); - } - } -} From c5c94038fb75e27e8abdcd020aa748eccf2730f4 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 1 Aug 2025 14:38:09 +0800 Subject: [PATCH 4/4] deletion --- ...icalDataRegionTsFileAndDeletionSource.java | 947 ------------------ 1 file changed, 947 deletions(-) delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java deleted file mode 100644 index af38d626f4e31..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java +++ /dev/null @@ -1,947 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.pipe.source.dataregion.historical; - -import org.apache.iotdb.commons.consensus.DataRegionId; -import org.apache.iotdb.commons.consensus.index.ProgressIndex; -import org.apache.iotdb.commons.consensus.index.ProgressIndexType; -import org.apache.iotdb.commons.consensus.index.impl.HybridProgressIndex; -import org.apache.iotdb.commons.consensus.index.impl.RecoverProgressIndex; -import org.apache.iotdb.commons.consensus.index.impl.StateProgressIndex; -import org.apache.iotdb.commons.consensus.index.impl.TimeWindowStateProgressIndex; -import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; -import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; -import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; -import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; -import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskExtractorRuntimeEnvironment; -import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; -import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; -import org.apache.iotdb.commons.pipe.datastructure.resource.PersistentResource; -import org.apache.iotdb.commons.pipe.event.ProgressReportEvent; -import org.apache.iotdb.commons.utils.PathUtils; -import org.apache.iotdb.consensus.pipe.PipeConsensus; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; -import org.apache.iotdb.db.pipe.consensus.ReplicateProgressDataNodeManager; -import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResource; -import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager; -import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent; -import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; -import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; -import org.apache.iotdb.db.pipe.processor.pipeconsensus.PipeConsensusProcessor; -import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; -import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter; -import org.apache.iotdb.db.storageengine.StorageEngine; -import org.apache.iotdb.db.storageengine.dataregion.DataRegion; -import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.iotdb.db.utils.DateTimeUtils; -import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration; -import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; -import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; -import org.apache.iotdb.pipe.api.event.Event; -import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; - -import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.PlainDeviceID; -import org.apache.tsfile.utils.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Queue; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_END_TIME_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_ENABLE_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_END_TIME_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_ALL_VALUE; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_PATH_VALUE; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_TIME_VALUE; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_HISTORY_START_TIME_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODE_STRICT_DEFAULT_VALUE; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODE_STRICT_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODS_DEFAULT_VALUE; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODS_ENABLE_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODS_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_START_TIME_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_END_TIME_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_HISTORY_ENABLE_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_HISTORY_END_TIME_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_HISTORY_LOOSE_RANGE_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_HISTORY_START_TIME_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_MODE_STRICT_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_MODS_ENABLE_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_MODS_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_START_TIME_KEY; -import static org.apache.iotdb.commons.pipe.source.IoTDBSource.getSkipIfNoPrivileges; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_ROOT; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; - -public class PipeHistoricalDataRegionTsFileAndDeletionSource - implements PipeHistoricalDataRegionSource { - - private static final Logger LOGGER = - LoggerFactory.getLogger(PipeHistoricalDataRegionTsFileAndDeletionSource.class); - - private static final Map DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP = new HashMap<>(); - - private static final String TREE_MODEL_EVENT_TABLE_NAME_PREFIX = PATH_ROOT + PATH_SEPARATOR; - - private String pipeName; - private long creationTime; - - private PipeTaskMeta pipeTaskMeta; - private ProgressIndex startIndex; - - private int dataRegionId; - - private TreePattern treePattern; - private TablePattern tablePattern; - - private boolean isModelDetected = false; - private boolean isTableModel; - private boolean isDbNameCoveredByPattern = false; - - private boolean isHistoricalExtractorEnabled = false; - private long historicalDataExtractionStartTime = Long.MIN_VALUE; // Event time - private long historicalDataExtractionEndTime = Long.MAX_VALUE; // Event time - - private boolean sloppyTimeRange; // true to disable time range filter after extraction - private boolean sloppyPattern; // true to disable pattern filter after extraction - - private Pair listeningOptionPair; - private boolean shouldExtractInsertion; - private boolean shouldExtractDeletion; - private boolean shouldTransferModFile; // Whether to transfer mods - protected String userName; - protected boolean skipIfNoPrivileges = true; - private boolean isTerminateSignalSent = false; - - private boolean isForwardingPipeRequests; - - private volatile boolean hasBeenStarted = false; - - private Queue pendingQueue; - private final Set filteredTsFileResources = new HashSet<>(); - - @Override - public void validate(final PipeParameterValidator validator) { - final PipeParameters parameters = validator.getParameters(); - - try { - listeningOptionPair = - DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters); - } catch (final Exception e) { - // compatible with the current validation framework - throw new PipeParameterNotValidException(e.getMessage()); - } - - if (parameters.hasAnyAttributes(EXTRACTOR_MODE_STRICT_KEY, SOURCE_MODE_STRICT_KEY)) { - final boolean isStrictMode = - parameters.getBooleanOrDefault( - Arrays.asList(EXTRACTOR_MODE_STRICT_KEY, SOURCE_MODE_STRICT_KEY), - EXTRACTOR_MODE_STRICT_DEFAULT_VALUE); - sloppyTimeRange = !isStrictMode; - sloppyPattern = !isStrictMode; - } else { - final String extractorHistoryLooseRangeValue = - parameters - .getStringOrDefault( - Arrays.asList(EXTRACTOR_HISTORY_LOOSE_RANGE_KEY, SOURCE_HISTORY_LOOSE_RANGE_KEY), - EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE) - .trim(); - if (EXTRACTOR_HISTORY_LOOSE_RANGE_ALL_VALUE.equalsIgnoreCase( - extractorHistoryLooseRangeValue)) { - sloppyTimeRange = true; - sloppyPattern = true; - } else { - final Set sloppyOptionSet = - Arrays.stream(extractorHistoryLooseRangeValue.split(",")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .map(String::toLowerCase) - .collect(Collectors.toSet()); - sloppyTimeRange = sloppyOptionSet.remove(EXTRACTOR_HISTORY_LOOSE_RANGE_TIME_VALUE); - sloppyPattern = sloppyOptionSet.remove(EXTRACTOR_HISTORY_LOOSE_RANGE_PATH_VALUE); - if (!sloppyOptionSet.isEmpty()) { - throw new PipeParameterNotValidException( - String.format( - "Parameters in set %s are not allowed in 'history.loose-range'", - sloppyOptionSet)); - } - } - } - - if (parameters.hasAnyAttributes( - SOURCE_START_TIME_KEY, - EXTRACTOR_START_TIME_KEY, - SOURCE_END_TIME_KEY, - EXTRACTOR_END_TIME_KEY)) { - isHistoricalExtractorEnabled = true; - - try { - historicalDataExtractionStartTime = - parameters.hasAnyAttributes(SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY) - ? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( - parameters.getStringByKeys(SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY)) - : Long.MIN_VALUE; - historicalDataExtractionEndTime = - parameters.hasAnyAttributes(SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY) - ? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( - parameters.getStringByKeys(SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY)) - : Long.MAX_VALUE; - if (historicalDataExtractionStartTime > historicalDataExtractionEndTime) { - throw new PipeParameterNotValidException( - String.format( - "%s (%s) [%s] should be less than or equal to %s (%s) [%s].", - SOURCE_START_TIME_KEY, - EXTRACTOR_START_TIME_KEY, - historicalDataExtractionStartTime, - SOURCE_END_TIME_KEY, - EXTRACTOR_END_TIME_KEY, - historicalDataExtractionEndTime)); - } - } catch (final PipeParameterNotValidException e) { - throw e; - } catch (final Exception e) { - // compatible with the current validation framework - throw new PipeParameterNotValidException(e.getMessage()); - } - - // return here - return; - } - - // Historical data extraction is enabled in the following cases: - // 1. System restarts the pipe. If the pipe is restarted but historical data extraction is not - // enabled, the pipe will lose some historical data. - // 2. User may set the EXTRACTOR_HISTORY_START_TIME and EXTRACTOR_HISTORY_END_TIME without - // enabling the historical data extraction, which may affect the realtime data extraction. - isHistoricalExtractorEnabled = - parameters.getBooleanOrDefault( - SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE) - || parameters.getBooleanOrDefault( - Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY), - EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE); - - try { - historicalDataExtractionStartTime = - parameters.hasAnyAttributes( - EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY) - ? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( - parameters.getStringByKeys( - EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY)) - : Long.MIN_VALUE; - historicalDataExtractionEndTime = - parameters.hasAnyAttributes(EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY) - ? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( - parameters.getStringByKeys( - EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY)) - : Long.MAX_VALUE; - if (historicalDataExtractionStartTime > historicalDataExtractionEndTime) { - throw new PipeParameterNotValidException( - String.format( - "%s (%s) [%s] should be less than or equal to %s (%s) [%s].", - EXTRACTOR_HISTORY_START_TIME_KEY, - SOURCE_HISTORY_START_TIME_KEY, - historicalDataExtractionStartTime, - EXTRACTOR_HISTORY_END_TIME_KEY, - SOURCE_HISTORY_END_TIME_KEY, - historicalDataExtractionEndTime)); - } - } catch (final Exception e) { - // Compatible with the current validation framework - throw new PipeParameterNotValidException(e.getMessage()); - } - } - - @Override - public void customize( - final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration) - throws IllegalPathException { - shouldExtractInsertion = listeningOptionPair.getLeft(); - shouldExtractDeletion = listeningOptionPair.getRight(); - // Do nothing if extract deletion - if (!shouldExtractInsertion) { - return; - } - - final PipeTaskExtractorRuntimeEnvironment environment = - (PipeTaskExtractorRuntimeEnvironment) configuration.getRuntimeEnvironment(); - - pipeName = environment.getPipeName(); - creationTime = environment.getCreationTime(); - pipeTaskMeta = environment.getPipeTaskMeta(); - if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { - startIndex = - tryToExtractLocalProgressIndexForIoTV2(environment.getPipeTaskMeta().getProgressIndex()); - } else { - startIndex = environment.getPipeTaskMeta().getProgressIndex(); - } - - dataRegionId = environment.getRegionId(); - synchronized (DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP) { - DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.putIfAbsent(dataRegionId, 0L); - } - - treePattern = TreePattern.parsePipePatternFromSourceParameters(parameters); - tablePattern = TablePattern.parsePipePatternFromSourceParameters(parameters); - - final DataRegion dataRegion = - StorageEngine.getInstance().getDataRegion(new DataRegionId(environment.getRegionId())); - if (Objects.nonNull(dataRegion)) { - final String databaseName = dataRegion.getDatabaseName(); - if (Objects.nonNull(databaseName)) { - isTableModel = PathUtils.isTableModelDatabase(databaseName); - isModelDetected = true; - if (isTableModel) { - isDbNameCoveredByPattern = tablePattern.coversDb(databaseName); - } else { - isDbNameCoveredByPattern = treePattern.coversDb(databaseName); - } - } - } - - if (parameters.hasAnyAttributes(EXTRACTOR_MODS_KEY, SOURCE_MODS_KEY)) { - shouldTransferModFile = - parameters.getBooleanOrDefault( - Arrays.asList(EXTRACTOR_MODS_KEY, SOURCE_MODS_KEY), - EXTRACTOR_MODS_DEFAULT_VALUE - || // Should extract deletion - listeningOptionPair.getRight()); - } else { - shouldTransferModFile = - parameters.getBooleanOrDefault( - Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY), - EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE - || // Should extract deletion - listeningOptionPair.getRight()); - } - - userName = - parameters.getStringByKeys( - PipeSourceConstant.EXTRACTOR_IOTDB_USER_KEY, - PipeSourceConstant.SOURCE_IOTDB_USER_KEY, - PipeSourceConstant.EXTRACTOR_IOTDB_USERNAME_KEY, - PipeSourceConstant.SOURCE_IOTDB_USERNAME_KEY); - - skipIfNoPrivileges = getSkipIfNoPrivileges(parameters); - - isForwardingPipeRequests = - parameters.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, - PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), - PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); - - if (LOGGER.isInfoEnabled()) { - LOGGER.info( - "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy pattern {}, sloppy time range {}, should transfer mod file {}, username: {}, skip if no privileges: {}, is forwarding pipe requests: {}", - pipeName, - dataRegionId, - DateTimeUtils.convertLongToDate(historicalDataExtractionStartTime), - historicalDataExtractionStartTime, - DateTimeUtils.convertLongToDate(historicalDataExtractionEndTime), - historicalDataExtractionEndTime, - sloppyPattern, - sloppyTimeRange, - shouldTransferModFile, - userName, - skipIfNoPrivileges, - isForwardingPipeRequests); - } - } - - /** - * IoTV2 will only resend event that contains un-replicated local write data. So we only extract - * ProgressIndex containing local writes for comparison to prevent misjudgment on whether - * high-level tsFiles with mixed progressIndexes need to be retransmitted - * - * @return recoverProgressIndex dedicated in local DataNodeId or origin for fallback. - */ - private ProgressIndex tryToExtractLocalProgressIndexForIoTV2(ProgressIndex origin) { - // There are only 2 cases: - // 1. origin is RecoverProgressIndex - if (origin instanceof RecoverProgressIndex) { - RecoverProgressIndex toBeTransformed = (RecoverProgressIndex) origin; - return extractRecoverProgressIndex(toBeTransformed); - } - // 2. origin is HybridProgressIndex - else if (origin instanceof HybridProgressIndex) { - HybridProgressIndex toBeTransformed = (HybridProgressIndex) origin; - // if hybridProgressIndex contains recoverProgressIndex, which is what we expected. - if (toBeTransformed - .getType2Index() - .containsKey(ProgressIndexType.RECOVER_PROGRESS_INDEX.getType())) { - // 2.1. transform recoverProgressIndex - RecoverProgressIndex specificToBeTransformed = - (RecoverProgressIndex) - toBeTransformed - .getType2Index() - .get(ProgressIndexType.RECOVER_PROGRESS_INDEX.getType()); - return extractRecoverProgressIndex(specificToBeTransformed); - } - // if hybridProgressIndex doesn't contain recoverProgressIndex, which is not what we expected, - // fallback. - return origin; - } else { - // fallback - LOGGER.warn( - "Pipe {}@{}: unexpected ProgressIndex type {}, fallback to origin {}.", - pipeName, - dataRegionId, - origin.getType(), - origin); - return origin; - } - } - - private ProgressIndex extractRecoverProgressIndex(RecoverProgressIndex toBeTransformed) { - return new RecoverProgressIndex( - toBeTransformed.getDataNodeId2LocalIndex().entrySet().stream() - .filter( - entry -> - entry - .getKey() - .equals(IoTDBDescriptor.getInstance().getConfig().getDataNodeId())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - } - - @Override - public synchronized void start() { - if (!shouldExtractInsertion) { - hasBeenStarted = true; - return; - } - if (!StorageEngine.getInstance().isReadyForNonReadWriteFunctions()) { - LOGGER.info( - "Pipe {}@{}: failed to start to extract historical TsFile, storage engine is not ready. Will retry later.", - pipeName, - dataRegionId); - return; - } - hasBeenStarted = true; - - final DataRegion dataRegion = - StorageEngine.getInstance().getDataRegion(new DataRegionId(dataRegionId)); - if (Objects.isNull(dataRegion)) { - pendingQueue = new ArrayDeque<>(); - return; - } - - final long startHistoricalExtractionTime = System.currentTimeMillis(); - dataRegion.writeLock( - "Pipe: start to extract historical TsFile and Deletion(if uses pipeConsensus)"); - try { - List originalResourceList = new ArrayList<>(); - - if (shouldExtractInsertion) { - flushTsFilesForExtraction(dataRegion); - extractTsFiles(dataRegion, startHistoricalExtractionTime, originalResourceList); - } - if (shouldExtractDeletion) { - Optional.ofNullable(DeletionResourceManager.getInstance(String.valueOf(dataRegionId))) - .ifPresent(manager -> extractDeletions(manager, originalResourceList)); - } - - // Sort tsFileResource and deletionResource - long startTime = System.currentTimeMillis(); - LOGGER.info("Pipe {}@{}: start to sort all extracted resources", pipeName, dataRegionId); - originalResourceList.sort( - (o1, o2) -> - startIndex instanceof TimeWindowStateProgressIndex - ? Long.compare(o1.getFileStartTime(), o2.getFileStartTime()) - : o1.getProgressIndex().topologicalCompareTo(o2.getProgressIndex())); - pendingQueue = new ArrayDeque<>(originalResourceList); - - LOGGER.info( - "Pipe {}@{}: finish to sort all extracted resources, took {} ms", - pipeName, - dataRegionId, - System.currentTimeMillis() - startTime); - } finally { - dataRegion.writeUnlock(); - } - } - - private void flushTsFilesForExtraction(DataRegion dataRegion) { - LOGGER.info("Pipe {}@{}: start to flush data region", pipeName, dataRegionId); - - // Consider the scenario: a consensus pipe comes to the same region, followed by another pipe - // **immediately**, the latter pipe will skip the flush operation. - // Since a large number of consensus pipes are not created at the same time, resulting in no - // serious waiting for locks. Therefore, the flush operation is always performed for the - // consensus pipe, and the lastFlushed timestamp is not updated here. - if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { - dataRegion.syncCloseAllWorkingTsFileProcessors(); - } else { - dataRegion.asyncCloseAllWorkingTsFileProcessors(); - } - } - - private void extractTsFiles( - final DataRegion dataRegion, - final long startHistoricalExtractionTime, - final List originalResourceList) { - final TsFileManager tsFileManager = dataRegion.getTsFileManager(); - tsFileManager.readLock(); - try { - final int originalSequenceTsFileCount = tsFileManager.size(true); - final int originalUnsequenceTsFileCount = tsFileManager.size(false); - LOGGER.info( - "Pipe {}@{}: start to extract historical TsFile, original sequence file count {}, " - + "original unsequence file count {}, start progress index {}", - pipeName, - dataRegionId, - originalSequenceTsFileCount, - originalUnsequenceTsFileCount, - startIndex); - - final Collection sequenceTsFileResources = - tsFileManager.getTsFileList(true).stream() - .peek(originalResourceList::add) - .filter( - resource -> - isHistoricalExtractorEnabled - && - // Some resource is marked as deleted but not removed from the list. - !resource.isDeleted() - // Some resource is generated by pipe. We ignore them if the pipe should - // not transfer pipe requests. - && (!resource.isGeneratedByPipe() || isForwardingPipeRequests) - && ( - // Some resource may not be closed due to the control of - // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them. - !resource.isClosed() - && Optional.ofNullable(resource.getProcessor()) - .map(TsFileProcessor::alreadyMarkedClosing) - .orElse(true) - || mayTsFileContainUnprocessedData(resource) - && isTsFileResourceOverlappedWithTimeRange(resource) - && mayTsFileResourceOverlappedWithPattern(resource))) - .collect(Collectors.toList()); - filteredTsFileResources.addAll(sequenceTsFileResources); - - final Collection unsequenceTsFileResources = - tsFileManager.getTsFileList(false).stream() - .peek(originalResourceList::add) - .filter( - resource -> - isHistoricalExtractorEnabled - && - // Some resource is marked as deleted but not removed from the list. - !resource.isDeleted() - // Some resource is generated by pipe. We ignore them if the pipe should - // not transfer pipe requests. - && (!resource.isGeneratedByPipe() || isForwardingPipeRequests) - && ( - // Some resource may not be closed due to the control of - // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them. - !resource.isClosed() - && Optional.ofNullable(resource.getProcessor()) - .map(TsFileProcessor::alreadyMarkedClosing) - .orElse(true) - || mayTsFileContainUnprocessedData(resource) - && isTsFileResourceOverlappedWithTimeRange(resource) - && mayTsFileResourceOverlappedWithPattern(resource))) - .collect(Collectors.toList()); - filteredTsFileResources.addAll(unsequenceTsFileResources); - - filteredTsFileResources.removeIf( - resource -> { - // Pin the resource, in case the file is removed by compaction or anything. - // Will unpin it after the PipeTsFileInsertionEvent is created and pinned. - try { - PipeDataNodeResourceManager.tsfile() - .pinTsFileResource(resource, shouldTransferModFile, pipeName); - return false; - } catch (final IOException e) { - LOGGER.warn("Pipe: failed to pin TsFileResource {}", resource.getTsFilePath(), e); - return true; - } - }); - - LOGGER.info( - "Pipe {}@{}: finish to extract historical TsFile, extracted sequence file count {}/{}, " - + "extracted unsequence file count {}/{}, extracted file count {}/{}, took {} ms", - pipeName, - dataRegionId, - sequenceTsFileResources.size(), - originalSequenceTsFileCount, - unsequenceTsFileResources.size(), - originalUnsequenceTsFileCount, - filteredTsFileResources.size(), - originalSequenceTsFileCount + originalUnsequenceTsFileCount, - System.currentTimeMillis() - startHistoricalExtractionTime); - } finally { - tsFileManager.readUnlock(); - } - } - - private boolean mayTsFileContainUnprocessedData(final TsFileResource resource) { - if (startIndex instanceof TimeWindowStateProgressIndex) { - // The resource is closed thus the TsFileResource#getFileEndTime() is safe to use - return ((TimeWindowStateProgressIndex) startIndex).getMinTime() <= resource.getFileEndTime(); - } - - if (startIndex instanceof StateProgressIndex) { - startIndex = ((StateProgressIndex) startIndex).getInnerProgressIndex(); - } - - if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { - // For consensus pipe, we only focus on the progressIndex that is generated from local write - // instead of replication or something else. - ProgressIndex dedicatedProgressIndex = - tryToExtractLocalProgressIndexForIoTV2(resource.getMaxProgressIndexAfterClose()); - return greaterThanStartIndex(resource, dedicatedProgressIndex); - } - return greaterThanStartIndex(resource, resource.getMaxProgressIndexAfterClose()); - } - - private boolean greaterThanStartIndex(PersistentResource resource, ProgressIndex progressIndex) { - if (!startIndex.isAfter(progressIndex) && !startIndex.equals(progressIndex)) { - LOGGER.info( - "Pipe {}@{}: resource {} meets mayTsFileContainUnprocessedData condition, extractor progressIndex: {}, resource ProgressIndex: {}", - pipeName, - dataRegionId, - resource, - startIndex, - progressIndex); - return true; - } - return false; - } - - private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource resource) { - final Set deviceSet; - try { - final Map deviceIsAlignedMap = - PipeDataNodeResourceManager.tsfile() - .getDeviceIsAlignedMapFromCache(resource.getTsFile(), false); - deviceSet = - Objects.nonNull(deviceIsAlignedMap) ? deviceIsAlignedMap.keySet() : resource.getDevices(); - } catch (final IOException e) { - LOGGER.warn( - "Pipe {}@{}: failed to get devices from TsFile {}, extract it anyway", - pipeName, - dataRegionId, - resource.getTsFilePath(), - e); - return true; - } - - return deviceSet.stream() - .anyMatch( - deviceID -> { - if (!isModelDetected) { - detectModel(resource, deviceID); - isModelDetected = true; - } - - return isTableModel - ? (tablePattern.isTableModelDataAllowedToBeCaptured() - && tablePattern.matchesDatabase(resource.getDatabaseName()) - && tablePattern.matchesTable(deviceID.getTableName())) - : (treePattern.isTreeModelDataAllowedToBeCaptured() - && treePattern.mayOverlapWithDevice(deviceID)); - }); - } - - private void detectModel(final TsFileResource resource, final IDeviceID deviceID) { - this.isTableModel = - !(deviceID instanceof PlainDeviceID - || deviceID.getTableName().startsWith(TREE_MODEL_EVENT_TABLE_NAME_PREFIX) - || deviceID.getTableName().equals(PATH_ROOT)); - - final String databaseName = resource.getDatabaseName(); - isDbNameCoveredByPattern = - isTableModel - ? tablePattern.isTableModelDataAllowedToBeCaptured() - && tablePattern.coversDb(databaseName) - : treePattern.isTreeModelDataAllowedToBeCaptured() - && treePattern.coversDb(databaseName); - } - - private boolean isTsFileResourceOverlappedWithTimeRange(final TsFileResource resource) { - return !(resource.getFileEndTime() < historicalDataExtractionStartTime - || historicalDataExtractionEndTime < resource.getFileStartTime()); - } - - private boolean isTsFileResourceCoveredByTimeRange(final TsFileResource resource) { - return historicalDataExtractionStartTime <= resource.getFileStartTime() - && historicalDataExtractionEndTime >= resource.getFileEndTime(); - } - - private void extractDeletions( - final DeletionResourceManager deletionResourceManager, - final List resourceList) { - LOGGER.info("Pipe {}@{}: start to extract deletions", pipeName, dataRegionId); - long startTime = System.currentTimeMillis(); - List allDeletionResources = deletionResourceManager.getAllDeletionResources(); - final int originalDeletionCount = allDeletionResources.size(); - // For deletions that are filtered and will not be sent, we should manually decrease its - // reference count. Because the initial value of referenceCount is `ReplicaNum - 1` - allDeletionResources.stream() - .filter( - resource -> { - ProgressIndex toBeCompared = resource.getProgressIndex(); - if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { - toBeCompared = tryToExtractLocalProgressIndexForIoTV2(toBeCompared); - } - return !greaterThanStartIndex(resource, toBeCompared); - }) - .forEach(DeletionResource::decreaseReference); - // Get deletions that should be sent. - allDeletionResources = - allDeletionResources.stream() - .filter( - resource -> { - ProgressIndex toBeCompared = resource.getProgressIndex(); - if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { - toBeCompared = tryToExtractLocalProgressIndexForIoTV2(toBeCompared); - } - return greaterThanStartIndex(resource, toBeCompared); - }) - .collect(Collectors.toList()); - resourceList.addAll(allDeletionResources); - LOGGER.info( - "Pipe {}@{}: finish to extract deletions, extract deletions count {}/{}, took {} ms", - pipeName, - dataRegionId, - allDeletionResources.size(), - originalDeletionCount, - System.currentTimeMillis() - startTime); - } - - @Override - public synchronized Event supply() { - if (!hasBeenStarted && StorageEngine.getInstance().isReadyForNonReadWriteFunctions()) { - start(); - } - - if (Objects.isNull(pendingQueue)) { - return null; - } - - final PersistentResource resource = pendingQueue.poll(); - if (resource == null) { - return supplyTerminateEvent(); - } else if (resource instanceof TsFileResource) { - return supplyTsFileEvent((TsFileResource) resource); - } else { - return supplyDeletionEvent((DeletionResource) resource); - } - } - - private Event supplyTerminateEvent() { - final PipeTerminateEvent terminateEvent = - new PipeTerminateEvent(pipeName, creationTime, pipeTaskMeta, dataRegionId); - if (!terminateEvent.increaseReferenceCount( - PipeHistoricalDataRegionTsFileAndDeletionSource.class.getName())) { - LOGGER.warn( - "Pipe {}@{}: failed to increase reference count for terminate event, will resend it", - pipeName, - dataRegionId); - return null; - } - isTerminateSignalSent = true; - return terminateEvent; - } - - private Event supplyTsFileEvent(final TsFileResource resource) { - if (!filteredTsFileResources.contains(resource)) { - final ProgressReportEvent progressReportEvent = - new ProgressReportEvent( - pipeName, - creationTime, - pipeTaskMeta, - treePattern, - tablePattern, - userName, - skipIfNoPrivileges, - historicalDataExtractionStartTime, - historicalDataExtractionEndTime); - progressReportEvent.bindProgressIndex(resource.getMaxProgressIndex()); - final boolean isReferenceCountIncreased = - progressReportEvent.increaseReferenceCount( - PipeHistoricalDataRegionTsFileAndDeletionSource.class.getName()); - if (!isReferenceCountIncreased) { - LOGGER.warn( - "The reference count of the event {} cannot be increased, skipping it.", - progressReportEvent); - } - return isReferenceCountIncreased ? progressReportEvent : null; - } - - filteredTsFileResources.remove(resource); - - final PipeTsFileInsertionEvent event = - new PipeTsFileInsertionEvent( - isModelDetected ? isTableModel : null, - resource.getDatabaseName(), - resource, - null, - shouldTransferModFile, - false, - true, - pipeName, - creationTime, - pipeTaskMeta, - treePattern, - tablePattern, - userName, - skipIfNoPrivileges, - historicalDataExtractionStartTime, - historicalDataExtractionEndTime); - - // if using IoTV2, assign a replicateIndex for this event - if (DataRegionConsensusImpl.getInstance() instanceof PipeConsensus - && PipeConsensusProcessor.isShouldReplicate(event)) { - event.setReplicateIndexForIoTV2( - ReplicateProgressDataNodeManager.assignReplicateIndexForIoTV2(pipeName)); - LOGGER.info( - "[{}]Set {} for historical event {}", pipeName, event.getReplicateIndexForIoTV2(), event); - } - - if (sloppyPattern || isDbNameCoveredByPattern) { - event.skipParsingPattern(); - } - if (sloppyTimeRange || isTsFileResourceCoveredByTimeRange(resource)) { - event.skipParsingTime(); - } - - try { - final boolean isReferenceCountIncreased = - event.increaseReferenceCount( - PipeHistoricalDataRegionTsFileAndDeletionSource.class.getName()); - if (!isReferenceCountIncreased) { - LOGGER.warn( - "Pipe {}@{}: failed to increase reference count for historical tsfile event {}, will discard it", - pipeName, - dataRegionId, - event); - } - return isReferenceCountIncreased ? event : null; - } finally { - try { - PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, pipeName); - } catch (final IOException e) { - LOGGER.warn( - "Pipe {}@{}: failed to unpin TsFileResource after creating event, original path: {}", - pipeName, - dataRegionId, - resource.getTsFilePath()); - } - } - } - - private Event supplyDeletionEvent(final DeletionResource deletionResource) { - final PipeDeleteDataNodeEvent event = - new PipeDeleteDataNodeEvent( - deletionResource.getDeleteDataNode(), - pipeName, - creationTime, - pipeTaskMeta, - treePattern, - tablePattern, - userName, - skipIfNoPrivileges, - false); - - if (sloppyPattern || isDbNameCoveredByPattern) { - event.skipParsingPattern(); - } - if (sloppyTimeRange) { - event.skipParsingTime(); - } - - final boolean isReferenceCountIncreased = - event.increaseReferenceCount( - PipeHistoricalDataRegionTsFileAndDeletionSource.class.getName()); - if (!isReferenceCountIncreased) { - LOGGER.warn( - "Pipe {}@{}: failed to increase reference count for historical deletion event {}, will discard it", - pipeName, - dataRegionId, - event); - } else { - Optional.ofNullable(DeletionResourceManager.getInstance(String.valueOf(dataRegionId))) - .ifPresent( - manager -> - event.setDeletionResource( - manager.getDeletionResource(event.getDeleteDataNode()))); - } - return isReferenceCountIncreased ? event : null; - } - - @Override - public synchronized boolean hasConsumedAll() { - // If the pendingQueue is null when the function is called, it implies that the extractor only - // extracts deletion thus the historical event has nothing to consume. - return hasBeenStarted - && (Objects.isNull(pendingQueue) || pendingQueue.isEmpty() && isTerminateSignalSent); - } - - @Override - public int getPendingQueueSize() { - return Objects.nonNull(pendingQueue) ? pendingQueue.size() : 0; - } - - @Override - public synchronized void close() { - if (Objects.nonNull(pendingQueue)) { - pendingQueue.forEach( - resource -> { - if (resource instanceof TsFileResource) { - try { - PipeDataNodeResourceManager.tsfile() - .unpinTsFileResource((TsFileResource) resource, pipeName); - } catch (final IOException e) { - LOGGER.warn( - "Pipe {}@{}: failed to unpin TsFileResource after dropping pipe, original path: {}", - pipeName, - dataRegionId, - ((TsFileResource) resource).getTsFilePath()); - } - } - }); - pendingQueue.clear(); - pendingQueue = null; - } - } -}