Skip to content

Commit ea8ee3a

Browse files
CaideyipiCaideyipi
authored
[To dev/1.3] Fixed multiple bugs of pipe
1 parent a0f52f6 commit ea8ee3a

170 files changed

Lines changed: 2753 additions & 5336 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,12 @@ public CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode) {
429429
return this;
430430
}
431431

432+
@Override
433+
public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
434+
setProperty("pipe_enable_memory_checked", String.valueOf(isPipeEnableMemoryCheck));
435+
return this;
436+
}
437+
432438
@Override
433439
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
434440
setProperty("pipe_air_gap_receiver_enabled", String.valueOf(isPipeAirGapReceiverEnabled));

‎integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,13 @@ public CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode) {
438438
return this;
439439
}
440440

441+
@Override
442+
public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
443+
dnConfig.setIsPipeEnableMemoryCheck(isPipeEnableMemoryCheck);
444+
cnConfig.setIsPipeEnableMemoryCheck(isPipeEnableMemoryCheck);
445+
return this;
446+
}
447+
441448
@Override
442449
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
443450
dnConfig.setPipeAirGapReceiverEnabled(isPipeAirGapReceiverEnabled);

‎integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,11 @@ public CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode) {
308308
return this;
309309
}
310310

311+
@Override
312+
public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
313+
return this;
314+
}
315+
311316
@Override
312317
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
313318
return this;

‎integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,8 @@ CommonConfig setEnableAutoLeaderBalanceForIoTConsensus(
138138

139139
CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode);
140140

141+
CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck);
142+
141143
CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled);
142144

143145
CommonConfig setDriverTaskExecutionTimeSliceInMs(long driverTaskExecutionTimeSliceInMs);

‎integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.junit.After;
3737
import org.junit.Assert;
3838
import org.junit.Before;
39+
import org.junit.Ignore;
3940
import org.junit.Test;
4041
import org.junit.experimental.categories.Category;
4142
import org.junit.runner.RunWith;
@@ -901,6 +902,7 @@ public void testLoadLocally() throws Exception {
901902
}
902903

903904
@Test
905+
@Ignore("Load with conversion is currently banned")
904906
public void testLoadWithConvertOnTypeMismatch() throws Exception {
905907

906908
List<Pair<MeasurementSchema, MeasurementSchema>> measurementSchemas =

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,15 @@ protected void setupConfig() {
4848
.getCommonConfig()
4949
.setAutoCreateSchemaEnabled(true)
5050
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
51-
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
51+
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
52+
.setIsPipeEnableMemoryCheck(false);
5253
receiverEnv
5354
.getConfig()
5455
.getCommonConfig()
5556
.setAutoCreateSchemaEnabled(true)
5657
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
57-
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
58+
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
59+
.setIsPipeEnableMemoryCheck(false);
5860

5961
// 10 min, assert that the operations will not time out
6062
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,16 @@ public void setUp() {
5959
.setAutoCreateSchemaEnabled(true)
6060
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
6161
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
62-
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
62+
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
63+
.setIsPipeEnableMemoryCheck(false);
6364
receiverEnv
6465
.getConfig()
6566
.getCommonConfig()
6667
.setAutoCreateSchemaEnabled(true)
6768
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
6869
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
69-
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
70+
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
71+
.setIsPipeEnableMemoryCheck(false);
7072

7173
// 10 min, assert that the operations will not time out
7274
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,8 @@ public void setUp() {
7474
.setAutoCreateSchemaEnabled(true)
7575
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
7676
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
77-
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
77+
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
78+
.setIsPipeEnableMemoryCheck(false);
7879

7980
receiverEnv
8081
.getConfig()
@@ -84,7 +85,8 @@ public void setUp() {
8485
.setSchemaReplicationFactor(3)
8586
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
8687
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
87-
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
88+
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
89+
.setIsPipeEnableMemoryCheck(false);
8890

8991
// 10 min, assert that the operations will not time out
9092
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,17 @@ public void setUp() {
6666
.getCommonConfig()
6767
.setAutoCreateSchemaEnabled(true)
6868
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
69-
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
69+
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
70+
.setIsPipeEnableMemoryCheck(false);
71+
7072
receiverEnv
7173
.getConfig()
7274
.getCommonConfig()
7375
.setAutoCreateSchemaEnabled(true)
7476
.setPipeAirGapReceiverEnabled(true)
7577
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
76-
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
78+
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
79+
.setIsPipeEnableMemoryCheck(false);
7780

7881
// 10 min, assert that the operations will not time out
7982
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -459,7 +459,6 @@ public void testAsyncLoadTsFileWithoutVerify() throws Exception {
459459
testLoadTsFileWithoutVerify("async");
460460
}
461461

462-
@Test
463462
private void testLoadTsFileWithoutVerify(final String loadTsFileStrategy) throws Exception {
464463
final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
465464

0 commit comments

Comments
 (0)