From 96335efd76b0ea8fb6d1e6eeff035df03ec6a523 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 13 Oct 2025 16:28:58 +0800 Subject: [PATCH 01/16] setup --- .../apache/iotdb/it/env/cluster/EnvUtils.java | 6 +- .../auto/basic/IoTDBTreePatternFormatIT.java | 326 +++++++++++++++++- .../TsFileInsertionEventParserProvider.java | 11 +- .../source/IoTDBDataRegionSourceTest.java | 2 + .../pattern/IoTDBTreePattern.java | 2 +- .../pattern/PrefixTreePattern.java | 2 +- .../pattern/SingleTreePattern.java | 54 +++ .../datastructure/pattern/TreePattern.java | 154 ++++++--- .../pattern/UnionTreePattern.java | 135 ++++++++ .../pipe/source/IoTDBNonDataRegionSource.java | 7 +- 10 files changed, 637 insertions(+), 62 deletions(-) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java index 651f5f409d276..0788843543d76 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java @@ -71,7 +71,7 @@ public class EnvUtils { public static int[] searchAvailablePorts() { int length = 10; while (true) { - int randomPortStart = 1000 + (int) (Math.random() * (1999 - 1000)); + int randomPortStart = 2000 + (int) (Math.random() * (2999 - 2000)); randomPortStart = randomPortStart * (length + 1) + 1; final String lockFilePath = getLockFilePath(randomPortStart); final File lockFile = new File(lockFilePath); @@ -233,7 +233,7 @@ public static Pair getNodeNum() { } return new Pair<>( getIntFromSysVar(DEFAULT_CONFIG_NODE_NUM, 1, 0), - getIntFromSysVar(DEFAULT_DATA_NODE_NUM, 3, 0)); + getIntFromSysVar(DEFAULT_DATA_NODE_NUM, 1, 0)); } public static Pair getNodeNum(final int index) { @@ -243,7 +243,7 @@ public static Pair getNodeNum(final int index) { } return new Pair<>( getIntFromSysVar(DEFAULT_CONFIG_NODE_NUM, 1, index), - getIntFromSysVar(DEFAULT_DATA_NODE_NUM, 3, index)); + getIntFromSysVar(DEFAULT_DATA_NODE_NUM, 1, index)); } public static String getFilePathFromSysVar(final String key, final int index) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java index e0b941ffc4772..4e2871067931b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java @@ -104,7 +104,7 @@ public void testPrefixPattern() throws Exception { } @Test - public void testIotdbPattern() throws Exception { + public void testIoTDBPattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); @@ -127,8 +127,6 @@ public void testIotdbPattern() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("extractor.path", "root.**.d1.s*"); - // When path is set, pattern should be ignored - extractorAttributes.put("extractor.pattern", "root"); extractorAttributes.put("extractor.inclusion", "data.insert"); connectorAttributes.put("connector", "iotdb-thrift-connector"); @@ -158,7 +156,7 @@ public void testIotdbPattern() throws Exception { } @Test - public void testIotdbPatternWithLegacySyntax() throws Exception { + public void testIoTDBPatternWithLegacySyntax() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); @@ -209,4 +207,324 @@ public void testIotdbPatternWithLegacySyntax() throws Exception { expectedResSet); } } + + @Test + public void testMultiplePrefixPatternHistoricalData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db.d1.s, root.db2.d1.s"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db.d2(time, s) values (2, 2)", + "insert into root.db2.d1(time, s) values (3, 3)"), + null); + awaitUntilFlush(senderEnv); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,null,1.0,1.0,"); + expectedResSet.add("3,3.0,null,null,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db2.**,root.db.**", + "Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,", + expectedResSet); + } + } + + @Test + public void testMultipleIoTDBPatternHistoricalData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.path", "root.db.**, root.db2.d1.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db.d2(time, s) values (2, 2)", + "insert into root.db2.d1(time, s, t) values (3, 3, 3)", + "insert into root.db3.d1(time, s) values (4, 4)"), + null); + awaitUntilFlush(senderEnv); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,null,null,1.0,1.0,null,"); + expectedResSet.add("2,null,null,null,null,2.0,"); + expectedResSet.add("3,3.0,3.0,null,null,null,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db2.**,root.db.**", + "Time,root.db2.d1.s,root.db2.d1.t,root.db.d1.s,root.db.d1.s1,root.db.d2.s,", + expectedResSet); + } + } + + @Test + public void testMultipleHybridPatternHistoricalData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.path", "root.db.d1.*"); + extractorAttributes.put("extractor.pattern", "root.db2.d1.s"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db2.d1(time, s) values (2, 2)", + "insert into root.db3.d1(time, s) values (3, 3)"), + null); + awaitUntilFlush(senderEnv); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,1.0,null,"); + expectedResSet.add("2,null,null,2.0,"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db.**,root.db2.**", + "Time,root.db.d1.s,root.db.d1.s1,root.db2.d1.s,", + expectedResSet); + } + } + + @Test + public void testMultiplePrefixPatternRealtimeData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db.d1.s, root.db2.d1.s"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db.d2(time, s) values (2, 2)", + "insert into root.db2.d1(time, s) values (3, 3)"), + null); + awaitUntilFlush(senderEnv); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,null,1.0,1.0,"); + expectedResSet.add("3,3.0,null,null,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db2.**,root.db.**", + "Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,", + expectedResSet); + } + } + + @Test + public void testMultipleIoTDBPatternRealtimeData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.path", "root.db.**, root.db2.d1.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db.d2(time, s) values (2, 2)", + "insert into root.db2.d1(time, s, t) values (3, 3, 3)", + "insert into root.db3.d1(time, s) values (4, 4)"), + null); + awaitUntilFlush(senderEnv); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,null,null,1.0,1.0,null,"); + expectedResSet.add("2,null,null,null,null,2.0,"); + expectedResSet.add("3,3.0,3.0,null,null,null,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db2.**,root.db.**", + "Time,root.db2.d1.s,root.db2.d1.t,root.db.d1.s,root.db.d1.s1,root.db.d2.s,", + expectedResSet); + } + } + + @Test + public void testMultipleHybridPatternRealtimeData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.path", "root.db.d1.*"); + extractorAttributes.put("extractor.pattern", "root.db2.d1.s"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db2.d1(time, s) values (2, 2)", + "insert into root.db3.d1(time, s) values (3, 3)"), + null); + awaitUntilFlush(senderEnv); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,1.0,null,"); + expectedResSet.add("2,null,null,2.0,"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db.**,root.db2.**", + "Time,root.db.d1.s,root.db.d1.s1,root.db2.d1.s,", + expectedResSet); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java index a965c817b994f..4ffe811baa37f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionTreePattern; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.query.TsFileInsertionEventQueryParser; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.scan.TsFileInsertionEventScanParser; @@ -99,8 +100,9 @@ public TsFileInsertionEventParser provide() throws IOException { // Use scan container to save memory if ((double) PipeDataNodeResourceManager.memory().getUsedMemorySizeInBytes() - / PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes() - > PipeTsFilePublicResource.MEMORY_SUFFICIENT_THRESHOLD) { + / PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes() + > PipeTsFilePublicResource.MEMORY_SUFFICIENT_THRESHOLD + && !(treePattern instanceof UnionTreePattern)) { return new TsFileInsertionEventScanParser( pipeName, creationTime, @@ -133,7 +135,7 @@ public TsFileInsertionEventParser provide() throws IOException { final Map deviceIsAlignedMap = PipeDataNodeResourceManager.tsfile().getDeviceIsAlignedMapFromCache(tsFile, false); - if (Objects.isNull(deviceIsAlignedMap)) { + if (Objects.isNull(deviceIsAlignedMap) && !(treePattern instanceof UnionTreePattern)) { // If we failed to get from cache, it indicates that the memory usage is high. // We use scan data container because it requires less memory. return new TsFileInsertionEventScanParser( @@ -152,7 +154,8 @@ public TsFileInsertionEventParser provide() throws IOException { filterDeviceIsAlignedMapByPattern(deviceIsAlignedMap); // Use scan data container if we need enough amount to data thus it's better to scan than query. return (double) filteredDeviceIsAlignedMap.size() / originalSize - > PipeConfig.getInstance().getPipeTsFileScanParsingThreshold() + > PipeConfig.getInstance().getPipeTsFileScanParsingThreshold() + && !(treePattern instanceof UnionTreePattern) ? new TsFileInsertionEventScanParser( pipeName, creationTime, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java index dd0558bc23805..fd1853fca1b74 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java @@ -82,6 +82,8 @@ public void testIoTDBDataRegionExtractorWithPattern() { Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root")); Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.`a-b`")); Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.1")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.a,root.b")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.a,root.b,root.db1.`a,b`.**")); } public Exception testIoTDBDataRegionExtractorWithPattern(final String pattern) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java index 9a1d817dc6061..a87eceb8eac67 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java @@ -37,7 +37,7 @@ import java.util.Objects; import java.util.stream.Collectors; -public class IoTDBTreePattern extends TreePattern { +public class IoTDBTreePattern extends SingleTreePattern { private final PartialPath patternPartialPath; private static volatile DevicePathGetter devicePathGetter = PartialPath::new; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/PrefixTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/PrefixTreePattern.java index 5d5f0586e12d8..d1c1d5e800d2f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/PrefixTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/PrefixTreePattern.java @@ -29,7 +29,7 @@ import java.util.Arrays; -public class PrefixTreePattern extends TreePattern { +public class PrefixTreePattern extends SingleTreePattern { public PrefixTreePattern(final boolean isTreeModelDataAllowedToBeCaptured, final String pattern) { super(isTreeModelDataAllowedToBeCaptured, pattern); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java new file mode 100644 index 0000000000000..af6fa5e15ea2d --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.datastructure.pattern; + +import java.util.Objects; + +public abstract class SingleTreePattern extends TreePattern { + + protected final String pattern; + + protected SingleTreePattern( + final boolean isTreeModelDataAllowedToBeCaptured, final String pattern) { + super(isTreeModelDataAllowedToBeCaptured); + this.pattern = pattern != null ? pattern : getDefaultPattern(); + } + + @Override + public String getPattern() { + return pattern; + } + + @Override + public boolean isRoot() { + return Objects.isNull(pattern) || this.pattern.equals(this.getDefaultPattern()); + } + + public abstract String getDefaultPattern(); + + @Override + public String toString() { + return "{pattern='" + + pattern + + "', isTreeModelDataAllowedToBeCaptured=" + + isTreeModelDataAllowedToBeCaptured + + '}'; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java index 7eef37ea3a618..492e662c0ee44 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java @@ -27,8 +27,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Objects; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATH_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATTERN_FORMAT_IOTDB_VALUE; @@ -44,25 +47,15 @@ public abstract class TreePattern { private static final Logger LOGGER = LoggerFactory.getLogger(TreePattern.class); protected final boolean isTreeModelDataAllowedToBeCaptured; - protected final String pattern; - protected TreePattern(final boolean isTreeModelDataAllowedToBeCaptured, final String pattern) { + protected TreePattern(final boolean isTreeModelDataAllowedToBeCaptured) { this.isTreeModelDataAllowedToBeCaptured = isTreeModelDataAllowedToBeCaptured; - this.pattern = pattern != null ? pattern : getDefaultPattern(); } public boolean isTreeModelDataAllowedToBeCaptured() { return isTreeModelDataAllowedToBeCaptured; } - public String getPattern() { - return pattern; - } - - public boolean isRoot() { - return Objects.isNull(pattern) || this.pattern.equals(this.getDefaultPattern()); - } - /** * Interpret from source parameters and get a {@link TreePattern}. * @@ -70,46 +63,118 @@ public boolean isRoot() { */ public static TreePattern parsePipePatternFromSourceParameters( final PipeParameters sourceParameters) { + final Function, TreePattern> listToTreePattern = + list -> list.size() == 1 ? list.get(0) : new UnionTreePattern(list); + final boolean isTreeModelDataAllowedToBeCaptured = isTreeModelDataAllowToBeCaptured(sourceParameters); final String path = sourceParameters.getStringByKeys(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY); + final String pattern = + sourceParameters.getStringByKeys(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY); - // 1. If "source.path" is specified, it will be interpreted as an IoTDB-style path, - // ignoring the other 2 parameters. - if (path != null) { - return new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, path); + // 1. If both "source.path" and "source.pattern" are specified, their union will be used. + if (path != null && pattern != null) { + final List result = new ArrayList<>(); + // Parse "source.path" as IoTDB-style path. + result.addAll( + parseMultiplePatterns( + path, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p))); + // Parse "source.pattern" using the helper method. + result.addAll( + parsePatternsFromPatternParameter( + pattern, sourceParameters, isTreeModelDataAllowedToBeCaptured)); + return listToTreePattern.apply(result); } - final String pattern = - sourceParameters.getStringByKeys(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY); + // 2. If only "source.path" is specified, it will be interpreted as an IoTDB-style path. + if (path != null) { + return listToTreePattern.apply( + parseMultiplePatterns( + path, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p))); + } - // 2. Otherwise, If "source.pattern" is specified, it will be interpreted - // according to "source.pattern.format". + // 3. If only "source.pattern" is specified, parse it using the helper method. if (pattern != null) { - final String patternFormat = - sourceParameters.getStringByKeys(EXTRACTOR_PATTERN_FORMAT_KEY, SOURCE_PATTERN_FORMAT_KEY); + return listToTreePattern.apply( + parsePatternsFromPatternParameter( + pattern, sourceParameters, isTreeModelDataAllowedToBeCaptured)); + } - // If "source.pattern.format" is not specified, use prefix format by default. - if (patternFormat == null) { - return new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, pattern); - } + // 4. If neither "source.path" nor "source.pattern" is specified, + // this pipe source will match all data. + return listToTreePattern.apply( + Collections.singletonList(new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, null))); + } - switch (patternFormat.toLowerCase()) { - case EXTRACTOR_PATTERN_FORMAT_IOTDB_VALUE: - return new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, pattern); - case EXTRACTOR_PATTERN_FORMAT_PREFIX_VALUE: - return new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, pattern); - default: - LOGGER.info( - "Unknown pattern format: {}, use prefix matching format by default.", patternFormat); - return new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, pattern); + /** + * A private helper method to parse a list of {@link TreePattern}s from the "pattern" parameter, + * considering its "format". + * + * @param pattern The pattern string to parse. + * @param sourceParameters The source parameters to read the format from. + * @param isTreeModelDataAllowedToBeCaptured A boolean flag passed to the TreePattern constructor. + * @return A list of parsed {@link TreePattern}s. + */ + private static List parsePatternsFromPatternParameter( + final String pattern, + final PipeParameters sourceParameters, + final boolean isTreeModelDataAllowedToBeCaptured) { + final String patternFormat = + sourceParameters.getStringByKeys(EXTRACTOR_PATTERN_FORMAT_KEY, SOURCE_PATTERN_FORMAT_KEY); + + // If "source.pattern.format" is not specified, use prefix format by default. + if (patternFormat == null) { + return parseMultiplePatterns( + pattern, p -> new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + } + + switch (patternFormat.toLowerCase()) { + case EXTRACTOR_PATTERN_FORMAT_IOTDB_VALUE: + return parseMultiplePatterns( + pattern, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + case EXTRACTOR_PATTERN_FORMAT_PREFIX_VALUE: + return parseMultiplePatterns( + pattern, p -> new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + default: + LOGGER.info( + "Unknown pattern format: {}, use prefix matching format by default.", patternFormat); + return parseMultiplePatterns( + pattern, p -> new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + } + } + + private static List parseMultiplePatterns( + final String pattern, final Function patternSupplier) { + if (pattern.isEmpty()) { + return Collections.singletonList(patternSupplier.apply(pattern)); + } + + final List patterns = new ArrayList<>(); + final StringBuilder currentPattern = new StringBuilder(); + boolean inBackticks = false; + + for (final char c : pattern.toCharArray()) { + if (c == '`') { + inBackticks = !inBackticks; + currentPattern.append(c); + } else if (c == ',' && !inBackticks) { + final String singlePattern = currentPattern.toString().trim(); + if (!singlePattern.isEmpty()) { + patterns.add(patternSupplier.apply(singlePattern)); + } + currentPattern.setLength(0); + } else { + currentPattern.append(c); } } - // 3. If neither "source.path" nor "source.pattern" is specified, - // this pipe source will match all data. - return new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, null); + final String lastPattern = currentPattern.toString().trim(); + if (!lastPattern.isEmpty()) { + patterns.add(patternSupplier.apply(lastPattern)); + } + + return patterns; } public static boolean isTreeModelDataAllowToBeCaptured(final PipeParameters sourceParameters) { @@ -128,7 +193,9 @@ public static boolean isTreeModelDataAllowToBeCaptured(final PipeParameters sour .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); } - public abstract String getDefaultPattern(); + public abstract String getPattern(); + + public abstract boolean isRoot(); /** Check if this pattern is legal. Different pattern type may have different rules. */ public abstract boolean isLegal(); @@ -162,13 +229,4 @@ public static boolean isTreeModelDataAllowToBeCaptured(final PipeParameters sour *

NOTE: this is only called when {@link TreePattern#mayOverlapWithDevice} is {@code true}. */ public abstract boolean matchesMeasurement(final IDeviceID device, final String measurement); - - @Override - public String toString() { - return "{pattern='" - + pattern - + "', isTreeModelDataAllowedToBeCaptured=" - + isTreeModelDataAllowedToBeCaptured - + '}'; - } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java new file mode 100644 index 0000000000000..48fcc9fa58490 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.datastructure.pattern; + +import org.apache.tsfile.file.metadata.IDeviceID; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Represents a union of multiple {@link TreePattern}s. A path is considered to match if it matches + * any of the patterns in the collection. + */ +public class UnionTreePattern extends TreePattern { + + private final List patterns; + + /** + * Constructs a {@link UnionTreePattern} with a list of {@link TreePattern}s. + * + * @param patterns A list of {@link TreePattern}s. Assumes all patterns share the same value for + * isTreeModelDataAllowedToBeCaptured. + */ + public UnionTreePattern(final List patterns) { + super(patterns.stream().anyMatch(pattern -> pattern.isTreeModelDataAllowedToBeCaptured)); + this.patterns = patterns; + } + + public TreePattern getFirstPattern() { + return patterns.get(0); + } + + @Override + public String getPattern() { + // TODO + return patterns.stream().map(TreePattern::getPattern).collect(Collectors.joining(",")); + } + + @Override + public boolean isRoot() { + for (final TreePattern pattern : patterns) { + if (pattern.isRoot()) { + return true; + } + } + return false; + } + + @Override + public boolean isLegal() { + for (final TreePattern pattern : patterns) { + if (!pattern.isLegal()) { + return false; + } + } + return true; + } + + @Override + public boolean coversDb(final String db) { + for (final TreePattern pattern : patterns) { + if (pattern.coversDb(db)) { + return true; + } + } + return false; + } + + @Override + public boolean coversDevice(final IDeviceID device) { + for (final TreePattern pattern : patterns) { + if (pattern.coversDevice(device)) { + return true; + } + } + return false; + } + + @Override + public boolean mayOverlapWithDb(final String db) { + for (final TreePattern pattern : patterns) { + if (pattern.mayOverlapWithDb(db)) { + return true; + } + } + return false; + } + + @Override + public boolean mayOverlapWithDevice(final IDeviceID device) { + for (final TreePattern pattern : patterns) { + if (pattern.mayOverlapWithDevice(device)) { + return true; + } + } + return false; + } + + @Override + public boolean matchesMeasurement(final IDeviceID device, final String measurement) { + for (final TreePattern pattern : patterns) { + if (pattern.matchesMeasurement(device, measurement)) { + return true; + } + } + return false; + } + + @Override + public String toString() { + return "UnionTreePattern{" + + "patterns=" + + patterns + + ", isTreeModelDataAllowedToBeCaptured=" + + isTreeModelDataAllowedToBeCaptured + + '}'; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java index bad0d1413ce69..722140962bb8a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionTreePattern; import org.apache.iotdb.commons.pipe.datastructure.queue.ConcurrentIterableLinkedQueue; import org.apache.iotdb.commons.pipe.datastructure.queue.listening.AbstractPipeListeningQueue; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; @@ -76,7 +77,11 @@ public void customize( throws Exception { super.customize(parameters, configuration); - final TreePattern pattern = TreePattern.parsePipePatternFromSourceParameters(parameters); + TreePattern pattern = TreePattern.parsePipePatternFromSourceParameters(parameters); + // TODO + if (pattern instanceof UnionTreePattern) { + pattern = ((UnionTreePattern) pattern).getFirstPattern(); + } if (!(pattern instanceof IoTDBTreePattern && (((IoTDBTreePattern) pattern).isPrefix() From 6bf5d0bb8c268e79f3554c8325212fc0c0a74eab Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 13 Oct 2025 16:30:05 +0800 Subject: [PATCH 02/16] reset --- .../main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java index 0788843543d76..651f5f409d276 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java @@ -71,7 +71,7 @@ public class EnvUtils { public static int[] searchAvailablePorts() { int length = 10; while (true) { - int randomPortStart = 2000 + (int) (Math.random() * (2999 - 2000)); + int randomPortStart = 1000 + (int) (Math.random() * (1999 - 1000)); randomPortStart = randomPortStart * (length + 1) + 1; final String lockFilePath = getLockFilePath(randomPortStart); final File lockFile = new File(lockFilePath); @@ -233,7 +233,7 @@ public static Pair getNodeNum() { } return new Pair<>( getIntFromSysVar(DEFAULT_CONFIG_NODE_NUM, 1, 0), - getIntFromSysVar(DEFAULT_DATA_NODE_NUM, 1, 0)); + getIntFromSysVar(DEFAULT_DATA_NODE_NUM, 3, 0)); } public static Pair getNodeNum(final int index) { @@ -243,7 +243,7 @@ public static Pair getNodeNum(final int index) { } return new Pair<>( getIntFromSysVar(DEFAULT_CONFIG_NODE_NUM, 1, index), - getIntFromSysVar(DEFAULT_DATA_NODE_NUM, 1, index)); + getIntFromSysVar(DEFAULT_DATA_NODE_NUM, 3, index)); } public static String getFilePathFromSysVar(final String key, final int index) { From 78e2c7dd91718d20b619d4de2078b92ae69b4a28 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 14 Oct 2025 16:27:08 +0800 Subject: [PATCH 03/16] support multi for meta sync --- .../protocol/IoTDBConfigNodeReceiver.java | 22 ++- .../pipe/source/IoTDBConfigRegionSource.java | 6 +- ...igPhysicalPlanTreePatternParseVisitor.java | 60 ++++--- ...ysicalPlanTreePatternParseVisitorTest.java | 7 +- .../task/connection/PipeEventCollector.java | 4 +- .../TsFileInsertionEventParserProvider.java | 13 +- .../thrift/IoTDBDataNodeReceiver.java | 16 +- .../PipeStatementTreePatternParseVisitor.java | 30 ++-- .../dataregion/IoTDBDataRegionSource.java | 8 +- .../PipePlanTreePatternParseVisitor.java | 53 +++--- ...eStatementTreePatternParseVisitorTest.java | 7 +- .../PipePlanTreePatternParseVisitorTest.java | 8 +- .../pattern/IoTDBTreePattern.java | 8 - .../pattern/SingleTreePattern.java | 5 + .../datastructure/pattern/TreePattern.java | 55 +++++- .../pattern/UnionIoTDBTreePattern.java | 160 ++++++++++++++++++ .../pattern/UnionTreePattern.java | 66 ++------ .../pipe/source/IoTDBNonDataRegionSource.java | 19 +-- 18 files changed, 366 insertions(+), 181 deletions(-) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java index 0e49b869e92c5..99dc6ae43cc30 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java @@ -28,6 +28,8 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.receiver.IoTDBFileReceiver; import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler; import org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapPseudoTPipeTransferRequest; @@ -1051,10 +1053,14 @@ protected TSStatus loadFileV2( final Set executionTypes = PipeConfigRegionSnapshotEvent.getConfigPhysicalPlanTypeSet( parameters.get(ColumnHeaderConstant.TYPE)); - final IoTDBTreePattern treePattern = - new IoTDBTreePattern( - parameters.containsKey(PipeTransferFileSealReqV2.TREE), - parameters.get(ColumnHeaderConstant.PATH_PATTERN)); + final boolean isTreeModelDataAllowedToBeCaptured = + parameters.containsKey(PipeTransferFileSealReqV2.TREE); + final List treePatterns = + TreePattern.parseMultiplePatterns( + parameters.get(ColumnHeaderConstant.PATH_PATTERN), + p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + final TreePattern treePattern = + TreePattern.buildUnionPattern(isTreeModelDataAllowedToBeCaptured, treePatterns); final TablePattern tablePattern = new TablePattern( parameters.containsKey(PipeTransferFileSealReqV2.TABLE), @@ -1062,11 +1068,15 @@ protected TSStatus loadFileV2( parameters.get(ColumnHeaderConstant.TABLE_NAME)); final List results = new ArrayList<>(); while (generator.hasNext()) { - IoTDBConfigRegionSource.parseConfigPlan(generator.next(), treePattern, tablePattern) + IoTDBConfigRegionSource.parseConfigPlan( + generator.next(), (UnionIoTDBTreePattern) treePattern, tablePattern) .filter( configPhysicalPlan -> IoTDBConfigRegionSource.isTypeListened( - configPhysicalPlan, executionTypes, treePattern, tablePattern)) + configPhysicalPlan, + executionTypes, + (UnionIoTDBTreePattern) treePattern, + tablePattern)) .ifPresent( configPhysicalPlan -> results.add(executePlanAndClassifyExceptions(configPhysicalPlan))); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java index fc965624c1d86..252f7331d0cd1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java @@ -25,8 +25,8 @@ import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.config.PipeConfig; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.queue.listening.AbstractPipeListeningQueue; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.event.PipeSnapshotEvent; @@ -253,7 +253,7 @@ protected Optional trimRealtimeEventByPipePattern( public static Optional parseConfigPlan( final ConfigPhysicalPlan plan, - final IoTDBTreePattern treePattern, + final UnionIoTDBTreePattern treePattern, final TablePattern tablePattern) { Optional result = Optional.of(plan); final Boolean isTableDatabasePlan = isTableDatabasePlan(plan); @@ -296,7 +296,7 @@ protected boolean isTypeListened(final PipeWritePlanEvent event) { public static boolean isTypeListened( final ConfigPhysicalPlan plan, final Set listenedTypeSet, - final IoTDBTreePattern treePattern, + final UnionIoTDBTreePattern treePattern, final TablePattern tablePattern) { final Boolean isTableDatabasePlan = isTableDatabasePlan(plan); return listenedTypeSet.contains(plan.getType()) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java index 992b74ca0e5b5..339ea27558988 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java @@ -22,7 +22,8 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; @@ -58,12 +59,12 @@ /** * The {@link PipeConfigPhysicalPlanTreePatternParseVisitor} will transform the schema {@link - * ConfigPhysicalPlan}s using {@link IoTDBTreePattern}. Rule: + * ConfigPhysicalPlan}s using {@link UnionIoTDBTreePattern}. Rule: * *

1. All patterns in the output {@link ConfigPhysicalPlan} will be the intersection of the - * original {@link ConfigPhysicalPlan}'s patterns and the given {@link IoTDBTreePattern}. + * original {@link ConfigPhysicalPlan}'s patterns and the given {@link UnionIoTDBTreePattern}. * - *

2. If a pattern does not intersect with the {@link IoTDBTreePattern}, it's dropped. + *

2. If a pattern does not intersect with the {@link UnionIoTDBTreePattern}, it's dropped. * *

3. If all the patterns in the {@link ConfigPhysicalPlan} is dropped, the {@link * ConfigPhysicalPlan} is dropped. @@ -72,13 +73,13 @@ * one is used in the {@link PipeConfigRegionWritePlanEvent} in {@link ConfigRegionListeningQueue}. */ public class PipeConfigPhysicalPlanTreePatternParseVisitor - extends ConfigPhysicalPlanVisitor, IoTDBTreePattern> { + extends ConfigPhysicalPlanVisitor, UnionIoTDBTreePattern> { private static final Logger LOGGER = LoggerFactory.getLogger(PipeConfigPhysicalPlanTreePatternParseVisitor.class); @Override public Optional visitPlan( - final ConfigPhysicalPlan plan, final IoTDBTreePattern pattern) { + final ConfigPhysicalPlan plan, final UnionIoTDBTreePattern pattern) { return Optional.of(plan); } @@ -90,7 +91,7 @@ public Optional visitPlan( // Other matches using "matchPrefixPath" are with the same principle. @Override public Optional visitCreateDatabase( - final DatabaseSchemaPlan createDatabasePlan, final IoTDBTreePattern pattern) { + final DatabaseSchemaPlan createDatabasePlan, final UnionIoTDBTreePattern pattern) { return pattern.matchPrefixPath(createDatabasePlan.getSchema().getName()) ? Optional.of(createDatabasePlan) : Optional.empty(); @@ -98,7 +99,7 @@ public Optional visitCreateDatabase( @Override public Optional visitAlterDatabase( - final DatabaseSchemaPlan alterDatabasePlan, final IoTDBTreePattern pattern) { + final DatabaseSchemaPlan alterDatabasePlan, final UnionIoTDBTreePattern pattern) { return pattern.matchPrefixPath(alterDatabasePlan.getSchema().getName()) ? Optional.of(alterDatabasePlan) : Optional.empty(); @@ -106,7 +107,7 @@ public Optional visitAlterDatabase( @Override public Optional visitDeleteDatabase( - final DeleteDatabasePlan deleteDatabasePlan, final IoTDBTreePattern pattern) { + final DeleteDatabasePlan deleteDatabasePlan, final UnionIoTDBTreePattern pattern) { return pattern.matchPrefixPath(deleteDatabasePlan.getName()) ? Optional.of(deleteDatabasePlan) : Optional.empty(); @@ -114,7 +115,8 @@ public Optional visitDeleteDatabase( @Override public Optional visitCreateSchemaTemplate( - final CreateSchemaTemplatePlan createSchemaTemplatePlan, final IoTDBTreePattern pattern) { + final CreateSchemaTemplatePlan createSchemaTemplatePlan, + final UnionIoTDBTreePattern pattern) { // This is a deserialized template and can be arbitrarily altered final Template template = createSchemaTemplatePlan.getTemplate(); template.getSchemaMap().keySet().removeIf(measurement -> !pattern.matchTailNode(measurement)); @@ -126,7 +128,7 @@ public Optional visitCreateSchemaTemplate( @Override public Optional visitCommitSetSchemaTemplate( final CommitSetSchemaTemplatePlan commitSetSchemaTemplatePlan, - final IoTDBTreePattern pattern) { + final UnionIoTDBTreePattern pattern) { return pattern.matchPrefixPath(commitSetSchemaTemplatePlan.getPath()) ? Optional.of(commitSetSchemaTemplatePlan) : Optional.empty(); @@ -135,7 +137,7 @@ public Optional visitCommitSetSchemaTemplate( @Override public Optional visitPipeUnsetSchemaTemplate( final PipeUnsetSchemaTemplatePlan pipeUnsetSchemaTemplatePlan, - final IoTDBTreePattern pattern) { + final UnionIoTDBTreePattern pattern) { return pattern.matchPrefixPath(pipeUnsetSchemaTemplatePlan.getPath()) ? Optional.of(pipeUnsetSchemaTemplatePlan) : Optional.empty(); @@ -143,7 +145,8 @@ public Optional visitPipeUnsetSchemaTemplate( @Override public Optional visitExtendSchemaTemplate( - final ExtendSchemaTemplatePlan extendSchemaTemplatePlan, final IoTDBTreePattern pattern) { + final ExtendSchemaTemplatePlan extendSchemaTemplatePlan, + final UnionIoTDBTreePattern pattern) { final TemplateExtendInfo extendInfo = extendSchemaTemplatePlan.getTemplateExtendInfo(); final int[] filteredIndexes = IntStream.range(0, extendInfo.getMeasurements().size()) @@ -154,41 +157,39 @@ public Optional visitExtendSchemaTemplate( new ExtendSchemaTemplatePlan( new TemplateExtendInfo( extendInfo.getTemplateName(), - IoTDBTreePattern.applyIndexesOnList( - filteredIndexes, extendInfo.getMeasurements()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getDataTypes()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getEncodings()), - IoTDBTreePattern.applyIndexesOnList( - filteredIndexes, extendInfo.getCompressors())))) + TreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getMeasurements()), + TreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getDataTypes()), + TreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getEncodings()), + TreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getCompressors())))) : Optional.empty(); } @Override public Optional visitGrantUser( - final AuthorTreePlan grantUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan grantUserPlan, final UnionIoTDBTreePattern pattern) { return visitTreeAuthorPlan(grantUserPlan, pattern); } @Override public Optional visitRevokeUser( - final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan revokeUserPlan, final UnionIoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } @Override public Optional visitGrantRole( - final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan revokeUserPlan, final UnionIoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } @Override public Optional visitRevokeRole( - final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan revokeUserPlan, final UnionIoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } private Optional visitTreeAuthorPlan( - final AuthorTreePlan pathRelatedAuthorTreePlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan pathRelatedAuthorTreePlan, final UnionIoTDBTreePattern pattern) { final List intersectedPaths = pathRelatedAuthorTreePlan.getNodeNameList().stream() .map(pattern::getIntersection) @@ -216,7 +217,8 @@ private Optional visitTreeAuthorPlan( @Override public Optional visitPipeDeleteTimeSeries( - final PipeDeleteTimeSeriesPlan pipeDeleteTimeSeriesPlan, final IoTDBTreePattern pattern) { + final PipeDeleteTimeSeriesPlan pipeDeleteTimeSeriesPlan, + final UnionIoTDBTreePattern pattern) { try { final PathPatternTree intersectedTree = pattern.getIntersection( @@ -234,7 +236,8 @@ public Optional visitPipeDeleteTimeSeries( @Override public Optional visitPipeDeleteLogicalView( - final PipeDeleteLogicalViewPlan pipeDeleteLogicalViewPlan, final IoTDBTreePattern pattern) { + final PipeDeleteLogicalViewPlan pipeDeleteLogicalViewPlan, + final UnionIoTDBTreePattern pattern) { try { final PathPatternTree intersectedTree = pattern.getIntersection( @@ -252,7 +255,8 @@ public Optional visitPipeDeleteLogicalView( @Override public Optional visitPipeDeactivateTemplate( - final PipeDeactivateTemplatePlan pipeDeactivateTemplatePlan, final IoTDBTreePattern pattern) { + final PipeDeactivateTemplatePlan pipeDeactivateTemplatePlan, + final UnionIoTDBTreePattern pattern) { final Map> newTemplateSetInfo = pipeDeactivateTemplatePlan.getTemplateSetInfo().entrySet().stream() .flatMap( @@ -275,7 +279,7 @@ public Optional visitPipeDeactivateTemplate( @Override public Optional visitTTL( - final SetTTLPlan setTTLPlan, final IoTDBTreePattern pattern) { + final SetTTLPlan setTTLPlan, final UnionIoTDBTreePattern pattern) { final PartialPath partialPath = new PartialPath(setTTLPlan.getPathPattern()); final List intersectionList = pattern.matchPrefixPath(partialPath.getFullPath()) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java index 6893cfa55230c..4bfc9a0a8cdbf 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; @@ -54,8 +55,10 @@ public class PipeConfigPhysicalPlanTreePatternParseVisitorTest { - private final IoTDBTreePattern prefixPathPattern = new IoTDBTreePattern("root.db.device.**"); - private final IoTDBTreePattern fullPathPattern = new IoTDBTreePattern("root.db.device.s1"); + private final UnionIoTDBTreePattern prefixPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); + private final UnionIoTDBTreePattern fullPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.s1")); @Test public void testCreateDatabase() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java index a4c9f0e8403cd..f979d4763b5da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.audit.UserEntity; import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.event.ProgressReportEvent; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; @@ -172,7 +172,7 @@ private void parseAndCollectEvent(final PipeDeleteDataNodeEvent deleteDataEvent) (deleteDataEvent.getDeleteDataNode() instanceof DeleteDataNode ? IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR.process( deleteDataEvent.getDeleteDataNode(), - (IoTDBTreePattern) deleteDataEvent.getTreePattern()) + (UnionIoTDBTreePattern) deleteDataEvent.getTreePattern()) : IoTDBSchemaRegionSource.TABLE_PATTERN_PARSE_VISITOR .process(deleteDataEvent.getDeleteDataNode(), deleteDataEvent.getTablePattern()) .flatMap( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java index 4ffe811baa37f..8ca2b159255b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java @@ -21,10 +21,9 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; -import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.query.TsFileInsertionEventQueryParser; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.scan.TsFileInsertionEventScanParser; @@ -102,7 +101,7 @@ public TsFileInsertionEventParser provide() throws IOException { if ((double) PipeDataNodeResourceManager.memory().getUsedMemorySizeInBytes() / PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes() > PipeTsFilePublicResource.MEMORY_SUFFICIENT_THRESHOLD - && !(treePattern instanceof UnionTreePattern)) { + && treePattern.isSingle()) { return new TsFileInsertionEventScanParser( pipeName, creationTime, @@ -114,8 +113,8 @@ public TsFileInsertionEventParser provide() throws IOException { sourceEvent); } - if (treePattern instanceof IoTDBTreePattern - && !((IoTDBTreePattern) treePattern).mayMatchMultipleTimeSeriesInOneDevice()) { + if (treePattern instanceof UnionIoTDBTreePattern + && !((UnionIoTDBTreePattern) treePattern).mayMatchMultipleTimeSeriesInOneDevice()) { // If the pattern matches only one time series in one device, use query container here // because there is no timestamps merge overhead. // @@ -135,7 +134,7 @@ public TsFileInsertionEventParser provide() throws IOException { final Map deviceIsAlignedMap = PipeDataNodeResourceManager.tsfile().getDeviceIsAlignedMapFromCache(tsFile, false); - if (Objects.isNull(deviceIsAlignedMap) && !(treePattern instanceof UnionTreePattern)) { + if (Objects.isNull(deviceIsAlignedMap) && treePattern.isSingle()) { // If we failed to get from cache, it indicates that the memory usage is high. // We use scan data container because it requires less memory. return new TsFileInsertionEventScanParser( @@ -155,7 +154,7 @@ public TsFileInsertionEventParser provide() throws IOException { // Use scan data container if we need enough amount to data thus it's better to scan than query. return (double) filteredDeviceIsAlignedMap.size() / originalSize > PipeConfig.getInstance().getPipeTsFileScanParsingThreshold() - && !(treePattern instanceof UnionTreePattern) + && treePattern.isSingle() ? new TsFileInsertionEventScanParser( pipeName, creationTime, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index e4093c260819c..ad503f5fe811c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -29,6 +29,8 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.receiver.IoTDBFileReceiver; import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; @@ -604,10 +606,14 @@ private TSStatus loadSchemaSnapShot( final Set executionTypes = PipeSchemaRegionSnapshotEvent.getStatementTypeSet( parameters.get(ColumnHeaderConstant.TYPE)); - final IoTDBTreePattern treePattern = - new IoTDBTreePattern( - parameters.containsKey(PipeTransferFileSealReqV2.TREE), - parameters.get(ColumnHeaderConstant.PATH_PATTERN)); + final boolean isTreeModelDataAllowedToBeCaptured = + parameters.containsKey(PipeTransferFileSealReqV2.TREE); + final List treePatterns = + TreePattern.parseMultiplePatterns( + parameters.get(ColumnHeaderConstant.PATH_PATTERN), + p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + final TreePattern treePattern = + TreePattern.buildUnionPattern(isTreeModelDataAllowedToBeCaptured, treePatterns); final TablePattern tablePattern = new TablePattern( parameters.containsKey(PipeTransferFileSealReqV2.TABLE), @@ -629,7 +635,7 @@ private TSStatus loadSchemaSnapShot( // Here we apply the statements as many as possible // Even if there are failed statements STATEMENT_TREE_PATTERN_PARSE_VISITOR - .process(originalStatement, treePattern) + .process(originalStatement, (UnionIoTDBTreePattern) treePattern) .flatMap(parsedStatement -> batchVisitor.process(parsedStatement, null)) .ifPresent(statement -> results.add(executeStatementAndClassifyExceptions(statement))); } else if (treeOrTableStatement diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTreePatternParseVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTreePatternParseVisitor.java index a369bb8a7c954..aa11fd69efa4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTreePatternParseVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTreePatternParseVisitor.java @@ -19,7 +19,8 @@ package org.apache.iotdb.db.pipe.receiver.visitor; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; @@ -36,12 +37,12 @@ /** * The {@link PipeStatementTreePatternParseVisitor} will transform the schema {@link Statement}s - * using {@link IoTDBTreePattern}. Rule: + * using {@link UnionIoTDBTreePattern}. Rule: * *

1. All patterns in the output {@link Statement} will be the intersection of the original - * {@link Statement}'s patterns and the given {@link IoTDBTreePattern}. + * {@link Statement}'s patterns and the given {@link UnionIoTDBTreePattern}. * - *

2. If a pattern does not intersect with the {@link IoTDBTreePattern}, it's dropped. + *

2. If a pattern does not intersect with the {@link UnionIoTDBTreePattern}, it's dropped. * *

3. If all the patterns in the {@link Statement} is dropped, the {@link Statement} is dropped. * @@ -49,16 +50,16 @@ * from the {@link SRStatementGenerator} and will no longer be used. */ public class PipeStatementTreePatternParseVisitor - extends StatementVisitor, IoTDBTreePattern> { + extends StatementVisitor, UnionIoTDBTreePattern> { @Override public Optional visitNode( - final StatementNode statement, final IoTDBTreePattern pattern) { + final StatementNode statement, final UnionIoTDBTreePattern pattern) { return Optional.of((Statement) statement); } @Override public Optional visitCreateTimeseries( - final CreateTimeSeriesStatement statement, final IoTDBTreePattern pattern) { + final CreateTimeSeriesStatement statement, final UnionIoTDBTreePattern pattern) { return pattern.matchesMeasurement( statement.getPath().getIDeviceID(), statement.getPath().getMeasurement()) ? Optional.of(statement) @@ -67,7 +68,7 @@ public Optional visitCreateTimeseries( @Override public Optional visitCreateAlignedTimeseries( - final CreateAlignedTimeSeriesStatement statement, final IoTDBTreePattern pattern) { + final CreateAlignedTimeSeriesStatement statement, final UnionIoTDBTreePattern pattern) { final int[] filteredIndexes = IntStream.range(0, statement.getMeasurements().size()) .filter( @@ -107,7 +108,8 @@ public Optional visitCreateAlignedTimeseries( // For logical view with tags/attributes @Override public Optional visitAlterTimeSeries( - final AlterTimeSeriesStatement alterTimeSeriesStatement, final IoTDBTreePattern pattern) { + final AlterTimeSeriesStatement alterTimeSeriesStatement, + final UnionIoTDBTreePattern pattern) { return pattern.matchesMeasurement( alterTimeSeriesStatement.getPath().getIDeviceID(), alterTimeSeriesStatement.getPath().getMeasurement()) @@ -117,7 +119,8 @@ public Optional visitAlterTimeSeries( @Override public Optional visitActivateTemplate( - final ActivateTemplateStatement activateTemplateStatement, final IoTDBTreePattern pattern) { + final ActivateTemplateStatement activateTemplateStatement, + final UnionIoTDBTreePattern pattern) { return pattern.matchDevice(activateTemplateStatement.getPath().getFullPath()) ? Optional.of(activateTemplateStatement) : Optional.empty(); @@ -125,7 +128,8 @@ public Optional visitActivateTemplate( @Override public Optional visitCreateLogicalView( - final CreateLogicalViewStatement createLogicalViewStatement, final IoTDBTreePattern pattern) { + final CreateLogicalViewStatement createLogicalViewStatement, + final UnionIoTDBTreePattern pattern) { final int[] filteredIndexes = IntStream.range(0, createLogicalViewStatement.getTargetPathList().size()) .filter( @@ -141,10 +145,10 @@ public Optional visitCreateLogicalView( return Optional.empty(); } createLogicalViewStatement.setTargetFullPaths( - IoTDBTreePattern.applyIndexesOnList( + TreePattern.applyIndexesOnList( filteredIndexes, createLogicalViewStatement.getTargetPathList())); createLogicalViewStatement.setViewExpressions( - IoTDBTreePattern.applyIndexesOnList( + TreePattern.applyIndexesOnList( filteredIndexes, createLogicalViewStatement.getViewExpressions())); return Optional.of(createLogicalViewStatement); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index b5fa3b2db40e6..a9d2e06d7562e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -22,8 +22,8 @@ import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.source.IoTDBSource; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -303,9 +303,9 @@ private void validatePattern(final TreePattern treePattern) { } if (shouldExtractDeletion - && !(treePattern instanceof IoTDBTreePattern - && (((IoTDBTreePattern) treePattern).isPrefix() - || ((IoTDBTreePattern) treePattern).isFullPath()))) { + && !(treePattern instanceof UnionIoTDBTreePattern + && (((UnionIoTDBTreePattern) treePattern).isPrefix() + || ((UnionIoTDBTreePattern) treePattern).isFullPath()))) { throw new IllegalArgumentException( String.format( "The path pattern %s is not valid for the source. Only prefix or full path is allowed.", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java index 4c0f0feb4f8a5..197c96cdee215 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -54,12 +54,12 @@ /** * The {@link PipePlanTreePatternParseVisitor} will transform the schema {@link PlanNode}s using - * {@link IoTDBTreePattern}. Rule: + * {@link UnionIoTDBTreePattern}. Rule: * *

1. All patterns in the output {@link PlanNode} will be the intersection of the original {@link - * PlanNode}'s patterns and the given {@link IoTDBTreePattern}. + * PlanNode}'s patterns and the given {@link UnionIoTDBTreePattern}. * - *

2. If a pattern does not intersect with the {@link IoTDBTreePattern}, it's dropped. + *

2. If a pattern does not intersect with the {@link UnionIoTDBTreePattern}, it's dropped. * *

3. If all the patterns in the {@link PlanNode} is dropped, the {@link PlanNode} is dropped. * @@ -67,15 +67,15 @@ * one is used in the {@link PipeSchemaRegionWritePlanEvent} in {@link SchemaRegionListeningQueue}. */ public class PipePlanTreePatternParseVisitor - extends PlanVisitor, IoTDBTreePattern> { + extends PlanVisitor, UnionIoTDBTreePattern> { @Override - public Optional visitPlan(final PlanNode node, final IoTDBTreePattern pattern) { + public Optional visitPlan(final PlanNode node, final UnionIoTDBTreePattern pattern) { return Optional.of(node); } @Override public Optional visitCreateTimeSeries( - final CreateTimeSeriesNode node, final IoTDBTreePattern pattern) { + final CreateTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { return pattern.matchesMeasurement( node.getPath().getIDeviceID(), node.getPath().getMeasurement()) ? Optional.of(node) @@ -84,7 +84,7 @@ public Optional visitCreateTimeSeries( @Override public Optional visitCreateAlignedTimeSeries( - final CreateAlignedTimeSeriesNode node, final IoTDBTreePattern pattern) { + final CreateAlignedTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { final int[] filteredIndexes = IntStream.range(0, node.getMeasurements().size()) .filter( @@ -98,19 +98,20 @@ public Optional visitCreateAlignedTimeSeries( new CreateAlignedTimeSeriesNode( node.getPlanNodeId(), node.getDevicePath(), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getMeasurements()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getDataTypes()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getEncodings()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getCompressors()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getAliasList()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getTagsList()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getAttributesList()))) + UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getMeasurements()), + UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getDataTypes()), + UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getEncodings()), + UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getCompressors()), + UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getAliasList()), + UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getTagsList()), + UnionIoTDBTreePattern.applyIndexesOnList( + filteredIndexes, node.getAttributesList()))) : Optional.empty(); } @Override public Optional visitCreateMultiTimeSeries( - final CreateMultiTimeSeriesNode node, final IoTDBTreePattern pattern) { + final CreateMultiTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { final Map filteredMeasurementGroupMap = node.getMeasurementGroupMap().entrySet().stream() .filter(entry -> pattern.matchPrefixPath(entry.getKey().getFullPath())) @@ -129,7 +130,7 @@ public Optional visitCreateMultiTimeSeries( } private static MeasurementGroup trimMeasurementGroup( - final IDeviceID device, final MeasurementGroup group, final IoTDBTreePattern pattern) { + final IDeviceID device, final MeasurementGroup group, final UnionIoTDBTreePattern pattern) { final int[] filteredIndexes = IntStream.range(0, group.size()) .filter(index -> pattern.matchesMeasurement(device, group.getMeasurements().get(index))) @@ -164,7 +165,7 @@ private static MeasurementGroup trimMeasurementGroup( @Override public Optional visitAlterTimeSeries( - final AlterTimeSeriesNode node, final IoTDBTreePattern pattern) { + final AlterTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { return pattern.matchesMeasurement( node.getPath().getIDeviceID(), node.getPath().getMeasurement()) ? Optional.of(node) @@ -173,7 +174,7 @@ public Optional visitAlterTimeSeries( @Override public Optional visitInternalCreateTimeSeries( - final InternalCreateTimeSeriesNode node, final IoTDBTreePattern pattern) { + final InternalCreateTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { final MeasurementGroup group = pattern.matchPrefixPath(node.getDevicePath().getFullPath()) ? trimMeasurementGroup( @@ -190,7 +191,7 @@ public Optional visitInternalCreateTimeSeries( @Override public Optional visitActivateTemplate( - final ActivateTemplateNode node, final IoTDBTreePattern pattern) { + final ActivateTemplateNode node, final UnionIoTDBTreePattern pattern) { return pattern.matchDevice(node.getActivatePath().getFullPath()) ? Optional.of(node) : Optional.empty(); @@ -198,7 +199,7 @@ public Optional visitActivateTemplate( @Override public Optional visitInternalBatchActivateTemplate( - final InternalBatchActivateTemplateNode node, final IoTDBTreePattern pattern) { + final InternalBatchActivateTemplateNode node, final UnionIoTDBTreePattern pattern) { final Map> filteredTemplateActivationMap = node.getTemplateActivationMap().entrySet().stream() .filter(entry -> pattern.matchDevice(entry.getKey().getFullPath())) @@ -212,7 +213,7 @@ public Optional visitInternalBatchActivateTemplate( @Override public Optional visitInternalCreateMultiTimeSeries( - final InternalCreateMultiTimeSeriesNode node, final IoTDBTreePattern pattern) { + final InternalCreateMultiTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { final Map> filteredDeviceMap = node.getDeviceMap().entrySet().stream() .filter(entry -> pattern.matchPrefixPath(entry.getKey().getFullPath())) @@ -236,7 +237,7 @@ public Optional visitInternalCreateMultiTimeSeries( @Override public Optional visitBatchActivateTemplate( - final BatchActivateTemplateNode node, final IoTDBTreePattern pattern) { + final BatchActivateTemplateNode node, final UnionIoTDBTreePattern pattern) { final Map> filteredTemplateActivationMap = node.getTemplateActivationMap().entrySet().stream() .filter(entry -> pattern.matchDevice(entry.getKey().getFullPath())) @@ -249,7 +250,7 @@ public Optional visitBatchActivateTemplate( @Override public Optional visitCreateLogicalView( - final CreateLogicalViewNode node, final IoTDBTreePattern pattern) { + final CreateLogicalViewNode node, final UnionIoTDBTreePattern pattern) { final Map filteredViewPathToSourceMap = node.getViewPathToSourceExpressionMap().entrySet().stream() .filter( @@ -264,7 +265,7 @@ public Optional visitCreateLogicalView( @Override public Optional visitAlterLogicalView( - final AlterLogicalViewNode node, final IoTDBTreePattern pattern) { + final AlterLogicalViewNode node, final UnionIoTDBTreePattern pattern) { final Map filteredViewPathToSourceMap = node.getViewPathToSourceMap().entrySet().stream() .filter( @@ -279,7 +280,7 @@ public Optional visitAlterLogicalView( @Override public Optional visitDeleteData( - final DeleteDataNode node, final IoTDBTreePattern pattern) { + final DeleteDataNode node, final UnionIoTDBTreePattern pattern) { final List intersectedPaths = node.getPathList().stream() .map(pattern::getIntersection) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java index c1619b64ad492..76b291cbf5e0b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.schema.view.viewExpression.leaf.TimeSeriesViewOperand; import org.apache.iotdb.db.pipe.receiver.visitor.PipeStatementTreePatternParseVisitor; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; @@ -43,8 +44,10 @@ public class PipeStatementTreePatternParseVisitorTest { - private final IoTDBTreePattern prefixPathPattern = new IoTDBTreePattern("root.db.device.**"); - private final IoTDBTreePattern fullPathPattern = new IoTDBTreePattern("root.db.device.s1"); + private final UnionIoTDBTreePattern prefixPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); + private final UnionIoTDBTreePattern fullPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.s1")); @Test public void testCreateTimeSeries() throws IllegalPathException { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java index b21942a241e96..31f019beee74d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.schema.view.viewExpression.leaf.TimeSeriesViewOperand; import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource; @@ -56,8 +57,11 @@ import java.util.Map; public class PipePlanTreePatternParseVisitorTest { - private final IoTDBTreePattern prefixPathPattern = new IoTDBTreePattern("root.db.device.**"); - private final IoTDBTreePattern fullPathPattern = new IoTDBTreePattern("root.db.device.s1"); + + private final UnionIoTDBTreePattern prefixPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); + private final UnionIoTDBTreePattern fullPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.s1")); @Test public void testCreateTimeSeries() throws IllegalPathException { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java index a87eceb8eac67..80e7df9ee5da2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java @@ -35,7 +35,6 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; public class IoTDBTreePattern extends SingleTreePattern { @@ -57,13 +56,6 @@ public IoTDBTreePattern(final String pattern) { this(true, pattern); } - public static List applyIndexesOnList( - final int[] filteredIndexes, final List originalList) { - return Objects.nonNull(originalList) - ? Arrays.stream(filteredIndexes).mapToObj(originalList::get).collect(Collectors.toList()) - : null; - } - @Override public String getDefaultPattern() { return PipeSourceConstant.EXTRACTOR_PATTERN_IOTDB_DEFAULT_VALUE; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java index af6fa5e15ea2d..b88e760d76a9d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java @@ -31,6 +31,11 @@ protected SingleTreePattern( this.pattern = pattern != null ? pattern : getDefaultPattern(); } + @Override + public boolean isSingle() { + return true; + } + @Override public String getPattern() { return pattern; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java index 492e662c0ee44..cc5997582d413 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java @@ -31,7 +31,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.function.Function; +import java.util.stream.Collectors; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATH_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATTERN_FORMAT_IOTDB_VALUE; @@ -56,6 +58,13 @@ public boolean isTreeModelDataAllowedToBeCaptured() { return isTreeModelDataAllowedToBeCaptured; } + public static List applyIndexesOnList( + final int[] filteredIndexes, final List originalList) { + return Objects.nonNull(originalList) + ? Arrays.stream(filteredIndexes).mapToObj(originalList::get).collect(Collectors.toList()) + : null; + } + /** * Interpret from source parameters and get a {@link TreePattern}. * @@ -63,9 +72,6 @@ public boolean isTreeModelDataAllowedToBeCaptured() { */ public static TreePattern parsePipePatternFromSourceParameters( final PipeParameters sourceParameters) { - final Function, TreePattern> listToTreePattern = - list -> list.size() == 1 ? list.get(0) : new UnionTreePattern(list); - final boolean isTreeModelDataAllowedToBeCaptured = isTreeModelDataAllowToBeCaptured(sourceParameters); @@ -84,26 +90,29 @@ public static TreePattern parsePipePatternFromSourceParameters( result.addAll( parsePatternsFromPatternParameter( pattern, sourceParameters, isTreeModelDataAllowedToBeCaptured)); - return listToTreePattern.apply(result); + return buildUnionPattern(isTreeModelDataAllowedToBeCaptured, result); } // 2. If only "source.path" is specified, it will be interpreted as an IoTDB-style path. if (path != null) { - return listToTreePattern.apply( + return buildUnionPattern( + isTreeModelDataAllowedToBeCaptured, parseMultiplePatterns( path, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p))); } // 3. If only "source.pattern" is specified, parse it using the helper method. if (pattern != null) { - return listToTreePattern.apply( + return buildUnionPattern( + isTreeModelDataAllowedToBeCaptured, parsePatternsFromPatternParameter( pattern, sourceParameters, isTreeModelDataAllowedToBeCaptured)); } // 4. If neither "source.path" nor "source.pattern" is specified, // this pipe source will match all data. - return listToTreePattern.apply( + return buildUnionPattern( + isTreeModelDataAllowedToBeCaptured, Collections.singletonList(new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, null))); } @@ -144,7 +153,7 @@ private static List parsePatternsFromPatternParameter( } } - private static List parseMultiplePatterns( + public static List parseMultiplePatterns( final String pattern, final Function patternSupplier) { if (pattern.isEmpty()) { return Collections.singletonList(patternSupplier.apply(pattern)); @@ -177,6 +186,34 @@ private static List parseMultiplePatterns( return patterns; } + /** + * A private helper method to build the most specific UnionTreePattern possible. If all patterns + * are IoTDBTreePattern, it returns an IoTDBUnionTreePattern. Otherwise, it returns a general + * UnionTreePattern. + */ + public static TreePattern buildUnionPattern( + final boolean isTreeModelDataAllowedToBeCaptured, final List patterns) { + // Check if all instances in the list are of type IoTDBTreePattern + boolean allIoTDB = true; + for (final TreePattern p : patterns) { + if (!(p instanceof IoTDBTreePattern)) { + allIoTDB = false; + break; + } + } + + if (allIoTDB) { + final List iotdbPatterns = new ArrayList<>(patterns.size()); + for (final TreePattern p : patterns) { + iotdbPatterns.add((IoTDBTreePattern) p); + } + return new UnionIoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, iotdbPatterns); + } else { + // If there's a mix of pattern types, use the general UnionTreePattern + return new UnionTreePattern(isTreeModelDataAllowedToBeCaptured, patterns); + } + } + public static boolean isTreeModelDataAllowToBeCaptured(final PipeParameters sourceParameters) { return sourceParameters.getBooleanOrDefault( Arrays.asList( @@ -193,6 +230,8 @@ public static boolean isTreeModelDataAllowToBeCaptured(final PipeParameters sour .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); } + public abstract boolean isSingle(); + public abstract String getPattern(); public abstract boolean isRoot(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java new file mode 100644 index 0000000000000..fbdf7e0312958 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.datastructure.pattern; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; + +import org.apache.tsfile.file.metadata.IDeviceID; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Represents a union of multiple {@link IoTDBTreePattern}s. This specialized class ensures type + * safety and provides access to methods specific to IoTDBTreePattern, such as getIntersection. + */ +public class UnionIoTDBTreePattern extends TreePattern { + + private final List patterns; + + public UnionIoTDBTreePattern( + final boolean isTreeModelDataAllowedToBeCaptured, final List patterns) { + super(isTreeModelDataAllowedToBeCaptured); + this.patterns = patterns; + } + + public UnionIoTDBTreePattern(final List patterns) { + super(true); + this.patterns = patterns; + } + + public UnionIoTDBTreePattern(final IoTDBTreePattern pattern) { + super(true); + this.patterns = Collections.singletonList(pattern); + } + + // ********************************************************************** + // IoTDBTreePattern-specific aggregated methods + // ********************************************************************** + + public boolean matchPrefixPath(final String path) { + return patterns.stream().anyMatch(p -> p.matchPrefixPath(path)); + } + + public boolean matchDevice(final String devicePath) { + return patterns.stream().anyMatch(p -> p.matchDevice(devicePath)); + } + + public boolean matchTailNode(final String tailNode) { + return patterns.stream().anyMatch(p -> p.matchTailNode(tailNode)); + } + + public List getIntersection(final PartialPath partialPath) { + final List allIntersections = new ArrayList<>(); + for (final IoTDBTreePattern pattern : patterns) { + allIntersections.addAll(pattern.getIntersection(partialPath)); + } + return allIntersections; + } + + public PathPatternTree getIntersection(final PathPatternTree patternTree) { + final PathPatternTree resultTree = new PathPatternTree(); + for (final IoTDBTreePattern pattern : patterns) { + final PathPatternTree intersection = pattern.getIntersection(patternTree); + intersection.getAllPathPatterns().forEach(resultTree::appendPathPattern); + } + resultTree.constructTree(); + return resultTree; + } + + public boolean isPrefix() { + return patterns.stream().allMatch(IoTDBTreePattern::isPrefix); + } + + public boolean isFullPath() { + return patterns.stream().allMatch(IoTDBTreePattern::isFullPath); + } + + public boolean mayMatchMultipleTimeSeriesInOneDevice() { + return patterns.stream().anyMatch(IoTDBTreePattern::mayMatchMultipleTimeSeriesInOneDevice); + } + + // ********************************************************************** + // Implementation of abstract methods from TreePattern + // ********************************************************************** + + @Override + public boolean isSingle() { + return patterns.size() == 1; + } + + @Override + public String getPattern() { + return patterns.stream().map(TreePattern::getPattern).collect(Collectors.joining(",")); + } + + @Override + public boolean isRoot() { + return patterns.stream().anyMatch(TreePattern::isRoot); + } + + @Override + public boolean isLegal() { + return patterns.stream().allMatch(TreePattern::isLegal); + } + + @Override + public boolean coversDb(final String db) { + return patterns.stream().anyMatch(p -> p.coversDb(db)); + } + + @Override + public boolean coversDevice(final IDeviceID device) { + return patterns.stream().anyMatch(p -> p.coversDevice(device)); + } + + @Override + public boolean mayOverlapWithDb(final String db) { + return patterns.stream().anyMatch(p -> p.mayOverlapWithDb(db)); + } + + @Override + public boolean mayOverlapWithDevice(final IDeviceID device) { + return patterns.stream().anyMatch(p -> p.mayOverlapWithDevice(device)); + } + + @Override + public boolean matchesMeasurement(final IDeviceID device, final String measurement) { + return patterns.stream().anyMatch(p -> p.matchesMeasurement(device, measurement)); + } + + @Override + public String toString() { + return "UnionIoTDBTreePattern{" + + "patterns=" + + patterns + + ", isTreeModelDataAllowedToBeCaptured=" + + isTreeModelDataAllowedToBeCaptured + + '}'; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java index 48fcc9fa58490..23b02a7f58192 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java @@ -32,95 +32,55 @@ public class UnionTreePattern extends TreePattern { private final List patterns; - /** - * Constructs a {@link UnionTreePattern} with a list of {@link TreePattern}s. - * - * @param patterns A list of {@link TreePattern}s. Assumes all patterns share the same value for - * isTreeModelDataAllowedToBeCaptured. - */ - public UnionTreePattern(final List patterns) { - super(patterns.stream().anyMatch(pattern -> pattern.isTreeModelDataAllowedToBeCaptured)); + public UnionTreePattern( + final boolean isTreeModelDataAllowedToBeCaptured, final List patterns) { + super(isTreeModelDataAllowedToBeCaptured); this.patterns = patterns; } - public TreePattern getFirstPattern() { - return patterns.get(0); + @Override + public boolean isSingle() { + return patterns.size() == 1; } @Override public String getPattern() { - // TODO return patterns.stream().map(TreePattern::getPattern).collect(Collectors.joining(",")); } @Override public boolean isRoot() { - for (final TreePattern pattern : patterns) { - if (pattern.isRoot()) { - return true; - } - } - return false; + return patterns.stream().anyMatch(TreePattern::isRoot); } @Override public boolean isLegal() { - for (final TreePattern pattern : patterns) { - if (!pattern.isLegal()) { - return false; - } - } - return true; + return patterns.stream().allMatch(TreePattern::isLegal); } @Override public boolean coversDb(final String db) { - for (final TreePattern pattern : patterns) { - if (pattern.coversDb(db)) { - return true; - } - } - return false; + return patterns.stream().anyMatch(p -> p.coversDb(db)); } @Override public boolean coversDevice(final IDeviceID device) { - for (final TreePattern pattern : patterns) { - if (pattern.coversDevice(device)) { - return true; - } - } - return false; + return patterns.stream().anyMatch(p -> p.coversDevice(device)); } @Override public boolean mayOverlapWithDb(final String db) { - for (final TreePattern pattern : patterns) { - if (pattern.mayOverlapWithDb(db)) { - return true; - } - } - return false; + return patterns.stream().anyMatch(p -> p.mayOverlapWithDb(db)); } @Override public boolean mayOverlapWithDevice(final IDeviceID device) { - for (final TreePattern pattern : patterns) { - if (pattern.mayOverlapWithDevice(device)) { - return true; - } - } - return false; + return patterns.stream().anyMatch(p -> p.mayOverlapWithDevice(device)); } @Override public boolean matchesMeasurement(final IDeviceID device, final String measurement) { - for (final TreePattern pattern : patterns) { - if (pattern.matchesMeasurement(device, measurement)) { - return true; - } - } - return false; + return patterns.stream().anyMatch(p -> p.matchesMeasurement(device, measurement)); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java index 722140962bb8a..83446a1371de6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java @@ -24,10 +24,9 @@ import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; -import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.queue.ConcurrentIterableLinkedQueue; import org.apache.iotdb.commons.pipe.datastructure.queue.listening.AbstractPipeListeningQueue; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; @@ -54,7 +53,7 @@ @TableModel public abstract class IoTDBNonDataRegionSource extends IoTDBSource { - protected IoTDBTreePattern treePattern; + protected UnionIoTDBTreePattern treePattern; protected TablePattern tablePattern; private List historicalEvents = new LinkedList<>(); @@ -77,21 +76,17 @@ public void customize( throws Exception { super.customize(parameters, configuration); - TreePattern pattern = TreePattern.parsePipePatternFromSourceParameters(parameters); - // TODO - if (pattern instanceof UnionTreePattern) { - pattern = ((UnionTreePattern) pattern).getFirstPattern(); - } + final TreePattern pattern = TreePattern.parsePipePatternFromSourceParameters(parameters); - if (!(pattern instanceof IoTDBTreePattern - && (((IoTDBTreePattern) pattern).isPrefix() - || ((IoTDBTreePattern) pattern).isFullPath()))) { + if (!(pattern instanceof UnionIoTDBTreePattern + && (((UnionIoTDBTreePattern) pattern).isPrefix() + || ((UnionIoTDBTreePattern) pattern).isFullPath()))) { throw new IllegalArgumentException( String.format( "The path pattern %s is not valid for the source. Only prefix or full path is allowed.", pattern.getPattern())); } - treePattern = (IoTDBTreePattern) pattern; + treePattern = (UnionIoTDBTreePattern) pattern; tablePattern = TablePattern.parsePipePatternFromSourceParameters(parameters); } From 06cd7f58539cb3ed20f352cb52497a5847fe05ed Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 10:50:04 +0800 Subject: [PATCH 04/16] apply review --- .../PipePlanTreePatternParseVisitor.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java index 197c96cdee215..b3f4aa3f88cb3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; @@ -98,14 +99,13 @@ public Optional visitCreateAlignedTimeSeries( new CreateAlignedTimeSeriesNode( node.getPlanNodeId(), node.getDevicePath(), - UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getMeasurements()), - UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getDataTypes()), - UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getEncodings()), - UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getCompressors()), - UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getAliasList()), - UnionIoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getTagsList()), - UnionIoTDBTreePattern.applyIndexesOnList( - filteredIndexes, node.getAttributesList()))) + TreePattern.applyIndexesOnList(filteredIndexes, node.getMeasurements()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getDataTypes()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getEncodings()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getCompressors()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getAliasList()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getTagsList()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getAttributesList()))) : Optional.empty(); } From 8fb8118f97de3bd5840c768d55c48d3d158c27c2 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 11:12:36 +0800 Subject: [PATCH 05/16] add tests for schema sync with patterns --- .../manual/IoTDBPipeInclusionIT.java | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java index c58a3ef60105b..8fead2c409ae2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java @@ -102,6 +102,143 @@ public void testPureSchemaInclusion() throws Exception { } } + @Test + public void testPureSchemaInclusionWithSinglePattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "schema"); + extractorAttributes.put("path", "root.ln.wf01.**"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + // TODO: add database creation after the database auto creating on receiver can be + // banned + "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "ALTER timeseries root.ln.wf01.wt01.status ADD TAGS tag3=v3", + "ALTER timeseries root.ln.wf01.wt01.status ADD ATTRIBUTES attr4=v4", + "create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "ALTER timeseries root.ln.wf02.wt01.status ADD TAGS tag3=v3", + "ALTER timeseries root.ln.wf02.wt01.status ADD ATTRIBUTES attr4=v4"), + null); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show timeseries root.ln.**", + "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", + Collections.singleton( + "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,PLAIN,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,")); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.ln.wf01.wt01(time, status) values(now(), false)", "flush"), + null); + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "select * from root.ln.**", "Time,", Collections.emptySet()); + } + } + + @Test + public void testPureSchemaInclusionWithMultiplePattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "schema"); + extractorAttributes.put("path", "root.ln.wf01.**, root.ln.wf02.**"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + // TODO: add database creation after the database auto creating on receiver can be + // banned + "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "ALTER timeseries root.ln.wf01.wt01.status ADD TAGS tag3=v3", + "ALTER timeseries root.ln.wf01.wt01.status ADD ATTRIBUTES attr4=v4", + "create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "ALTER timeseries root.ln.wf02.wt01.status ADD TAGS tag3=v3", + "ALTER timeseries root.ln.wf02.wt01.status ADD ATTRIBUTES attr4=v4", + "create timeseries root.ln.wf03.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "ALTER timeseries root.ln.wf03.wt01.status ADD TAGS tag3=v3", + "ALTER timeseries root.ln.wf03.wt01.status ADD ATTRIBUTES attr4=v4"), + null); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show timeseries root.ln.**", + "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", + new HashSet() { + { + add( + "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,PLAIN,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,"); + add( + "root.ln.wf02.wt01.status,null,root.ln,BOOLEAN,PLAIN,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,"); + } + }); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.ln.wf01.wt01(time, status) values(now(), false)", "flush"), + null); + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "select * from root.ln.**", "Time,", Collections.emptySet()); + } + } + @Test public void testAuthExclusion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); From 30bc1f0157e3a2a15692f65e80d6196d23bfc317 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 15:56:06 +0800 Subject: [PATCH 06/16] consider "isPrefixOrFullPath" --- .../db/pipe/source/dataregion/IoTDBDataRegionSource.java | 3 +-- .../pipe/datastructure/pattern/UnionIoTDBTreePattern.java | 8 ++------ .../commons/pipe/source/IoTDBNonDataRegionSource.java | 3 +-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index a9d2e06d7562e..9d2b629a67dcb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -304,8 +304,7 @@ private void validatePattern(final TreePattern treePattern) { if (shouldExtractDeletion && !(treePattern instanceof UnionIoTDBTreePattern - && (((UnionIoTDBTreePattern) treePattern).isPrefix() - || ((UnionIoTDBTreePattern) treePattern).isFullPath()))) { + && (((UnionIoTDBTreePattern) treePattern).isPrefixOrFullPath()))) { throw new IllegalArgumentException( String.format( "The path pattern %s is not valid for the source. Only prefix or full path is allowed.", diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java index fbdf7e0312958..f4c6ab8874e30 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java @@ -87,12 +87,8 @@ public PathPatternTree getIntersection(final PathPatternTree patternTree) { return resultTree; } - public boolean isPrefix() { - return patterns.stream().allMatch(IoTDBTreePattern::isPrefix); - } - - public boolean isFullPath() { - return patterns.stream().allMatch(IoTDBTreePattern::isFullPath); + public boolean isPrefixOrFullPath() { + return patterns.stream().allMatch(p -> p.isPrefix() || p.isFullPath()); } public boolean mayMatchMultipleTimeSeriesInOneDevice() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java index 83446a1371de6..6fe235352c8d1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java @@ -79,8 +79,7 @@ public void customize( final TreePattern pattern = TreePattern.parsePipePatternFromSourceParameters(parameters); if (!(pattern instanceof UnionIoTDBTreePattern - && (((UnionIoTDBTreePattern) pattern).isPrefix() - || ((UnionIoTDBTreePattern) pattern).isFullPath()))) { + && (((UnionIoTDBTreePattern) pattern).isPrefixOrFullPath()))) { throw new IllegalArgumentException( String.format( "The path pattern %s is not valid for the source. Only prefix or full path is allowed.", From 57ae2ded8ae81e337e07d3c41aebf0df0316c24b Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 16:14:43 +0800 Subject: [PATCH 07/16] add more cases for PipePlanTreePatternParseVisitorTest --- .../PipePlanTreePatternParseVisitorTest.java | 250 +++++++++++++++++- 1 file changed, 238 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java index 31f019beee74d..16ede1b603b4b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -50,7 +50,6 @@ import org.junit.Assert; import org.junit.Test; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -62,6 +61,11 @@ public class PipePlanTreePatternParseVisitorTest { new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); private final UnionIoTDBTreePattern fullPathPattern = new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.s1")); + private final UnionIoTDBTreePattern multiplePathPattern = + new UnionIoTDBTreePattern( + Arrays.asList( + new IoTDBTreePattern("root.db.device.s1"), + new IoTDBTreePattern("root.db.device.s2"))); @Test public void testCreateTimeSeries() throws IllegalPathException { @@ -87,6 +91,17 @@ public void testCreateTimeSeries() throws IllegalPathException { Collections.emptyMap(), Collections.emptyMap(), "a1"); + final CreateTimeSeriesNode createTimeSeriesNode2 = + new CreateTimeSeriesNode( + new PlanNodeId("2024-04-30-3"), + new MeasurementPath("root.db.device.s2"), + TSDataType.FLOAT, + TSEncoding.RLE, + CompressionType.SNAPPY, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + "a2"); Assert.assertEquals( createTimeSeriesNode, @@ -97,6 +112,21 @@ public void testCreateTimeSeries() throws IllegalPathException { IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR .visitCreateTimeSeries(createTimeSeriesNodeToFilter, prefixPathPattern) .isPresent()); + + Assert.assertEquals( + createTimeSeriesNode, + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateTimeSeries(createTimeSeriesNode, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertEquals( + createTimeSeriesNode2, + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateTimeSeries(createTimeSeriesNode2, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateTimeSeries(createTimeSeriesNodeToFilter, multiplePathPattern) + .isPresent()); } @Test @@ -126,6 +156,35 @@ public void testCreateAlignedTimeSeries() throws IllegalPathException { Arrays.asList(Collections.emptyMap(), Collections.emptyMap())), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new CreateAlignedTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + new PartialPath("root.db.device"), + Arrays.asList("s1", "s2"), + Arrays.asList(TSDataType.FLOAT, TSDataType.BOOLEAN), + Arrays.asList(TSEncoding.RLE, TSEncoding.PLAIN), + Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY), + Arrays.asList("a1", "a2"), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap())), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateAlignedTimeSeries( + new CreateAlignedTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + new PartialPath("root.db.device"), + Arrays.asList("s1", "s2", "s3"), + Arrays.asList(TSDataType.FLOAT, TSDataType.BOOLEAN, TSDataType.INT32), + Arrays.asList(TSEncoding.RLE, TSEncoding.PLAIN, TSEncoding.RLE), + Arrays.asList( + CompressionType.SNAPPY, CompressionType.SNAPPY, CompressionType.SNAPPY), + Arrays.asList("a1", "a2", "a3"), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap())), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -157,6 +216,40 @@ public void testCreateMultiTimeSeries() throws IllegalPathException { Arrays.asList(Collections.emptyMap(), Collections.emptyMap())), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new CreateMultiTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + Arrays.asList( + new MeasurementPath("root.db.device.s1"), new MeasurementPath("root.db.device.s2")), + Arrays.asList(TSDataType.FLOAT, TSDataType.BOOLEAN), + Arrays.asList(TSEncoding.RLE, TSEncoding.PLAIN), + Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList("a1", "a2"), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap())), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateMultiTimeSeries( + new CreateMultiTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + Arrays.asList( + new MeasurementPath("root.db.device.s1"), + new MeasurementPath("root.db.device.s2"), + new MeasurementPath("root.db1.device.s1")), + Arrays.asList(TSDataType.FLOAT, TSDataType.BOOLEAN, TSDataType.INT32), + Arrays.asList(TSEncoding.RLE, TSEncoding.PLAIN, TSEncoding.RLE), + Arrays.asList( + CompressionType.SNAPPY, CompressionType.SNAPPY, CompressionType.SNAPPY), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList("a1", "a2", "a3"), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap())), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -182,6 +275,16 @@ public void testAlterTimeSeries() throws IllegalPathException { Collections.emptyMap(), attributesMap, false); + final AlterTimeSeriesNode alterTimeSeriesNode2 = + new AlterTimeSeriesNode( + new PlanNodeId("2024-04-30-3"), + new MeasurementPath("root.db.device.s2"), + AlterTimeSeriesStatement.AlterType.ADD_ATTRIBUTES, + attributesMap, + "", + Collections.emptyMap(), + attributesMap, + false); Assert.assertEquals( alterTimeSeriesNode, @@ -192,6 +295,21 @@ public void testAlterTimeSeries() throws IllegalPathException { IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR .visitAlterTimeSeries(alterTimeSeriesNodeToFilter, prefixPathPattern) .isPresent()); + + Assert.assertEquals( + alterTimeSeriesNode, + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitAlterTimeSeries(alterTimeSeriesNode, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertEquals( + alterTimeSeriesNode2, + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitAlterTimeSeries(alterTimeSeriesNode2, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitAlterTimeSeries(alterTimeSeriesNodeToFilter, multiplePathPattern) + .isPresent()); } @Test @@ -204,12 +322,13 @@ public void testInternalCreateTimeSeries() throws IllegalPathException { expectedMeasurementGroup.addTags(Collections.emptyMap()); expectedMeasurementGroup.addAttributes(Collections.emptyMap()); - final ByteBuffer byteBuffer = ByteBuffer.allocate(1000); - expectedMeasurementGroup.serialize(byteBuffer); - byteBuffer.flip(); final MeasurementGroup originalMeasurementGroup = new MeasurementGroup(); - originalMeasurementGroup.deserialize(byteBuffer); - + originalMeasurementGroup.addMeasurement( + "s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY); + originalMeasurementGroup.addProps(Collections.emptyMap()); + originalMeasurementGroup.addAlias("a1"); + originalMeasurementGroup.addTags(Collections.emptyMap()); + originalMeasurementGroup.addAttributes(Collections.emptyMap()); originalMeasurementGroup.addMeasurement( "s2", TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY); originalMeasurementGroup.addProps(Collections.emptyMap()); @@ -232,6 +351,22 @@ public void testInternalCreateTimeSeries() throws IllegalPathException { true), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new InternalCreateTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + new PartialPath("root.db.device"), + originalMeasurementGroup, + true), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitInternalCreateTimeSeries( + new InternalCreateTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + new PartialPath("root.db.device"), + originalMeasurementGroup, + true), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -251,6 +386,16 @@ public void testActivateTemplate() throws IllegalPathException { IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR .visitActivateTemplate(activateTemplateNodeToFilter, prefixPathPattern) .isPresent()); + + Assert.assertEquals( + activateTemplateNode, + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitActivateTemplate(activateTemplateNode, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitActivateTemplate(activateTemplateNodeToFilter, multiplePathPattern) + .isPresent()); } @Test @@ -271,6 +416,23 @@ public void testInternalBatchActivateTemplate() throws IllegalPathException { }), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new InternalBatchActivateTemplateNode( + new PlanNodeId("2024-04-30-2"), + Collections.singletonMap(new PartialPath("root.db.device"), new Pair<>(1, 1))), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitInternalBatchActivateTemplate( + new InternalBatchActivateTemplateNode( + new PlanNodeId("2024-04-30-2"), + new HashMap>() { + { + put(new PartialPath("root.db.device"), new Pair<>(1, 1)); + put(new PartialPath("root.db1"), new Pair<>(2, 2)); + } + }), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -283,12 +445,13 @@ public void testInternalCreateMultiTimeSeries() throws IllegalPathException { expectedMeasurementGroup.addTags(Collections.emptyMap()); expectedMeasurementGroup.addAttributes(Collections.emptyMap()); - final ByteBuffer byteBuffer = ByteBuffer.allocate(1000); - expectedMeasurementGroup.serialize(byteBuffer); - byteBuffer.flip(); final MeasurementGroup originalMeasurementGroup = new MeasurementGroup(); - originalMeasurementGroup.deserialize(byteBuffer); - + originalMeasurementGroup.addMeasurement( + "s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY); + originalMeasurementGroup.addProps(Collections.emptyMap()); + originalMeasurementGroup.addAlias("a1"); + originalMeasurementGroup.addTags(Collections.emptyMap()); + originalMeasurementGroup.addAttributes(Collections.emptyMap()); originalMeasurementGroup.addMeasurement( "s2", TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY); originalMeasurementGroup.addProps(Collections.emptyMap()); @@ -296,6 +459,10 @@ public void testInternalCreateMultiTimeSeries() throws IllegalPathException { originalMeasurementGroup.addTags(Collections.emptyMap()); originalMeasurementGroup.addAttributes(Collections.emptyMap()); + final MeasurementGroup anotherMeasurementGroup = new MeasurementGroup(); + anotherMeasurementGroup.addMeasurement( + "s3", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY); + Assert.assertEquals( new InternalCreateMultiTimeSeriesNode( new PlanNodeId("2024-04-30-1"), @@ -317,6 +484,28 @@ public void testInternalCreateMultiTimeSeries() throws IllegalPathException { }), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new InternalCreateMultiTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + Collections.singletonMap( + new PartialPath("root.db.device"), new Pair<>(false, originalMeasurementGroup))), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitInternalCreateMultiTimeSeries( + new InternalCreateMultiTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + new HashMap>() { + { + put( + new PartialPath("root.db.device"), + new Pair<>(false, originalMeasurementGroup)); + put( + new PartialPath("root.db1.device"), + new Pair<>(false, anotherMeasurementGroup)); + } + }), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -337,6 +526,23 @@ public void testBatchActivateTemplate() throws IllegalPathException { }), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new BatchActivateTemplateNode( + new PlanNodeId("2024-04-30-2"), + Collections.singletonMap(new PartialPath("root.db.device"), new Pair<>(1, 1))), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitBatchActivateTemplate( + new BatchActivateTemplateNode( + new PlanNodeId("2024-04-30-2"), + new HashMap>() { + { + put(new PartialPath("root.db.device"), new Pair<>(1, 1)); + put(new PartialPath("root.db"), new Pair<>(2, 2)); + } + }), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -408,5 +614,25 @@ public void testDeleteData() throws IllegalPathException { Long.MAX_VALUE), prefixPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new DeleteDataNode( + new PlanNodeId("2024-04-30-2"), + Arrays.asList( + new MeasurementPath("root.db.device.s1"), new MeasurementPath("root.db.device.s2")), + Long.MIN_VALUE, + Long.MAX_VALUE), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitDeleteData( + new DeleteDataNode( + new PlanNodeId("2024-04-30-2"), + Arrays.asList( + new MeasurementPath("root.db.device.s1"), + new MeasurementPath("root.db.device.s2"), + new MeasurementPath("root.db.device.s3")), + Long.MIN_VALUE, + Long.MAX_VALUE), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } } From 2966019a856ae4e94ab0c6e8cbcb2c97c4279f2e Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 16:24:45 +0800 Subject: [PATCH 08/16] add more cases for PipeStatementTreePatternParseVisitorTest --- ...eStatementTreePatternParseVisitorTest.java | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java index 76b291cbf5e0b..c33f90df948e5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -48,6 +48,11 @@ public class PipeStatementTreePatternParseVisitorTest { new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); private final UnionIoTDBTreePattern fullPathPattern = new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.s1")); + private final UnionIoTDBTreePattern multiplePathPattern = + new UnionIoTDBTreePattern( + Arrays.asList( + new IoTDBTreePattern("root.db.device.s1"), + new IoTDBTreePattern("root.db.device.s2"))); @Test public void testCreateTimeSeries() throws IllegalPathException { @@ -81,6 +86,15 @@ public void testCreateTimeSeries() throws IllegalPathException { new PipeStatementTreePatternParseVisitor() .visitCreateTimeseries(createTimeSeriesStatementToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + createTimeSeriesStatement, + new PipeStatementTreePatternParseVisitor() + .visitCreateTimeseries(createTimeSeriesStatement, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + new PipeStatementTreePatternParseVisitor() + .visitCreateTimeseries(createTimeSeriesStatementToFilter, multiplePathPattern) + .isPresent()); } @Test @@ -122,6 +136,13 @@ public void testCreateAlignedTimeSeries() throws IllegalPathException { new PipeStatementTreePatternParseVisitor() .visitCreateAlignedTimeseries(originalCreateAlignedTimeSeriesStatement, fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + originalCreateAlignedTimeSeriesStatement, + new PipeStatementTreePatternParseVisitor() + .visitCreateAlignedTimeseries( + originalCreateAlignedTimeSeriesStatement, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -153,6 +174,15 @@ public void testAlterTimeSeries() throws IllegalPathException { new PipeStatementTreePatternParseVisitor() .visitAlterTimeSeries(alterTimeSeriesStatementToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + alterTimeSeriesStatement, + new PipeStatementTreePatternParseVisitor() + .visitAlterTimeSeries(alterTimeSeriesStatement, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + new PipeStatementTreePatternParseVisitor() + .visitAlterTimeSeries(alterTimeSeriesStatementToFilter, multiplePathPattern) + .isPresent()); } @Test @@ -171,6 +201,15 @@ public void testActivateTemplate() throws IllegalPathException { new PipeStatementTreePatternParseVisitor() .visitActivateTemplate(activateTemplateStatementToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + activateTemplateStatement, + new PipeStatementTreePatternParseVisitor() + .visitActivateTemplate(activateTemplateStatement, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + new PipeStatementTreePatternParseVisitor() + .visitActivateTemplate(activateTemplateStatementToFilter, multiplePathPattern) + .isPresent()); } @Test From 85b2362ff70e0dcf5779a4dba0ac3a93d06b5cf3 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 16:47:54 +0800 Subject: [PATCH 09/16] add more cases for PipeConfigPhysicalPlanTreePatternParseVisitorTest --- ...ysicalPlanTreePatternParseVisitorTest.java | 72 +++++++++++++++++++ .../pattern/UnionIoTDBTreePattern.java | 11 ++- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java index 4bfc9a0a8cdbf..442351cc19d2b 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java @@ -59,6 +59,11 @@ public class PipeConfigPhysicalPlanTreePatternParseVisitorTest { new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); private final UnionIoTDBTreePattern fullPathPattern = new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.s1")); + private final UnionIoTDBTreePattern multiplePathPattern = + new UnionIoTDBTreePattern( + Arrays.asList( + new IoTDBTreePattern("root.db.device.s1"), + new IoTDBTreePattern("root.db.device.s2"))); @Test public void testCreateDatabase() { @@ -78,6 +83,11 @@ public void testCreateDatabase() { IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR .visitCreateDatabase(createDatabasePlanToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + createDatabasePlan, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateDatabase(createDatabasePlan, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -98,6 +108,11 @@ public void testAlterDatabase() { IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR .visitAlterDatabase(alterDatabasePlanToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + alterDatabasePlan, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitAlterDatabase(alterDatabasePlan, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -114,6 +129,11 @@ public void testDeleteDatabase() { IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR .visitDeleteDatabase(deleteDatabasePlanToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + deleteDatabasePlan, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitDeleteDatabase(deleteDatabasePlan, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -145,6 +165,15 @@ public void testCreateSchemaTemplate() throws IllegalPathException { Assert.assertEquals( createSchemaTemplatePlan.getTemplate().getSchemaMap().get("s1"), parsedTemplatePlan.getTemplate().getSchemaMap().get("s1")); + + final CreateSchemaTemplatePlan parsedTemplatePlan2 = + (CreateSchemaTemplatePlan) + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateSchemaTemplate(createSchemaTemplatePlan, multiplePathPattern) + .orElseThrow(AssertionError::new); + Assert.assertEquals( + new HashSet<>(Arrays.asList("s1", "s2")), + parsedTemplatePlan2.getTemplate().getSchemaMap().keySet()); } @Test @@ -164,6 +193,11 @@ public void testCommitSetSchemaTemplate() { IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR .visitCommitSetSchemaTemplate(setSchemaTemplatePlanOnFullPath, fullPathPattern) .orElseThrow(AssertionError::new)); + Assert.assertEquals( + setSchemaTemplatePlanOnFullPath, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCommitSetSchemaTemplate(setSchemaTemplatePlanOnFullPath, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -183,6 +217,12 @@ public void testPipeUnsetSchemaTemplate() { IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR .visitPipeUnsetSchemaTemplate(pipeUnsetSchemaTemplatePlanOrFullPath, fullPathPattern) .orElseThrow(AssertionError::new)); + Assert.assertEquals( + pipeUnsetSchemaTemplatePlanOrFullPath, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitPipeUnsetSchemaTemplate( + pipeUnsetSchemaTemplatePlanOrFullPath, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -216,6 +256,14 @@ public void testExtendSchemaTemplate() { Assert.assertEquals( extendSchemaTemplatePlan.getTemplateExtendInfo().getCompressors().get(0), parsedTemplatePlan.getTemplateExtendInfo().getCompressors().get(0)); + + final ExtendSchemaTemplatePlan parsedTemplatePlan2 = + (ExtendSchemaTemplatePlan) + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitExtendSchemaTemplate(extendSchemaTemplatePlan, multiplePathPattern) + .orElseThrow(AssertionError::new); + Assert.assertEquals( + Arrays.asList("s1", "s2"), parsedTemplatePlan2.getTemplateExtendInfo().getMeasurements()); } @Test @@ -323,6 +371,18 @@ public void testPipeDeleteTimeSeries() throws IllegalPathException, IOException .orElseThrow(AssertionError::new)) .getPatternTreeBytes()) .getAllPathPatterns()); + + Assert.assertEquals( + Collections.singletonList(new PartialPath("root.db.device.s1")), + PathPatternTree.deserialize( + ((PipeDeleteTimeSeriesPlan) + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitPipeDeleteTimeSeries( + new PipeDeleteTimeSeriesPlan(patternTree.serialize()), + multiplePathPattern) + .orElseThrow(AssertionError::new)) + .getPatternTreeBytes()) + .getAllPathPatterns()); } @Test @@ -342,6 +402,18 @@ public void testPipeDeleteLogicalView() throws IllegalPathException, IOException .orElseThrow(AssertionError::new)) .getPatternTreeBytes()) .getAllPathPatterns()); + + Assert.assertEquals( + Collections.singletonList(new PartialPath("root.db.device.s1")), + PathPatternTree.deserialize( + ((PipeDeleteLogicalViewPlan) + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitPipeDeleteLogicalView( + new PipeDeleteLogicalViewPlan(patternTree.serialize()), + multiplePathPattern) + .orElseThrow(AssertionError::new)) + .getPatternTreeBytes()) + .getAllPathPatterns()); } @Test diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java index f4c6ab8874e30..6e3ebc316c27d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java @@ -26,7 +26,9 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** @@ -70,17 +72,20 @@ public boolean matchTailNode(final String tailNode) { } public List getIntersection(final PartialPath partialPath) { - final List allIntersections = new ArrayList<>(); + final Set uniqueIntersections = new LinkedHashSet<>(); for (final IoTDBTreePattern pattern : patterns) { - allIntersections.addAll(pattern.getIntersection(partialPath)); + uniqueIntersections.addAll(pattern.getIntersection(partialPath)); } - return allIntersections; + return new ArrayList<>(uniqueIntersections); } public PathPatternTree getIntersection(final PathPatternTree patternTree) { final PathPatternTree resultTree = new PathPatternTree(); for (final IoTDBTreePattern pattern : patterns) { final PathPatternTree intersection = pattern.getIntersection(patternTree); + if (intersection.isEmpty()) { + continue; + } intersection.getAllPathPatterns().forEach(resultTree::appendPathPattern); } resultTree.constructTree(); From b721fa060f65c3db703e93cc33826aea8ea0aff2 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 16:53:23 +0800 Subject: [PATCH 10/16] apply review --- .../manual/IoTDBPipeInclusionIT.java | 67 +------------------ 1 file changed, 2 insertions(+), 65 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java index 8fead2c409ae2..3e7d87e592e06 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java @@ -102,70 +102,6 @@ public void testPureSchemaInclusion() throws Exception { } } - @Test - public void testPureSchemaInclusionWithSinglePattern() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.inclusion", "schema"); - extractorAttributes.put("path", "root.ln.wf01.**"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList( - // TODO: add database creation after the database auto creating on receiver can be - // banned - "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN", - "ALTER timeseries root.ln.wf01.wt01.status ADD TAGS tag3=v3", - "ALTER timeseries root.ln.wf01.wt01.status ADD ATTRIBUTES attr4=v4", - "create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN", - "ALTER timeseries root.ln.wf02.wt01.status ADD TAGS tag3=v3", - "ALTER timeseries root.ln.wf02.wt01.status ADD ATTRIBUTES attr4=v4"), - null); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "show timeseries root.ln.**", - "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", - Collections.singleton( - "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,PLAIN,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,")); - - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList( - "insert into root.ln.wf01.wt01(time, status) values(now(), false)", "flush"), - null); - - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "select * from root.ln.**", "Time,", Collections.emptySet()); - } - } - @Test public void testPureSchemaInclusionWithMultiplePattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -180,7 +116,8 @@ public void testPureSchemaInclusionWithMultiplePattern() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("extractor.inclusion", "schema"); - extractorAttributes.put("path", "root.ln.wf01.**, root.ln.wf02.**"); + extractorAttributes.put("pattern", "root.ln.wf01.wt01.status"); + extractorAttributes.put("path", "root.ln.wf02.**"); connectorAttributes.put("connector", "iotdb-thrift-connector"); connectorAttributes.put("connector.ip", receiverIp); From 121c8cde675f004114758545effd914d5c7453dc Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 16:56:33 +0800 Subject: [PATCH 11/16] apply review --- .../parser/TsFileInsertionEventParserProvider.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java index 8ca2b159255b0..d6a276b9f25e8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java @@ -99,9 +99,8 @@ public TsFileInsertionEventParser provide() throws IOException { // Use scan container to save memory if ((double) PipeDataNodeResourceManager.memory().getUsedMemorySizeInBytes() - / PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes() - > PipeTsFilePublicResource.MEMORY_SUFFICIENT_THRESHOLD - && treePattern.isSingle()) { + / PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes() + > PipeTsFilePublicResource.MEMORY_SUFFICIENT_THRESHOLD) { return new TsFileInsertionEventScanParser( pipeName, creationTime, @@ -134,7 +133,7 @@ public TsFileInsertionEventParser provide() throws IOException { final Map deviceIsAlignedMap = PipeDataNodeResourceManager.tsfile().getDeviceIsAlignedMapFromCache(tsFile, false); - if (Objects.isNull(deviceIsAlignedMap) && treePattern.isSingle()) { + if (Objects.isNull(deviceIsAlignedMap)) { // If we failed to get from cache, it indicates that the memory usage is high. // We use scan data container because it requires less memory. return new TsFileInsertionEventScanParser( @@ -153,8 +152,7 @@ public TsFileInsertionEventParser provide() throws IOException { filterDeviceIsAlignedMapByPattern(deviceIsAlignedMap); // Use scan data container if we need enough amount to data thus it's better to scan than query. return (double) filteredDeviceIsAlignedMap.size() / originalSize - > PipeConfig.getInstance().getPipeTsFileScanParsingThreshold() - && treePattern.isSingle() + > PipeConfig.getInstance().getPipeTsFileScanParsingThreshold() ? new TsFileInsertionEventScanParser( pipeName, creationTime, From f76f1df171b320c08d89693ef0b21ba13fc5a978 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 17:02:56 +0800 Subject: [PATCH 12/16] reset --- .../db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java index c33f90df948e5..85ae98ac26a85 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an From 9047d059f01a1f4498cd5db8c763c6df9356d578 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 17:04:16 +0800 Subject: [PATCH 13/16] reset --- .../db/pipe/source/PipePlanTreePatternParseVisitorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java index 16ede1b603b4b..f4aa8f56a7efa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an From 0a1a56be6949066f2719486c06ef61e767e8d71a Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 17:06:09 +0800 Subject: [PATCH 14/16] fix license header --- .../pipe/datastructure/pattern/UnionIoTDBTreePattern.java | 2 +- .../commons/pipe/datastructure/pattern/UnionTreePattern.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java index 6e3ebc316c27d..a44f4c9d6479b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java index 23b02a7f58192..e43a18ad73855 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an From 99d5e078517f7ec62c3da81d9e072056e5bb2ab4 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 15 Oct 2025 17:55:16 +0800 Subject: [PATCH 15/16] fix tests --- .../pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java index 3e7d87e592e06..e2536f3467988 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java @@ -116,8 +116,7 @@ public void testPureSchemaInclusionWithMultiplePattern() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("extractor.inclusion", "schema"); - extractorAttributes.put("pattern", "root.ln.wf01.wt01.status"); - extractorAttributes.put("path", "root.ln.wf02.**"); + extractorAttributes.put("path", "root.ln.wf01.wt01.status,root.ln.wf02.**"); connectorAttributes.put("connector", "iotdb-thrift-connector"); connectorAttributes.put("connector.ip", receiverIp); From 4051e0f36a698a3d1430786f1414b7bd2fdb4377 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 16 Oct 2025 18:52:48 +0800 Subject: [PATCH 16/16] remove TODO --- .../pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java index e2536f3467988..565631fb1e94a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java @@ -138,8 +138,6 @@ public void testPureSchemaInclusionWithMultiplePattern() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - // TODO: add database creation after the database auto creating on receiver can be - // banned "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN", "ALTER timeseries root.ln.wf01.wt01.status ADD TAGS tag3=v3", "ALTER timeseries root.ln.wf01.wt01.status ADD ATTRIBUTES attr4=v4",