diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java index e0b941ffc4772..4e2871067931b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java @@ -104,7 +104,7 @@ public void testPrefixPattern() throws Exception { } @Test - public void testIotdbPattern() throws Exception { + public void testIoTDBPattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); @@ -127,8 +127,6 @@ public void testIotdbPattern() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("extractor.path", "root.**.d1.s*"); - // When path is set, pattern should be ignored - extractorAttributes.put("extractor.pattern", "root"); extractorAttributes.put("extractor.inclusion", "data.insert"); connectorAttributes.put("connector", "iotdb-thrift-connector"); @@ -158,7 +156,7 @@ public void testIotdbPattern() throws Exception { } @Test - public void testIotdbPatternWithLegacySyntax() throws Exception { + public void testIoTDBPatternWithLegacySyntax() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); @@ -209,4 +207,324 @@ public void testIotdbPatternWithLegacySyntax() throws Exception { expectedResSet); } } + + @Test + public void testMultiplePrefixPatternHistoricalData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db.d1.s, root.db2.d1.s"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db.d2(time, s) values (2, 2)", + "insert into root.db2.d1(time, s) values (3, 3)"), + null); + awaitUntilFlush(senderEnv); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,null,1.0,1.0,"); + expectedResSet.add("3,3.0,null,null,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db2.**,root.db.**", + "Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,", + expectedResSet); + } + } + + @Test + public void testMultipleIoTDBPatternHistoricalData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.path", "root.db.**, root.db2.d1.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db.d2(time, s) values (2, 2)", + "insert into root.db2.d1(time, s, t) values (3, 3, 3)", + "insert into root.db3.d1(time, s) values (4, 4)"), + null); + awaitUntilFlush(senderEnv); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,null,null,1.0,1.0,null,"); + expectedResSet.add("2,null,null,null,null,2.0,"); + expectedResSet.add("3,3.0,3.0,null,null,null,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db2.**,root.db.**", + "Time,root.db2.d1.s,root.db2.d1.t,root.db.d1.s,root.db.d1.s1,root.db.d2.s,", + expectedResSet); + } + } + + @Test + public void testMultipleHybridPatternHistoricalData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.path", "root.db.d1.*"); + extractorAttributes.put("extractor.pattern", "root.db2.d1.s"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db2.d1(time, s) values (2, 2)", + "insert into root.db3.d1(time, s) values (3, 3)"), + null); + awaitUntilFlush(senderEnv); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,1.0,null,"); + expectedResSet.add("2,null,null,2.0,"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db.**,root.db2.**", + "Time,root.db.d1.s,root.db.d1.s1,root.db2.d1.s,", + expectedResSet); + } + } + + @Test + public void testMultiplePrefixPatternRealtimeData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.pattern", "root.db.d1.s, root.db2.d1.s"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db.d2(time, s) values (2, 2)", + "insert into root.db2.d1(time, s) values (3, 3)"), + null); + awaitUntilFlush(senderEnv); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,null,1.0,1.0,"); + expectedResSet.add("3,3.0,null,null,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db2.**,root.db.**", + "Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,", + expectedResSet); + } + } + + @Test + public void testMultipleIoTDBPatternRealtimeData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.path", "root.db.**, root.db2.d1.*"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db.d2(time, s) values (2, 2)", + "insert into root.db2.d1(time, s, t) values (3, 3, 3)", + "insert into root.db3.d1(time, s) values (4, 4)"), + null); + awaitUntilFlush(senderEnv); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,null,null,1.0,1.0,null,"); + expectedResSet.add("2,null,null,null,null,2.0,"); + expectedResSet.add("3,3.0,3.0,null,null,null,"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db2.**,root.db.**", + "Time,root.db2.d1.s,root.db2.d1.t,root.db.d1.s,root.db.d1.s1,root.db.d2.s,", + expectedResSet); + } + } + + @Test + public void testMultipleHybridPatternRealtimeData() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.path", "root.db.d1.*"); + extractorAttributes.put("extractor.pattern", "root.db2.d1.s"); + extractorAttributes.put("extractor.inclusion", "data.insert"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1(time, s, s1) values (1, 1, 1)", + "insert into root.db2.d1(time, s) values (2, 2)", + "insert into root.db3.d1(time, s) values (3, 3)"), + null); + awaitUntilFlush(senderEnv); + + final Set expectedResSet = new HashSet<>(); + expectedResSet.add("1,1.0,1.0,null,"); + expectedResSet.add("2,null,null,2.0,"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.db.**,root.db2.**", + "Time,root.db.d1.s,root.db.d1.s1,root.db2.d1.s,", + expectedResSet); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java index 4d98ebee37f48..84c90a667eecc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java @@ -102,6 +102,77 @@ public void testPureSchemaInclusion() throws Exception { } } + @Test + public void testPureSchemaInclusionWithMultiplePattern() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "schema"); + extractorAttributes.put("path", "root.ln.wf01.wt01.status,root.ln.wf02.**"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "ALTER timeseries root.ln.wf01.wt01.status ADD TAGS tag3=v3", + "ALTER timeseries root.ln.wf01.wt01.status ADD ATTRIBUTES attr4=v4", + "create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "ALTER timeseries root.ln.wf02.wt01.status ADD TAGS tag3=v3", + "ALTER timeseries root.ln.wf02.wt01.status ADD ATTRIBUTES attr4=v4", + "create timeseries root.ln.wf03.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "ALTER timeseries root.ln.wf03.wt01.status ADD TAGS tag3=v3", + "ALTER timeseries root.ln.wf03.wt01.status ADD ATTRIBUTES attr4=v4"), + null); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show timeseries root.ln.**", + "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", + new HashSet() { + { + add( + "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,PLAIN,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,"); + add( + "root.ln.wf02.wt01.status,null,root.ln,BOOLEAN,PLAIN,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,"); + } + }); + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.ln.wf01.wt01(time, status) values(now(), false)", "flush"), + null); + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "select * from root.ln.**", "Time,", Collections.emptySet()); + } + } + @Test public void testAuthExclusion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java index a8c90c6ced834..fb864d449bd22 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java @@ -28,6 +28,8 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.receiver.IoTDBFileReceiver; import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler; import org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapPseudoTPipeTransferRequest; @@ -1065,10 +1067,14 @@ protected TSStatus loadFileV2( final Set executionTypes = PipeConfigRegionSnapshotEvent.getConfigPhysicalPlanTypeSet( parameters.get(ColumnHeaderConstant.TYPE)); - final IoTDBTreePattern treePattern = - new IoTDBTreePattern( - parameters.containsKey(PipeTransferFileSealReqV2.TREE), - parameters.get(ColumnHeaderConstant.PATH_PATTERN)); + final boolean isTreeModelDataAllowedToBeCaptured = + parameters.containsKey(PipeTransferFileSealReqV2.TREE); + final List treePatterns = + TreePattern.parseMultiplePatterns( + parameters.get(ColumnHeaderConstant.PATH_PATTERN), + p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + final TreePattern treePattern = + TreePattern.buildUnionPattern(isTreeModelDataAllowedToBeCaptured, treePatterns); final TablePattern tablePattern = new TablePattern( parameters.containsKey(PipeTransferFileSealReqV2.TABLE), @@ -1076,11 +1082,15 @@ protected TSStatus loadFileV2( parameters.get(ColumnHeaderConstant.TABLE_NAME)); final List results = new ArrayList<>(); while (generator.hasNext()) { - IoTDBConfigRegionSource.parseConfigPlan(generator.next(), treePattern, tablePattern) + IoTDBConfigRegionSource.parseConfigPlan( + generator.next(), (UnionIoTDBTreePattern) treePattern, tablePattern) .filter( configPhysicalPlan -> IoTDBConfigRegionSource.isTypeListened( - configPhysicalPlan, executionTypes, treePattern, tablePattern)) + configPhysicalPlan, + executionTypes, + (UnionIoTDBTreePattern) treePattern, + tablePattern)) .ifPresent( configPhysicalPlan -> results.add(executePlanAndClassifyExceptions(configPhysicalPlan))); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java index fc965624c1d86..252f7331d0cd1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java @@ -25,8 +25,8 @@ import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.config.PipeConfig; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.queue.listening.AbstractPipeListeningQueue; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.event.PipeSnapshotEvent; @@ -253,7 +253,7 @@ protected Optional trimRealtimeEventByPipePattern( public static Optional parseConfigPlan( final ConfigPhysicalPlan plan, - final IoTDBTreePattern treePattern, + final UnionIoTDBTreePattern treePattern, final TablePattern tablePattern) { Optional result = Optional.of(plan); final Boolean isTableDatabasePlan = isTableDatabasePlan(plan); @@ -296,7 +296,7 @@ protected boolean isTypeListened(final PipeWritePlanEvent event) { public static boolean isTypeListened( final ConfigPhysicalPlan plan, final Set listenedTypeSet, - final IoTDBTreePattern treePattern, + final UnionIoTDBTreePattern treePattern, final TablePattern tablePattern) { final Boolean isTableDatabasePlan = isTableDatabasePlan(plan); return listenedTypeSet.contains(plan.getType()) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java index 992b74ca0e5b5..339ea27558988 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java @@ -22,7 +22,8 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; @@ -58,12 +59,12 @@ /** * The {@link PipeConfigPhysicalPlanTreePatternParseVisitor} will transform the schema {@link - * ConfigPhysicalPlan}s using {@link IoTDBTreePattern}. Rule: + * ConfigPhysicalPlan}s using {@link UnionIoTDBTreePattern}. Rule: * *

1. All patterns in the output {@link ConfigPhysicalPlan} will be the intersection of the - * original {@link ConfigPhysicalPlan}'s patterns and the given {@link IoTDBTreePattern}. + * original {@link ConfigPhysicalPlan}'s patterns and the given {@link UnionIoTDBTreePattern}. * - *

2. If a pattern does not intersect with the {@link IoTDBTreePattern}, it's dropped. + *

2. If a pattern does not intersect with the {@link UnionIoTDBTreePattern}, it's dropped. * *

3. If all the patterns in the {@link ConfigPhysicalPlan} is dropped, the {@link * ConfigPhysicalPlan} is dropped. @@ -72,13 +73,13 @@ * one is used in the {@link PipeConfigRegionWritePlanEvent} in {@link ConfigRegionListeningQueue}. */ public class PipeConfigPhysicalPlanTreePatternParseVisitor - extends ConfigPhysicalPlanVisitor, IoTDBTreePattern> { + extends ConfigPhysicalPlanVisitor, UnionIoTDBTreePattern> { private static final Logger LOGGER = LoggerFactory.getLogger(PipeConfigPhysicalPlanTreePatternParseVisitor.class); @Override public Optional visitPlan( - final ConfigPhysicalPlan plan, final IoTDBTreePattern pattern) { + final ConfigPhysicalPlan plan, final UnionIoTDBTreePattern pattern) { return Optional.of(plan); } @@ -90,7 +91,7 @@ public Optional visitPlan( // Other matches using "matchPrefixPath" are with the same principle. @Override public Optional visitCreateDatabase( - final DatabaseSchemaPlan createDatabasePlan, final IoTDBTreePattern pattern) { + final DatabaseSchemaPlan createDatabasePlan, final UnionIoTDBTreePattern pattern) { return pattern.matchPrefixPath(createDatabasePlan.getSchema().getName()) ? Optional.of(createDatabasePlan) : Optional.empty(); @@ -98,7 +99,7 @@ public Optional visitCreateDatabase( @Override public Optional visitAlterDatabase( - final DatabaseSchemaPlan alterDatabasePlan, final IoTDBTreePattern pattern) { + final DatabaseSchemaPlan alterDatabasePlan, final UnionIoTDBTreePattern pattern) { return pattern.matchPrefixPath(alterDatabasePlan.getSchema().getName()) ? Optional.of(alterDatabasePlan) : Optional.empty(); @@ -106,7 +107,7 @@ public Optional visitAlterDatabase( @Override public Optional visitDeleteDatabase( - final DeleteDatabasePlan deleteDatabasePlan, final IoTDBTreePattern pattern) { + final DeleteDatabasePlan deleteDatabasePlan, final UnionIoTDBTreePattern pattern) { return pattern.matchPrefixPath(deleteDatabasePlan.getName()) ? Optional.of(deleteDatabasePlan) : Optional.empty(); @@ -114,7 +115,8 @@ public Optional visitDeleteDatabase( @Override public Optional visitCreateSchemaTemplate( - final CreateSchemaTemplatePlan createSchemaTemplatePlan, final IoTDBTreePattern pattern) { + final CreateSchemaTemplatePlan createSchemaTemplatePlan, + final UnionIoTDBTreePattern pattern) { // This is a deserialized template and can be arbitrarily altered final Template template = createSchemaTemplatePlan.getTemplate(); template.getSchemaMap().keySet().removeIf(measurement -> !pattern.matchTailNode(measurement)); @@ -126,7 +128,7 @@ public Optional visitCreateSchemaTemplate( @Override public Optional visitCommitSetSchemaTemplate( final CommitSetSchemaTemplatePlan commitSetSchemaTemplatePlan, - final IoTDBTreePattern pattern) { + final UnionIoTDBTreePattern pattern) { return pattern.matchPrefixPath(commitSetSchemaTemplatePlan.getPath()) ? Optional.of(commitSetSchemaTemplatePlan) : Optional.empty(); @@ -135,7 +137,7 @@ public Optional visitCommitSetSchemaTemplate( @Override public Optional visitPipeUnsetSchemaTemplate( final PipeUnsetSchemaTemplatePlan pipeUnsetSchemaTemplatePlan, - final IoTDBTreePattern pattern) { + final UnionIoTDBTreePattern pattern) { return pattern.matchPrefixPath(pipeUnsetSchemaTemplatePlan.getPath()) ? Optional.of(pipeUnsetSchemaTemplatePlan) : Optional.empty(); @@ -143,7 +145,8 @@ public Optional visitPipeUnsetSchemaTemplate( @Override public Optional visitExtendSchemaTemplate( - final ExtendSchemaTemplatePlan extendSchemaTemplatePlan, final IoTDBTreePattern pattern) { + final ExtendSchemaTemplatePlan extendSchemaTemplatePlan, + final UnionIoTDBTreePattern pattern) { final TemplateExtendInfo extendInfo = extendSchemaTemplatePlan.getTemplateExtendInfo(); final int[] filteredIndexes = IntStream.range(0, extendInfo.getMeasurements().size()) @@ -154,41 +157,39 @@ public Optional visitExtendSchemaTemplate( new ExtendSchemaTemplatePlan( new TemplateExtendInfo( extendInfo.getTemplateName(), - IoTDBTreePattern.applyIndexesOnList( - filteredIndexes, extendInfo.getMeasurements()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getDataTypes()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getEncodings()), - IoTDBTreePattern.applyIndexesOnList( - filteredIndexes, extendInfo.getCompressors())))) + TreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getMeasurements()), + TreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getDataTypes()), + TreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getEncodings()), + TreePattern.applyIndexesOnList(filteredIndexes, extendInfo.getCompressors())))) : Optional.empty(); } @Override public Optional visitGrantUser( - final AuthorTreePlan grantUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan grantUserPlan, final UnionIoTDBTreePattern pattern) { return visitTreeAuthorPlan(grantUserPlan, pattern); } @Override public Optional visitRevokeUser( - final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan revokeUserPlan, final UnionIoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } @Override public Optional visitGrantRole( - final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan revokeUserPlan, final UnionIoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } @Override public Optional visitRevokeRole( - final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan revokeUserPlan, final UnionIoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } private Optional visitTreeAuthorPlan( - final AuthorTreePlan pathRelatedAuthorTreePlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan pathRelatedAuthorTreePlan, final UnionIoTDBTreePattern pattern) { final List intersectedPaths = pathRelatedAuthorTreePlan.getNodeNameList().stream() .map(pattern::getIntersection) @@ -216,7 +217,8 @@ private Optional visitTreeAuthorPlan( @Override public Optional visitPipeDeleteTimeSeries( - final PipeDeleteTimeSeriesPlan pipeDeleteTimeSeriesPlan, final IoTDBTreePattern pattern) { + final PipeDeleteTimeSeriesPlan pipeDeleteTimeSeriesPlan, + final UnionIoTDBTreePattern pattern) { try { final PathPatternTree intersectedTree = pattern.getIntersection( @@ -234,7 +236,8 @@ public Optional visitPipeDeleteTimeSeries( @Override public Optional visitPipeDeleteLogicalView( - final PipeDeleteLogicalViewPlan pipeDeleteLogicalViewPlan, final IoTDBTreePattern pattern) { + final PipeDeleteLogicalViewPlan pipeDeleteLogicalViewPlan, + final UnionIoTDBTreePattern pattern) { try { final PathPatternTree intersectedTree = pattern.getIntersection( @@ -252,7 +255,8 @@ public Optional visitPipeDeleteLogicalView( @Override public Optional visitPipeDeactivateTemplate( - final PipeDeactivateTemplatePlan pipeDeactivateTemplatePlan, final IoTDBTreePattern pattern) { + final PipeDeactivateTemplatePlan pipeDeactivateTemplatePlan, + final UnionIoTDBTreePattern pattern) { final Map> newTemplateSetInfo = pipeDeactivateTemplatePlan.getTemplateSetInfo().entrySet().stream() .flatMap( @@ -275,7 +279,7 @@ public Optional visitPipeDeactivateTemplate( @Override public Optional visitTTL( - final SetTTLPlan setTTLPlan, final IoTDBTreePattern pattern) { + final SetTTLPlan setTTLPlan, final UnionIoTDBTreePattern pattern) { final PartialPath partialPath = new PartialPath(setTTLPlan.getPathPattern()); final List intersectionList = pattern.matchPrefixPath(partialPath.getFullPath()) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java index 6893cfa55230c..442351cc19d2b 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; @@ -54,8 +55,15 @@ public class PipeConfigPhysicalPlanTreePatternParseVisitorTest { - private final IoTDBTreePattern prefixPathPattern = new IoTDBTreePattern("root.db.device.**"); - private final IoTDBTreePattern fullPathPattern = new IoTDBTreePattern("root.db.device.s1"); + private final UnionIoTDBTreePattern prefixPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); + private final UnionIoTDBTreePattern fullPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.s1")); + private final UnionIoTDBTreePattern multiplePathPattern = + new UnionIoTDBTreePattern( + Arrays.asList( + new IoTDBTreePattern("root.db.device.s1"), + new IoTDBTreePattern("root.db.device.s2"))); @Test public void testCreateDatabase() { @@ -75,6 +83,11 @@ public void testCreateDatabase() { IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR .visitCreateDatabase(createDatabasePlanToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + createDatabasePlan, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateDatabase(createDatabasePlan, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -95,6 +108,11 @@ public void testAlterDatabase() { IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR .visitAlterDatabase(alterDatabasePlanToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + alterDatabasePlan, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitAlterDatabase(alterDatabasePlan, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -111,6 +129,11 @@ public void testDeleteDatabase() { IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR .visitDeleteDatabase(deleteDatabasePlanToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + deleteDatabasePlan, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitDeleteDatabase(deleteDatabasePlan, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -142,6 +165,15 @@ public void testCreateSchemaTemplate() throws IllegalPathException { Assert.assertEquals( createSchemaTemplatePlan.getTemplate().getSchemaMap().get("s1"), parsedTemplatePlan.getTemplate().getSchemaMap().get("s1")); + + final CreateSchemaTemplatePlan parsedTemplatePlan2 = + (CreateSchemaTemplatePlan) + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateSchemaTemplate(createSchemaTemplatePlan, multiplePathPattern) + .orElseThrow(AssertionError::new); + Assert.assertEquals( + new HashSet<>(Arrays.asList("s1", "s2")), + parsedTemplatePlan2.getTemplate().getSchemaMap().keySet()); } @Test @@ -161,6 +193,11 @@ public void testCommitSetSchemaTemplate() { IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR .visitCommitSetSchemaTemplate(setSchemaTemplatePlanOnFullPath, fullPathPattern) .orElseThrow(AssertionError::new)); + Assert.assertEquals( + setSchemaTemplatePlanOnFullPath, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCommitSetSchemaTemplate(setSchemaTemplatePlanOnFullPath, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -180,6 +217,12 @@ public void testPipeUnsetSchemaTemplate() { IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR .visitPipeUnsetSchemaTemplate(pipeUnsetSchemaTemplatePlanOrFullPath, fullPathPattern) .orElseThrow(AssertionError::new)); + Assert.assertEquals( + pipeUnsetSchemaTemplatePlanOrFullPath, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitPipeUnsetSchemaTemplate( + pipeUnsetSchemaTemplatePlanOrFullPath, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -213,6 +256,14 @@ public void testExtendSchemaTemplate() { Assert.assertEquals( extendSchemaTemplatePlan.getTemplateExtendInfo().getCompressors().get(0), parsedTemplatePlan.getTemplateExtendInfo().getCompressors().get(0)); + + final ExtendSchemaTemplatePlan parsedTemplatePlan2 = + (ExtendSchemaTemplatePlan) + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitExtendSchemaTemplate(extendSchemaTemplatePlan, multiplePathPattern) + .orElseThrow(AssertionError::new); + Assert.assertEquals( + Arrays.asList("s1", "s2"), parsedTemplatePlan2.getTemplateExtendInfo().getMeasurements()); } @Test @@ -320,6 +371,18 @@ public void testPipeDeleteTimeSeries() throws IllegalPathException, IOException .orElseThrow(AssertionError::new)) .getPatternTreeBytes()) .getAllPathPatterns()); + + Assert.assertEquals( + Collections.singletonList(new PartialPath("root.db.device.s1")), + PathPatternTree.deserialize( + ((PipeDeleteTimeSeriesPlan) + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitPipeDeleteTimeSeries( + new PipeDeleteTimeSeriesPlan(patternTree.serialize()), + multiplePathPattern) + .orElseThrow(AssertionError::new)) + .getPatternTreeBytes()) + .getAllPathPatterns()); } @Test @@ -339,6 +402,18 @@ public void testPipeDeleteLogicalView() throws IllegalPathException, IOException .orElseThrow(AssertionError::new)) .getPatternTreeBytes()) .getAllPathPatterns()); + + Assert.assertEquals( + Collections.singletonList(new PartialPath("root.db.device.s1")), + PathPatternTree.deserialize( + ((PipeDeleteLogicalViewPlan) + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitPipeDeleteLogicalView( + new PipeDeleteLogicalViewPlan(patternTree.serialize()), + multiplePathPattern) + .orElseThrow(AssertionError::new)) + .getPatternTreeBytes()) + .getAllPathPatterns()); } @Test diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java index a4c9f0e8403cd..f979d4763b5da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.audit.UserEntity; import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.event.ProgressReportEvent; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; @@ -172,7 +172,7 @@ private void parseAndCollectEvent(final PipeDeleteDataNodeEvent deleteDataEvent) (deleteDataEvent.getDeleteDataNode() instanceof DeleteDataNode ? IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR.process( deleteDataEvent.getDeleteDataNode(), - (IoTDBTreePattern) deleteDataEvent.getTreePattern()) + (UnionIoTDBTreePattern) deleteDataEvent.getTreePattern()) : IoTDBSchemaRegionSource.TABLE_PATTERN_PARSE_VISITOR .process(deleteDataEvent.getDeleteDataNode(), deleteDataEvent.getTablePattern()) .flatMap( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java index a965c817b994f..d6a276b9f25e8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java @@ -21,9 +21,9 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.query.TsFileInsertionEventQueryParser; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.scan.TsFileInsertionEventScanParser; @@ -112,8 +112,8 @@ public TsFileInsertionEventParser provide() throws IOException { sourceEvent); } - if (treePattern instanceof IoTDBTreePattern - && !((IoTDBTreePattern) treePattern).mayMatchMultipleTimeSeriesInOneDevice()) { + if (treePattern instanceof UnionIoTDBTreePattern + && !((UnionIoTDBTreePattern) treePattern).mayMatchMultipleTimeSeriesInOneDevice()) { // If the pattern matches only one time series in one device, use query container here // because there is no timestamps merge overhead. // diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index e4093c260819c..ad503f5fe811c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -29,6 +29,8 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.receiver.IoTDBFileReceiver; import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; @@ -604,10 +606,14 @@ private TSStatus loadSchemaSnapShot( final Set executionTypes = PipeSchemaRegionSnapshotEvent.getStatementTypeSet( parameters.get(ColumnHeaderConstant.TYPE)); - final IoTDBTreePattern treePattern = - new IoTDBTreePattern( - parameters.containsKey(PipeTransferFileSealReqV2.TREE), - parameters.get(ColumnHeaderConstant.PATH_PATTERN)); + final boolean isTreeModelDataAllowedToBeCaptured = + parameters.containsKey(PipeTransferFileSealReqV2.TREE); + final List treePatterns = + TreePattern.parseMultiplePatterns( + parameters.get(ColumnHeaderConstant.PATH_PATTERN), + p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + final TreePattern treePattern = + TreePattern.buildUnionPattern(isTreeModelDataAllowedToBeCaptured, treePatterns); final TablePattern tablePattern = new TablePattern( parameters.containsKey(PipeTransferFileSealReqV2.TABLE), @@ -629,7 +635,7 @@ private TSStatus loadSchemaSnapShot( // Here we apply the statements as many as possible // Even if there are failed statements STATEMENT_TREE_PATTERN_PARSE_VISITOR - .process(originalStatement, treePattern) + .process(originalStatement, (UnionIoTDBTreePattern) treePattern) .flatMap(parsedStatement -> batchVisitor.process(parsedStatement, null)) .ifPresent(statement -> results.add(executeStatementAndClassifyExceptions(statement))); } else if (treeOrTableStatement diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTreePatternParseVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTreePatternParseVisitor.java index a369bb8a7c954..aa11fd69efa4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTreePatternParseVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTreePatternParseVisitor.java @@ -19,7 +19,8 @@ package org.apache.iotdb.db.pipe.receiver.visitor; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; @@ -36,12 +37,12 @@ /** * The {@link PipeStatementTreePatternParseVisitor} will transform the schema {@link Statement}s - * using {@link IoTDBTreePattern}. Rule: + * using {@link UnionIoTDBTreePattern}. Rule: * *

1. All patterns in the output {@link Statement} will be the intersection of the original - * {@link Statement}'s patterns and the given {@link IoTDBTreePattern}. + * {@link Statement}'s patterns and the given {@link UnionIoTDBTreePattern}. * - *

2. If a pattern does not intersect with the {@link IoTDBTreePattern}, it's dropped. + *

2. If a pattern does not intersect with the {@link UnionIoTDBTreePattern}, it's dropped. * *

3. If all the patterns in the {@link Statement} is dropped, the {@link Statement} is dropped. * @@ -49,16 +50,16 @@ * from the {@link SRStatementGenerator} and will no longer be used. */ public class PipeStatementTreePatternParseVisitor - extends StatementVisitor, IoTDBTreePattern> { + extends StatementVisitor, UnionIoTDBTreePattern> { @Override public Optional visitNode( - final StatementNode statement, final IoTDBTreePattern pattern) { + final StatementNode statement, final UnionIoTDBTreePattern pattern) { return Optional.of((Statement) statement); } @Override public Optional visitCreateTimeseries( - final CreateTimeSeriesStatement statement, final IoTDBTreePattern pattern) { + final CreateTimeSeriesStatement statement, final UnionIoTDBTreePattern pattern) { return pattern.matchesMeasurement( statement.getPath().getIDeviceID(), statement.getPath().getMeasurement()) ? Optional.of(statement) @@ -67,7 +68,7 @@ public Optional visitCreateTimeseries( @Override public Optional visitCreateAlignedTimeseries( - final CreateAlignedTimeSeriesStatement statement, final IoTDBTreePattern pattern) { + final CreateAlignedTimeSeriesStatement statement, final UnionIoTDBTreePattern pattern) { final int[] filteredIndexes = IntStream.range(0, statement.getMeasurements().size()) .filter( @@ -107,7 +108,8 @@ public Optional visitCreateAlignedTimeseries( // For logical view with tags/attributes @Override public Optional visitAlterTimeSeries( - final AlterTimeSeriesStatement alterTimeSeriesStatement, final IoTDBTreePattern pattern) { + final AlterTimeSeriesStatement alterTimeSeriesStatement, + final UnionIoTDBTreePattern pattern) { return pattern.matchesMeasurement( alterTimeSeriesStatement.getPath().getIDeviceID(), alterTimeSeriesStatement.getPath().getMeasurement()) @@ -117,7 +119,8 @@ public Optional visitAlterTimeSeries( @Override public Optional visitActivateTemplate( - final ActivateTemplateStatement activateTemplateStatement, final IoTDBTreePattern pattern) { + final ActivateTemplateStatement activateTemplateStatement, + final UnionIoTDBTreePattern pattern) { return pattern.matchDevice(activateTemplateStatement.getPath().getFullPath()) ? Optional.of(activateTemplateStatement) : Optional.empty(); @@ -125,7 +128,8 @@ public Optional visitActivateTemplate( @Override public Optional visitCreateLogicalView( - final CreateLogicalViewStatement createLogicalViewStatement, final IoTDBTreePattern pattern) { + final CreateLogicalViewStatement createLogicalViewStatement, + final UnionIoTDBTreePattern pattern) { final int[] filteredIndexes = IntStream.range(0, createLogicalViewStatement.getTargetPathList().size()) .filter( @@ -141,10 +145,10 @@ public Optional visitCreateLogicalView( return Optional.empty(); } createLogicalViewStatement.setTargetFullPaths( - IoTDBTreePattern.applyIndexesOnList( + TreePattern.applyIndexesOnList( filteredIndexes, createLogicalViewStatement.getTargetPathList())); createLogicalViewStatement.setViewExpressions( - IoTDBTreePattern.applyIndexesOnList( + TreePattern.applyIndexesOnList( filteredIndexes, createLogicalViewStatement.getViewExpressions())); return Optional.of(createLogicalViewStatement); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index b5fa3b2db40e6..9d2b629a67dcb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -22,8 +22,8 @@ import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.source.IoTDBSource; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -303,9 +303,8 @@ private void validatePattern(final TreePattern treePattern) { } if (shouldExtractDeletion - && !(treePattern instanceof IoTDBTreePattern - && (((IoTDBTreePattern) treePattern).isPrefix() - || ((IoTDBTreePattern) treePattern).isFullPath()))) { + && !(treePattern instanceof UnionIoTDBTreePattern + && (((UnionIoTDBTreePattern) treePattern).isPrefixOrFullPath()))) { throw new IllegalArgumentException( String.format( "The path pattern %s is not valid for the source. Only prefix or full path is allowed.", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java index 4c0f0feb4f8a5..b3f4aa3f88cb3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePatternParseVisitor.java @@ -21,7 +21,8 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -54,12 +55,12 @@ /** * The {@link PipePlanTreePatternParseVisitor} will transform the schema {@link PlanNode}s using - * {@link IoTDBTreePattern}. Rule: + * {@link UnionIoTDBTreePattern}. Rule: * *

1. All patterns in the output {@link PlanNode} will be the intersection of the original {@link - * PlanNode}'s patterns and the given {@link IoTDBTreePattern}. + * PlanNode}'s patterns and the given {@link UnionIoTDBTreePattern}. * - *

2. If a pattern does not intersect with the {@link IoTDBTreePattern}, it's dropped. + *

2. If a pattern does not intersect with the {@link UnionIoTDBTreePattern}, it's dropped. * *

3. If all the patterns in the {@link PlanNode} is dropped, the {@link PlanNode} is dropped. * @@ -67,15 +68,15 @@ * one is used in the {@link PipeSchemaRegionWritePlanEvent} in {@link SchemaRegionListeningQueue}. */ public class PipePlanTreePatternParseVisitor - extends PlanVisitor, IoTDBTreePattern> { + extends PlanVisitor, UnionIoTDBTreePattern> { @Override - public Optional visitPlan(final PlanNode node, final IoTDBTreePattern pattern) { + public Optional visitPlan(final PlanNode node, final UnionIoTDBTreePattern pattern) { return Optional.of(node); } @Override public Optional visitCreateTimeSeries( - final CreateTimeSeriesNode node, final IoTDBTreePattern pattern) { + final CreateTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { return pattern.matchesMeasurement( node.getPath().getIDeviceID(), node.getPath().getMeasurement()) ? Optional.of(node) @@ -84,7 +85,7 @@ public Optional visitCreateTimeSeries( @Override public Optional visitCreateAlignedTimeSeries( - final CreateAlignedTimeSeriesNode node, final IoTDBTreePattern pattern) { + final CreateAlignedTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { final int[] filteredIndexes = IntStream.range(0, node.getMeasurements().size()) .filter( @@ -98,19 +99,19 @@ public Optional visitCreateAlignedTimeSeries( new CreateAlignedTimeSeriesNode( node.getPlanNodeId(), node.getDevicePath(), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getMeasurements()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getDataTypes()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getEncodings()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getCompressors()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getAliasList()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getTagsList()), - IoTDBTreePattern.applyIndexesOnList(filteredIndexes, node.getAttributesList()))) + TreePattern.applyIndexesOnList(filteredIndexes, node.getMeasurements()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getDataTypes()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getEncodings()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getCompressors()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getAliasList()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getTagsList()), + TreePattern.applyIndexesOnList(filteredIndexes, node.getAttributesList()))) : Optional.empty(); } @Override public Optional visitCreateMultiTimeSeries( - final CreateMultiTimeSeriesNode node, final IoTDBTreePattern pattern) { + final CreateMultiTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { final Map filteredMeasurementGroupMap = node.getMeasurementGroupMap().entrySet().stream() .filter(entry -> pattern.matchPrefixPath(entry.getKey().getFullPath())) @@ -129,7 +130,7 @@ public Optional visitCreateMultiTimeSeries( } private static MeasurementGroup trimMeasurementGroup( - final IDeviceID device, final MeasurementGroup group, final IoTDBTreePattern pattern) { + final IDeviceID device, final MeasurementGroup group, final UnionIoTDBTreePattern pattern) { final int[] filteredIndexes = IntStream.range(0, group.size()) .filter(index -> pattern.matchesMeasurement(device, group.getMeasurements().get(index))) @@ -164,7 +165,7 @@ private static MeasurementGroup trimMeasurementGroup( @Override public Optional visitAlterTimeSeries( - final AlterTimeSeriesNode node, final IoTDBTreePattern pattern) { + final AlterTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { return pattern.matchesMeasurement( node.getPath().getIDeviceID(), node.getPath().getMeasurement()) ? Optional.of(node) @@ -173,7 +174,7 @@ public Optional visitAlterTimeSeries( @Override public Optional visitInternalCreateTimeSeries( - final InternalCreateTimeSeriesNode node, final IoTDBTreePattern pattern) { + final InternalCreateTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { final MeasurementGroup group = pattern.matchPrefixPath(node.getDevicePath().getFullPath()) ? trimMeasurementGroup( @@ -190,7 +191,7 @@ public Optional visitInternalCreateTimeSeries( @Override public Optional visitActivateTemplate( - final ActivateTemplateNode node, final IoTDBTreePattern pattern) { + final ActivateTemplateNode node, final UnionIoTDBTreePattern pattern) { return pattern.matchDevice(node.getActivatePath().getFullPath()) ? Optional.of(node) : Optional.empty(); @@ -198,7 +199,7 @@ public Optional visitActivateTemplate( @Override public Optional visitInternalBatchActivateTemplate( - final InternalBatchActivateTemplateNode node, final IoTDBTreePattern pattern) { + final InternalBatchActivateTemplateNode node, final UnionIoTDBTreePattern pattern) { final Map> filteredTemplateActivationMap = node.getTemplateActivationMap().entrySet().stream() .filter(entry -> pattern.matchDevice(entry.getKey().getFullPath())) @@ -212,7 +213,7 @@ public Optional visitInternalBatchActivateTemplate( @Override public Optional visitInternalCreateMultiTimeSeries( - final InternalCreateMultiTimeSeriesNode node, final IoTDBTreePattern pattern) { + final InternalCreateMultiTimeSeriesNode node, final UnionIoTDBTreePattern pattern) { final Map> filteredDeviceMap = node.getDeviceMap().entrySet().stream() .filter(entry -> pattern.matchPrefixPath(entry.getKey().getFullPath())) @@ -236,7 +237,7 @@ public Optional visitInternalCreateMultiTimeSeries( @Override public Optional visitBatchActivateTemplate( - final BatchActivateTemplateNode node, final IoTDBTreePattern pattern) { + final BatchActivateTemplateNode node, final UnionIoTDBTreePattern pattern) { final Map> filteredTemplateActivationMap = node.getTemplateActivationMap().entrySet().stream() .filter(entry -> pattern.matchDevice(entry.getKey().getFullPath())) @@ -249,7 +250,7 @@ public Optional visitBatchActivateTemplate( @Override public Optional visitCreateLogicalView( - final CreateLogicalViewNode node, final IoTDBTreePattern pattern) { + final CreateLogicalViewNode node, final UnionIoTDBTreePattern pattern) { final Map filteredViewPathToSourceMap = node.getViewPathToSourceExpressionMap().entrySet().stream() .filter( @@ -264,7 +265,7 @@ public Optional visitCreateLogicalView( @Override public Optional visitAlterLogicalView( - final AlterLogicalViewNode node, final IoTDBTreePattern pattern) { + final AlterLogicalViewNode node, final UnionIoTDBTreePattern pattern) { final Map filteredViewPathToSourceMap = node.getViewPathToSourceMap().entrySet().stream() .filter( @@ -279,7 +280,7 @@ public Optional visitAlterLogicalView( @Override public Optional visitDeleteData( - final DeleteDataNode node, final IoTDBTreePattern pattern) { + final DeleteDataNode node, final UnionIoTDBTreePattern pattern) { final List intersectedPaths = node.getPathList().stream() .map(pattern::getIntersection) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java index c1619b64ad492..85ae98ac26a85 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.schema.view.viewExpression.leaf.TimeSeriesViewOperand; import org.apache.iotdb.db.pipe.receiver.visitor.PipeStatementTreePatternParseVisitor; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; @@ -43,8 +44,15 @@ public class PipeStatementTreePatternParseVisitorTest { - private final IoTDBTreePattern prefixPathPattern = new IoTDBTreePattern("root.db.device.**"); - private final IoTDBTreePattern fullPathPattern = new IoTDBTreePattern("root.db.device.s1"); + private final UnionIoTDBTreePattern prefixPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); + private final UnionIoTDBTreePattern fullPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.s1")); + private final UnionIoTDBTreePattern multiplePathPattern = + new UnionIoTDBTreePattern( + Arrays.asList( + new IoTDBTreePattern("root.db.device.s1"), + new IoTDBTreePattern("root.db.device.s2"))); @Test public void testCreateTimeSeries() throws IllegalPathException { @@ -78,6 +86,15 @@ public void testCreateTimeSeries() throws IllegalPathException { new PipeStatementTreePatternParseVisitor() .visitCreateTimeseries(createTimeSeriesStatementToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + createTimeSeriesStatement, + new PipeStatementTreePatternParseVisitor() + .visitCreateTimeseries(createTimeSeriesStatement, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + new PipeStatementTreePatternParseVisitor() + .visitCreateTimeseries(createTimeSeriesStatementToFilter, multiplePathPattern) + .isPresent()); } @Test @@ -119,6 +136,13 @@ public void testCreateAlignedTimeSeries() throws IllegalPathException { new PipeStatementTreePatternParseVisitor() .visitCreateAlignedTimeseries(originalCreateAlignedTimeSeriesStatement, fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + originalCreateAlignedTimeSeriesStatement, + new PipeStatementTreePatternParseVisitor() + .visitCreateAlignedTimeseries( + originalCreateAlignedTimeSeriesStatement, multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -150,6 +174,15 @@ public void testAlterTimeSeries() throws IllegalPathException { new PipeStatementTreePatternParseVisitor() .visitAlterTimeSeries(alterTimeSeriesStatementToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + alterTimeSeriesStatement, + new PipeStatementTreePatternParseVisitor() + .visitAlterTimeSeries(alterTimeSeriesStatement, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + new PipeStatementTreePatternParseVisitor() + .visitAlterTimeSeries(alterTimeSeriesStatementToFilter, multiplePathPattern) + .isPresent()); } @Test @@ -168,6 +201,15 @@ public void testActivateTemplate() throws IllegalPathException { new PipeStatementTreePatternParseVisitor() .visitActivateTemplate(activateTemplateStatementToFilter, prefixPathPattern) .isPresent()); + Assert.assertEquals( + activateTemplateStatement, + new PipeStatementTreePatternParseVisitor() + .visitActivateTemplate(activateTemplateStatement, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + new PipeStatementTreePatternParseVisitor() + .visitActivateTemplate(activateTemplateStatementToFilter, multiplePathPattern) + .isPresent()); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java index dd0558bc23805..fd1853fca1b74 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java @@ -82,6 +82,8 @@ public void testIoTDBDataRegionExtractorWithPattern() { Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root")); Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.`a-b`")); Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.1")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.a,root.b")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.a,root.b,root.db1.`a,b`.**")); } public Exception testIoTDBDataRegionExtractorWithPattern(final String pattern) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java index b21942a241e96..f4aa8f56a7efa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.schema.view.viewExpression.leaf.TimeSeriesViewOperand; import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource; @@ -49,15 +50,22 @@ import org.junit.Assert; import org.junit.Test; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; public class PipePlanTreePatternParseVisitorTest { - private final IoTDBTreePattern prefixPathPattern = new IoTDBTreePattern("root.db.device.**"); - private final IoTDBTreePattern fullPathPattern = new IoTDBTreePattern("root.db.device.s1"); + + private final UnionIoTDBTreePattern prefixPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); + private final UnionIoTDBTreePattern fullPathPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.s1")); + private final UnionIoTDBTreePattern multiplePathPattern = + new UnionIoTDBTreePattern( + Arrays.asList( + new IoTDBTreePattern("root.db.device.s1"), + new IoTDBTreePattern("root.db.device.s2"))); @Test public void testCreateTimeSeries() throws IllegalPathException { @@ -83,6 +91,17 @@ public void testCreateTimeSeries() throws IllegalPathException { Collections.emptyMap(), Collections.emptyMap(), "a1"); + final CreateTimeSeriesNode createTimeSeriesNode2 = + new CreateTimeSeriesNode( + new PlanNodeId("2024-04-30-3"), + new MeasurementPath("root.db.device.s2"), + TSDataType.FLOAT, + TSEncoding.RLE, + CompressionType.SNAPPY, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + "a2"); Assert.assertEquals( createTimeSeriesNode, @@ -93,6 +112,21 @@ public void testCreateTimeSeries() throws IllegalPathException { IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR .visitCreateTimeSeries(createTimeSeriesNodeToFilter, prefixPathPattern) .isPresent()); + + Assert.assertEquals( + createTimeSeriesNode, + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateTimeSeries(createTimeSeriesNode, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertEquals( + createTimeSeriesNode2, + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateTimeSeries(createTimeSeriesNode2, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateTimeSeries(createTimeSeriesNodeToFilter, multiplePathPattern) + .isPresent()); } @Test @@ -122,6 +156,35 @@ public void testCreateAlignedTimeSeries() throws IllegalPathException { Arrays.asList(Collections.emptyMap(), Collections.emptyMap())), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new CreateAlignedTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + new PartialPath("root.db.device"), + Arrays.asList("s1", "s2"), + Arrays.asList(TSDataType.FLOAT, TSDataType.BOOLEAN), + Arrays.asList(TSEncoding.RLE, TSEncoding.PLAIN), + Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY), + Arrays.asList("a1", "a2"), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap())), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateAlignedTimeSeries( + new CreateAlignedTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + new PartialPath("root.db.device"), + Arrays.asList("s1", "s2", "s3"), + Arrays.asList(TSDataType.FLOAT, TSDataType.BOOLEAN, TSDataType.INT32), + Arrays.asList(TSEncoding.RLE, TSEncoding.PLAIN, TSEncoding.RLE), + Arrays.asList( + CompressionType.SNAPPY, CompressionType.SNAPPY, CompressionType.SNAPPY), + Arrays.asList("a1", "a2", "a3"), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap())), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -153,6 +216,40 @@ public void testCreateMultiTimeSeries() throws IllegalPathException { Arrays.asList(Collections.emptyMap(), Collections.emptyMap())), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new CreateMultiTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + Arrays.asList( + new MeasurementPath("root.db.device.s1"), new MeasurementPath("root.db.device.s2")), + Arrays.asList(TSDataType.FLOAT, TSDataType.BOOLEAN), + Arrays.asList(TSEncoding.RLE, TSEncoding.PLAIN), + Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList("a1", "a2"), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap())), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateMultiTimeSeries( + new CreateMultiTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + Arrays.asList( + new MeasurementPath("root.db.device.s1"), + new MeasurementPath("root.db.device.s2"), + new MeasurementPath("root.db1.device.s1")), + Arrays.asList(TSDataType.FLOAT, TSDataType.BOOLEAN, TSDataType.INT32), + Arrays.asList(TSEncoding.RLE, TSEncoding.PLAIN, TSEncoding.RLE), + Arrays.asList( + CompressionType.SNAPPY, CompressionType.SNAPPY, CompressionType.SNAPPY), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList("a1", "a2", "a3"), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap())), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -178,6 +275,16 @@ public void testAlterTimeSeries() throws IllegalPathException { Collections.emptyMap(), attributesMap, false); + final AlterTimeSeriesNode alterTimeSeriesNode2 = + new AlterTimeSeriesNode( + new PlanNodeId("2024-04-30-3"), + new MeasurementPath("root.db.device.s2"), + AlterTimeSeriesStatement.AlterType.ADD_ATTRIBUTES, + attributesMap, + "", + Collections.emptyMap(), + attributesMap, + false); Assert.assertEquals( alterTimeSeriesNode, @@ -188,6 +295,21 @@ public void testAlterTimeSeries() throws IllegalPathException { IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR .visitAlterTimeSeries(alterTimeSeriesNodeToFilter, prefixPathPattern) .isPresent()); + + Assert.assertEquals( + alterTimeSeriesNode, + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitAlterTimeSeries(alterTimeSeriesNode, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertEquals( + alterTimeSeriesNode2, + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitAlterTimeSeries(alterTimeSeriesNode2, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitAlterTimeSeries(alterTimeSeriesNodeToFilter, multiplePathPattern) + .isPresent()); } @Test @@ -200,12 +322,13 @@ public void testInternalCreateTimeSeries() throws IllegalPathException { expectedMeasurementGroup.addTags(Collections.emptyMap()); expectedMeasurementGroup.addAttributes(Collections.emptyMap()); - final ByteBuffer byteBuffer = ByteBuffer.allocate(1000); - expectedMeasurementGroup.serialize(byteBuffer); - byteBuffer.flip(); final MeasurementGroup originalMeasurementGroup = new MeasurementGroup(); - originalMeasurementGroup.deserialize(byteBuffer); - + originalMeasurementGroup.addMeasurement( + "s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY); + originalMeasurementGroup.addProps(Collections.emptyMap()); + originalMeasurementGroup.addAlias("a1"); + originalMeasurementGroup.addTags(Collections.emptyMap()); + originalMeasurementGroup.addAttributes(Collections.emptyMap()); originalMeasurementGroup.addMeasurement( "s2", TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY); originalMeasurementGroup.addProps(Collections.emptyMap()); @@ -228,6 +351,22 @@ public void testInternalCreateTimeSeries() throws IllegalPathException { true), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new InternalCreateTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + new PartialPath("root.db.device"), + originalMeasurementGroup, + true), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitInternalCreateTimeSeries( + new InternalCreateTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + new PartialPath("root.db.device"), + originalMeasurementGroup, + true), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -247,6 +386,16 @@ public void testActivateTemplate() throws IllegalPathException { IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR .visitActivateTemplate(activateTemplateNodeToFilter, prefixPathPattern) .isPresent()); + + Assert.assertEquals( + activateTemplateNode, + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitActivateTemplate(activateTemplateNode, multiplePathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitActivateTemplate(activateTemplateNodeToFilter, multiplePathPattern) + .isPresent()); } @Test @@ -267,6 +416,23 @@ public void testInternalBatchActivateTemplate() throws IllegalPathException { }), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new InternalBatchActivateTemplateNode( + new PlanNodeId("2024-04-30-2"), + Collections.singletonMap(new PartialPath("root.db.device"), new Pair<>(1, 1))), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitInternalBatchActivateTemplate( + new InternalBatchActivateTemplateNode( + new PlanNodeId("2024-04-30-2"), + new HashMap>() { + { + put(new PartialPath("root.db.device"), new Pair<>(1, 1)); + put(new PartialPath("root.db1"), new Pair<>(2, 2)); + } + }), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -279,12 +445,13 @@ public void testInternalCreateMultiTimeSeries() throws IllegalPathException { expectedMeasurementGroup.addTags(Collections.emptyMap()); expectedMeasurementGroup.addAttributes(Collections.emptyMap()); - final ByteBuffer byteBuffer = ByteBuffer.allocate(1000); - expectedMeasurementGroup.serialize(byteBuffer); - byteBuffer.flip(); final MeasurementGroup originalMeasurementGroup = new MeasurementGroup(); - originalMeasurementGroup.deserialize(byteBuffer); - + originalMeasurementGroup.addMeasurement( + "s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY); + originalMeasurementGroup.addProps(Collections.emptyMap()); + originalMeasurementGroup.addAlias("a1"); + originalMeasurementGroup.addTags(Collections.emptyMap()); + originalMeasurementGroup.addAttributes(Collections.emptyMap()); originalMeasurementGroup.addMeasurement( "s2", TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY); originalMeasurementGroup.addProps(Collections.emptyMap()); @@ -292,6 +459,10 @@ public void testInternalCreateMultiTimeSeries() throws IllegalPathException { originalMeasurementGroup.addTags(Collections.emptyMap()); originalMeasurementGroup.addAttributes(Collections.emptyMap()); + final MeasurementGroup anotherMeasurementGroup = new MeasurementGroup(); + anotherMeasurementGroup.addMeasurement( + "s3", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY); + Assert.assertEquals( new InternalCreateMultiTimeSeriesNode( new PlanNodeId("2024-04-30-1"), @@ -313,6 +484,28 @@ public void testInternalCreateMultiTimeSeries() throws IllegalPathException { }), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new InternalCreateMultiTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + Collections.singletonMap( + new PartialPath("root.db.device"), new Pair<>(false, originalMeasurementGroup))), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitInternalCreateMultiTimeSeries( + new InternalCreateMultiTimeSeriesNode( + new PlanNodeId("2024-04-30-2"), + new HashMap>() { + { + put( + new PartialPath("root.db.device"), + new Pair<>(false, originalMeasurementGroup)); + put( + new PartialPath("root.db1.device"), + new Pair<>(false, anotherMeasurementGroup)); + } + }), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -333,6 +526,23 @@ public void testBatchActivateTemplate() throws IllegalPathException { }), fullPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new BatchActivateTemplateNode( + new PlanNodeId("2024-04-30-2"), + Collections.singletonMap(new PartialPath("root.db.device"), new Pair<>(1, 1))), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitBatchActivateTemplate( + new BatchActivateTemplateNode( + new PlanNodeId("2024-04-30-2"), + new HashMap>() { + { + put(new PartialPath("root.db.device"), new Pair<>(1, 1)); + put(new PartialPath("root.db"), new Pair<>(2, 2)); + } + }), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } @Test @@ -404,5 +614,25 @@ public void testDeleteData() throws IllegalPathException { Long.MAX_VALUE), prefixPathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new DeleteDataNode( + new PlanNodeId("2024-04-30-2"), + Arrays.asList( + new MeasurementPath("root.db.device.s1"), new MeasurementPath("root.db.device.s2")), + Long.MIN_VALUE, + Long.MAX_VALUE), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitDeleteData( + new DeleteDataNode( + new PlanNodeId("2024-04-30-2"), + Arrays.asList( + new MeasurementPath("root.db.device.s1"), + new MeasurementPath("root.db.device.s2"), + new MeasurementPath("root.db.device.s3")), + Long.MIN_VALUE, + Long.MAX_VALUE), + multiplePathPattern) + .orElseThrow(AssertionError::new)); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java index 9a1d817dc6061..80e7df9ee5da2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java @@ -35,9 +35,8 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; -public class IoTDBTreePattern extends TreePattern { +public class IoTDBTreePattern extends SingleTreePattern { private final PartialPath patternPartialPath; private static volatile DevicePathGetter devicePathGetter = PartialPath::new; @@ -57,13 +56,6 @@ public IoTDBTreePattern(final String pattern) { this(true, pattern); } - public static List applyIndexesOnList( - final int[] filteredIndexes, final List originalList) { - return Objects.nonNull(originalList) - ? Arrays.stream(filteredIndexes).mapToObj(originalList::get).collect(Collectors.toList()) - : null; - } - @Override public String getDefaultPattern() { return PipeSourceConstant.EXTRACTOR_PATTERN_IOTDB_DEFAULT_VALUE; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/PrefixTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/PrefixTreePattern.java index 5d5f0586e12d8..d1c1d5e800d2f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/PrefixTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/PrefixTreePattern.java @@ -29,7 +29,7 @@ import java.util.Arrays; -public class PrefixTreePattern extends TreePattern { +public class PrefixTreePattern extends SingleTreePattern { public PrefixTreePattern(final boolean isTreeModelDataAllowedToBeCaptured, final String pattern) { super(isTreeModelDataAllowedToBeCaptured, pattern); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java new file mode 100644 index 0000000000000..b88e760d76a9d --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/SingleTreePattern.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.datastructure.pattern; + +import java.util.Objects; + +public abstract class SingleTreePattern extends TreePattern { + + protected final String pattern; + + protected SingleTreePattern( + final boolean isTreeModelDataAllowedToBeCaptured, final String pattern) { + super(isTreeModelDataAllowedToBeCaptured); + this.pattern = pattern != null ? pattern : getDefaultPattern(); + } + + @Override + public boolean isSingle() { + return true; + } + + @Override + public String getPattern() { + return pattern; + } + + @Override + public boolean isRoot() { + return Objects.isNull(pattern) || this.pattern.equals(this.getDefaultPattern()); + } + + public abstract String getDefaultPattern(); + + @Override + public String toString() { + return "{pattern='" + + pattern + + "', isTreeModelDataAllowedToBeCaptured=" + + isTreeModelDataAllowedToBeCaptured + + '}'; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java index 7eef37ea3a618..cc5997582d413 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java @@ -27,8 +27,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATH_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATTERN_FORMAT_IOTDB_VALUE; @@ -44,23 +49,20 @@ public abstract class TreePattern { private static final Logger LOGGER = LoggerFactory.getLogger(TreePattern.class); protected final boolean isTreeModelDataAllowedToBeCaptured; - protected final String pattern; - protected TreePattern(final boolean isTreeModelDataAllowedToBeCaptured, final String pattern) { + protected TreePattern(final boolean isTreeModelDataAllowedToBeCaptured) { this.isTreeModelDataAllowedToBeCaptured = isTreeModelDataAllowedToBeCaptured; - this.pattern = pattern != null ? pattern : getDefaultPattern(); } public boolean isTreeModelDataAllowedToBeCaptured() { return isTreeModelDataAllowedToBeCaptured; } - public String getPattern() { - return pattern; - } - - public boolean isRoot() { - return Objects.isNull(pattern) || this.pattern.equals(this.getDefaultPattern()); + public static List applyIndexesOnList( + final int[] filteredIndexes, final List originalList) { + return Objects.nonNull(originalList) + ? Arrays.stream(filteredIndexes).mapToObj(originalList::get).collect(Collectors.toList()) + : null; } /** @@ -74,42 +76,142 @@ public static TreePattern parsePipePatternFromSourceParameters( isTreeModelDataAllowToBeCaptured(sourceParameters); final String path = sourceParameters.getStringByKeys(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY); + final String pattern = + sourceParameters.getStringByKeys(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY); - // 1. If "source.path" is specified, it will be interpreted as an IoTDB-style path, - // ignoring the other 2 parameters. - if (path != null) { - return new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, path); + // 1. If both "source.path" and "source.pattern" are specified, their union will be used. + if (path != null && pattern != null) { + final List result = new ArrayList<>(); + // Parse "source.path" as IoTDB-style path. + result.addAll( + parseMultiplePatterns( + path, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p))); + // Parse "source.pattern" using the helper method. + result.addAll( + parsePatternsFromPatternParameter( + pattern, sourceParameters, isTreeModelDataAllowedToBeCaptured)); + return buildUnionPattern(isTreeModelDataAllowedToBeCaptured, result); } - final String pattern = - sourceParameters.getStringByKeys(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY); + // 2. If only "source.path" is specified, it will be interpreted as an IoTDB-style path. + if (path != null) { + return buildUnionPattern( + isTreeModelDataAllowedToBeCaptured, + parseMultiplePatterns( + path, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p))); + } - // 2. Otherwise, If "source.pattern" is specified, it will be interpreted - // according to "source.pattern.format". + // 3. If only "source.pattern" is specified, parse it using the helper method. if (pattern != null) { - final String patternFormat = - sourceParameters.getStringByKeys(EXTRACTOR_PATTERN_FORMAT_KEY, SOURCE_PATTERN_FORMAT_KEY); + return buildUnionPattern( + isTreeModelDataAllowedToBeCaptured, + parsePatternsFromPatternParameter( + pattern, sourceParameters, isTreeModelDataAllowedToBeCaptured)); + } - // If "source.pattern.format" is not specified, use prefix format by default. - if (patternFormat == null) { - return new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, pattern); + // 4. If neither "source.path" nor "source.pattern" is specified, + // this pipe source will match all data. + return buildUnionPattern( + isTreeModelDataAllowedToBeCaptured, + Collections.singletonList(new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, null))); + } + + /** + * A private helper method to parse a list of {@link TreePattern}s from the "pattern" parameter, + * considering its "format". + * + * @param pattern The pattern string to parse. + * @param sourceParameters The source parameters to read the format from. + * @param isTreeModelDataAllowedToBeCaptured A boolean flag passed to the TreePattern constructor. + * @return A list of parsed {@link TreePattern}s. + */ + private static List parsePatternsFromPatternParameter( + final String pattern, + final PipeParameters sourceParameters, + final boolean isTreeModelDataAllowedToBeCaptured) { + final String patternFormat = + sourceParameters.getStringByKeys(EXTRACTOR_PATTERN_FORMAT_KEY, SOURCE_PATTERN_FORMAT_KEY); + + // If "source.pattern.format" is not specified, use prefix format by default. + if (patternFormat == null) { + return parseMultiplePatterns( + pattern, p -> new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + } + + switch (patternFormat.toLowerCase()) { + case EXTRACTOR_PATTERN_FORMAT_IOTDB_VALUE: + return parseMultiplePatterns( + pattern, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + case EXTRACTOR_PATTERN_FORMAT_PREFIX_VALUE: + return parseMultiplePatterns( + pattern, p -> new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + default: + LOGGER.info( + "Unknown pattern format: {}, use prefix matching format by default.", patternFormat); + return parseMultiplePatterns( + pattern, p -> new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, p)); + } + } + + public static List parseMultiplePatterns( + final String pattern, final Function patternSupplier) { + if (pattern.isEmpty()) { + return Collections.singletonList(patternSupplier.apply(pattern)); + } + + final List patterns = new ArrayList<>(); + final StringBuilder currentPattern = new StringBuilder(); + boolean inBackticks = false; + + for (final char c : pattern.toCharArray()) { + if (c == '`') { + inBackticks = !inBackticks; + currentPattern.append(c); + } else if (c == ',' && !inBackticks) { + final String singlePattern = currentPattern.toString().trim(); + if (!singlePattern.isEmpty()) { + patterns.add(patternSupplier.apply(singlePattern)); + } + currentPattern.setLength(0); + } else { + currentPattern.append(c); } + } + + final String lastPattern = currentPattern.toString().trim(); + if (!lastPattern.isEmpty()) { + patterns.add(patternSupplier.apply(lastPattern)); + } + + return patterns; + } - switch (patternFormat.toLowerCase()) { - case EXTRACTOR_PATTERN_FORMAT_IOTDB_VALUE: - return new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, pattern); - case EXTRACTOR_PATTERN_FORMAT_PREFIX_VALUE: - return new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, pattern); - default: - LOGGER.info( - "Unknown pattern format: {}, use prefix matching format by default.", patternFormat); - return new PrefixTreePattern(isTreeModelDataAllowedToBeCaptured, pattern); + /** + * A private helper method to build the most specific UnionTreePattern possible. If all patterns + * are IoTDBTreePattern, it returns an IoTDBUnionTreePattern. Otherwise, it returns a general + * UnionTreePattern. + */ + public static TreePattern buildUnionPattern( + final boolean isTreeModelDataAllowedToBeCaptured, final List patterns) { + // Check if all instances in the list are of type IoTDBTreePattern + boolean allIoTDB = true; + for (final TreePattern p : patterns) { + if (!(p instanceof IoTDBTreePattern)) { + allIoTDB = false; + break; } } - // 3. If neither "source.path" nor "source.pattern" is specified, - // this pipe source will match all data. - return new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, null); + if (allIoTDB) { + final List iotdbPatterns = new ArrayList<>(patterns.size()); + for (final TreePattern p : patterns) { + iotdbPatterns.add((IoTDBTreePattern) p); + } + return new UnionIoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, iotdbPatterns); + } else { + // If there's a mix of pattern types, use the general UnionTreePattern + return new UnionTreePattern(isTreeModelDataAllowedToBeCaptured, patterns); + } } public static boolean isTreeModelDataAllowToBeCaptured(final PipeParameters sourceParameters) { @@ -128,7 +230,11 @@ public static boolean isTreeModelDataAllowToBeCaptured(final PipeParameters sour .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); } - public abstract String getDefaultPattern(); + public abstract boolean isSingle(); + + public abstract String getPattern(); + + public abstract boolean isRoot(); /** Check if this pattern is legal. Different pattern type may have different rules. */ public abstract boolean isLegal(); @@ -162,13 +268,4 @@ public static boolean isTreeModelDataAllowToBeCaptured(final PipeParameters sour *

NOTE: this is only called when {@link TreePattern#mayOverlapWithDevice} is {@code true}. */ public abstract boolean matchesMeasurement(final IDeviceID device, final String measurement); - - @Override - public String toString() { - return "{pattern='" - + pattern - + "', isTreeModelDataAllowedToBeCaptured=" - + isTreeModelDataAllowedToBeCaptured - + '}'; - } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java new file mode 100644 index 0000000000000..a44f4c9d6479b --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionIoTDBTreePattern.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.datastructure.pattern; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; + +import org.apache.tsfile.file.metadata.IDeviceID; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Represents a union of multiple {@link IoTDBTreePattern}s. This specialized class ensures type + * safety and provides access to methods specific to IoTDBTreePattern, such as getIntersection. + */ +public class UnionIoTDBTreePattern extends TreePattern { + + private final List patterns; + + public UnionIoTDBTreePattern( + final boolean isTreeModelDataAllowedToBeCaptured, final List patterns) { + super(isTreeModelDataAllowedToBeCaptured); + this.patterns = patterns; + } + + public UnionIoTDBTreePattern(final List patterns) { + super(true); + this.patterns = patterns; + } + + public UnionIoTDBTreePattern(final IoTDBTreePattern pattern) { + super(true); + this.patterns = Collections.singletonList(pattern); + } + + // ********************************************************************** + // IoTDBTreePattern-specific aggregated methods + // ********************************************************************** + + public boolean matchPrefixPath(final String path) { + return patterns.stream().anyMatch(p -> p.matchPrefixPath(path)); + } + + public boolean matchDevice(final String devicePath) { + return patterns.stream().anyMatch(p -> p.matchDevice(devicePath)); + } + + public boolean matchTailNode(final String tailNode) { + return patterns.stream().anyMatch(p -> p.matchTailNode(tailNode)); + } + + public List getIntersection(final PartialPath partialPath) { + final Set uniqueIntersections = new LinkedHashSet<>(); + for (final IoTDBTreePattern pattern : patterns) { + uniqueIntersections.addAll(pattern.getIntersection(partialPath)); + } + return new ArrayList<>(uniqueIntersections); + } + + public PathPatternTree getIntersection(final PathPatternTree patternTree) { + final PathPatternTree resultTree = new PathPatternTree(); + for (final IoTDBTreePattern pattern : patterns) { + final PathPatternTree intersection = pattern.getIntersection(patternTree); + if (intersection.isEmpty()) { + continue; + } + intersection.getAllPathPatterns().forEach(resultTree::appendPathPattern); + } + resultTree.constructTree(); + return resultTree; + } + + public boolean isPrefixOrFullPath() { + return patterns.stream().allMatch(p -> p.isPrefix() || p.isFullPath()); + } + + public boolean mayMatchMultipleTimeSeriesInOneDevice() { + return patterns.stream().anyMatch(IoTDBTreePattern::mayMatchMultipleTimeSeriesInOneDevice); + } + + // ********************************************************************** + // Implementation of abstract methods from TreePattern + // ********************************************************************** + + @Override + public boolean isSingle() { + return patterns.size() == 1; + } + + @Override + public String getPattern() { + return patterns.stream().map(TreePattern::getPattern).collect(Collectors.joining(",")); + } + + @Override + public boolean isRoot() { + return patterns.stream().anyMatch(TreePattern::isRoot); + } + + @Override + public boolean isLegal() { + return patterns.stream().allMatch(TreePattern::isLegal); + } + + @Override + public boolean coversDb(final String db) { + return patterns.stream().anyMatch(p -> p.coversDb(db)); + } + + @Override + public boolean coversDevice(final IDeviceID device) { + return patterns.stream().anyMatch(p -> p.coversDevice(device)); + } + + @Override + public boolean mayOverlapWithDb(final String db) { + return patterns.stream().anyMatch(p -> p.mayOverlapWithDb(db)); + } + + @Override + public boolean mayOverlapWithDevice(final IDeviceID device) { + return patterns.stream().anyMatch(p -> p.mayOverlapWithDevice(device)); + } + + @Override + public boolean matchesMeasurement(final IDeviceID device, final String measurement) { + return patterns.stream().anyMatch(p -> p.matchesMeasurement(device, measurement)); + } + + @Override + public String toString() { + return "UnionIoTDBTreePattern{" + + "patterns=" + + patterns + + ", isTreeModelDataAllowedToBeCaptured=" + + isTreeModelDataAllowedToBeCaptured + + '}'; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java new file mode 100644 index 0000000000000..e43a18ad73855 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/UnionTreePattern.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.datastructure.pattern; + +import org.apache.tsfile.file.metadata.IDeviceID; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Represents a union of multiple {@link TreePattern}s. A path is considered to match if it matches + * any of the patterns in the collection. + */ +public class UnionTreePattern extends TreePattern { + + private final List patterns; + + public UnionTreePattern( + final boolean isTreeModelDataAllowedToBeCaptured, final List patterns) { + super(isTreeModelDataAllowedToBeCaptured); + this.patterns = patterns; + } + + @Override + public boolean isSingle() { + return patterns.size() == 1; + } + + @Override + public String getPattern() { + return patterns.stream().map(TreePattern::getPattern).collect(Collectors.joining(",")); + } + + @Override + public boolean isRoot() { + return patterns.stream().anyMatch(TreePattern::isRoot); + } + + @Override + public boolean isLegal() { + return patterns.stream().allMatch(TreePattern::isLegal); + } + + @Override + public boolean coversDb(final String db) { + return patterns.stream().anyMatch(p -> p.coversDb(db)); + } + + @Override + public boolean coversDevice(final IDeviceID device) { + return patterns.stream().anyMatch(p -> p.coversDevice(device)); + } + + @Override + public boolean mayOverlapWithDb(final String db) { + return patterns.stream().anyMatch(p -> p.mayOverlapWithDb(db)); + } + + @Override + public boolean mayOverlapWithDevice(final IDeviceID device) { + return patterns.stream().anyMatch(p -> p.mayOverlapWithDevice(device)); + } + + @Override + public boolean matchesMeasurement(final IDeviceID device, final String measurement) { + return patterns.stream().anyMatch(p -> p.matchesMeasurement(device, measurement)); + } + + @Override + public String toString() { + return "UnionTreePattern{" + + "patterns=" + + patterns + + ", isTreeModelDataAllowedToBeCaptured=" + + isTreeModelDataAllowedToBeCaptured + + '}'; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java index 97987adb7770a..acb0ae825ac44 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java @@ -24,9 +24,9 @@ import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; -import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.queue.ConcurrentIterableLinkedQueue; import org.apache.iotdb.commons.pipe.datastructure.queue.listening.AbstractPipeListeningQueue; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; @@ -53,7 +53,7 @@ @TableModel public abstract class IoTDBNonDataRegionSource extends IoTDBSource { - protected IoTDBTreePattern treePattern; + protected UnionIoTDBTreePattern treePattern; protected TablePattern tablePattern; private List historicalEvents = new LinkedList<>(); @@ -78,15 +78,14 @@ public void customize( final TreePattern pattern = TreePattern.parsePipePatternFromSourceParameters(parameters); - if (!(pattern instanceof IoTDBTreePattern - && (((IoTDBTreePattern) pattern).isPrefix() - || ((IoTDBTreePattern) pattern).isFullPath()))) { + if (!(pattern instanceof UnionIoTDBTreePattern + && (((UnionIoTDBTreePattern) pattern).isPrefixOrFullPath()))) { throw new IllegalArgumentException( String.format( "The path pattern %s is not valid for the source. Only prefix or full path is allowed.", pattern.getPattern())); } - treePattern = (IoTDBTreePattern) pattern; + treePattern = (UnionIoTDBTreePattern) pattern; tablePattern = TablePattern.parsePipePatternFromSourceParameters(parameters); }