Skip to content

Commit 66d0c07

Browse files
Wei-hao-LiWei-hao-Li
authored
Remove useless ProjectNode after SortElimination (#17806)
1 parent 2fd70b9 commit 66d0c07

3 files changed

Lines changed: 141 additions & 3 deletions

File tree

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@
2828
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode;
2929
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode;
3030
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode;
31+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode;
3132
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode;
33+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode;
3234

3335
import java.util.ArrayList;
3436
import java.util.LinkedHashMap;
@@ -91,7 +93,51 @@ public static Optional<PlanNode> pruneColumns(TableScanNode node, Set<Symbol> re
9193
.forEach(
9294
symbol -> newAssignments.put(symbol, node.getAssignments().get(symbol))));
9395

94-
if (node instanceof TreeDeviceViewScanNode) {
96+
if (node instanceof TreeAlignedDeviceViewScanNode) {
97+
TreeAlignedDeviceViewScanNode treeDeviceViewScanNode =
98+
(TreeAlignedDeviceViewScanNode) deviceTableScanNode;
99+
TreeAlignedDeviceViewScanNode prunedNode =
100+
new TreeAlignedDeviceViewScanNode(
101+
deviceTableScanNode.getPlanNodeId(),
102+
deviceTableScanNode.getQualifiedObjectName(),
103+
newOutputs,
104+
newAssignments,
105+
deviceTableScanNode.getDeviceEntries(),
106+
deviceTableScanNode.getTagAndAttributeIndexMap(),
107+
deviceTableScanNode.getScanOrder(),
108+
deviceTableScanNode.getTimePredicate().orElse(null),
109+
deviceTableScanNode.getPushDownPredicate(),
110+
deviceTableScanNode.getPushDownLimit(),
111+
deviceTableScanNode.getPushDownOffset(),
112+
deviceTableScanNode.isPushLimitToEachDevice(),
113+
deviceTableScanNode.containsNonAlignedDevice(),
114+
treeDeviceViewScanNode.getTreeDBName(),
115+
treeDeviceViewScanNode.getMeasurementColumnNameMap());
116+
prunedNode.setRegionReplicaSet(deviceTableScanNode.getRegionReplicaSet());
117+
return Optional.of(prunedNode);
118+
} else if (node instanceof TreeNonAlignedDeviceViewScanNode) {
119+
TreeNonAlignedDeviceViewScanNode treeDeviceViewScanNode =
120+
(TreeNonAlignedDeviceViewScanNode) deviceTableScanNode;
121+
TreeNonAlignedDeviceViewScanNode prunedNode =
122+
new TreeNonAlignedDeviceViewScanNode(
123+
deviceTableScanNode.getPlanNodeId(),
124+
deviceTableScanNode.getQualifiedObjectName(),
125+
newOutputs,
126+
newAssignments,
127+
deviceTableScanNode.getDeviceEntries(),
128+
deviceTableScanNode.getTagAndAttributeIndexMap(),
129+
deviceTableScanNode.getScanOrder(),
130+
deviceTableScanNode.getTimePredicate().orElse(null),
131+
deviceTableScanNode.getPushDownPredicate(),
132+
deviceTableScanNode.getPushDownLimit(),
133+
deviceTableScanNode.getPushDownOffset(),
134+
deviceTableScanNode.isPushLimitToEachDevice(),
135+
deviceTableScanNode.containsNonAlignedDevice(),
136+
treeDeviceViewScanNode.getTreeDBName(),
137+
treeDeviceViewScanNode.getMeasurementColumnNameMap());
138+
prunedNode.setRegionReplicaSet(deviceTableScanNode.getRegionReplicaSet());
139+
return Optional.of(prunedNode);
140+
} else if (node instanceof TreeDeviceViewScanNode) {
95141
TreeDeviceViewScanNode treeDeviceViewScanNode =
96142
(TreeDeviceViewScanNode) deviceTableScanNode;
97143
return Optional.of(
@@ -112,7 +158,7 @@ public static Optional<PlanNode> pruneColumns(TableScanNode node, Set<Symbol> re
112158
treeDeviceViewScanNode.getTreeDBName(),
113159
treeDeviceViewScanNode.getMeasurementColumnNameMap()));
114160
} else {
115-
return Optional.of(
161+
DeviceTableScanNode prunedNode =
116162
new DeviceTableScanNode(
117163
deviceTableScanNode.getPlanNodeId(),
118164
deviceTableScanNode.getQualifiedObjectName(),
@@ -126,7 +172,9 @@ public static Optional<PlanNode> pruneColumns(TableScanNode node, Set<Symbol> re
126172
deviceTableScanNode.getPushDownLimit(),
127173
deviceTableScanNode.getPushDownOffset(),
128174
deviceTableScanNode.isPushLimitToEachDevice(),
129-
deviceTableScanNode.containsNonAlignedDevice()));
175+
deviceTableScanNode.containsNonAlignedDevice());
176+
prunedNode.setRegionReplicaSet(deviceTableScanNode.getRegionReplicaSet());
177+
return Optional.of(prunedNode);
130178
}
131179
} else if (node instanceof InformationSchemaTableScanNode) {
132180
// For the convenience of process in execution stage, column-prune for

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,21 @@
2525
import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.FillNode;
2626
import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GapFillNode;
2727
import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.PatternRecognitionNode;
28+
import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.ProjectNode;
2829
import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode;
2930
import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.StreamSortNode;
3031
import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.ValueFillNode;
3132
import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.WindowNode;
3233
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
34+
import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneTableScanColumns;
3335
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode;
3436

37+
import com.google.common.collect.ImmutableSet;
38+
3539
import java.util.Collections;
40+
import java.util.List;
41+
import java.util.Optional;
42+
import java.util.Set;
3643

3744
/**
3845
* <b>Optimization phase:</b> Distributed plan planning.
@@ -42,6 +49,8 @@
4249
* SortNode can be eliminated.
4350
* <li>When order by all IDColumns and time, the SortNode can be eliminated.
4451
* <li>When StreamSortIndex==OrderBy size()-1, remove this StreamSortNode
52+
* <li>After SortNode elimination, visitProject will remove redundant identity ProjectNodes above
53+
* TableScan and pushes column pruning into the scan.
4554
*/
4655
public class SortElimination implements PlanOptimizer {
4756

@@ -64,6 +73,43 @@ public PlanNode visitPlan(PlanNode node, Context context) {
6473
return newNode;
6574
}
6675

76+
@Override
77+
public PlanNode visitProject(ProjectNode node, Context context) {
78+
Context newContext = new Context();
79+
PlanNode child = node.getChild().accept(this, newContext);
80+
context.setCannotEliminateSort(newContext.cannotEliminateSort);
81+
82+
// Remove useless ProjectNode and prune columns of TableScanNode
83+
return eliminateProjectOverTableScan(node, child)
84+
.orElseGet(() -> node.replaceChildren(Collections.singletonList(child)));
85+
}
86+
87+
private static Optional<PlanNode> eliminateProjectOverTableScan(
88+
ProjectNode project, PlanNode child) {
89+
if (!(child instanceof DeviceTableScanNode) || !project.isIdentity()) {
90+
return Optional.empty();
91+
}
92+
93+
// Notice that SortNode may have been eliminated in TableDistributedPlanGenerator
94+
DeviceTableScanNode tableScan = (DeviceTableScanNode) child;
95+
int projectOutputsSize = project.getOutputSymbols().size();
96+
int scanOutputsSize = tableScan.getOutputSymbols().size();
97+
if (projectOutputsSize > scanOutputsSize) {
98+
return Optional.empty();
99+
}
100+
101+
List<Symbol> projectOutputs = project.getOutputSymbols();
102+
Set<Symbol> scanOutputs = ImmutableSet.copyOf(tableScan.getOutputSymbols());
103+
if (!scanOutputs.containsAll(projectOutputs)) {
104+
return Optional.empty();
105+
}
106+
107+
if (projectOutputsSize == scanOutputsSize) {
108+
return Optional.of(tableScan);
109+
}
110+
return PruneTableScanColumns.pruneColumns(tableScan, ImmutableSet.copyOf(projectOutputs));
111+
}
112+
67113
@Override
68114
public PlanNode visitSort(SortNode node, Context context) {
69115
Context newContext = new Context();

‎iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,16 @@
3535
import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan;
3636
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode;
3737
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
38+
import org.apache.iotdb.db.queryengine.plan.relational.planner.PlanTester;
3839
import org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolAllocator;
3940
import org.apache.iotdb.db.queryengine.plan.relational.planner.TableLogicalPlanner;
4041
import org.apache.iotdb.db.queryengine.plan.relational.planner.distribute.TableDistributedPlanner;
4142
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode;
4243
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode;
4344
import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
4445

46+
import com.google.common.collect.ImmutableList;
47+
import com.google.common.collect.ImmutableSet;
4548
import org.junit.BeforeClass;
4649
import org.junit.Test;
4750

@@ -58,6 +61,12 @@
5861
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.TEST_MATADATA;
5962
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.assertTableScan;
6063
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.getChildrenNode;
64+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan;
65+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.exchange;
66+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.mergeSort;
67+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output;
68+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.project;
69+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.tableScan;
6170
import static org.apache.iotdb.db.queryengine.plan.statement.component.Ordering.ASC;
6271
import static org.apache.iotdb.db.queryengine.plan.statement.component.Ordering.DESC;
6372
import static org.junit.Assert.assertEquals;
@@ -767,4 +776,39 @@ public void assertTopKNoFilter(
767776
expectedPushDownOffset,
768777
isPushLimitToEachDevice);
769778
}
779+
780+
@Test
781+
public void singleDeviceOrderByAllIdsAndTimeDescTest() {
782+
PlanTester planTester = new PlanTester();
783+
planTester.createPlan(
784+
"SELECT s1, s2, s3 FROM table1 "
785+
+ "WHERE time >= 1000 AND time <= 2000 AND tag1='beijing' AND tag2='A1'"
786+
+ "ORDER BY tag1, tag2, tag3, time DESC");
787+
assertPlan(
788+
planTester.getFragmentPlan(0),
789+
output(
790+
tableScan(
791+
"testdb.table1",
792+
ImmutableList.of("s1", "s2", "s3"),
793+
ImmutableSet.of("time", "s1", "s2", "s3"))));
794+
}
795+
796+
@Test
797+
public void multiDeviceOrderByAllIdsAndTimeDescTest() {
798+
PlanTester planTester = new PlanTester();
799+
planTester.createPlan(
800+
"SELECT s1, s2, s3 FROM table1 "
801+
+ "WHERE time >= 1000 AND time <= 2000 AND tag1='beijing' "
802+
+ "ORDER BY tag1, tag2, tag3, time DESC");
803+
assertPlan(
804+
planTester.getFragmentPlan(0),
805+
output(project(mergeSort(exchange(), exchange(), exchange()))));
806+
// Device in multi-region
807+
assertPlan(
808+
planTester.getFragmentPlan(1),
809+
tableScan(
810+
"testdb.table1",
811+
ImmutableList.of("time", "tag1", "tag2", "tag3", "s1", "s2", "s3"),
812+
ImmutableSet.of("time", "tag1", "tag2", "tag3", "s1", "s2", "s3")));
813+
}
770814
}

0 commit comments

Comments
 (0)