Skip to content

Commit e7e8c6d

Browse files
Jibing-LiJibing-Li
committed
Agg table set preagg on when doing sample analyzing.
1 parent b8eb186 commit e7e8c6d

File tree

20 files changed

+659
-97
lines changed

20 files changed

+659
-97
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ public enum TableFrom {
158158
private final List<Expression> joinFilters = new ArrayList<>();
159159

160160
private final List<Hint> hints = new ArrayList<>();
161+
private boolean hintForcePreAggOn = false;
161162

162163
// the columns in Plan.getExpressions(), such as columns in join condition or filter condition, group by expression
163164
private final Set<SlotReference> keySlots = Sets.newHashSet();
@@ -252,6 +253,14 @@ public void setNeedLockTables(boolean needLockTables) {
252253
this.needLockTables = needLockTables;
253254
}
254255

256+
public void setHintForcePreAggOn(boolean preAggOn) {
257+
this.hintForcePreAggOn = preAggOn;
258+
}
259+
260+
public boolean isHintForcePreAggOn() {
261+
return hintForcePreAggOn;
262+
}
263+
255264
/**
256265
* cache view info to avoid view's def and sql mode changed before lock it.
257266
*

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.doris.nereids.rules.analysis.CompressedMaterialize;
3333
import org.apache.doris.nereids.rules.analysis.EliminateDistinctConstant;
3434
import org.apache.doris.nereids.rules.analysis.EliminateGroupByConstant;
35+
import org.apache.doris.nereids.rules.analysis.EliminateLogicalCommonHint;
3536
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
3637
import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots;
3738
import org.apache.doris.nereids.rules.analysis.FillUpQualifyMissingSlot;
@@ -97,7 +98,8 @@ private static List<RewriteJob> buildAnalyzerJobs() {
9798
return jobs(
9899
// we should eliminate hint before "Subquery unnesting".
99100
topDown(new AnalyzeCTE()),
100-
topDown(new EliminateLogicalSelectHint()),
101+
topDown(new EliminateLogicalSelectHint(),
102+
new EliminateLogicalCommonHint()),
101103
bottomUp(
102104
new BindRelation(),
103105
new CheckPolicy()

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 86 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -813,6 +813,7 @@
813813
import org.apache.doris.nereids.trees.plans.commands.use.UseCommand;
814814
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
815815
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
816+
import org.apache.doris.nereids.trees.plans.logical.LogicalCommonHint;
816817
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
817818
import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink;
818819
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
@@ -2028,12 +2029,15 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte
20282029
return selectPlan;
20292030
}
20302031
List<ParserRuleContext> selectHintContexts = Lists.newArrayList();
2032+
List<ParserRuleContext> commonHintContexts = Lists.newArrayList();
20312033
for (Integer key : selectHintMap.keySet()) {
20322034
if (key > selectCtx.getStart().getStopIndex() && key < selectCtx.getStop().getStartIndex()) {
20332035
selectHintContexts.add(selectHintMap.get(key));
2036+
} else {
2037+
commonHintContexts.add(selectHintMap.get(key));
20342038
}
20352039
}
2036-
return withSelectHint(selectPlan, selectHintContexts);
2040+
return withHints(selectPlan, selectHintContexts, commonHintContexts);
20372041
});
20382042
}
20392043

@@ -3689,82 +3693,97 @@ private List<List<String>> getTableList(List<MultipartIdentifierContext> ctx) {
36893693
return tableList;
36903694
}
36913695

3692-
private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleContext> hintContexts) {
3693-
if (hintContexts.isEmpty()) {
3696+
private LogicalPlan withHints(LogicalPlan logicalPlan, List<ParserRuleContext> selectHintContexts,
3697+
List<ParserRuleContext> commonHintContexts) {
3698+
if (selectHintContexts.isEmpty() && commonHintContexts.isEmpty()) {
36943699
return logicalPlan;
36953700
}
3696-
ImmutableList.Builder<SelectHint> hints = ImmutableList.builder();
3697-
for (ParserRuleContext hintContext : hintContexts) {
3698-
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3699-
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3700-
if (hintStatement.USE_MV() != null) {
3701-
hints.add(new SelectHintUseMv("USE_MV", getTableList(hintStatement.tableList), true));
3702-
continue;
3703-
} else if (hintStatement.NO_USE_MV() != null) {
3704-
hints.add(new SelectHintUseMv("NO_USE_MV", getTableList(hintStatement.tableList), false));
3705-
continue;
3706-
}
3707-
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3708-
switch (hintName) {
3709-
case "set_var":
3710-
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3711-
for (HintAssignmentContext kv : hintStatement.parameters) {
3712-
if (kv.key != null) {
3713-
String parameterName = visitIdentifierOrText(kv.key);
3714-
Optional<String> value = Optional.empty();
3715-
if (kv.constantValue != null) {
3716-
Literal literal = (Literal) visit(kv.constantValue);
3717-
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3718-
} else if (kv.identifierValue != null) {
3719-
// maybe we should throw exception when the identifierValue is quoted identifier
3720-
value = Optional.ofNullable(kv.identifierValue.getText());
3701+
LogicalPlan newPlan = logicalPlan;
3702+
if (!selectHintContexts.isEmpty()) {
3703+
ImmutableList.Builder<SelectHint> hints = ImmutableList.builder();
3704+
for (ParserRuleContext hintContext : selectHintContexts) {
3705+
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3706+
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3707+
if (hintStatement.USE_MV() != null) {
3708+
hints.add(new SelectHintUseMv("USE_MV", getTableList(hintStatement.tableList), true));
3709+
continue;
3710+
} else if (hintStatement.NO_USE_MV() != null) {
3711+
hints.add(new SelectHintUseMv("NO_USE_MV", getTableList(hintStatement.tableList), false));
3712+
continue;
3713+
}
3714+
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3715+
switch (hintName) {
3716+
case "set_var":
3717+
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3718+
for (HintAssignmentContext kv : hintStatement.parameters) {
3719+
if (kv.key != null) {
3720+
String parameterName = visitIdentifierOrText(kv.key);
3721+
Optional<String> value = Optional.empty();
3722+
if (kv.constantValue != null) {
3723+
Literal literal = (Literal) visit(kv.constantValue);
3724+
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3725+
} else if (kv.identifierValue != null) {
3726+
// maybe we should throw exception when the identifierValue is quoted identifier
3727+
value = Optional.ofNullable(kv.identifierValue.getText());
3728+
}
3729+
parameters.put(parameterName, value);
37213730
}
3722-
parameters.put(parameterName, value);
37233731
}
3724-
}
3725-
SelectHintSetVar setVar = new SelectHintSetVar(hintName, parameters);
3726-
setVar.setVarOnceInSql(ConnectContext.get().getStatementContext());
3727-
hints.add(setVar);
3728-
break;
3729-
case "leading":
3730-
List<String> leadingParameters = new ArrayList<>();
3731-
for (HintAssignmentContext kv : hintStatement.parameters) {
3732-
if (kv.key != null) {
3733-
String parameterName = visitIdentifierOrText(kv.key);
3734-
leadingParameters.add(parameterName);
3732+
SelectHintSetVar setVar = new SelectHintSetVar(hintName, parameters);
3733+
setVar.setVarOnceInSql(ConnectContext.get().getStatementContext());
3734+
hints.add(setVar);
3735+
break;
3736+
case "leading":
3737+
List<String> leadingParameters = new ArrayList<>();
3738+
for (HintAssignmentContext kv : hintStatement.parameters) {
3739+
if (kv.key != null) {
3740+
String parameterName = visitIdentifierOrText(kv.key);
3741+
leadingParameters.add(parameterName);
3742+
}
37353743
}
3736-
}
3737-
hints.add(new SelectHintLeading(hintName, leadingParameters));
3738-
break;
3739-
case "ordered":
3740-
hints.add(new SelectHintOrdered(hintName));
3741-
break;
3742-
case "use_cbo_rule":
3743-
List<String> useRuleParameters = new ArrayList<>();
3744-
for (HintAssignmentContext kv : hintStatement.parameters) {
3745-
if (kv.key != null) {
3746-
String parameterName = visitIdentifierOrText(kv.key);
3747-
useRuleParameters.add(parameterName);
3744+
hints.add(new SelectHintLeading(hintName, leadingParameters));
3745+
break;
3746+
case "ordered":
3747+
hints.add(new SelectHintOrdered(hintName));
3748+
break;
3749+
case "use_cbo_rule":
3750+
List<String> useRuleParameters = new ArrayList<>();
3751+
for (HintAssignmentContext kv : hintStatement.parameters) {
3752+
if (kv.key != null) {
3753+
String parameterName = visitIdentifierOrText(kv.key);
3754+
useRuleParameters.add(parameterName);
3755+
}
37483756
}
3749-
}
3750-
hints.add(new SelectHintUseCboRule(hintName, useRuleParameters, false));
3751-
break;
3752-
case "no_use_cbo_rule":
3753-
List<String> noUseRuleParameters = new ArrayList<>();
3754-
for (HintAssignmentContext kv : hintStatement.parameters) {
3755-
String parameterName = visitIdentifierOrText(kv.key);
3756-
if (kv.key != null) {
3757-
noUseRuleParameters.add(parameterName);
3757+
hints.add(new SelectHintUseCboRule(hintName, useRuleParameters, false));
3758+
break;
3759+
case "no_use_cbo_rule":
3760+
List<String> noUseRuleParameters = new ArrayList<>();
3761+
for (HintAssignmentContext kv : hintStatement.parameters) {
3762+
String parameterName = visitIdentifierOrText(kv.key);
3763+
if (kv.key != null) {
3764+
noUseRuleParameters.add(parameterName);
3765+
}
37583766
}
3759-
}
3760-
hints.add(new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3761-
break;
3762-
default:
3763-
break;
3767+
hints.add(new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3768+
break;
3769+
default:
3770+
break;
3771+
}
3772+
}
3773+
}
3774+
newPlan = new LogicalSelectHint<>(hints.build(), newPlan);
3775+
}
3776+
if (!commonHintContexts.isEmpty()) {
3777+
for (ParserRuleContext hintContext : commonHintContexts) {
3778+
SelectHintContext commonHintContext = (SelectHintContext) hintContext;
3779+
String text = commonHintContext.hintStatement.hintName.getText();
3780+
if (text.equalsIgnoreCase("PREAGGOPEN")) {
3781+
newPlan = new LogicalCommonHint<>(newPlan);
3782+
break;
37643783
}
37653784
}
37663785
}
3767-
return new LogicalSelectHint<>(hints.build(), logicalPlan);
3786+
return newPlan;
37683787
}
37693788

37703789
@Override

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ public enum RuleType {
138138
ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE),
139139

140140
ELIMINATE_LOGICAL_SELECT_HINT(RuleTypeClass.REWRITE),
141+
ELIMINATE_LOGICAL_COMMON_HINT(RuleTypeClass.REWRITE),
141142
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
142143
ELIMINATE_ORDER_BY_UNDER_SUBQUERY(RuleTypeClass.REWRITE),
143144
ELIMINATE_ORDER_BY_UNDER_VIEW(RuleTypeClass.REWRITE),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@
9696
import com.google.common.collect.Lists;
9797
import com.google.common.collect.Sets;
9898
import org.apache.commons.collections.CollectionUtils;
99+
import org.apache.logging.log4j.LogManager;
100+
import org.apache.logging.log4j.Logger;
99101

100102
import java.util.ArrayList;
101103
import java.util.List;
@@ -105,6 +107,7 @@
105107
* Rule to bind relations in query plan.
106108
*/
107109
public class BindRelation extends OneAnalysisRuleFactory {
110+
private static final Logger LOG = LogManager.getLogger(StatementContext.class);
108111

109112
public BindRelation() {}
110113

@@ -179,7 +182,8 @@ private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboun
179182
return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
180183
}
181184

182-
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier) {
185+
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier,
186+
CascadesContext cascadesContext) {
183187
LogicalOlapScan scan;
184188
List<Long> partIds = getPartitionIds(table, unboundRelation, qualifier);
185189
List<Long> tabletIds = unboundRelation.getTabletIds();
@@ -217,6 +221,9 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
217221
// This tabletIds is set manually, so need to set specifiedTabletIds
218222
scan = scan.withManuallySpecifiedTabletIds(tabletIds);
219223
}
224+
if (cascadesContext.getStatementContext().isHintForcePreAggOn()) {
225+
return scan.withPreAggStatus(PreAggStatus.on());
226+
}
220227
if (needGenerateLogicalAggForRandomDistAggTable(scan)) {
221228
// it's a random distribution agg table
222229
// add agg on olap scan
@@ -383,7 +390,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
383390
switch (table.getType()) {
384391
case OLAP:
385392
case MATERIALIZED_VIEW:
386-
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName);
393+
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName, cascadesContext);
387394
case VIEW:
388395
View view = (View) table;
389396
isView = true;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalCommonHint.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.analysis;
19+
20+
import org.apache.doris.nereids.rules.Rule;
21+
import org.apache.doris.nereids.rules.RuleType;
22+
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
23+
import org.apache.doris.nereids.trees.plans.Plan;
24+
import org.apache.doris.nereids.trees.plans.logical.LogicalCommonHint;
25+
26+
import com.google.common.base.Preconditions;
27+
28+
/**
29+
* eliminate logical common hint and set them to cascade context
30+
*/
31+
public class EliminateLogicalCommonHint extends OneRewriteRuleFactory {
32+
33+
@Override
34+
public Rule build() {
35+
return logicalCommonHint().thenApply(ctx -> {
36+
LogicalCommonHint<Plan> commonHintPlan = ctx.root;
37+
Preconditions.checkArgument(commonHintPlan.preAggEnabledByHint(), "common hint only support PREAGGOPEN");
38+
ctx.statementContext.setHintForcePreAggOn(true);
39+
return commonHintPlan.child();
40+
}).toRule(RuleType.ELIMINATE_LOGICAL_SELECT_HINT);
41+
}
42+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public enum PlanType {
6262
LOGICAL_APPLY,
6363
LOGICAL_ASSERT_NUM_ROWS,
6464
LOGICAL_CHECK_POLICY,
65+
LOGICAL_COMMON_HINT,
6566
LOGICAL_CTE,
6667
LOGICAL_CTE_ANCHOR,
6768
LOGICAL_CTE_PRODUCER,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.doris.nereids.rules.analysis.BindExpression;
4242
import org.apache.doris.nereids.rules.analysis.BindRelation;
4343
import org.apache.doris.nereids.rules.analysis.CheckPolicy;
44+
import org.apache.doris.nereids.rules.analysis.EliminateLogicalCommonHint;
4445
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
4546
import org.apache.doris.nereids.trees.expressions.Expression;
4647
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -263,7 +264,8 @@ public List<RewriteJob> getJobs() {
263264

264265
private static List<RewriteJob> buildAnalyzeViewJobsForStar() {
265266
return jobs(
266-
topDown(new EliminateLogicalSelectHint()),
267+
topDown(new EliminateLogicalSelectHint(),
268+
new EliminateLogicalCommonHint()),
267269
topDown(new AnalyzeCTE()),
268270
bottomUp(
269271
new BindRelation(),

0 commit comments

Comments
 (0)