Skip to content

Commit 2a092a7

Browse files
Jibing-LiJibing-Li
committed
Agg table set preagg on when doing sample analyzing.
1 parent c679c83 commit 2a092a7

File tree

22 files changed

+672
-149
lines changed

22 files changed

+672
-149
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ public enum TableFrom {
158158
private final List<Expression> joinFilters = new ArrayList<>();
159159

160160
private final List<Hint> hints = new ArrayList<>();
161+
private boolean hintForcePreAggOn = false;
161162

162163
// the columns in Plan.getExpressions(), such as columns in join condition or filter condition, group by expression
163164
private final Set<SlotReference> keySlots = Sets.newHashSet();
@@ -252,6 +253,14 @@ public void setNeedLockTables(boolean needLockTables) {
252253
this.needLockTables = needLockTables;
253254
}
254255

256+
public void setHintForcePreAggOn(boolean preAggOn) {
257+
this.hintForcePreAggOn = preAggOn;
258+
}
259+
260+
public boolean isHintForcePreAggOn() {
261+
return hintForcePreAggOn;
262+
}
263+
255264
/**
256265
* cache view info to avoid view's def and sql mode changed before lock it.
257266
*

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.apache.doris.nereids.rules.analysis.CollectSubQueryAlias;
3232
import org.apache.doris.nereids.rules.analysis.CompressedMaterialize;
3333
import org.apache.doris.nereids.rules.analysis.EliminateDistinctConstant;
34+
import org.apache.doris.nereids.rules.analysis.EliminateLogicalCommonHint;
3435
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
3536
import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots;
3637
import org.apache.doris.nereids.rules.analysis.FillUpQualifyMissingSlot;
@@ -96,7 +97,8 @@ private static List<RewriteJob> buildAnalyzerJobs() {
9697
return jobs(
9798
// we should eliminate hint before "Subquery unnesting".
9899
topDown(new AnalyzeCTE()),
99-
topDown(new EliminateLogicalSelectHint()),
100+
topDown(new EliminateLogicalSelectHint(),
101+
new EliminateLogicalCommonHint()),
100102
bottomUp(
101103
new BindRelation(),
102104
new CheckPolicy()

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 89 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -829,6 +829,7 @@
829829
import org.apache.doris.nereids.trees.plans.commands.use.UseCommand;
830830
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
831831
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
832+
import org.apache.doris.nereids.trees.plans.logical.LogicalCommonHint;
832833
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
833834
import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink;
834835
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
@@ -2041,12 +2042,15 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte
20412042
return selectPlan;
20422043
}
20432044
List<ParserRuleContext> selectHintContexts = Lists.newArrayList();
2045+
List<ParserRuleContext> commonHintContexts = Lists.newArrayList();
20442046
for (Integer key : selectHintMap.keySet()) {
20452047
if (key > selectCtx.getStart().getStopIndex() && key < selectCtx.getStop().getStartIndex()) {
20462048
selectHintContexts.add(selectHintMap.get(key));
2049+
} else {
2050+
commonHintContexts.add(selectHintMap.get(key));
20472051
}
20482052
}
2049-
return withSelectHint(selectPlan, selectHintContexts);
2053+
return withHints(selectPlan, selectHintContexts, commonHintContexts);
20502054
});
20512055
}
20522056

@@ -3702,82 +3706,101 @@ private List<List<String>> getTableList(List<MultipartIdentifierContext> ctx) {
37023706
return tableList;
37033707
}
37043708

3705-
private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleContext> hintContexts) {
3706-
if (hintContexts.isEmpty()) {
3709+
private LogicalPlan withHints(LogicalPlan logicalPlan, List<ParserRuleContext> selectHintContexts,
3710+
List<ParserRuleContext> commonHintContexts) {
3711+
if (selectHintContexts.isEmpty() && commonHintContexts.isEmpty()) {
37073712
return logicalPlan;
37083713
}
3709-
ImmutableList.Builder<SelectHint> hints = ImmutableList.builder();
3710-
for (ParserRuleContext hintContext : hintContexts) {
3711-
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3712-
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3713-
if (hintStatement.USE_MV() != null) {
3714-
hints.add(new SelectHintUseMv("USE_MV", getTableList(hintStatement.tableList), true));
3715-
continue;
3716-
} else if (hintStatement.NO_USE_MV() != null) {
3717-
hints.add(new SelectHintUseMv("NO_USE_MV", getTableList(hintStatement.tableList), false));
3718-
continue;
3719-
}
3720-
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3721-
switch (hintName) {
3722-
case "set_var":
3723-
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3724-
for (HintAssignmentContext kv : hintStatement.parameters) {
3725-
if (kv.key != null) {
3726-
String parameterName = visitIdentifierOrText(kv.key);
3727-
Optional<String> value = Optional.empty();
3728-
if (kv.constantValue != null) {
3729-
Literal literal = (Literal) visit(kv.constantValue);
3730-
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3731-
} else if (kv.identifierValue != null) {
3732-
// maybe we should throw exception when the identifierValue is quoted identifier
3733-
value = Optional.ofNullable(kv.identifierValue.getText());
3714+
LogicalPlan newPlan = logicalPlan;
3715+
if (!selectHintContexts.isEmpty()) {
3716+
ImmutableList.Builder<SelectHint> hints = ImmutableList.builder();
3717+
for (ParserRuleContext hintContext : selectHintContexts) {
3718+
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3719+
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3720+
if (hintStatement.USE_MV() != null) {
3721+
hints.add(new SelectHintUseMv("USE_MV", getTableList(hintStatement.tableList), true));
3722+
continue;
3723+
} else if (hintStatement.NO_USE_MV() != null) {
3724+
hints.add(new SelectHintUseMv("NO_USE_MV", getTableList(hintStatement.tableList), false));
3725+
continue;
3726+
}
3727+
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3728+
switch (hintName) {
3729+
case "set_var":
3730+
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3731+
for (HintAssignmentContext kv : hintStatement.parameters) {
3732+
if (kv.key != null) {
3733+
String parameterName = visitIdentifierOrText(kv.key);
3734+
Optional<String> value = Optional.empty();
3735+
if (kv.constantValue != null) {
3736+
Literal literal = (Literal) visit(kv.constantValue);
3737+
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3738+
} else if (kv.identifierValue != null) {
3739+
// maybe we should throw exception when the identifierValue is quoted identifier
3740+
value = Optional.ofNullable(kv.identifierValue.getText());
3741+
}
3742+
parameters.put(parameterName, value);
37343743
}
3735-
parameters.put(parameterName, value);
37363744
}
3737-
}
3738-
SelectHintSetVar setVar = new SelectHintSetVar(hintName, parameters);
3739-
setVar.setVarOnceInSql(ConnectContext.get().getStatementContext());
3740-
hints.add(setVar);
3741-
break;
3742-
case "leading":
3743-
List<String> leadingParameters = new ArrayList<>();
3744-
for (HintAssignmentContext kv : hintStatement.parameters) {
3745-
if (kv.key != null) {
3746-
String parameterName = visitIdentifierOrText(kv.key);
3747-
leadingParameters.add(parameterName);
3745+
SelectHintSetVar setVar = new SelectHintSetVar(hintName, parameters);
3746+
setVar.setVarOnceInSql(ConnectContext.get().getStatementContext());
3747+
hints.add(setVar);
3748+
break;
3749+
case "leading":
3750+
List<String> leadingParameters = new ArrayList<>();
3751+
for (HintAssignmentContext kv : hintStatement.parameters) {
3752+
if (kv.key != null) {
3753+
String parameterName = visitIdentifierOrText(kv.key);
3754+
leadingParameters.add(parameterName);
3755+
}
37483756
}
3749-
}
3750-
hints.add(new SelectHintLeading(hintName, leadingParameters));
3751-
break;
3752-
case "ordered":
3753-
hints.add(new SelectHintOrdered(hintName));
3754-
break;
3755-
case "use_cbo_rule":
3756-
List<String> useRuleParameters = new ArrayList<>();
3757-
for (HintAssignmentContext kv : hintStatement.parameters) {
3758-
if (kv.key != null) {
3759-
String parameterName = visitIdentifierOrText(kv.key);
3760-
useRuleParameters.add(parameterName);
3757+
hints.add(new SelectHintLeading(hintName, leadingParameters));
3758+
break;
3759+
case "ordered":
3760+
hints.add(new SelectHintOrdered(hintName));
3761+
break;
3762+
case "use_cbo_rule":
3763+
List<String> useRuleParameters = new ArrayList<>();
3764+
for (HintAssignmentContext kv : hintStatement.parameters) {
3765+
if (kv.key != null) {
3766+
String parameterName = visitIdentifierOrText(kv.key);
3767+
useRuleParameters.add(parameterName);
3768+
}
37613769
}
3762-
}
3763-
hints.add(new SelectHintUseCboRule(hintName, useRuleParameters, false));
3764-
break;
3765-
case "no_use_cbo_rule":
3766-
List<String> noUseRuleParameters = new ArrayList<>();
3767-
for (HintAssignmentContext kv : hintStatement.parameters) {
3768-
String parameterName = visitIdentifierOrText(kv.key);
3769-
if (kv.key != null) {
3770-
noUseRuleParameters.add(parameterName);
3770+
hints.add(new SelectHintUseCboRule(hintName, useRuleParameters, false));
3771+
break;
3772+
case "no_use_cbo_rule":
3773+
List<String> noUseRuleParameters = new ArrayList<>();
3774+
for (HintAssignmentContext kv : hintStatement.parameters) {
3775+
String parameterName = visitIdentifierOrText(kv.key);
3776+
if (kv.key != null) {
3777+
noUseRuleParameters.add(parameterName);
3778+
}
37713779
}
3780+
hints.add(new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3781+
break;
3782+
default:
3783+
break;
3784+
}
3785+
}
3786+
}
3787+
newPlan = new LogicalSelectHint<>(hints.build(), newPlan);
3788+
}
3789+
if (!commonHintContexts.isEmpty()) {
3790+
for (ParserRuleContext hintContext : commonHintContexts) {
3791+
if (hintContext instanceof SelectHintContext) {
3792+
SelectHintContext commonHintContext = (SelectHintContext) hintContext;
3793+
if (commonHintContext.hintStatement != null && commonHintContext.hintStatement.hintName != null) {
3794+
String text = commonHintContext.hintStatement.hintName.getText();
3795+
if (text.equalsIgnoreCase("PREAGGOPEN")) {
3796+
newPlan = new LogicalCommonHint<>(newPlan);
3797+
break;
37723798
}
3773-
hints.add(new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3774-
break;
3775-
default:
3776-
break;
3799+
}
37773800
}
37783801
}
37793802
}
3780-
return new LogicalSelectHint<>(hints.build(), logicalPlan);
3803+
return newPlan;
37813804
}
37823805

37833806
@Override

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ public enum RuleType {
138138
ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE),
139139

140140
ELIMINATE_LOGICAL_SELECT_HINT(RuleTypeClass.REWRITE),
141+
ELIMINATE_LOGICAL_COMMON_HINT(RuleTypeClass.REWRITE),
141142
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
142143
ELIMINATE_ORDER_BY_UNDER_SUBQUERY(RuleTypeClass.REWRITE),
143144
ELIMINATE_ORDER_BY_UNDER_VIEW(RuleTypeClass.REWRITE),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@
9696
import com.google.common.collect.Lists;
9797
import com.google.common.collect.Sets;
9898
import org.apache.commons.collections.CollectionUtils;
99+
import org.apache.logging.log4j.LogManager;
100+
import org.apache.logging.log4j.Logger;
99101

100102
import java.util.ArrayList;
101103
import java.util.List;
@@ -105,6 +107,7 @@
105107
* Rule to bind relations in query plan.
106108
*/
107109
public class BindRelation extends OneAnalysisRuleFactory {
110+
private static final Logger LOG = LogManager.getLogger(StatementContext.class);
108111

109112
public BindRelation() {}
110113

@@ -179,7 +182,8 @@ private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboun
179182
return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
180183
}
181184

182-
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier) {
185+
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier,
186+
CascadesContext cascadesContext) {
183187
LogicalOlapScan scan;
184188
List<Long> partIds = getPartitionIds(table, unboundRelation, qualifier);
185189
List<Long> tabletIds = unboundRelation.getTabletIds();
@@ -217,6 +221,9 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
217221
// This tabletIds is set manually, so need to set specifiedTabletIds
218222
scan = scan.withManuallySpecifiedTabletIds(tabletIds);
219223
}
224+
if (cascadesContext.getStatementContext().isHintForcePreAggOn()) {
225+
return scan.withPreAggStatus(PreAggStatus.on());
226+
}
220227
if (needGenerateLogicalAggForRandomDistAggTable(scan)) {
221228
// it's a random distribution agg table
222229
// add agg on olap scan
@@ -383,7 +390,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
383390
switch (table.getType()) {
384391
case OLAP:
385392
case MATERIALIZED_VIEW:
386-
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName);
393+
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName, cascadesContext);
387394
case VIEW:
388395
View view = (View) table;
389396
isView = true;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalCommonHint.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.analysis;
19+
20+
import org.apache.doris.nereids.rules.Rule;
21+
import org.apache.doris.nereids.rules.RuleType;
22+
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
23+
import org.apache.doris.nereids.trees.plans.Plan;
24+
import org.apache.doris.nereids.trees.plans.logical.LogicalCommonHint;
25+
26+
import com.google.common.base.Preconditions;
27+
28+
/**
29+
* eliminate logical common hint and set them to cascade context
30+
*/
31+
public class EliminateLogicalCommonHint extends OneRewriteRuleFactory {
32+
33+
@Override
34+
public Rule build() {
35+
return logicalCommonHint().thenApply(ctx -> {
36+
LogicalCommonHint<Plan> commonHintPlan = ctx.root;
37+
Preconditions.checkArgument(commonHintPlan.preAggEnabledByHint(), "common hint only support PREAGGOPEN");
38+
ctx.statementContext.setHintForcePreAggOn(true);
39+
return commonHintPlan.child();
40+
}).toRule(RuleType.ELIMINATE_LOGICAL_SELECT_HINT);
41+
}
42+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public enum PlanType {
6262
LOGICAL_APPLY,
6363
LOGICAL_ASSERT_NUM_ROWS,
6464
LOGICAL_CHECK_POLICY,
65+
LOGICAL_COMMON_HINT,
6566
LOGICAL_CTE,
6667
LOGICAL_CTE_ANCHOR,
6768
LOGICAL_CTE_PRODUCER,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.doris.nereids.rules.analysis.BindExpression;
4242
import org.apache.doris.nereids.rules.analysis.BindRelation;
4343
import org.apache.doris.nereids.rules.analysis.CheckPolicy;
44+
import org.apache.doris.nereids.rules.analysis.EliminateLogicalCommonHint;
4445
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
4546
import org.apache.doris.nereids.trees.expressions.Expression;
4647
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -263,7 +264,8 @@ public List<RewriteJob> getJobs() {
263264

264265
private static List<RewriteJob> buildAnalyzeViewJobsForStar() {
265266
return jobs(
266-
topDown(new EliminateLogicalSelectHint()),
267+
topDown(new EliminateLogicalSelectHint(),
268+
new EliminateLogicalCommonHint()),
267269
topDown(new AnalyzeCTE()),
268270
bottomUp(
269271
new BindRelation(),

0 commit comments

Comments
 (0)