Skip to content

Commit ce42bb5

Browse files
Jibing-LiJibing-Li
authored
branch-3.0:[improvement](statistics)Agg table set preagg on when doing sample analyzing. (#49918) (#51672)
backport: #49918
1 parent c381a0b commit ce42bb5

23 files changed

+730
-147
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ public enum TableFrom {
156156
private final List<Expression> joinFilters = new ArrayList<>();
157157

158158
private final List<Hint> hints = new ArrayList<>();
159+
private boolean hintForcePreAggOn = false;
159160

160161
// the columns in Plan.getExpressions(), such as columns in join condition or filter condition, group by expression
161162
private final Set<SlotReference> keySlots = Sets.newHashSet();
@@ -246,6 +247,14 @@ public void setNeedLockTables(boolean needLockTables) {
246247
this.needLockTables = needLockTables;
247248
}
248249

250+
public void setHintForcePreAggOn(boolean preAggOn) {
251+
this.hintForcePreAggOn = preAggOn;
252+
}
253+
254+
public boolean isHintForcePreAggOn() {
255+
return hintForcePreAggOn;
256+
}
257+
249258
/**
250259
* cache view info to avoid view's def and sql mode changed before lock it.
251260
*

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.doris.nereids.rules.analysis.CollectJoinConstraint;
3131
import org.apache.doris.nereids.rules.analysis.CollectSubQueryAlias;
3232
import org.apache.doris.nereids.rules.analysis.EliminateDistinctConstant;
33+
import org.apache.doris.nereids.rules.analysis.EliminateLogicalPreAggOnHint;
3334
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
3435
import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots;
3536
import org.apache.doris.nereids.rules.analysis.HavingToFilter;
@@ -93,7 +94,8 @@ private static List<RewriteJob> buildAnalyzerJobs() {
9394
return jobs(
9495
// we should eliminate hint before "Subquery unnesting".
9596
topDown(new AnalyzeCTE()),
96-
topDown(new EliminateLogicalSelectHint()),
97+
topDown(new EliminateLogicalSelectHint(),
98+
new EliminateLogicalPreAggOnHint()),
9799
bottomUp(
98100
new BindRelation(),
99101
new CheckPolicy()

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 81 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -459,6 +459,7 @@
459459
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
460460
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
461461
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
462+
import org.apache.doris.nereids.trees.plans.logical.LogicalPreAggOnHint;
462463
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
463464
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
464465
import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
@@ -1460,12 +1461,15 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte
14601461
return selectPlan;
14611462
}
14621463
List<ParserRuleContext> selectHintContexts = Lists.newArrayList();
1464+
List<ParserRuleContext> preAggOnHintContexts = Lists.newArrayList();
14631465
for (Integer key : selectHintMap.keySet()) {
14641466
if (key > selectCtx.getStart().getStopIndex() && key < selectCtx.getStop().getStartIndex()) {
14651467
selectHintContexts.add(selectHintMap.get(key));
1468+
} else {
1469+
preAggOnHintContexts.add(selectHintMap.get(key));
14661470
}
14671471
}
1468-
return withSelectHint(selectPlan, selectHintContexts);
1472+
return withHints(selectPlan, selectHintContexts, preAggOnHintContexts);
14691473
});
14701474
}
14711475

@@ -3274,73 +3278,93 @@ private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) {
32743278
return last;
32753279
}
32763280

3277-
private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleContext> hintContexts) {
3278-
if (hintContexts.isEmpty()) {
3281+
private LogicalPlan withHints(LogicalPlan logicalPlan, List<ParserRuleContext> selectHintContexts,
3282+
List<ParserRuleContext> preAggOnHintContexts) {
3283+
if (selectHintContexts.isEmpty() && preAggOnHintContexts.isEmpty()) {
32793284
return logicalPlan;
32803285
}
3281-
Map<String, SelectHint> hints = Maps.newLinkedHashMap();
3282-
for (ParserRuleContext hintContext : hintContexts) {
3283-
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3284-
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3285-
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3286-
switch (hintName) {
3287-
case "set_var":
3288-
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3289-
for (HintAssignmentContext kv : hintStatement.parameters) {
3290-
if (kv.key != null) {
3291-
String parameterName = visitIdentifierOrText(kv.key);
3292-
Optional<String> value = Optional.empty();
3293-
if (kv.constantValue != null) {
3294-
Literal literal = (Literal) visit(kv.constantValue);
3295-
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3296-
} else if (kv.identifierValue != null) {
3297-
// maybe we should throw exception when the identifierValue is quoted identifier
3298-
value = Optional.ofNullable(kv.identifierValue.getText());
3286+
LogicalPlan newPlan = logicalPlan;
3287+
if (!selectHintContexts.isEmpty()) {
3288+
Map<String, SelectHint> hints = Maps.newLinkedHashMap();
3289+
for (ParserRuleContext hintContext : selectHintContexts) {
3290+
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3291+
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3292+
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3293+
switch (hintName) {
3294+
case "set_var":
3295+
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3296+
for (HintAssignmentContext kv : hintStatement.parameters) {
3297+
if (kv.key != null) {
3298+
String parameterName = visitIdentifierOrText(kv.key);
3299+
Optional<String> value = Optional.empty();
3300+
if (kv.constantValue != null) {
3301+
Literal literal = (Literal) visit(kv.constantValue);
3302+
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3303+
} else if (kv.identifierValue != null) {
3304+
// maybe we should throw exception when the identifierValue is quoted identifier
3305+
value = Optional.ofNullable(kv.identifierValue.getText());
3306+
}
3307+
parameters.put(parameterName, value);
32993308
}
3300-
parameters.put(parameterName, value);
33013309
}
3302-
}
3303-
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
3304-
break;
3305-
case "leading":
3306-
List<String> leadingParameters = new ArrayList<>();
3307-
for (HintAssignmentContext kv : hintStatement.parameters) {
3308-
if (kv.key != null) {
3309-
String parameterName = visitIdentifierOrText(kv.key);
3310-
leadingParameters.add(parameterName);
3310+
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
3311+
break;
3312+
case "leading":
3313+
List<String> leadingParameters = new ArrayList<>();
3314+
for (HintAssignmentContext kv : hintStatement.parameters) {
3315+
if (kv.key != null) {
3316+
String parameterName = visitIdentifierOrText(kv.key);
3317+
leadingParameters.add(parameterName);
3318+
}
33113319
}
3312-
}
3313-
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
3314-
break;
3315-
case "ordered":
3316-
hints.put(hintName, new SelectHintOrdered(hintName));
3317-
break;
3318-
case "use_cbo_rule":
3319-
List<String> useRuleParameters = new ArrayList<>();
3320-
for (HintAssignmentContext kv : hintStatement.parameters) {
3321-
if (kv.key != null) {
3322-
String parameterName = visitIdentifierOrText(kv.key);
3323-
useRuleParameters.add(parameterName);
3320+
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
3321+
break;
3322+
case "ordered":
3323+
hints.put(hintName, new SelectHintOrdered(hintName));
3324+
break;
3325+
case "use_cbo_rule":
3326+
List<String> useRuleParameters = new ArrayList<>();
3327+
for (HintAssignmentContext kv : hintStatement.parameters) {
3328+
if (kv.key != null) {
3329+
String parameterName = visitIdentifierOrText(kv.key);
3330+
useRuleParameters.add(parameterName);
3331+
}
33243332
}
3325-
}
3326-
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
3327-
break;
3328-
case "no_use_cbo_rule":
3329-
List<String> noUseRuleParameters = new ArrayList<>();
3330-
for (HintAssignmentContext kv : hintStatement.parameters) {
3331-
String parameterName = visitIdentifierOrText(kv.key);
3332-
if (kv.key != null) {
3333-
noUseRuleParameters.add(parameterName);
3333+
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
3334+
break;
3335+
case "no_use_cbo_rule":
3336+
List<String> noUseRuleParameters = new ArrayList<>();
3337+
for (HintAssignmentContext kv : hintStatement.parameters) {
3338+
String parameterName = visitIdentifierOrText(kv.key);
3339+
if (kv.key != null) {
3340+
noUseRuleParameters.add(parameterName);
3341+
}
33343342
}
3343+
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3344+
break;
3345+
default:
3346+
break;
3347+
}
3348+
}
3349+
}
3350+
newPlan = new LogicalSelectHint<>(hints, newPlan);
3351+
}
3352+
if (!preAggOnHintContexts.isEmpty()) {
3353+
for (ParserRuleContext hintContext : preAggOnHintContexts) {
3354+
if (hintContext instanceof SelectHintContext) {
3355+
SelectHintContext preAggOnHintContext = (SelectHintContext) hintContext;
3356+
if (preAggOnHintContext.hintStatement != null
3357+
&& preAggOnHintContext.hintStatement.hintName != null) {
3358+
String text = preAggOnHintContext.hintStatement.hintName.getText();
3359+
if (text.equalsIgnoreCase("PREAGGOPEN")) {
3360+
newPlan = new LogicalPreAggOnHint<>(newPlan);
3361+
break;
33353362
}
3336-
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3337-
break;
3338-
default:
3339-
break;
3363+
}
33403364
}
33413365
}
33423366
}
3343-
return new LogicalSelectHint<>(hints, logicalPlan);
3367+
return newPlan;
33443368
}
33453369

33463370
@Override

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ public enum RuleType {
122122
ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE),
123123

124124
ELIMINATE_LOGICAL_SELECT_HINT(RuleTypeClass.REWRITE),
125+
ELIMINATE_LOGICAL_PRE_AGG_ON_HINT(RuleTypeClass.REWRITE),
125126
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
126127
ELIMINATE_ORDER_BY_UNDER_SUBQUERY(RuleTypeClass.REWRITE),
127128
ELIMINATE_ORDER_BY_UNDER_VIEW(RuleTypeClass.REWRITE),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@
9595
import com.google.common.collect.Lists;
9696
import com.google.common.collect.Sets;
9797
import org.apache.commons.collections.CollectionUtils;
98+
import org.apache.logging.log4j.LogManager;
99+
import org.apache.logging.log4j.Logger;
98100

99101
import java.util.ArrayList;
100102
import java.util.List;
@@ -104,6 +106,7 @@
104106
* Rule to bind relations in query plan.
105107
*/
106108
public class BindRelation extends OneAnalysisRuleFactory {
109+
private static final Logger LOG = LogManager.getLogger(StatementContext.class);
107110

108111
public BindRelation() {}
109112

@@ -178,7 +181,8 @@ private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboun
178181
return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
179182
}
180183

181-
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier) {
184+
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier,
185+
CascadesContext cascadesContext) {
182186
LogicalOlapScan scan;
183187
List<Long> partIds = getPartitionIds(table, unboundRelation, qualifier);
184188
List<Long> tabletIds = unboundRelation.getTabletIds();
@@ -214,6 +218,9 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
214218
// This tabletIds is set manually, so need to set specifiedTabletIds
215219
scan = scan.withManuallySpecifiedTabletIds(tabletIds);
216220
}
221+
if (cascadesContext.getStatementContext().isHintForcePreAggOn()) {
222+
return scan.withPreAggStatus(PreAggStatus.on());
223+
}
217224
if (needGenerateLogicalAggForRandomDistAggTable(scan)) {
218225
// it's a random distribution agg table
219226
// add agg on olap scan
@@ -380,7 +387,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
380387
switch (table.getType()) {
381388
case OLAP:
382389
case MATERIALIZED_VIEW:
383-
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName);
390+
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName, cascadesContext);
384391
case VIEW:
385392
View view = (View) table;
386393
isView = true;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalPreAggOnHint.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.analysis;
19+
20+
import org.apache.doris.nereids.rules.Rule;
21+
import org.apache.doris.nereids.rules.RuleType;
22+
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
23+
import org.apache.doris.nereids.trees.plans.Plan;
24+
import org.apache.doris.nereids.trees.plans.logical.LogicalPreAggOnHint;
25+
26+
/**
27+
* eliminate logical common hint and set them to cascade context
28+
*/
29+
public class EliminateLogicalPreAggOnHint extends OneRewriteRuleFactory {
30+
31+
@Override
32+
public Rule build() {
33+
return logicalPreAggOnHint().thenApply(ctx -> {
34+
LogicalPreAggOnHint<Plan> preAggHintPlan = ctx.root;
35+
ctx.statementContext.setHintForcePreAggOn(true);
36+
return preAggHintPlan.child();
37+
}).toRule(RuleType.ELIMINATE_LOGICAL_PRE_AGG_ON_HINT);
38+
}
39+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public enum PlanType {
6262
LOGICAL_APPLY,
6363
LOGICAL_ASSERT_NUM_ROWS,
6464
LOGICAL_CHECK_POLICY,
65+
LOGICAL_COMMON_HINT,
6566
LOGICAL_CTE,
6667
LOGICAL_CTE_ANCHOR,
6768
LOGICAL_CTE_PRODUCER,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.doris.nereids.rules.analysis.BindExpression;
4242
import org.apache.doris.nereids.rules.analysis.BindRelation;
4343
import org.apache.doris.nereids.rules.analysis.CheckPolicy;
44+
import org.apache.doris.nereids.rules.analysis.EliminateLogicalPreAggOnHint;
4445
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
4546
import org.apache.doris.nereids.trees.expressions.Expression;
4647
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -271,7 +272,8 @@ public List<RewriteJob> getJobs() {
271272

272273
private static List<RewriteJob> buildAnalyzeViewJobsForStar() {
273274
return jobs(
274-
topDown(new EliminateLogicalSelectHint()),
275+
topDown(new EliminateLogicalSelectHint(),
276+
new EliminateLogicalPreAggOnHint()),
275277
topDown(new AnalyzeCTE()),
276278
bottomUp(
277279
new BindRelation(),

0 commit comments

Comments
 (0)