Skip to content

Commit 90cbfc5

Browse files
rijeshkprijeshkp
authored
[Enhancement] (nereids)implement showDataSkewCommand in nereids #42755 (#44704)
Issue Number: close #42755
1 parent 9e49bdf commit 90cbfc5

7 files changed

Lines changed: 175 additions & 2 deletions

File tree

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,7 @@ supportedShowStatement
265265
| SHOW WHITELIST #showWhitelist
266266
| SHOW TABLETS BELONG
267267
tabletIds+=INTEGER_VALUE (COMMA tabletIds+=INTEGER_VALUE)* #showTabletsBelong
268+
| SHOW DATA SKEW FROM baseTableRef #showDataSkew
268269
;
269270

270271
supportedLoadStatement
@@ -331,7 +332,6 @@ unsupportedShowStatement
331332
| SHOW ALTER TABLE (ROLLUP | (MATERIALIZED VIEW) | COLUMN)
332333
((FROM | IN) database=multipartIdentifier)? wildWhere?
333334
sortClause? limitClause? #showAlterTable
334-
| SHOW DATA SKEW FROM baseTableRef #showDataSkew
335335
| SHOW DATA (ALL)? (FROM tableName=multipartIdentifier)?
336336
sortClause? propertyClause? #showData
337337
| SHOW TEMPORARY? PARTITIONS FROM tableName=multipartIdentifier

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,7 @@ public static List<List<String>> getDataSkew(ShowDataSkewStmt stmt) throws DdlEx
401401
return getDataSkew(stmt.getDbName(), stmt.getTblName(), stmt.getPartitionNames());
402402
}
403403

404-
private static List<List<String>> getDataSkew(String dbName, String tblName, PartitionNames partitionNames)
404+
public static List<List<String>> getDataSkew(String dbName, String tblName, PartitionNames partitionNames)
405405
throws DdlException {
406406
DecimalFormat df = new DecimalFormat("00.00 %");
407407

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,7 @@
246246
import org.apache.doris.nereids.DorisParser.ShowCreateRepositoryContext;
247247
import org.apache.doris.nereids.DorisParser.ShowCreateTableContext;
248248
import org.apache.doris.nereids.DorisParser.ShowCreateViewContext;
249+
import org.apache.doris.nereids.DorisParser.ShowDataSkewContext;
249250
import org.apache.doris.nereids.DorisParser.ShowDatabaseIdContext;
250251
import org.apache.doris.nereids.DorisParser.ShowDeleteContext;
251252
import org.apache.doris.nereids.DorisParser.ShowDiagnoseTabletContext;
@@ -548,6 +549,7 @@
548549
import org.apache.doris.nereids.trees.plans.commands.ShowCreateRepositoryCommand;
549550
import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
550551
import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
552+
import org.apache.doris.nereids.trees.plans.commands.ShowDataSkewCommand;
551553
import org.apache.doris.nereids.trees.plans.commands.ShowDatabaseIdCommand;
552554
import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
553555
import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand;
@@ -4966,5 +4968,11 @@ public LogicalPlan visitAdminCheckTablets(AdminCheckTabletsContext ctx) {
49664968
: Maps.newHashMap();
49674969
return new AdminCheckTabletsCommand(tabletIdLists, properties);
49684970
}
4971+
4972+
@Override
4973+
public LogicalPlan visitShowDataSkew(ShowDataSkewContext ctx) {
4974+
TableRefInfo tableRefInfo = visitBaseTableRefContext(ctx.baseTableRef());
4975+
return new ShowDataSkewCommand(tableRefInfo);
4976+
}
49694977
}
49704978

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ public enum PlanType {
212212
SHOW_CREATE_TABLE_COMMAND,
213213
SHOW_CREATE_VIEW_COMMAND,
214214
SHOW_DATABASE_ID_COMMAND,
215+
SHOW_DATA_SKEW_COMMAND,
215216
SHOW_DELETE_COMMAND,
216217
SHOW_DIAGNOSE_TABLET_COMMAND,
217218
SHOW_DYNAMIC_PARTITION_COMMAND,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.trees.plans.commands;
19+
20+
import org.apache.doris.analysis.PartitionNames;
21+
import org.apache.doris.analysis.RedirectStatus;
22+
import org.apache.doris.catalog.Column;
23+
import org.apache.doris.catalog.Env;
24+
import org.apache.doris.catalog.MetadataViewer;
25+
import org.apache.doris.catalog.ScalarType;
26+
import org.apache.doris.common.AnalysisException;
27+
import org.apache.doris.common.DdlException;
28+
import org.apache.doris.common.ErrorCode;
29+
import org.apache.doris.common.ErrorReport;
30+
import org.apache.doris.common.util.Util;
31+
import org.apache.doris.mysql.privilege.PrivPredicate;
32+
import org.apache.doris.nereids.trees.plans.PlanType;
33+
import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo;
34+
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
35+
import org.apache.doris.qe.ConnectContext;
36+
import org.apache.doris.qe.ShowResultSet;
37+
import org.apache.doris.qe.ShowResultSetMetaData;
38+
import org.apache.doris.qe.StmtExecutor;
39+
40+
import com.google.common.collect.ImmutableList;
41+
42+
import java.util.List;
43+
44+
/**
45+
* show table id command
46+
*/
47+
public class ShowDataSkewCommand extends ShowCommand {
48+
public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
49+
.add("PartitionName").add("BucketIdx").add("AvgRowCount").add("AvgDataSize")
50+
.add("Graph").add("Percent")
51+
.build();
52+
53+
private final TableRefInfo tableRefInfo;
54+
55+
/**
56+
* constructor
57+
*/
58+
public ShowDataSkewCommand(TableRefInfo tableRefInfo) {
59+
super(PlanType.SHOW_DATA_SKEW_COMMAND);
60+
this.tableRefInfo = tableRefInfo;
61+
}
62+
63+
@Override
64+
public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
65+
tableRefInfo.analyze(ctx);
66+
// disallow external catalog
67+
Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(), this.getClass().getSimpleName());
68+
if (!Env.getCurrentEnv().getAccessManager()
69+
.checkTblPriv(ctx, tableRefInfo.getTableNameInfo().getCtl(),
70+
tableRefInfo.getTableNameInfo().getDb(), tableRefInfo.getTableNameInfo().getTbl(),
71+
PrivPredicate.SHOW)) {
72+
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW DATA SKEW",
73+
ctx.getQualifiedUser(),
74+
ctx.getRemoteIP(),
75+
tableRefInfo.getTableNameInfo().getDb() + "." + tableRefInfo.getTableNameInfo().getTbl());
76+
}
77+
try {
78+
tableRefInfo.analyze(ctx);
79+
Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(), this.getClass().getSimpleName());
80+
81+
PartitionNames partitionNames = (tableRefInfo.getPartitionNamesInfo() != null)
82+
? tableRefInfo.getPartitionNamesInfo().translateToLegacyPartitionNames() : null;
83+
List<List<String>> results = MetadataViewer.getDataSkew(tableRefInfo.getTableNameInfo().getDb(),
84+
tableRefInfo.getTableNameInfo().getTbl(), partitionNames);
85+
return new ShowResultSet(getMetaData(), results);
86+
} catch (DdlException e) {
87+
throw new AnalysisException(e.getMessage());
88+
}
89+
90+
}
91+
92+
public ShowResultSetMetaData getMetaData() {
93+
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
94+
for (String title : TITLE_NAMES) {
95+
builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
96+
}
97+
return builder.build();
98+
}
99+
100+
@Override
101+
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
102+
return visitor.visitShowDataSkewCommand(this, context);
103+
}
104+
105+
@Override
106+
public RedirectStatus toRedirectStatus() {
107+
return RedirectStatus.FORWARD_NO_SYNC;
108+
}
109+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@
9292
import org.apache.doris.nereids.trees.plans.commands.ShowCreateRepositoryCommand;
9393
import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
9494
import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
95+
import org.apache.doris.nereids.trees.plans.commands.ShowDataSkewCommand;
9596
import org.apache.doris.nereids.trees.plans.commands.ShowDatabaseIdCommand;
9697
import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
9798
import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand;
@@ -600,4 +601,8 @@ default R visitShowProcessListCommand(ShowProcessListCommand showProcessListComm
600601
default R visitAdminCheckTabletsCommand(AdminCheckTabletsCommand adminCheckTabletsCommand, C context) {
601602
return visitCommand(adminCheckTabletsCommand, context);
602603
}
604+
605+
default R visitShowDataSkewCommand(ShowDataSkewCommand showDataSkewCommand, C context) {
606+
return visitCommand(showDataSkewCommand, context);
607+
}
603608
}

‎regression-test/suites/nereids_p0/show/test_nereids_show_data_skew.groovy‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
19+
suite("test_nereids_show_data_skew") {
20+
def table = "test_nereids_show_data_skew"
21+
String dbName = "${table}_db"
22+
sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
23+
sql "use ${dbName}"
24+
// create table and insert data
25+
sql """ drop table if exists ${table} force"""
26+
sql """
27+
CREATE TABLE ${table} (
28+
id int,
29+
name string,
30+
pdate DATETIME)
31+
PARTITION BY RANGE(pdate) (
32+
FROM ("2023-04-16") TO ("2023-04-20") INTERVAL 1 DAY
33+
) DISTRIBUTED BY HASH(id) BUCKETS 5
34+
properties("replication_num" = "1");
35+
"""
36+
37+
checkNereidsExecute("show data skew from ${dbName}.test_nereids_show_data_skew;")
38+
def result = sql """show data skew from ${dbName}.test_nereids_show_data_skew;"""
39+
assertTrue(result.size() == 20)
40+
41+
checkNereidsExecute("show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416);")
42+
def result2 = sql """show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416);"""
43+
assertTrue(result2.size() == 5)
44+
45+
checkNereidsExecute("show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416, p_20230418);")
46+
def result3 = sql """show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416, p_20230418);"""
47+
assertTrue(result3.size() == 10)
48+
49+
50+
}

0 commit comments

Comments
 (0)