diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index fd40e2c14007177..f1dec27ff196414 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -263,6 +263,7 @@ supportedShowStatement | SHOW WHITELIST #showWhitelist | SHOW TABLETS BELONG tabletIds+=INTEGER_VALUE (COMMA tabletIds+=INTEGER_VALUE)* #showTabletsBelong + | SHOW DATA SKEW FROM baseTableRef #showDataSkew ; supportedLoadStatement @@ -329,7 +330,6 @@ unsupportedShowStatement | SHOW ALTER TABLE (ROLLUP | (MATERIALIZED VIEW) | COLUMN) ((FROM | IN) database=multipartIdentifier)? wildWhere? sortClause? limitClause? #showAlterTable - | SHOW DATA SKEW FROM baseTableRef #showDataSkew | SHOW DATA (ALL)? (FROM tableName=multipartIdentifier)? sortClause? propertyClause? #showData | SHOW TEMPORARY? PARTITIONS FROM tableName=multipartIdentifier diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java index 501d49e4add8d9b..61853bea3d48b7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java @@ -401,7 +401,7 @@ public static List> getDataSkew(ShowDataSkewStmt stmt) throws DdlEx return getDataSkew(stmt.getDbName(), stmt.getTblName(), stmt.getPartitionNames()); } - private static List> getDataSkew(String dbName, String tblName, PartitionNames partitionNames) + public static List> getDataSkew(String dbName, String tblName, PartitionNames partitionNames) throws DdlException { DecimalFormat df = new DecimalFormat("00.00 %"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 9ed352b65e587a6..10bd2b9fb4b1010 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -244,6 +244,7 @@ import org.apache.doris.nereids.DorisParser.ShowCreateRepositoryContext; import org.apache.doris.nereids.DorisParser.ShowCreateTableContext; import org.apache.doris.nereids.DorisParser.ShowCreateViewContext; +import org.apache.doris.nereids.DorisParser.ShowDataSkewContext; import org.apache.doris.nereids.DorisParser.ShowDatabaseIdContext; import org.apache.doris.nereids.DorisParser.ShowDeleteContext; import org.apache.doris.nereids.DorisParser.ShowDiagnoseTabletContext; @@ -544,6 +545,7 @@ import org.apache.doris.nereids.trees.plans.commands.ShowCreateRepositoryCommand; import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand; import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowDataSkewCommand; import org.apache.doris.nereids.trees.plans.commands.ShowDatabaseIdCommand; import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand; import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand; @@ -4943,4 +4945,10 @@ public LogicalPlan visitShowCollation(ShowCollationContext ctx) { } return new ShowCollationCommand(wild); } + + @Override + public LogicalPlan visitShowDataSkew(ShowDataSkewContext ctx) { + TableRefInfo tableRefInfo = visitBaseTableRefContext(ctx.baseTableRef()); + return new ShowDataSkewCommand(tableRefInfo); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index aaa93d70c355aca..4dae72a0d577ff4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -210,6 +210,7 @@ public enum PlanType { SHOW_CREATE_TABLE_COMMAND, SHOW_CREATE_VIEW_COMMAND, SHOW_DATABASE_ID_COMMAND, + SHOW_DATA_SKEW_COMMAND, SHOW_DELETE_COMMAND, SHOW_DIAGNOSE_TABLET_COMMAND, SHOW_DYNAMIC_PARTITION_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java new file mode 100644 index 000000000000000..a623df025b6f452 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.PartitionNames; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MetadataViewer; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.util.Util; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * show table id command + */ +public class ShowDataSkewCommand extends ShowCommand { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("PartitionName").add("BucketIdx").add("AvgRowCount").add("AvgDataSize") + .add("Graph").add("Percent") + .build(); + + private final TableRefInfo tableRefInfo; + + /** + * constructor + */ + public ShowDataSkewCommand(TableRefInfo tableRefInfo) { + super(PlanType.SHOW_DATA_SKEW_COMMAND); + this.tableRefInfo = tableRefInfo; + } + + @Override + public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + tableRefInfo.analyze(ctx); + // disallow external catalog + Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(), this.getClass().getSimpleName()); + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ctx, tableRefInfo.getTableNameInfo().getCtl(), + tableRefInfo.getTableNameInfo().getDb(), tableRefInfo.getTableNameInfo().getTbl(), + PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW DATA SKEW", + ctx.getQualifiedUser(), + ctx.getRemoteIP(), + tableRefInfo.getTableNameInfo().getDb() + "." + tableRefInfo.getTableNameInfo().getTbl()); + } + try { + tableRefInfo.analyze(ctx); + Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(), this.getClass().getSimpleName()); + + PartitionNames partitionNames = (tableRefInfo.getPartitionNamesInfo() != null) + ? tableRefInfo.getPartitionNamesInfo().translateToLegacyPartitionNames() : null; + List> results = MetadataViewer.getDataSkew(tableRefInfo.getTableNameInfo().getDb(), + tableRefInfo.getTableNameInfo().getTbl(), partitionNames); + return new ShowResultSet(getMetaData(), results); + } catch (DdlException e) { + throw new AnalysisException(e.getMessage()); + } + + } + + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + for (String title : TITLE_NAMES) { + builder.addColumn(new Column(title, ScalarType.createVarchar(30))); + } + return builder.build(); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitShowDataSkewCommand(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index 339bb4e4c6a5f0f..6733de82d7062da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -90,6 +90,7 @@ import org.apache.doris.nereids.trees.plans.commands.ShowCreateRepositoryCommand; import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand; import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowDataSkewCommand; import org.apache.doris.nereids.trees.plans.commands.ShowDatabaseIdCommand; import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand; import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand; @@ -590,4 +591,8 @@ default R visitCreateRoutineLoadCommand(CreateRoutineLoadCommand createRoutineLo default R visitShowProcessListCommand(ShowProcessListCommand showProcessListCommand, C context) { return visitCommand(showProcessListCommand, context); } + + default R visitShowDataSkewCommand(ShowDataSkewCommand showDataSkewCommand, C context) { + return visitCommand(showDataSkewCommand, context); + } } diff --git a/regression-test/suites/nereids_p0/show/test_nereids_show_data_skew.groovy b/regression-test/suites/nereids_p0/show/test_nereids_show_data_skew.groovy new file mode 100644 index 000000000000000..5e316f22afcf170 --- /dev/null +++ b/regression-test/suites/nereids_p0/show/test_nereids_show_data_skew.groovy @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_nereids_show_data_skew") { + def table = "test_nereids_show_data_skew" + String dbName = "${table}_db" + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "use ${dbName}" + // create table and insert data + sql """ drop table if exists ${table} force""" + sql """ + CREATE TABLE ${table} ( + id int, + name string, + pdate DATETIME) + PARTITION BY RANGE(pdate) ( + FROM ("2023-04-16") TO ("2023-04-20") INTERVAL 1 DAY + ) DISTRIBUTED BY HASH(id) BUCKETS 5 + properties("replication_num" = "1"); + """ + + checkNereidsExecute("show data skew from ${dbName}.test_nereids_show_data_skew;") + def result = sql """show data skew from ${dbName}.test_nereids_show_data_skew;""" + assertTrue(result.size() == 20) + + checkNereidsExecute("show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416);") + def result2 = sql """show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416);""" + assertTrue(result2.size() == 5) + + checkNereidsExecute("show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416, p_20230418);") + def result3 = sql """show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416, p_20230418);""" + assertTrue(result3.size() == 10) + + +}