From ca610b76435dc2ea39e9d561aa80228b56a14f54 Mon Sep 17 00:00:00 2001 From: lichi <12095047@qq.com> Date: Thu, 19 Dec 2024 21:55:16 +0800 Subject: [PATCH 1/2] [feature](nereids)support CreateMaterializedViewCommand in nereids --- .../java/org/apache/doris/alter/Alter.java | 13 + .../doris/alter/MaterializedViewHandler.java | 235 ++++++ .../apache/doris/analysis/MVColumnItem.java | 12 +- .../java/org/apache/doris/catalog/Env.java | 6 + .../translator/PlanTranslatorContext.java | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 12 +- .../trees/expressions/SlotReference.java | 4 + .../doris/nereids/trees/plans/PlanType.java | 1 + .../CreateMaterializedViewCommand.java | 691 ++++++++++++++++++ .../trees/plans/visitor/CommandVisitor.java | 5 + .../mv_p0/agg_state/order_by/order_by.groovy | 2 +- .../testProjectionMV1.groovy | 6 +- 12 files changed, 972 insertions(+), 17 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index c8cd28c8617506..ec3bf5848033f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -69,6 +69,7 @@ import org.apache.doris.common.util.PropertyAnalyzer.RewriteProperty; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.persist.AlterMTMV; import org.apache.doris.persist.AlterViewInfo; @@ -131,6 +132,18 @@ public void processCreateMaterializedView(CreateMaterializedViewStmt stmt) ((MaterializedViewHandler) materializedViewHandler).processCreateMaterializedView(stmt, db, olapTable); } + public void processCreateMaterializedView(CreateMaterializedViewCommand command) + throws DdlException, AnalysisException, MetaNotFoundException { + String tableName = command.getBaseIndexName(); + // check db + String dbName = command.getDBName(); + Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName); + Env.getCurrentInternalCatalog().checkAvailableCapacity(db); + + OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableName, TableType.OLAP); + ((MaterializedViewHandler) materializedViewHandler).processCreateMaterializedView(command, db, olapTable); + } + public void processDropMaterializedView(DropMaterializedViewStmt stmt) throws DdlException, MetaNotFoundException { TableName tableName = stmt.getTableName(); String dbName = tableName.getDb(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index a6f1cae9987678..2870983fa255e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -58,6 +58,7 @@ import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; import org.apache.doris.persist.BatchDropInfo; import org.apache.doris.persist.DropInfo; import org.apache.doris.persist.EditLog; @@ -232,6 +233,65 @@ public void processCreateMaterializedView(CreateMaterializedViewStmt addMVClause } } + /** + * There are 2 main steps in this function. + * Step1: validate the request. + * Step1.1: semantic analysis: the name of olapTable must be same as the base table name in createMvCommand. + * Step1.2: base table validation: the status of base table and partition could be NORMAL. + * Step1.3: materialized view validation: the name and columns of mv is checked. + * Step2: create mv job + * @param createMvCommand + * @param db + * @param olapTable + * @throws DdlException + */ + public void processCreateMaterializedView(CreateMaterializedViewCommand createMvCommand, Database db, + OlapTable olapTable) throws DdlException, AnalysisException { + // wait wal delete + Env.getCurrentEnv().getGroupCommitManager().blockTable(olapTable.getId()); + Env.getCurrentEnv().getGroupCommitManager().waitWalFinished(olapTable.getId()); + olapTable.writeLockOrDdlException(); + try { + olapTable.checkNormalStateForAlter(); + if (olapTable.existTempPartitions()) { + throw new DdlException("Can not alter table when there are temp partitions in table"); + } + + // Step1.1: semantic analysis + // TODO(ML): support the materialized view as base index + if (!createMvCommand.getBaseIndexName().equals(olapTable.getName())) { + throw new DdlException("The name of table in from clause must be same as the name of alter table"); + } + // Step1.2: base table validation + String baseIndexName = createMvCommand.getBaseIndexName(); + String mvIndexName = createMvCommand.getMVName(); + LOG.info("process add materialized view[{}] based on [{}]", mvIndexName, baseIndexName); + + // avoid conflict against with batch add rollup job + Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL); + + long baseIndexId = checkAndGetBaseIndex(baseIndexName, olapTable); + // Step1.3: mv clause validation + List mvColumns = checkAndPrepareMaterializedView(createMvCommand, olapTable); + + // Step2: create mv job + RollupJobV2 rollupJobV2 = + createMaterializedViewJob(null, mvIndexName, baseIndexName, mvColumns, + createMvCommand.getWhereClauseItemColumn(olapTable), + createMvCommand.getProperties(), olapTable, db, baseIndexId, + createMvCommand.getMVKeysType(), createMvCommand.getOriginStatement()); + + addAlterJobV2(rollupJobV2); + + olapTable.setState(OlapTableState.ROLLUP); + + Env.getCurrentEnv().getEditLog().logAlterJob(rollupJobV2); + LOG.info("finished to create materialized view job: {}", rollupJobV2.getJobId()); + } finally { + olapTable.writeUnlock(); + } + } + /** * There are 2 main steps. * Step1: validate the request @@ -479,6 +539,181 @@ private RollupJobV2 createMaterializedViewJob(String rawSql, String mvName, Stri return mvJob; } + private List checkAndPrepareMaterializedView(CreateMaterializedViewCommand createMvCommand, + OlapTable olapTable) + throws DdlException { + // check if mv index already exists + if (olapTable.hasMaterializedIndex(createMvCommand.getMVName())) { + throw new DdlException("Materialized view[" + createMvCommand.getMVName() + "] already exists"); + } + if (olapTable.getRowStoreCol() != null) { + throw new DdlException("RowStore table can't create materialized view."); + } + // check if mv columns are valid + // a. Aggregate table: + // 1. all slot's aggregationType must same with value mv column + // 2. all slot's isKey must same with mv column + // 3. value column'define expr must be slot (except all slot belong replace family) + // b. Unique table: + // 1. mv must not contain group expr + // 2. all slot's isKey same with mv column + // 3. mv must contain all key column + // c. Duplicate table: + // 1. Columns resolved by semantics are legal + // 2. Key column not allow float/double type. + + // update mv columns + List mvColumnItemList = createMvCommand.getMVColumnItemList(); + List newMVColumns = Lists.newArrayList(); + + if (olapTable.getKeysType().isAggregationFamily()) { + if (olapTable.getKeysType() == KeysType.AGG_KEYS && createMvCommand.getMVKeysType() != KeysType.AGG_KEYS) { + throw new DdlException("The materialized view of aggregation table must has grouping columns"); + } + if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS + && createMvCommand.getMVKeysType() == KeysType.AGG_KEYS) { + // check b.1 + throw new DdlException("The materialized view of unique table must not has grouping columns"); + } + + for (MVColumnItem mvColumnItem : mvColumnItemList) { + if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS) { + mvColumnItem.setIsKey(false); + for (String slotName : mvColumnItem.getBaseColumnNames()) { + if (olapTable + .getColumn(MaterializedIndexMeta + .normalizeName(CreateMaterializedViewStmt.mvColumnBreaker(slotName))) + .isKey()) { + mvColumnItem.setIsKey(true); + } + } + if (!mvColumnItem.isKey()) { + mvColumnItem.setAggregationType(AggregateType.REPLACE, true); + } + } + + // check a.2 and b.2 + for (String slotName : mvColumnItem.getBaseColumnNames()) { + if (olapTable + .getColumn(MaterializedIndexMeta + .normalizeName(CreateMaterializedViewStmt.mvColumnBreaker(slotName))) + .isKey() != mvColumnItem.isKey()) { + throw new DdlException("The mvItem[" + mvColumnItem.getName() + + "]'s isKey must same with all slot, mvItem.isKey=" + + (mvColumnItem.isKey() ? "true" : "false")); + } + } + + if (!mvColumnItem.isKey() && olapTable.getKeysType() == KeysType.AGG_KEYS) { + // check a.1 + for (String slotName : mvColumnItem.getBaseColumnNames()) { + if (olapTable + .getColumn(MaterializedIndexMeta + .normalizeName(CreateMaterializedViewStmt.mvColumnBreaker(slotName))) + .getAggregationType() != mvColumnItem.getAggregationType()) { + throw new DdlException("The mvItem[" + mvColumnItem.getName() + + "]'s AggregationType must same with all slot"); + } + } + + // check a.3 + if (!mvColumnItem.getAggregationType().isReplaceFamily() + && !(mvColumnItem.getDefineExpr() instanceof SlotRef) + && !((mvColumnItem.getDefineExpr() instanceof CastExpr) + && mvColumnItem.getDefineExpr().getChild(0) instanceof SlotRef)) { + throw new DdlException( + "The mvItem[" + mvColumnItem.getName() + "] require slot because it is value column"); + } + } + newMVColumns.add(mvColumnItem.toMVColumn(olapTable)); + } + } else { + for (MVColumnItem mvColumnItem : mvColumnItemList) { + Set names = mvColumnItem.getBaseColumnNames(); + if (names == null) { + throw new DdlException("Base columns is null"); + } + + newMVColumns.add(mvColumnItem.toMVColumn(olapTable)); + } + } + + for (Column column : newMVColumns) { + // check c.2 + if (column.isKey() && column.getType().isFloatingPointType()) { + throw new DdlException("Do not support float/double type on key column, you can change it to decimal"); + } + } + + // check b.3 + if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS) { + Set originColumns = new TreeSet(String.CASE_INSENSITIVE_ORDER); + for (Column column : newMVColumns) { + originColumns.add(CreateMaterializedViewStmt.mvColumnBreaker(column.getName())); + } + for (Column column : olapTable.getBaseSchema()) { + if (column.isKey() && !originColumns.contains(column.getName())) { + throw new DdlException("The materialized view of uniq table must contain all key columns. column:" + + column.getName()); + } + } + } + + if (newMVColumns.size() == olapTable.getBaseSchema().size() + && createMvCommand.getMVKeysType() == olapTable.getKeysType()) { + boolean allKeysMatch = true; + for (int i = 0; i < newMVColumns.size(); i++) { + if (!CreateMaterializedViewStmt.mvColumnBreaker(newMVColumns.get(i).getName()) + .equalsIgnoreCase(olapTable.getBaseSchema().get(i).getName())) { + allKeysMatch = false; + break; + } + } + if (allKeysMatch) { + throw new DdlException("MV same with base table is useless."); + } + } + + if (KeysType.UNIQUE_KEYS == olapTable.getKeysType() && olapTable.hasDeleteSign()) { + newMVColumns.add(new Column(olapTable.getDeleteSignColumn())); + } + if (KeysType.UNIQUE_KEYS == olapTable.getKeysType() && olapTable.hasSequenceCol()) { + newMVColumns.add(new Column(olapTable.getSequenceCol())); + } + if (olapTable.storeRowColumn()) { + Column newColumn = new Column(olapTable.getRowStoreCol()); + newColumn.setAggregationType(AggregateType.NONE, true); + newMVColumns.add(newColumn); + } + // if the column is complex type, we forbid to create materialized view + for (Column column : newMVColumns) { + if (column.getDataType().isComplexType() || column.getDataType().isJsonbType()) { + throw new DdlException("The " + column.getDataType() + " column[" + column + "] not support " + + "to create materialized view"); + } + if (createMvCommand.getMVKeysType() != KeysType.AGG_KEYS + && (column.getType().isBitmapType() || column.getType().isHllType())) { + throw new DdlException("Bitmap/HLL type only support aggregate table"); + } + } + + if (olapTable.getEnableLightSchemaChange()) { + int nextColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE + 1; + for (Column column : newMVColumns) { + column.setUniqueId(nextColUniqueId); + nextColUniqueId++; + } + } else { + newMVColumns.forEach(column -> { + column.setUniqueId(Column.COLUMN_UNIQUE_ID_INIT_VALUE); + }); + } + if (LOG.isDebugEnabled()) { + LOG.debug("lightSchemaChange:{}, newMVColumns:{}", olapTable.getEnableLightSchemaChange(), newMVColumns); + } + return newMVColumns; + } + private List checkAndPrepareMaterializedView(CreateMaterializedViewStmt addMVClause, OlapTable olapTable) throws DdlException { // check if mv index already exists diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java index a9e11458582547..2b712f1d3653e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java @@ -69,10 +69,8 @@ public MVColumnItem(String name, Type type, AggregateType aggregateType, boolean baseColumnNames = new HashSet<>(); Map> tableIdToColumnNames = defineExpr.getTableIdToColumnNames(); - if (defineExpr instanceof SlotRef) { - baseColumnNames = new HashSet<>(); - baseColumnNames.add(this.name); - } else if (tableIdToColumnNames.size() == 1) { + + if (tableIdToColumnNames.size() == 1) { for (Map.Entry> entry : tableIdToColumnNames.entrySet()) { baseColumnNames = entry.getValue(); } @@ -104,10 +102,8 @@ public MVColumnItem(Expr defineExpr) throws AnalysisException { } Map> tableIdToColumnNames = defineExpr.getTableIdToColumnNames(); - if (defineExpr instanceof SlotRef) { - baseColumnNames = new HashSet<>(); - baseColumnNames.add(this.name); - } else if (tableIdToColumnNames.size() == 1) { + + if (tableIdToColumnNames.size() == 1) { for (Map.Entry> entry : tableIdToColumnNames.entrySet()) { baseColumnNames = entry.getValue(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 7bca89fc1a1d3a..f2ad083bf3aef7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -201,6 +201,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.jobs.load.LabelProcessor; import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType; import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVPropertyInfo; import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVRefreshInfo; @@ -4753,6 +4754,11 @@ public void createMaterializedView(CreateMaterializedViewStmt stmt) this.alter.processCreateMaterializedView(stmt); } + public void createMaterializedView(CreateMaterializedViewCommand command) + throws AnalysisException, DdlException, MetaNotFoundException { + this.alter.processCreateMaterializedView(command); + } + public void dropMaterializedView(DropMaterializedViewStmt stmt) throws DdlException, MetaNotFoundException { this.alter.processDropMaterializedView(stmt); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index f40a4e2e8f47c6..9c1a1f40dc2043 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -302,7 +302,7 @@ public SlotDescriptor createSlotDesc(TupleDescriptor tupleDesc, SlotReference sl } } slotRef.setTable(table); - slotRef.setLabel(slotReference.getName()); + slotRef.setLabel("`" + slotReference.getName() + "`"); if (column.isPresent()) { slotDescriptor.setAutoInc(column.get().isAutoInc()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 25c315225d21d4..b30eb8782a2aaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -558,6 +558,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand; import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand; import org.apache.doris.nereids.trees.plans.commands.CreateRoleCommand; @@ -1024,8 +1025,7 @@ public Command visitCreateMTMV(CreateMTMVContext ctx) { if (ctx.buildMode() == null && ctx.refreshMethod() == null && ctx.refreshTrigger() == null && ctx.cols == null && ctx.keys == null && ctx.HASH() == null && ctx.RANDOM() == null && ctx.BUCKETS() == null) { - // TODO support create sync mv - return new UnsupportedCommand(); + return visitCreateSyncMvCommand(ctx); } List nameParts = visitMultipartIdentifier(ctx.mvName); @@ -1062,6 +1062,14 @@ public Command visitCreateMTMV(CreateMTMVContext ctx) { )); } + private Command visitCreateSyncMvCommand(CreateMTMVContext ctx) { + List nameParts = visitMultipartIdentifier(ctx.mvName); + LogicalPlan logicalPlan = new UnboundResultSink<>(visitQuery(ctx.query())); + Map properties = ctx.propertyClause() != null + ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap(); + return new CreateMaterializedViewCommand(new TableNameInfo(nameParts), logicalPlan, properties); + } + /** * get MTMVPartitionDefinition * diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 890fbdfdb96f72..1aac5a3d587380 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -337,4 +337,8 @@ private static Supplier> buildInternalName( public String getQualifiedNameWithBackquote() throws UnboundException { return Utils.qualifiedNameWithBackquote(getQualifier(), getName()); } + + public boolean hasAutoInc() { + return column != null ? column.isAutoInc() : false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index c71a2c8b442fcf..5cad262cc1ffc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -147,6 +147,7 @@ public enum PlanType { SELECT_INTO_OUTFILE_COMMAND, UPDATE_COMMAND, CREATE_MTMV_COMMAND, + CREATE_MATERIALIZED_VIEW_COMMAND, CREATE_JOB_COMMAND, PAUSE_JOB_COMMAND, CANCEL_JOB_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java new file mode 100644 index 00000000000000..d040001665f7bb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java @@ -0,0 +1,691 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.CreateMaterializedViewStmt; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.MVColumnItem; +import org.apache.doris.analysis.StmtType; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.glue.translator.ExpressionTranslator; +import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.BinaryArithmetic; +import org.apache.doris.nereids.trees.expressions.CaseWhen; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.expressions.functions.FunctionTrait; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.HllUnion; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.combinator.StateCombinator; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmapWithCheck; +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.StmtExecutor; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * create synchronized materialized view + */ +public class CreateMaterializedViewCommand extends Command implements ForwardWithSync { + private static final String SYNC_MV_PLANER_DISABLE_RULES = "OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION," + + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, HAVING_TO_FILTER, MERGE_PERCENTILE_TO_ARRAY"; + private final TableNameInfo name; + private final LogicalPlan logicalPlan; + private Map properties; + private List mvColumnItemList; + private MVColumnItem whereClauseItem; + private String baseIndexName; + private KeysType mvKeysType; + private OriginStatement originStatement; + + public CreateMaterializedViewCommand(TableNameInfo name, LogicalPlan logicalPlan, + Map properties) { + super(PlanType.CREATE_MATERIALIZED_VIEW_COMMAND); + this.name = name; + this.logicalPlan = logicalPlan; + this.properties = properties; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCreateMaterializedViewCommand(this, context); + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + originStatement = executor.getOriginStmt(); + validate(ctx); + ctx.getEnv().createMaterializedView(this); + } + + @Override + public StmtType stmtType() { + return StmtType.CREATE; + } + + public String getMVName() { + return name.getTbl(); + } + + public List getMVColumnItemList() { + return mvColumnItemList; + } + + public String getBaseIndexName() { + return baseIndexName; + } + + public Map getProperties() { + return properties; + } + + public String getDBName() { + return name.getDb(); + } + + public KeysType getMVKeysType() { + return mvKeysType; + } + + public OriginStatement getOriginStatement() { + return originStatement; + } + + public Column getWhereClauseItemColumn(OlapTable olapTable) throws DdlException { + if (whereClauseItem == null) { + return null; + } + return whereClauseItem.toMVColumn(olapTable); + } + + private void validate(ConnectContext ctx) throws AnalysisException { + name.analyze(ctx); + Pair result = analyzeAndRewriteLogicalPlan(logicalPlan, ctx); + PlanValidator planValidator = new PlanValidator(); + planValidator.validate(result.first, result.second); + mvColumnItemList = planValidator.context.selectItems; + whereClauseItem = planValidator.context.filterItem; + mvKeysType = planValidator.context.keysType; + baseIndexName = planValidator.context.baseIndexName; + } + + private Pair analyzeAndRewriteLogicalPlan(LogicalPlan unboundPlan, + ConnectContext ctx) { + StatementContext statementContext = ctx.getStatementContext(); + NereidsPlanner planner = new NereidsPlanner(statementContext); + Set tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames(); + ctx.getSessionVariable().setDisableNereidsRules(SYNC_MV_PLANER_DISABLE_RULES); + ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + LogicalPlan plan; + try { + // disable rbo sync mv rewrite + ctx.getSessionVariable().setVarOnce(SessionVariable.ENABLE_SYNC_MV_COST_BASED_REWRITE, "true"); + // disable constant fold + ctx.getSessionVariable().setVarOnce(SessionVariable.DEBUG_SKIP_FOLD_CONSTANT, "true"); + plan = (LogicalPlan) planner.planWithLock(unboundPlan, PhysicalProperties.ANY, + ExplainCommand.ExplainLevel.REWRITTEN_PLAN); + } finally { + // after operate, roll back the disable rules + ctx.getSessionVariable().setDisableNereidsRules(String.join(",", tempDisableRules)); + ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + } + return Pair.of(plan, planner.getCascadesContext()); + } + + private class ValidateContext { + public List selectItems; + public MVColumnItem filterItem; + public String baseIndexName; + public KeysType keysType; + private final PlanTranslatorContext planTranslatorContext; + private Map groupByExprs; + private List orderByExprs; + private Map exprReplaceMap = Maps.newHashMap(); + + public ValidateContext(CascadesContext cascadesContext) { + this.planTranslatorContext = new PlanTranslatorContext(cascadesContext); + } + } + + private class PlanValidator extends DefaultPlanRewriter { + public ValidateContext context; + + public Plan validate(LogicalPlan plan, CascadesContext cascadesContext) { + context = new ValidateContext(cascadesContext); + return plan.accept(this, context); + } + + @Override + public Plan visit(Plan plan, ValidateContext context) { + throw new AnalysisException(String.format("%s is not supported", plan.getClass().getSimpleName())); + } + + @Override + public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, ValidateContext validateContext) { + OlapTable olapTable = olapScan.getTable(); + validateContext.baseIndexName = olapTable.getName(); + validateContext.keysType = olapTable.getKeysType(); + PlanTranslatorContext translatorContext = validateContext.planTranslatorContext; + TupleDescriptor tupleDescriptor = validateContext.planTranslatorContext.generateTupleDesc(); + tupleDescriptor.setTable(olapTable); + for (Slot slot : olapScan.getOutput()) { + translatorContext.createSlotDesc(tupleDescriptor, (SlotReference) slot, olapTable); + } + return olapScan; + } + + @Override + public Plan visitLogicalFilter(LogicalFilter filter, ValidateContext context) { + super.visit(filter, context); + if (context.filterItem != null) { + throw new AnalysisException( + String.format("Only support one filter node, the second is %s", filter.getPredicate())); + } + checkNoNondeterministicFunction(filter); + Set conjuncts = filter.getConjuncts().stream().filter(expr -> { + Set slots = expr.getInputSlots(); + for (Slot slot : slots) { + if (slot instanceof SlotReference) { + Column column = ((SlotReference) slot).getColumn().orElse(null); + if (column != null) { + if (column.isVisible()) { + AggregateType aggregateType = column.getAggregationType(); + if (aggregateType != null && aggregateType != AggregateType.NONE) { + throw new AnalysisException(String.format( + "The where clause contained aggregate column is not supported, expr is %s", + expr)); + } + } else { + return false; + } + } + } + } + return true; + }).collect(Collectors.toSet()); + + if (!conjuncts.isEmpty()) { + Expression predicate = ExpressionUtils.and(conjuncts); + if (!context.exprReplaceMap.isEmpty()) { + predicate = ExpressionUtils.replace(predicate, context.exprReplaceMap); + } + try { + context.filterItem = new MVColumnItem( + translateToLegacyExpr(predicate, context.planTranslatorContext)); + } catch (Exception ex) { + throw new AnalysisException(ex.getMessage()); + } + } + return filter.withConjuncts(conjuncts); + } + + @Override + public Plan visitLogicalAggregate(LogicalAggregate aggregate, ValidateContext context) { + super.visit(aggregate, context); + if (context.groupByExprs != null) { + throw new AnalysisException(String.format("Only support one agg node, the second is %s", aggregate)); + } + context.keysType = KeysType.AGG_KEYS; + checkNoNondeterministicFunction(aggregate); + for (AggregateFunction aggregateFunction : aggregate.getAggregateFunctions()) { + validateAggFunnction(aggregateFunction); + } + List outputs = aggregate.getOutputs(); + if (!context.exprReplaceMap.isEmpty()) { + outputs = ExpressionUtils.replaceNamedExpressions(outputs, context.exprReplaceMap); + } + int groupByExprCount = aggregate.getGroupByExpressions().size(); + context.groupByExprs = Maps.newHashMap(); + for (int i = 0; i < groupByExprCount; ++i) { + context.groupByExprs.put(outputs.get(i).getExprId(), outputs.get(i)); + } + context.exprReplaceMap.putAll(ExpressionUtils.generateReplaceMap(outputs)); + return aggregate; + } + + @Override + public Plan visitLogicalSort(LogicalSort sort, ValidateContext context) { + super.visit(sort, context); + if (context.orderByExprs != null) { + throw new AnalysisException(String.format("Only support one sort node, the second is %s", sort)); + } + checkNoNondeterministicFunction(sort); + context.orderByExprs = (List) sort.getExpressions(); + if (!context.exprReplaceMap.isEmpty()) { + context.orderByExprs = ExpressionUtils.replaceNamedExpressions(context.orderByExprs, + context.exprReplaceMap); + context.orderByExprs = context.orderByExprs.stream() + .map(expr -> expr instanceof Alias && ((Alias) expr).child() instanceof SlotReference + ? (SlotReference) ((Alias) expr).child() + : expr) + .collect(Collectors.toList()); + } + return sort; + } + + @Override + public Plan visitLogicalProject(LogicalProject project, ValidateContext context) { + super.visit(project, context); + checkNoNondeterministicFunction(project); + List outputs = project.getOutputs(); + if (!context.exprReplaceMap.isEmpty()) { + outputs = ExpressionUtils.replaceNamedExpressions(outputs, context.exprReplaceMap); + } + context.exprReplaceMap.putAll(ExpressionUtils.generateReplaceMap(outputs)); + return project; + } + + @Override + public Plan visitLogicalResultSink(LogicalResultSink resultSink, ValidateContext context) { + super.visit(resultSink, context); + List outputs = resultSink.getOutputExprs(); + if (!context.exprReplaceMap.isEmpty()) { + outputs = ExpressionUtils.replaceNamedExpressions(outputs, context.exprReplaceMap); + outputs = outputs.stream() + .map(expr -> expr instanceof Alias && ((Alias) expr).child() instanceof SlotReference + ? (SlotReference) ((Alias) expr).child() + : expr) + .collect(Collectors.toList()); + } + + Set outputExprIds = outputs.stream().map(NamedExpression::getExprId).collect(Collectors.toSet()); + if (outputExprIds.size() != outputs.size()) { + throw new AnalysisException("The select expr is duplicated."); + } + if (context.groupByExprs != null) { + for (ExprId exprId : context.groupByExprs.keySet()) { + if (!outputExprIds.contains(exprId)) { + throw new AnalysisException(String.format("The group expr %s not in select list", + context.groupByExprs.get(exprId))); + } + } + } else { + if (context.keysType == KeysType.AGG_KEYS) { + throw new AnalysisException("agg mv must has group by clause"); + } + } + + if (context.orderByExprs != null) { + int orderByExprCount = context.orderByExprs.size(); + if (outputs.size() < orderByExprCount) { + throw new AnalysisException("The number of columns in order clause must be less than " + + "the number of columns in select clause"); + } + if (context.groupByExprs != null && context.groupByExprs.size() != orderByExprCount) { + throw new AnalysisException("The key of columns in mv must be all of group by columns"); + } + for (int i = 0; i < orderByExprCount; ++i) { + if (outputs.get(i).getExprId() != context.orderByExprs.get(i).getExprId()) { + throw new AnalysisException(String.format( + "The order of columns in order by clause must be same as the order of columns" + + "in select list, %s vs %s", outputs.get(i), context.orderByExprs.get(i))); + } + } + } + + outputs = ExpressionUtils.rewriteDownShortCircuit(outputs, e -> { + if (e instanceof ToBitmap) { + return new ToBitmapWithCheck(e.child(0)); + } else { + return e; + } + }); + context.selectItems = new ArrayList<>(outputs.size()); + boolean meetAggFunction = false; + boolean meetNoneAggExpr = false; + for (NamedExpression output : outputs) { + Expression expr = output; + if (output instanceof Alias) { + expr = ((Alias) output).child(); + } + Expression ignoreCastExpr = expr instanceof Cast ? ((Cast) expr).child() : expr; + if (!(ignoreCastExpr instanceof SlotReference || ignoreCastExpr instanceof BinaryArithmetic + || ignoreCastExpr instanceof BoundFunction)) { + throw new AnalysisException( + String.format( + "The materialized view only support the single column or function expr. " + + "Error column: %s", ignoreCastExpr)); + } + List usedSlots = expr.collectToList(SlotReference.class::isInstance); + for (SlotReference slot : usedSlots) { + if (slot.hasAutoInc()) { + throw new AnalysisException("The materialized view can not involved auto increment column"); + } + } + if (expr.containsType(AggregateFunction.class)) { + meetAggFunction = true; + if (expr instanceof AggregateFunction) { + context.selectItems.add(buildMVColumnItem((AggregateFunction) expr, context)); + } else { + throw new AnalysisException(String.format( + "The materialized view's expr calculations cannot be included outside" + + "aggregate functions, expr: %s", expr)); + } + } else { + if (meetAggFunction) { + throw new AnalysisException("The aggregate column should be after none agg column"); + } + meetNoneAggExpr = true; + try { + context.selectItems + .add(new MVColumnItem(translateToLegacyExpr(expr, context.planTranslatorContext))); + } catch (Exception ex) { + throw new AnalysisException(ex.getMessage()); + } + } + } + if (!meetNoneAggExpr) { + throw new AnalysisException("The materialized view must contain at least one key column"); + } + setKeyForSelectItems(context.selectItems, context); + return resultSink; + } + + private void setKeyForSelectItems(List selectItems, ValidateContext ctx) { + if (ctx.orderByExprs != null) { + int size = ctx.orderByExprs.size(); + for (int i = 0; i < size; ++i) { + MVColumnItem mvColumnItem = selectItems.get(i); + Preconditions.checkState(mvColumnItem.getAggregationType() == null, String.format( + "key column's agg type should be null, but it's %s", mvColumnItem.getAggregationType())); + selectItems.get(i).setIsKey(true); + } + for (int i = size; i < selectItems.size(); ++i) { + MVColumnItem mvColumnItem = selectItems.get(i); + if (mvColumnItem.getAggregationType() != null) { + break; + } + mvColumnItem.setAggregationType(AggregateType.NONE, true); + } + } else { + /* + The keys type of Materialized view is aggregation. + All of group by columns are keys of materialized view. + */ + if (context.keysType == KeysType.DUP_KEYS) { + /* + There is no aggregation function in materialized view. + Supplement key of MV columns + The key is same as the short key in duplicate table + For example: select k1, k2 ... kn from t1 + The default key columns are first 36 bytes of the columns in define order. + If the number of columns in the first 36 is more than 3, the first 3 columns will be used. + column: k1, k2, k3. The key is true. + Supplement non-key of MV columns + column: k4... kn. The key is false, aggregation type is none, isAggregationTypeImplicit is true. + */ + int theBeginIndexOfValue = 0; + // supply key + int keySizeByte = 0; + for (; theBeginIndexOfValue < selectItems.size(); theBeginIndexOfValue++) { + MVColumnItem column = selectItems.get(theBeginIndexOfValue); + keySizeByte += column.getType().getIndexSize(); + if (theBeginIndexOfValue + 1 > FeConstants.shortkey_max_column_count + || keySizeByte > FeConstants.shortkey_maxsize_bytes) { + if (theBeginIndexOfValue == 0 && column.getType().getPrimitiveType().isCharFamily()) { + column.setIsKey(true); + theBeginIndexOfValue++; + } + break; + } + if (column.getType().isFloatingPointType()) { + break; + } + if (column.getType().getPrimitiveType() == PrimitiveType.VARCHAR) { + column.setIsKey(true); + theBeginIndexOfValue++; + break; + } + column.setIsKey(true); + } + if (theBeginIndexOfValue == 0) { + throw new AnalysisException( + "The first column could not be float or double type, use decimal instead"); + } + // supply value + for (; theBeginIndexOfValue < selectItems.size(); theBeginIndexOfValue++) { + MVColumnItem mvColumnItem = selectItems.get(theBeginIndexOfValue); + mvColumnItem.setAggregationType(AggregateType.NONE, true); + } + } else { + for (MVColumnItem mvColumnItem : selectItems) { + if (mvColumnItem.getAggregationType() != null) { + break; + } + mvColumnItem.setIsKey(true); + } + } + } + } + + private MVColumnItem buildMVColumnItem(AggregateFunction aggregateFunction, ValidateContext ctx) + throws AnalysisException { + Expression defineExpr = getAggFunctionFirstParam(aggregateFunction); + DataType paramDataType = defineExpr.getDataType(); + DataType mvDataType = aggregateFunction.getDataType(); + AggregateType mvAggType; + if (aggregateFunction instanceof Sum) { + mvAggType = AggregateType.SUM; + if (mvDataType != paramDataType) { + defineExpr = new Cast(defineExpr, mvDataType, true); + } + } else if (aggregateFunction instanceof Min) { + mvAggType = AggregateType.MIN; + } else if (aggregateFunction instanceof Max) { + mvAggType = AggregateType.MAX; + } else if (aggregateFunction instanceof Count) { + mvAggType = AggregateType.SUM; + List whenClauses = new ArrayList<>(1); + whenClauses.add(new WhenClause(new IsNull(defineExpr), new BigIntLiteral(0))); + defineExpr = new CaseWhen(whenClauses, new BigIntLiteral(1)); + } else if (aggregateFunction instanceof BitmapUnion) { + mvAggType = AggregateType.BITMAP_UNION; + } else if (aggregateFunction instanceof HllUnion) { + mvAggType = AggregateType.HLL_UNION; + } else { + mvAggType = AggregateType.GENERIC; + defineExpr = StateCombinator.create(aggregateFunction); + mvDataType = defineExpr.getDataType(); + } + Expr expr = translateToLegacyExpr(defineExpr, ctx.planTranslatorContext); + return new MVColumnItem(mvDataType.toCatalogDataType(), mvAggType, expr, + CreateMaterializedViewStmt.mvColumnBuilder(expr.toSql())); + } + + private Expr translateToLegacyExpr(Expression expression, PlanTranslatorContext context) { + Expr expr = ExpressionTranslator.translate(expression, context); + expr.setDisableTableName(true); + return expr; + } + + private Expression getAggFunctionFirstParam(AggregateFunction aggregateFunction) { + if (aggregateFunction instanceof Count && ((Count) aggregateFunction).isStar()) { + return new BigIntLiteral(1); + } + if (aggregateFunction.children().isEmpty()) { + throw new AnalysisException(String.format("%s must have a param", aggregateFunction.getName())); + } + return aggregateFunction.child(0); + } + + private void checkNoNondeterministicFunction(Plan plan) { + for (Expression expression : plan.getExpressions()) { + Set nondeterministicFunctions = expression + .collect(expr -> !((ExpressionTrait) expr).isDeterministic() + && expr instanceof FunctionTrait); + if (!nondeterministicFunctions.isEmpty()) { + throw new AnalysisException(String.format( + "can not contain nonDeterministic expression, the expression is %s ", expression)); + } + } + } + + private void validateAggFunnction(AggregateFunction aggregateFunction) { + // if aggregate function use a value column of agg table, + // the value columns' agg type must be consistent with aggregate function + // we do it in two steps: + // 1. if aggregate function use a value column param, find the value column's agg type or else get null + // 2. check the value column's agg type is consistent with aggregate function. + // if no value column used in aggregate function, we check the column type is valid for aggregate functions + Set inputSlots = aggregateFunction.getInputSlots(); + AggregateType aggregateType = null; + Slot aggParamSlot = null; + // try to find a value column + for (Slot slot : inputSlots) { + aggregateType = getAggTypeFromSlot(slot); + if (aggregateType != null && aggregateType != AggregateType.NONE) { + aggParamSlot = slot; + break; + } + } + if (aggParamSlot != null) { + // if aggregate function use a value column param, the value column must be the one and only param + if (aggregateFunction.children().size() != 1 || aggregateFunction.child(0) != aggParamSlot) { + throw new AnalysisException( + String.format("only allow %s as %s's param", aggParamSlot, aggregateFunction.getName())); + } + // check the value columns' agg type is consistent with aggregate function + if (aggregateFunction instanceof Sum) { + if (aggregateType != AggregateType.SUM) { + throw new AnalysisException(String.format( + "Aggregate function require same with slot aggregate type, input: %s, required: SUM", + aggregateType)); + } + } else if (aggregateFunction instanceof Min) { + if (aggregateType != AggregateType.MIN) { + throw new AnalysisException(String.format( + "Aggregate function require same with slot aggregate type, input: %s, required: MIN", + aggregateType)); + } + } else if (aggregateFunction instanceof Max) { + if (aggregateType != AggregateType.MAX) { + throw new AnalysisException(String.format( + "Aggregate function require same with slot aggregate type, input: %s, required: MAX", + aggregateType)); + } + } else if (aggregateFunction instanceof Count) { + if (aggregateType != AggregateType.SUM) { + throw new AnalysisException(String.format( + "Aggregate function require same with slot aggregate type, input: %s, required: SUM", + aggregateType)); + } + } else if (aggregateFunction instanceof BitmapUnion) { + if (aggregateType != AggregateType.BITMAP_UNION) { + throw new AnalysisException(String.format( + "Aggregate function require same with slot aggregate type, input: %s, " + + "required: BITMAP_UNION", aggregateType)); + } + } else if (aggregateFunction instanceof HllUnion) { + if (aggregateType != AggregateType.HLL_UNION) { + throw new AnalysisException(String.format( + "Aggregate function require same with slot aggregate type, input: %s, " + + "required: HLL_UNION", aggregateType)); + } + } else { + if (aggregateType != AggregateType.GENERIC) { + throw new AnalysisException(String.format( + "Aggregate function require same with slot aggregate type, input: %s, " + + "required: GENERIC", aggregateType)); + } + } + } else { + // no value column used in aggregate function, we check the param's column type is valid + DataType paramDataType = getAggFunctionFirstParam(aggregateFunction).getDataType(); + if (aggregateFunction instanceof BitmapUnion) { + if (!paramDataType.isBitmapType()) { + throw new AnalysisException(String.format( + "BITMAP_UNION need input a bitmap column, but input %s", paramDataType)); + } + } else if (aggregateFunction instanceof HllUnion) { + if (!paramDataType.isHllType()) { + throw new AnalysisException(String.format( + "HLL_UNION need input a hll column, but input %s", paramDataType)); + } + } else if (aggregateFunction instanceof Sum || aggregateFunction instanceof Max + || aggregateFunction instanceof Min || aggregateFunction instanceof Count) { + if (paramDataType.isAggStateType()) { + throw new AnalysisException(String.format( + "% can not use agg_state as its param", aggregateFunction.getName())); + } + } + } + } + + private AggregateType getAggTypeFromSlot(Slot slot) { + if (slot instanceof SlotReference) { + Column column = ((SlotReference) slot).getColumn().orElse(null); + if (column != null && column.isVisible()) { + return column.getAggregationType(); + } + } + return null; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index 85d5d320f50afe..82b734184d4a18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -47,6 +47,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand; import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand; import org.apache.doris.nereids.trees.plans.commands.CreateRoleCommand; @@ -221,6 +222,10 @@ default R visitCreateMTMVCommand(CreateMTMVCommand createMTMVCommand, C context) return visitCommand(createMTMVCommand, context); } + default R visitCreateMaterializedViewCommand(CreateMaterializedViewCommand createSyncMVCommand, C context) { + return visitCommand(createSyncMVCommand, context); + } + default R visitCreateJobCommand(CreateJobCommand createJobCommand, C context) { return visitCommand(createJobCommand, context); } diff --git a/regression-test/suites/mv_p0/agg_state/order_by/order_by.groovy b/regression-test/suites/mv_p0/agg_state/order_by/order_by.groovy index 94f4151890217b..61318444f0d11b 100644 --- a/regression-test/suites/mv_p0/agg_state/order_by/order_by.groovy +++ b/regression-test/suites/mv_p0/agg_state/order_by/order_by.groovy @@ -35,6 +35,6 @@ suite ("order_by") { test { sql "create materialized view mv1_2 as select k1, group_concat(cast(abs(k2) as varchar) order by abs(k2)) from d_table group by k1 order by k1;" - exception "The materialized-view do not support aggregate with order by elements." + exception "group_concat_state doesn't support order by expression" } } diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy index d578cbddba945d..48c750d8efaa0a 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy @@ -34,12 +34,8 @@ suite ("testProjectionMV1") { sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" - test { - sql "create materialized view emps_mv as select deptno, empid from emps t order by deptno;" - exception "errCode = 2," - } - createMV("create materialized view emps_mv as select deptno, empid from emps order by deptno;") + createMV("create materialized view emps_mv as select deptno, empid from emps t order by deptno;") sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" From 506ae67bff3d4a1df6a13e469e50d998ff6dbb98 Mon Sep 17 00:00:00 2001 From: lichi Date: Fri, 20 Dec 2024 14:35:00 +0800 Subject: [PATCH 2/2] fix case --- .../apache/doris/analysis/MVColumnItem.java | 5 ++- .../translator/PlanTranslatorContext.java | 2 +- .../CreateMaterializedViewCommand.java | 34 ++++++++++++++++--- .../alter/MaterializedViewHandlerTest.java | 1 + .../create_view_use_mv.groovy | 12 +++---- 5 files changed, 41 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java index 2b712f1d3653e7..43a40566daf484 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java @@ -66,7 +66,6 @@ public MVColumnItem(String name, Type type, AggregateType aggregateType, boolean this.aggregationType = aggregateType; this.isAggregationTypeImplicit = isAggregationTypeImplicit; this.defineExpr = defineExpr; - baseColumnNames = new HashSet<>(); Map> tableIdToColumnNames = defineExpr.getTableIdToColumnNames(); @@ -74,6 +73,8 @@ public MVColumnItem(String name, Type type, AggregateType aggregateType, boolean for (Map.Entry> entry : tableIdToColumnNames.entrySet()) { baseColumnNames = entry.getValue(); } + } else { + baseColumnNames = new HashSet<>(); } } @@ -107,6 +108,8 @@ public MVColumnItem(Expr defineExpr) throws AnalysisException { for (Map.Entry> entry : tableIdToColumnNames.entrySet()) { baseColumnNames = entry.getValue(); } + } else { + baseColumnNames = new HashSet<>(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index 9c1a1f40dc2043..f40a4e2e8f47c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -302,7 +302,7 @@ public SlotDescriptor createSlotDesc(TupleDescriptor tupleDesc, SlotReference sl } } slotRef.setTable(table); - slotRef.setLabel("`" + slotReference.getName() + "`"); + slotRef.setLabel(slotReference.getName()); if (column.isPresent()) { slotDescriptor.setAutoInc(column.get().isAutoInc()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java index d040001665f7bb..ef75d476b8674d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java @@ -20,16 +20,22 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.MVColumnItem; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StmtType; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; @@ -94,13 +100,16 @@ * create synchronized materialized view */ public class CreateMaterializedViewCommand extends Command implements ForwardWithSync { - private static final String SYNC_MV_PLANER_DISABLE_RULES = "OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION," - + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, HAVING_TO_FILTER, MERGE_PERCENTILE_TO_ARRAY"; + private static final String SYNC_MV_PLANER_DISABLE_RULES = "OLAP_SCAN_PARTITION_PRUNE, PRUNE_EMPTY_PARTITION, " + + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, HAVING_TO_FILTER, " + + "MERGE_PERCENTILE_TO_ARRAY, VARIANT_SUB_PATH_PRUNING"; private final TableNameInfo name; + private final LogicalPlan logicalPlan; private Map properties; private List mvColumnItemList; private MVColumnItem whereClauseItem; + private String dbName; private String baseIndexName; private KeysType mvKeysType; private OriginStatement originStatement; @@ -147,7 +156,7 @@ public Map getProperties() { } public String getDBName() { - return name.getDb(); + return dbName; } public KeysType getMVKeysType() { @@ -165,7 +174,7 @@ public Column getWhereClauseItemColumn(OlapTable olapTable) throws DdlException return whereClauseItem.toMVColumn(olapTable); } - private void validate(ConnectContext ctx) throws AnalysisException { + private void validate(ConnectContext ctx) throws Exception { name.analyze(ctx); Pair result = analyzeAndRewriteLogicalPlan(logicalPlan, ctx); PlanValidator planValidator = new PlanValidator(); @@ -173,7 +182,13 @@ private void validate(ConnectContext ctx) throws AnalysisException { mvColumnItemList = planValidator.context.selectItems; whereClauseItem = planValidator.context.filterItem; mvKeysType = planValidator.context.keysType; + dbName = planValidator.context.dbName; baseIndexName = planValidator.context.baseIndexName; + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), InternalCatalog.INTERNAL_CATALOG_NAME, dbName, baseIndexName, + PrivPredicate.ALTER)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ALTER"); + } } private Pair analyzeAndRewriteLogicalPlan(LogicalPlan unboundPlan, @@ -202,6 +217,7 @@ private Pair analyzeAndRewriteLogicalPlan(LogicalP private class ValidateContext { public List selectItems; public MVColumnItem filterItem; + public String dbName; public String baseIndexName; public KeysType keysType; private final PlanTranslatorContext planTranslatorContext; @@ -231,12 +247,16 @@ public Plan visit(Plan plan, ValidateContext context) { public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, ValidateContext validateContext) { OlapTable olapTable = olapScan.getTable(); validateContext.baseIndexName = olapTable.getName(); + validateContext.dbName = olapTable.getDBName(); validateContext.keysType = olapTable.getKeysType(); PlanTranslatorContext translatorContext = validateContext.planTranslatorContext; TupleDescriptor tupleDescriptor = validateContext.planTranslatorContext.generateTupleDesc(); tupleDescriptor.setTable(olapTable); for (Slot slot : olapScan.getOutput()) { translatorContext.createSlotDesc(tupleDescriptor, (SlotReference) slot, olapTable); + SlotRef slotRef = translatorContext.findSlotRef(slot.getExprId()); + slotRef.setLabel("`" + slot.getName() + "`"); + slotRef.setDisableTableName(true); } return olapScan; } @@ -405,6 +425,10 @@ public Plan visitLogicalResultSink(LogicalResultSink resultSink, if (output instanceof Alias) { expr = ((Alias) output).child(); } + if (expr.isConstant()) { + throw new AnalysisException(String.format( + "The materialized view contain constant expr is disallowed, expr: %s", expr)); + } Expression ignoreCastExpr = expr instanceof Cast ? ((Cast) expr).child() : expr; if (!(ignoreCastExpr instanceof SlotReference || ignoreCastExpr instanceof BinaryArithmetic || ignoreCastExpr instanceof BoundFunction)) { @@ -426,7 +450,7 @@ public Plan visitLogicalResultSink(LogicalResultSink resultSink, } else { throw new AnalysisException(String.format( "The materialized view's expr calculations cannot be included outside" - + "aggregate functions, expr: %s", expr)); + + " aggregate functions, expr: %s", expr)); } } else { if (meetAggFunction) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java index d4df410d75fb09..b8fc46a621edda 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java @@ -215,6 +215,7 @@ public void testDuplicateTable(@Injectable CreateMaterializedViewStmt createMate } mvColumnItem.setIsKey(true); mvColumnItem.setAggregationType(null, false); + mvColumnItem.getBaseColumnNames().add(columnName1); List list = Lists.newArrayList(mvColumnItem); new Expectations() { { diff --git a/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy b/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy index 295b195aa58954..669ae6b3793d78 100644 --- a/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy +++ b/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy @@ -62,13 +62,13 @@ suite("create_view_use_mv") { sql "drop view if exists t_mv_v_view" sql """CREATE VIEW t_mv_v_view (k1, k2, k3, k4, k5, k6, v1, v2, v3, v4, v5, v6) as - select `mv_o_orderkey` as k1, `mva_SUM__``o_totalprice``` as k2, `mva_MAX__``o_totalprice``` as k3, + select `mv_o_orderkey` as k1, `mva_SUM__CAST(``o_totalprice`` AS decimalv3(38,2))` as k2, `mva_MAX__``o_totalprice``` as k3, `mva_MIN__``o_totalprice``` as k4, `mva_SUM__CASE WHEN 1 IS NULL THEN 0 ELSE 1 END` as k5, l_orderkey, sum(`mv_o_orderkey`) as sum_total, max(`mv_o_orderkey`) as max_total, min(`mv_o_orderkey`) as min_total, - count(`mva_SUM__``o_totalprice```) as count_all, - bitmap_union(to_bitmap(case when mv_o_orderkey > 1 then `mva_SUM__``o_totalprice``` else null end)) cnt_1, + count(`mva_SUM__CAST(``o_totalprice`` AS decimalv3(38,2))`) as count_all, + bitmap_union(to_bitmap(case when mv_o_orderkey > 1 then `mva_SUM__CAST(``o_totalprice`` AS decimalv3(38,2))` else null end)) cnt_1, bitmap_union(to_bitmap(case when mv_o_orderkey > 2 then `mva_MAX__``o_totalprice``` else null end)) as cnt_2 from orders index t_mv_mv left join lineitem on lineitem.l_orderkey = orders.mv_o_orderkey @@ -79,13 +79,13 @@ suite("create_view_use_mv") { sql "drop view if exists v_for_alter" sql "CREATE VIEW v_for_alter AS SELECT * FROM orders" sql """ALTER VIEW v_for_alter as - select `mv_o_orderkey` as k1, `mva_SUM__``o_totalprice``` as k2, `mva_MAX__``o_totalprice``` as k3, + select `mv_o_orderkey` as k1, `mva_SUM__CAST(``o_totalprice`` AS decimalv3(38,2))` as k2, `mva_MAX__``o_totalprice``` as k3, `mva_MIN__``o_totalprice``` as k4, `mva_SUM__CASE WHEN 1 IS NULL THEN 0 ELSE 1 END` as k5, l_orderkey, sum(`mv_o_orderkey`) as sum_total, max(`mv_o_orderkey`) as max_total, min(`mv_o_orderkey`) as min_total, - count(`mva_SUM__``o_totalprice```) as count_all, - bitmap_union(to_bitmap(case when mv_o_orderkey > 1 then `mva_SUM__``o_totalprice``` else null end)) cnt_1, + count(`mva_SUM__CAST(``o_totalprice`` AS decimalv3(38,2))`) as count_all, + bitmap_union(to_bitmap(case when mv_o_orderkey > 1 then `mva_SUM__CAST(``o_totalprice`` AS decimalv3(38,2))` else null end)) cnt_1, bitmap_union(to_bitmap(case when mv_o_orderkey > 2 then `mva_MAX__``o_totalprice``` else null end)) as cnt_2 from orders index t_mv_mv left join lineitem on lineitem.l_orderkey = orders.mv_o_orderkey