From 3068c591e436fdb91ac0086580e20d9748bdc85e Mon Sep 17 00:00:00 2001 From: srlch Date: Fri, 6 Sep 2024 15:26:50 +0800 Subject: [PATCH] [Enhancement] Support Generated Column rewrite in complex Query In this pr: 1. Refactor Generated Column rewrite code: a. Collect generated column expr mapping From QueryAnalyzer. b. Introduce ReplaceScalarOperatorRule to replace the ScalarOperator match the generated column expression. 2. Supprot View, JOIN with Subquery and CTE for Generated Column rewriting. 3. Support simple equivalent expression rewrite for generated column. Note: All query structure support Generated Column rewriting obey a simple rule: the Generated Column and its ref columns output directly into the outer scope. Signed-off-by: srlch --- .../starrocks/sql/analyzer/AnalyzeState.java | 11 - .../starrocks/sql/analyzer/QueryAnalyzer.java | 281 +++++++++++------- .../java/com/starrocks/sql/ast/Relation.java | 18 ++ .../com/starrocks/sql/ast/SelectRelation.java | 14 - .../rewrite/ScalarOperatorRewriter.java | 9 + .../scalar/ReplaceScalarOperatorRule.java | 39 +++ .../transformer/ExpressionMapping.java | 9 + .../optimizer/transformer/OptExprBuilder.java | 4 + .../transformer/QueryTransformer.java | 11 +- .../SqlToScalarOperatorTranslator.java | 3 + .../sql/plan/GeneratedColumnTest.java | 4 +- .../R/test_generated_column_rewrite | 154 ++++++++-- .../T/test_generated_column_rewrite | 92 ++++-- 13 files changed, 469 insertions(+), 180 deletions(-) create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/ReplaceScalarOperatorRule.java diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeState.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeState.java index 1fc9212a1da09..3f091b61feff5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeState.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeState.java @@ -20,7 +20,6 @@ import com.starrocks.analysis.FunctionCallExpr; import com.starrocks.analysis.LimitElement; import com.starrocks.analysis.OrderByElement; -import com.starrocks.analysis.SlotRef; import com.starrocks.common.IdGenerator; import com.starrocks.sql.ast.Relation; import com.starrocks.sql.ast.SelectRelation; @@ -56,8 +55,6 @@ public class AnalyzeState { private Scope orderScope; private List orderSourceExpressions; - private Map generatedExprToColumnRef = new HashMap<>(); - /** * outputExprInOrderByScope is used to record which expressions in outputExpression are to be * recorded in the first level of OrderByScope (order by expressions can refer to columns in output) @@ -257,12 +254,4 @@ public ExprId getNextNondeterministicId() { public List getColumnNotInGroupBy() { return columnNotInGroupBy; } - - public void setGeneratedExprToColumnRef(Map generatedExprToColumnRef) { - this.generatedExprToColumnRef = generatedExprToColumnRef; - } - - public Map getGeneratedExprToColumnRef() { - return generatedExprToColumnRef; - } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/QueryAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/QueryAnalyzer.java index 69aa2de7bd411..56662da3554c2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/QueryAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/QueryAnalyzer.java @@ -128,6 +128,154 @@ public void analyze(StatementBase node, Scope parent) { new Visitor().process(node, parent); } + private class GeneratedColumnExprMappingCollector implements AstVisitor { + public GeneratedColumnExprMappingCollector() { + } + + public Void process(ParseNode node, Scope scope) { + return node.accept(this, scope); + } + + private void reAnalyzeExpressionBasedOnCurrentScope(SelectRelation childSelectRelation, Scope scope, + Map resultGeneratedExprToColumnRef) { + if (childSelectRelation.getGeneratedExprToColumnRef() == null || + childSelectRelation.getGeneratedExprToColumnRef().isEmpty()) { + return; + } + // 1. get all available generated column from child selectRelation + // available means that: + // a. generated column output from child selectRelation directly. + // b. all reference column of generated column output from child selectRelation directly. + List outputSlotRef = childSelectRelation.getOutputExpression() + .stream().filter(e -> e instanceof SlotRef) + .map(e -> (SlotRef) e).collect(Collectors.toList()); + boolean hasStar = childSelectRelation.getSelectList() + .getItems().stream().anyMatch(SelectListItem::isStar); + Map generatedExprToColumnRef = new HashMap<>(); + for (Map.Entry entry : childSelectRelation.getGeneratedExprToColumnRef().entrySet()) { + List allRefColumns = Lists.newArrayList(); + entry.getKey().collect(SlotRef.class, allRefColumns); + allRefColumns.add(entry.getValue()); + if (hasStar || outputSlotRef.containsAll(allRefColumns)) { + generatedExprToColumnRef.put(entry.getKey().clone(), (SlotRef) entry.getValue().clone()); + } + } + + // 2. rewrite(rename slotRef) generated column expression(unAnalyzed) using alias in current scope + Map slotRefToAlias = new HashMap<>(); + for (SelectListItem item : childSelectRelation.getSelectList().getItems()) { + if (item.isStar()) { + slotRefToAlias.clear(); + break; + } + + if (!(item.getExpr() instanceof SlotRef) || (item.getAlias() == null || item.getAlias().isEmpty())) { + continue; + } + + slotRefToAlias.put(((SlotRef) item.getExpr()).toSql(), item.getAlias()); + } + List allRefSlotRefs = new ArrayList<>(); + for (Map.Entry entry : generatedExprToColumnRef.entrySet()) { + List refColumns = Lists.newArrayList(); + entry.getKey().collect(SlotRef.class, refColumns); + + allRefSlotRefs.addAll(refColumns); + allRefSlotRefs.add(entry.getValue()); + } + for (SlotRef slotRef : allRefSlotRefs) { + if (!slotRefToAlias.isEmpty()) { + String alias = slotRefToAlias.get(slotRef.toSql()); + if (alias != null) { + slotRef.setColumnName(alias); + } + } + slotRef.setTblName(null); + } + + // 3. analyze generated column expression based on current scope + for (Map.Entry entry : generatedExprToColumnRef.entrySet()) { + entry.getKey().reset(); + entry.getValue().reset(); + + try { + ExpressionAnalyzer.analyzeExpression(entry.getKey(), new AnalyzeState(), scope, session); + ExpressionAnalyzer.analyzeExpression(entry.getValue(), new AnalyzeState(), scope, session); + } catch (Exception ignore) { + // ignore generated column rewrite if hit any exception + generatedExprToColumnRef.clear(); + } + } + resultGeneratedExprToColumnRef.putAll(generatedExprToColumnRef); + } + + @Override + public Void visitTable(TableRelation tableRelation, Scope scope) { + Table table = tableRelation.getTable(); + Map generatedExprToColumnRef = new HashMap<>(); + for (Column column : table.getBaseSchema()) { + Expr generatedColumnExpression = column.getGeneratedColumnExpr(table.getIdToColumn()); + if (generatedColumnExpression != null) { + SlotRef slotRef = new SlotRef(null, column.getName()); + ExpressionAnalyzer.analyzeExpression(generatedColumnExpression, new AnalyzeState(), scope, session); + ExpressionAnalyzer.analyzeExpression(slotRef, new AnalyzeState(), scope, session); + generatedExprToColumnRef.put(generatedColumnExpression, slotRef); + } + } + tableRelation.setGeneratedExprToColumnRef(generatedExprToColumnRef); + return null; + } + + @Override + public Void visitSelect(SelectRelation selectRelation, Scope scope) { + selectRelation.setGeneratedExprToColumnRef(selectRelation.getRelation().getGeneratedExprToColumnRef()); + return null; + } + + @Override + public Void visitSubquery(SubqueryRelation subquery, Scope scope) { + QueryRelation queryRelation = subquery.getQueryStatement().getQueryRelation(); + if (queryRelation instanceof SelectRelation) { + SelectRelation childSelectRelation = (SelectRelation) queryRelation; + reAnalyzeExpressionBasedOnCurrentScope(childSelectRelation, scope, subquery.getGeneratedExprToColumnRef()); + } + return null; + } + + @Override + public Void visitJoin(JoinRelation joinRelation, Scope scope) { + Relation leftRelation = joinRelation.getLeft(); + Relation rightRelation = joinRelation.getRight(); + joinRelation.getGeneratedExprToColumnRef().putAll(leftRelation.getGeneratedExprToColumnRef()); + joinRelation.getGeneratedExprToColumnRef().putAll(rightRelation.getGeneratedExprToColumnRef()); + return null; + } + + @Override + public Void visitView(ViewRelation node, Scope scope) { + QueryRelation queryRelation = node.getQueryStatement().getQueryRelation(); + if (queryRelation instanceof SubqueryRelation) { + node.setGeneratedExprToColumnRef(queryRelation.getGeneratedExprToColumnRef()); + } else if (queryRelation instanceof SelectRelation) { + SelectRelation childSelectRelation = (SelectRelation) queryRelation; + reAnalyzeExpressionBasedOnCurrentScope(childSelectRelation, scope, node.getGeneratedExprToColumnRef()); + } + return null; + } + + @Override + public Void visitCTE(CTERelation cteRelation, Scope scope) { + QueryRelation queryRelation = cteRelation.getCteQueryStatement().getQueryRelation(); + if (queryRelation instanceof SubqueryRelation) { + cteRelation.setGeneratedExprToColumnRef(queryRelation.getGeneratedExprToColumnRef()); + } else if (queryRelation instanceof SelectRelation) { + SelectRelation childSelectRelation = (SelectRelation) queryRelation; + reAnalyzeExpressionBasedOnCurrentScope(childSelectRelation, scope, cteRelation.getGeneratedExprToColumnRef()); + } + return null; + } + } + private class Visitor implements AstVisitor { public Visitor() { } @@ -218,108 +366,6 @@ public Scope visitSelect(SelectRelation selectRelation, Scope scope) { Scope sourceScope = process(resolvedRelation, scope); sourceScope.setParent(scope); - Map generatedExprToColumnRef = new HashMap<>(); - new AstVisitor() { - @Override - public Void visitTable(TableRelation tableRelation, Void context) { - generatedExprToColumnRef.putAll(tableRelation.getGeneratedExprToColumnRef()); - return null; - } - - @Override - public Void visitSubquery(SubqueryRelation subquery, Void context) { - QueryRelation queryRelation = subquery.getQueryStatement().getQueryRelation(); - if (queryRelation instanceof SelectRelation) { - SelectRelation childSelectRelation = (SelectRelation) queryRelation; - if (childSelectRelation.getGeneratedExprToColumnRef() == null || - childSelectRelation.getGeneratedExprToColumnRef().isEmpty()) { - return null; - } - // 1. get all available generated column from subquery - // available means that: - // a. generated column output from subquery directly. - // b. all reference column of generated column output from subquery directly. - List outputSlotRef = childSelectRelation.getOutputExpression() - .stream().filter(e -> e instanceof SlotRef) - .map(e -> (SlotRef) e).collect(Collectors.toList()); - boolean hasStar = childSelectRelation.getSelectList() - .getItems().stream().anyMatch(SelectListItem::isStar); - - for (Map.Entry entry : childSelectRelation.getGeneratedExprToColumnRef().entrySet()) { - List allRefColumns = Lists.newArrayList(); - entry.getKey().collect(SlotRef.class, allRefColumns); - allRefColumns.add(entry.getValue()); - if (hasStar || outputSlotRef.containsAll(allRefColumns)) { - generatedExprToColumnRef.put(entry.getKey().clone(), (SlotRef) entry.getValue().clone()); - } - } - - // 2. rewrite(rename slotRef) generated column expression(unAnalyzed) using alias in current scope - Map slotRefToAlias = new HashMap<>(); - for (SelectListItem item : childSelectRelation.getSelectList().getItems()) { - if (item.isStar()) { - slotRefToAlias.clear(); - break; - } - - if (!(item.getExpr() instanceof SlotRef) || (item.getAlias() == null || item.getAlias().isEmpty())) { - continue; - } - - slotRefToAlias.put(((SlotRef) item.getExpr()).toSql(), item.getAlias()); - } - List allRefSlotRefs = new ArrayList<>(); - for (Map.Entry entry : generatedExprToColumnRef.entrySet()) { - List refColumns = Lists.newArrayList(); - entry.getKey().collect(SlotRef.class, refColumns); - - allRefSlotRefs.addAll(refColumns); - allRefSlotRefs.add(entry.getValue()); - } - for (SlotRef slotRef : allRefSlotRefs) { - if (!slotRefToAlias.isEmpty()) { - String alias = slotRefToAlias.get(slotRef.toSql()); - if (alias != null) { - slotRef.setColumnName(alias); - } - } - slotRef.setTblName(null); - } - - // 3. analyze generated column expression based on current scope - for (Map.Entry entry : generatedExprToColumnRef.entrySet()) { - entry.getKey().reset(); - entry.getValue().reset(); - - try { - ExpressionAnalyzer.analyzeExpression(entry.getKey(), new AnalyzeState(), sourceScope, session); - ExpressionAnalyzer.analyzeExpression(entry.getValue(), new AnalyzeState(), sourceScope, session); - } catch (Exception ignore) { - // ignore generated column rewrite if hit any exception - generatedExprToColumnRef.clear(); - } - } - } - return null; - } - - // Do not support rewrite like JOIN wiht {left: Subquery, right: Relation} - @Override - public Void visitJoin(JoinRelation joinRelation, Void context) { - Relation leftRelation = joinRelation.getLeft(); - Relation rightRelation = joinRelation.getRight(); - if (leftRelation instanceof TableRelation && rightRelation instanceof TableRelation) { - TableRelation leftTableRelation = (TableRelation) leftRelation; - TableRelation rightTableRelation = (TableRelation) rightRelation; - - generatedExprToColumnRef.putAll(leftTableRelation.getGeneratedExprToColumnRef()); - generatedExprToColumnRef.putAll(rightTableRelation.getGeneratedExprToColumnRef()); - } - return null; - } - }.visit(resolvedRelation); - analyzeState.setGeneratedExprToColumnRef(generatedExprToColumnRef); - selectRelation.accept(new RewriteAliasVisitor(sourceScope, session), null); SelectAnalyzer selectAnalyzer = new SelectAnalyzer(session); selectAnalyzer.analyze( @@ -334,6 +380,8 @@ public Void visitJoin(JoinRelation joinRelation, Void context) { selectRelation.getLimit()); selectRelation.fillResolvedAST(analyzeState); + GeneratedColumnExprMappingCollector collector = new GeneratedColumnExprMappingCollector(); + collector.process(selectRelation, sourceScope); return analyzeState.getOutputScope(); } @@ -595,17 +643,8 @@ public Scope visitTable(TableRelation node, Scope outerScope) { Scope scope = new Scope(RelationId.of(node), new RelationFields(fields.build())); node.setScope(scope); - Map getGeneratedExprToColumnRef = new HashMap<>(); - for (Column column : table.getBaseSchema()) { - Expr generatedColumnExpression = column.getGeneratedColumnExpr(table.getIdToColumn()); - if (generatedColumnExpression != null) { - SlotRef slotRef = new SlotRef(null, column.getName()); - ExpressionAnalyzer.analyzeExpression(generatedColumnExpression, new AnalyzeState(), scope, session); - ExpressionAnalyzer.analyzeExpression(slotRef, new AnalyzeState(), scope, session); - getGeneratedExprToColumnRef.put(generatedColumnExpression, slotRef); - } - } - node.setGeneratedExprToColumnRef(getGeneratedExprToColumnRef); + GeneratedColumnExprMappingCollector collector = new GeneratedColumnExprMappingCollector(); + collector.process(node, scope); return scope; } @@ -656,6 +695,10 @@ public Scope visitCTE(CTERelation cteRelation, Scope context) { } Scope scope = new Scope(RelationId.of(cteRelation), new RelationFields(outputFields.build())); cteRelation.setScope(scope); + + GeneratedColumnExprMappingCollector collector = new GeneratedColumnExprMappingCollector(); + collector.process(cteRelation, scope); + return scope; } @@ -760,6 +803,10 @@ public Scope visitJoin(JoinRelation join, Scope parentScope) { leftScope.getRelationFields().joinWith(rightScope.getRelationFields())); } join.setScope(scope); + + GeneratedColumnExprMappingCollector collector = new GeneratedColumnExprMappingCollector(); + collector.process(join, scope); + return scope; } @@ -881,6 +928,10 @@ public Scope visitSubquery(SubqueryRelation subquery, Scope context) { analyzeOrderByClause(subquery, scope); subquery.setScope(scope); + + GeneratedColumnExprMappingCollector collector = new GeneratedColumnExprMappingCollector(); + collector.process(subquery, scope); + return scope; } @@ -958,6 +1009,10 @@ public Scope visitView(ViewRelation node, Scope scope) { Scope viewScope = new Scope(RelationId.of(node), new RelationFields(fields)); node.setScope(viewScope); + + GeneratedColumnExprMappingCollector collector = new GeneratedColumnExprMappingCollector(); + collector.process(node, viewScope); + return viewScope; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/Relation.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/Relation.java index 8dc6cd8cd94f2..3e3e24662a774 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/Relation.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/Relation.java @@ -14,7 +14,9 @@ package com.starrocks.sql.ast; +import com.starrocks.analysis.Expr; import com.starrocks.analysis.ParseNode; +import com.starrocks.analysis.SlotRef; import com.starrocks.analysis.TableName; import com.starrocks.sql.analyzer.RelationFields; import com.starrocks.sql.analyzer.Scope; @@ -22,7 +24,9 @@ import com.starrocks.sql.common.StarRocksPlannerException; import com.starrocks.sql.parser.NodePosition; +import java.util.HashMap; import java.util.List; +import java.util.Map; public abstract class Relation implements ParseNode { private Scope scope; @@ -39,6 +43,12 @@ public abstract class Relation implements ParseNode { // generated by Security Policy rewriting does not perform permission verification. private boolean createByPolicyRewritten = false; + /** + * generatedExprToColumnRef stores the mapping relationship + * between generated expressions and generated columns + */ + private Map generatedExprToColumnRef = new HashMap<>(); + protected final NodePosition pos; protected Relation(NodePosition pos) { @@ -109,6 +119,14 @@ public List getExplicitColumnNames() { return explicitColumnNames; } + public void setGeneratedExprToColumnRef(Map generatedExprToColumnRef) { + this.generatedExprToColumnRef = generatedExprToColumnRef; + } + + public Map getGeneratedExprToColumnRef() { + return generatedExprToColumnRef; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitRelation(this, context); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/SelectRelation.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/SelectRelation.java index 077504b8117fd..3b4b038b38194 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/SelectRelation.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/SelectRelation.java @@ -20,13 +20,11 @@ import com.starrocks.analysis.GroupByClause; import com.starrocks.analysis.LimitElement; import com.starrocks.analysis.OrderByElement; -import com.starrocks.analysis.SlotRef; import com.starrocks.sql.analyzer.AnalyzeState; import com.starrocks.sql.analyzer.FieldId; import com.starrocks.sql.analyzer.Scope; import com.starrocks.sql.parser.NodePosition; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -79,12 +77,6 @@ public class SelectRelation extends QueryRelation { private Map columnReferences; - /** - * materializeExpressionToColumnRef stores the mapping relationship - * between generated expressions and generated columns - */ - private Map generatedExprToColumnRef = new HashMap<>(); - public SelectRelation( SelectList selectList, Relation fromRelation, @@ -160,8 +152,6 @@ public void fillResolvedAST(AnalyzeState analyzeState) { this.columnReferences = analyzeState.getColumnReferences(); - this.generatedExprToColumnRef = analyzeState.getGeneratedExprToColumnRef(); - this.setScope(analyzeState.getOutputScope()); } @@ -307,8 +297,4 @@ public boolean hasAnalyticInfo() { public List getOutputExpression() { return outputExpr; } - - public Map getGeneratedExprToColumnRef() { - return generatedExprToColumnRef; - } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorRewriter.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorRewriter.java index e0f9e4e259685..d37080c42b1b9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorRewriter.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorRewriter.java @@ -18,6 +18,7 @@ import com.starrocks.common.Config; import com.starrocks.sql.common.ErrorType; import com.starrocks.sql.common.StarRocksPlannerException; +import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rewrite.scalar.ArithmeticCommutativeRule; import com.starrocks.sql.optimizer.rewrite.scalar.ConsolidateLikesRule; @@ -28,12 +29,14 @@ import com.starrocks.sql.optimizer.rewrite.scalar.NormalizePredicateRule; import com.starrocks.sql.optimizer.rewrite.scalar.PruneTediousPredicateRule; import com.starrocks.sql.optimizer.rewrite.scalar.ReduceCastRule; +import com.starrocks.sql.optimizer.rewrite.scalar.ReplaceScalarOperatorRule; import com.starrocks.sql.optimizer.rewrite.scalar.ScalarOperatorRewriteRule; import com.starrocks.sql.optimizer.rewrite.scalar.SimplifiedCaseWhenRule; import com.starrocks.sql.optimizer.rewrite.scalar.SimplifiedPredicateRule; import com.starrocks.sql.optimizer.rewrite.scalar.SimplifiedScanColumnRule; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class ScalarOperatorRewriter { @@ -157,4 +160,10 @@ public static ScalarOperator simplifyCaseWhen(ScalarOperator predicates) { // simplify case-when return new ScalarOperatorRewriter().rewrite(predicates, CASE_WHEN_PREDICATE_RULE); } + + public static ScalarOperator replaceScalarOperatorByColumnRef(ScalarOperator operator, + Map translateMap) { + ReplaceScalarOperatorRule rule = new ReplaceScalarOperatorRule(translateMap); + return new ScalarOperatorRewriter().rewrite(operator, Lists.newArrayList(rule)); + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/ReplaceScalarOperatorRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/ReplaceScalarOperatorRule.java new file mode 100644 index 0000000000000..ae66ed76cb206 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/ReplaceScalarOperatorRule.java @@ -0,0 +1,39 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer.rewrite.scalar; + +import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; +import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; +import com.starrocks.sql.optimizer.rewrite.ScalarOperatorRewriteContext; + +import java.util.Map; + +public class ReplaceScalarOperatorRule extends BottomUpScalarOperatorRewriteRule { + private Map translateMap; + + public ReplaceScalarOperatorRule(Map translateMap) { + this.translateMap = translateMap; + } + + @Override + public ScalarOperator visit(ScalarOperator scalarOperator, ScalarOperatorRewriteContext context) { + for (Map.Entry m : translateMap.entrySet()) { + if (ScalarOperator.isEquivalent(m.getKey(), scalarOperator)) { + return m.getValue(); + } + } + return scalarOperator; + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/ExpressionMapping.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/ExpressionMapping.java index b67c0bb470be3..11f8ea8082c44 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/ExpressionMapping.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/ExpressionMapping.java @@ -54,6 +54,7 @@ public class ExpressionMapping { // record columnRefOp which is generated by const expr in project node // if this columnRefOp is referenced by upper node, we should replace it with const expr in upper node private Map columnRefToConstOperators = new HashMap<>(); + private Map generatedColumnExprOpToColumnRef = new HashMap<>(); public ExpressionMapping(Scope scope, List fieldMappings) { this.scope = scope; @@ -202,4 +203,12 @@ public Map getColumnRefToConstOperators() { public void addColumnRefToConstOperators(Map columnRefToConstOperators) { this.columnRefToConstOperators.putAll(columnRefToConstOperators); } + + public Map getGeneratedColumnExprOpToColumnRef() { + return generatedColumnExprOpToColumnRef; + } + + public void addGeneratedColumnExprOpToColumnRef(Map generatedColumnExprOpToColumnRef) { + this.generatedColumnExprOpToColumnRef.putAll(generatedColumnExprOpToColumnRef); + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/OptExprBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/OptExprBuilder.java index 59fe20b66b8cc..466702f137cfc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/OptExprBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/OptExprBuilder.java @@ -58,6 +58,10 @@ public Map getColumnRefToConstOperators() { return expressionMapping.getColumnRefToConstOperators(); } + public Map getGeneratedColumnExprOpToColumnRef() { + return expressionMapping.getGeneratedColumnExprOpToColumnRef(); + } + public void setExpressionMapping(ExpressionMapping expressionMapping) { this.expressionMapping = expressionMapping; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/QueryTransformer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/QueryTransformer.java index b24d8664047ef..1078abae1f9ea 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/QueryTransformer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/QueryTransformer.java @@ -88,12 +88,15 @@ public LogicalPlan plan(SelectRelation queryBlock, ExpressionMapping outer) { builder.getColumnRefToConstOperators())); Map generatedExprToColumnRef = queryBlock.getGeneratedExprToColumnRef(); - ExpressionMapping expressionMapping = builder.getExpressionMapping(); + Map generatedColumnExprOpToColumnRef = new HashMap<>(); for (Map.Entry m : generatedExprToColumnRef.entrySet()) { - ScalarOperator scalarOperator = SqlToScalarOperatorTranslator.translate(m.getValue(), + ScalarOperator scalarOperator = SqlToScalarOperatorTranslator.translate(m.getKey(), builder.getExpressionMapping(), columnRefFactory); - expressionMapping.put(m.getKey(), (ColumnRefOperator) scalarOperator); + ColumnRefOperator columnRefOp = (ColumnRefOperator) SqlToScalarOperatorTranslator.translate(m.getValue(), + builder.getExpressionMapping(), columnRefFactory); + generatedColumnExprOpToColumnRef.put(scalarOperator, columnRefOp); } + builder.getExpressionMapping().addGeneratedColumnExprOpToColumnRef(generatedColumnExprOpToColumnRef); builder = filter(builder, queryBlock.getPredicate()); builder = aggregate(builder, queryBlock.getGroupBy(), queryBlock.getAggregate(), @@ -246,6 +249,7 @@ private OptExprBuilder projectForOrder(OptExprBuilder subOpt, outputTranslations.addExpressionToColumns(subOpt.getExpressionMapping().getExpressionToColumns()); outputTranslations.addColumnRefToConstOperators(subOpt.getColumnRefToConstOperators()); + outputTranslations.addGeneratedColumnExprOpToColumnRef(subOpt.getGeneratedColumnExprOpToColumnRef()); LogicalProjectOperator projectOperator = new LogicalProjectOperator(projections); return new OptExprBuilder(projectOperator, Lists.newArrayList(subOpt), outputTranslations); @@ -279,6 +283,7 @@ private OptExprBuilder project(OptExprBuilder subOpt, Iterable expressions outputTranslations.addExpressionToColumns(subOpt.getExpressionMapping().getExpressionToColumns()); outputTranslations.addColumnRefToConstOperators(subOpt.getColumnRefToConstOperators()); + outputTranslations.addGeneratedColumnExprOpToColumnRef(subOpt.getGeneratedColumnExprOpToColumnRef()); LogicalProjectOperator projectOperator = new LogicalProjectOperator(projections, limit); return new OptExprBuilder(projectOperator, Lists.newArrayList(subOpt), outputTranslations); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/SqlToScalarOperatorTranslator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/SqlToScalarOperatorTranslator.java index fbd46aee8e5b5..b7255a276c759 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/SqlToScalarOperatorTranslator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/SqlToScalarOperatorTranslator.java @@ -191,6 +191,9 @@ public static ScalarOperator translate(Expr expression, ExpressionMapping expres ScalarOperatorRewriter scalarRewriter = new ScalarOperatorRewriter(); result = scalarRewriter.rewrite(result, ScalarOperatorRewriter.DEFAULT_REWRITE_RULES); + result = ScalarOperatorRewriter.replaceScalarOperatorByColumnRef(result, + expressionMapping.getGeneratedColumnExprOpToColumnRef()); + requireNonNull(result, "translated expression is null"); return result; } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/GeneratedColumnTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/GeneratedColumnTest.java index f5aae667e48da..c36b10efdb7de 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/GeneratedColumnTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/GeneratedColumnTest.java @@ -139,11 +139,11 @@ public void test() throws Exception { sql = " select tmc.v1 + 1 from tmc as v,tmc2 as tmc"; plan = getFragmentPlan(sql); - assertContains(plan, " : 5: v1 + 1"); + assertContains(plan, " : 3: v3"); sql = " select tmc.v1 + 1 from tmc as v,tmc2 as tmc"; plan = getFragmentPlan(sql); - assertContains(plan, " : 5: v1 + 1"); + assertContains(plan, " : 3: v3"); sql = " select * from view_1"; plan = getFragmentPlan(sql); diff --git a/test/sql/test_materialized_column/R/test_generated_column_rewrite b/test/sql/test_materialized_column/R/test_generated_column_rewrite index 56d6dfcb1f913..d10c684b644b5 100644 --- a/test/sql/test_materialized_column/R/test_generated_column_rewrite +++ b/test/sql/test_materialized_column/R/test_generated_column_rewrite @@ -35,8 +35,8 @@ PLAN FRAGMENT 0 DROP DATABASE test_generated_column_rewrite; -- result: -- !result --- name: test_generated_column_subquery_rewrite -CREATE TABLE `t_generated_column_subquery_rewrite_1` ( +-- name: test_generated_column_complex_rewrite +CREATE TABLE `t_generated_column_complex_rewrite_1` ( `id` bigint(20) NOT NULL COMMENT "", `col` STRING AS CONCAT(CAST(id AS STRING), "_abc") ) ENGINE=OLAP @@ -52,7 +52,7 @@ PROPERTIES ( ); -- result: -- !result -CREATE TABLE `t_generated_column_subquery_rewrite_2` ( +CREATE TABLE `t_generated_column_complex_rewrite_2` ( `id` bigint(20) NOT NULL COMMENT "", `col` STRING AS CONCAT(CAST(id AS STRING), "_abc") ) ENGINE=OLAP @@ -68,60 +68,178 @@ PROPERTIES ( ); -- result: -- !result -INSERT INTO t_generated_column_subquery_rewrite_1 VALUES (1); +INSERT INTO t_generated_column_complex_rewrite_1 VALUES (1); -- result: -- !result -INSERT INTO t_generated_column_subquery_rewrite_2 VALUES (1); +INSERT INTO t_generated_column_complex_rewrite_2 VALUES (1); -- result: -- !result -INSERT INTO t_generated_column_subquery_rewrite_2 VALUES (2); +INSERT INTO t_generated_column_complex_rewrite_2 VALUES (2); -- result: -- !result -function: assert_explain_not_contains('SELECT CONCAT(CAST(id AS STRING), "_abc") FROM t_generated_column_subquery_rewrite_1', "abc") +function: assert_explain_not_contains('SELECT CONCAT(CAST(id AS STRING), "_abc") FROM t_generated_column_complex_rewrite_1', "abc") -- result: None -- !result -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") -- result: None -- !result -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") -- result: None -- !result -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_subquery_rewrite_1 where id = 1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_complex_rewrite_1 where id = 1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") -- result: None -- !result -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_subquery_rewrite_1 where id = 1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_complex_rewrite_1 where id = 1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") -- result: None -- !result -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col AS col1, id AS id1 FROM t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id1 AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col AS col1, id AS id1 FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id1 AS STRING), "_abc") IS NOT NULL', "abc") -- result: None -- !result -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM (SELECT * FROM t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', 'abc') +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM (SELECT * FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', 'abc') -- result: None -- !result -function: assert_explain_contains('SELECT COUNT(*) FROM (SELECT col AS id FROM t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', 'abc') +function: assert_explain_contains('SELECT COUNT(*) FROM (SELECT col AS id FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', 'abc') -- result: None -- !result -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_subquery_rewrite_1 where id = 1) result WHERE CONCAT(CAST(result.id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_complex_rewrite_1 where id = 1) result WHERE CONCAT(CAST(result.id AS STRING), "_abc") IS NOT NULL', "abc") -- result: None -- !result -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_subquery_rewrite_1 where id = 1) result WHERE CONCAT(CAST(result.id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_complex_rewrite_1 where id = 1) result WHERE CONCAT(CAST(result.id AS STRING), "_abc") IS NOT NULL', "abc") -- result: None -- !result -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 FROM t_generated_column_subquery_rewrite_1 t1, t_generated_column_subquery_rewrite_2 t2 WHERE t1.id = t2.id) result WHERE CONCAT(CAST(result.col1 AS STRING), "_abc") = CONCAT(CAST(result.col3 AS STRING), "_abc")', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 FROM t_generated_column_complex_rewrite_1 t1, t_generated_column_complex_rewrite_2 t2 WHERE t1.id = t2.id) result WHERE CONCAT(CAST(result.col1 AS STRING), "_abc") = CONCAT(CAST(result.col3 AS STRING), "_abc")', "abc") -- result: None -- !result -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 FROM t_generated_column_subquery_rewrite_1 t1, t_generated_column_subquery_rewrite_2 t2 WHERE CONCAT(CAST(t1.id AS STRING), "_abc") = CONCAT(CAST(t2.id AS STRING), "_abc")) result where CONCAT(CAST(result.col1 AS STRING), "_abc") = CONCAT(CAST(result.col3 AS STRING), "_abc")', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 FROM t_generated_column_complex_rewrite_1 t1, t_generated_column_complex_rewrite_2 t2 WHERE CONCAT(CAST(t1.id AS STRING), "_abc") = CONCAT(CAST(t2.id AS STRING), "_abc")) result where CONCAT(CAST(result.col1 AS STRING), "_abc") = CONCAT(CAST(result.col3 AS STRING), "_abc")', "abc") -- result: None +-- !result +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT * FROM t_generated_column_complex_rewrite_1; +-- result: +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +-- result: +None +-- !result +DROP VIEW t_generated_column_complex_rewrite_view; +-- result: +-- !result +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT * FROM (SELECT * FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1; +-- result: +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +-- result: +None +-- !result +DROP VIEW t_generated_column_complex_rewrite_view; +-- result: +-- !result +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT * FROM (SELECT * FROM t_generated_column_complex_rewrite_1) result; +-- result: +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +-- result: +None +-- !result +DROP VIEW t_generated_column_complex_rewrite_view; +-- result: +-- !result +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT * FROM (SELECT id as id1, col as col1 FROM t_generated_column_complex_rewrite_1) result; +-- result: +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(id1 AS STRING), "_abc") IS NOT NULL', "abc") +-- result: +None +-- !result +DROP VIEW t_generated_column_complex_rewrite_view; +-- result: +-- !result +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 from t_generated_column_complex_rewrite_1 t1 join t_generated_column_complex_rewrite_2 t2 on t1.id = t2.id; +-- result: +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(col1 AS STRING), "_abc") IS NOT NULL', "abc") +-- result: +None +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM (SELECT * FROM t_generated_column_complex_rewrite_view) t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(col1 AS STRING), "_abc") IS NOT NULL', "abc") +-- result: +None +-- !result +DROP VIEW t_generated_column_complex_rewrite_view; +-- result: +-- !result +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 from t_generated_column_complex_rewrite_1 t1 join (SELECT * FROM t_generated_column_complex_rewrite_2) t2 on t1.id = t2.id; +-- result: +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(col1 AS STRING), "_abc") IS NOT NULL', "abc") +-- result: +None +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM (SELECT * FROM t_generated_column_complex_rewrite_view) t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(col1 AS STRING), "_abc") IS NOT NULL', "abc") +-- result: +None +-- !result +DROP VIEW t_generated_column_complex_rewrite_view; +-- result: +-- !result +CREATE VIEW t_generated_column_complex_rewrite_view AS WITH tmp as (SELECT * FROM (SELECT * FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL) select * from tmp; +-- result: +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view', "abc") +-- result: +None +-- !result +DROP VIEW t_generated_column_complex_rewrite_view; +-- result: +-- !result +CREATE VIEW t_generated_column_complex_rewrite_view AS WITH tmp as (SELECT * FROM (SELECT * FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL) select * from tmp where CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL; +-- result: +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view', "abc") +-- result: +None +-- !result +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view where CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +-- result: +None +-- !result +DROP VIEW t_generated_column_complex_rewrite_view; +-- result: +-- !result +CREATE TABLE `t_generated_column_complex_rewrite_3` ( + `id` bigint(20) NOT NULL COMMENT "", + `col` STRING AS cast(cast(cast(id + 10 as string) as string) as string) COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`id`) +DISTRIBUTED BY RANDOM BUCKETS 1 +PROPERTIES ( +"replication_num" = "1", +"in_memory" = "false", +"enable_persistent_index" = "false", +"replicated_storage" = "true", +"fast_schema_evolution" = "true", +"compression" = "LZ4" +); +-- result: +-- !result +INSERT INTO t_generated_column_complex_rewrite_3 VALUES (1); +-- result: +-- !result +function: assert_explain_contains('SELECT COUNT(*) FROM t_generated_column_complex_rewrite_3 WHERE cast(id + 10 as string) IS NOT NULL', 'col') +-- result: +None +-- !result +DROP table t_generated_column_complex_rewrite_3; +-- result: -- !result \ No newline at end of file diff --git a/test/sql/test_materialized_column/T/test_generated_column_rewrite b/test/sql/test_materialized_column/T/test_generated_column_rewrite index 6cf9d35720f17..e249df6292740 100644 --- a/test/sql/test_materialized_column/T/test_generated_column_rewrite +++ b/test/sql/test_materialized_column/T/test_generated_column_rewrite @@ -9,8 +9,8 @@ insert into t1 values(1); DROP DATABASE test_generated_column_rewrite; --- name: test_generated_column_subquery_rewrite -CREATE TABLE `t_generated_column_subquery_rewrite_1` ( +-- name: test_generated_column_complex_rewrite +CREATE TABLE `t_generated_column_complex_rewrite_1` ( `id` bigint(20) NOT NULL COMMENT "", `col` STRING AS CONCAT(CAST(id AS STRING), "_abc") ) ENGINE=OLAP @@ -25,7 +25,7 @@ PROPERTIES ( "compression" = "LZ4" ); -CREATE TABLE `t_generated_column_subquery_rewrite_2` ( +CREATE TABLE `t_generated_column_complex_rewrite_2` ( `id` bigint(20) NOT NULL COMMENT "", `col` STRING AS CONCAT(CAST(id AS STRING), "_abc") ) ENGINE=OLAP @@ -40,19 +40,73 @@ PROPERTIES ( "compression" = "LZ4" ); -INSERT INTO t_generated_column_subquery_rewrite_1 VALUES (1); -INSERT INTO t_generated_column_subquery_rewrite_2 VALUES (1); -INSERT INTO t_generated_column_subquery_rewrite_2 VALUES (2); - -function: assert_explain_not_contains('SELECT CONCAT(CAST(id AS STRING), "_abc") FROM t_generated_column_subquery_rewrite_1', "abc") -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_subquery_rewrite_1 where id = 1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_subquery_rewrite_1 where id = 1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col AS col1, id AS id1 FROM t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id1 AS STRING), "_abc") IS NOT NULL', "abc") -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM (SELECT * FROM t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', 'abc') -function: assert_explain_contains('SELECT COUNT(*) FROM (SELECT col AS id FROM t_generated_column_subquery_rewrite_1) t_generated_column_subquery_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', 'abc') -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_subquery_rewrite_1 where id = 1) result WHERE CONCAT(CAST(result.id AS STRING), "_abc") IS NOT NULL', "abc") -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_subquery_rewrite_1 where id = 1) result WHERE CONCAT(CAST(result.id AS STRING), "_abc") IS NOT NULL', "abc") -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 FROM t_generated_column_subquery_rewrite_1 t1, t_generated_column_subquery_rewrite_2 t2 WHERE t1.id = t2.id) result WHERE CONCAT(CAST(result.col1 AS STRING), "_abc") = CONCAT(CAST(result.col3 AS STRING), "_abc")', "abc") -function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 FROM t_generated_column_subquery_rewrite_1 t1, t_generated_column_subquery_rewrite_2 t2 WHERE CONCAT(CAST(t1.id AS STRING), "_abc") = CONCAT(CAST(t2.id AS STRING), "_abc")) result where CONCAT(CAST(result.col1 AS STRING), "_abc") = CONCAT(CAST(result.col3 AS STRING), "_abc")', "abc") +INSERT INTO t_generated_column_complex_rewrite_1 VALUES (1); +INSERT INTO t_generated_column_complex_rewrite_2 VALUES (1); +INSERT INTO t_generated_column_complex_rewrite_2 VALUES (2); + +function: assert_explain_not_contains('SELECT CONCAT(CAST(id AS STRING), "_abc") FROM t_generated_column_complex_rewrite_1', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_complex_rewrite_1 where id = 1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_complex_rewrite_1 where id = 1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col AS col1, id AS id1 FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id1 AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM (SELECT * FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', 'abc') +function: assert_explain_contains('SELECT COUNT(*) FROM (SELECT col AS id FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', 'abc') +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT * FROM t_generated_column_complex_rewrite_1 where id = 1) result WHERE CONCAT(CAST(result.id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT col, id FROM t_generated_column_complex_rewrite_1 where id = 1) result WHERE CONCAT(CAST(result.id AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 FROM t_generated_column_complex_rewrite_1 t1, t_generated_column_complex_rewrite_2 t2 WHERE t1.id = t2.id) result WHERE CONCAT(CAST(result.col1 AS STRING), "_abc") = CONCAT(CAST(result.col3 AS STRING), "_abc")', "abc") +function: assert_explain_not_contains('SELECT COUNT(*) FROM (SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 FROM t_generated_column_complex_rewrite_1 t1, t_generated_column_complex_rewrite_2 t2 WHERE CONCAT(CAST(t1.id AS STRING), "_abc") = CONCAT(CAST(t2.id AS STRING), "_abc")) result where CONCAT(CAST(result.col1 AS STRING), "_abc") = CONCAT(CAST(result.col3 AS STRING), "_abc")', "abc") + +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT * FROM t_generated_column_complex_rewrite_1; +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +DROP VIEW t_generated_column_complex_rewrite_view; + +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT * FROM (SELECT * FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1; +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +DROP VIEW t_generated_column_complex_rewrite_view; + +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT * FROM (SELECT * FROM t_generated_column_complex_rewrite_1) result; +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +DROP VIEW t_generated_column_complex_rewrite_view; + +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT * FROM (SELECT id as id1, col as col1 FROM t_generated_column_complex_rewrite_1) result; +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(id1 AS STRING), "_abc") IS NOT NULL', "abc") +DROP VIEW t_generated_column_complex_rewrite_view; + +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 from t_generated_column_complex_rewrite_1 t1 join t_generated_column_complex_rewrite_2 t2 on t1.id = t2.id; +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(col1 AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT count(*) FROM (SELECT * FROM t_generated_column_complex_rewrite_view) t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(col1 AS STRING), "_abc") IS NOT NULL', "abc") +DROP VIEW t_generated_column_complex_rewrite_view; + +CREATE VIEW t_generated_column_complex_rewrite_view AS SELECT t1.id as col1, t1.col as col2, t2.id as col3, t2.col as col4 from t_generated_column_complex_rewrite_1 t1 join (SELECT * FROM t_generated_column_complex_rewrite_2) t2 on t1.id = t2.id; +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(col1 AS STRING), "_abc") IS NOT NULL', "abc") +function: assert_explain_not_contains('SELECT count(*) FROM (SELECT * FROM t_generated_column_complex_rewrite_view) t_generated_column_complex_rewrite_view WHERE CONCAT(CAST(col1 AS STRING), "_abc") IS NOT NULL', "abc") +DROP VIEW t_generated_column_complex_rewrite_view; + +CREATE VIEW t_generated_column_complex_rewrite_view AS WITH tmp as (SELECT * FROM (SELECT * FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL) select * from tmp; +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view', "abc") +DROP VIEW t_generated_column_complex_rewrite_view; + +CREATE VIEW t_generated_column_complex_rewrite_view AS WITH tmp as (SELECT * FROM (SELECT * FROM t_generated_column_complex_rewrite_1) t_generated_column_complex_rewrite_1 WHERE CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL) select * from tmp where CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL; +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view', "abc") +function: assert_explain_not_contains('SELECT count(*) FROM t_generated_column_complex_rewrite_view where CONCAT(CAST(id AS STRING), "_abc") IS NOT NULL', "abc") +DROP VIEW t_generated_column_complex_rewrite_view; + +CREATE TABLE `t_generated_column_complex_rewrite_3` ( + `id` bigint(20) NOT NULL COMMENT "", + `col` STRING AS cast(cast(cast(id + 10 as string) as string) as string) COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`id`) +DISTRIBUTED BY RANDOM BUCKETS 1 +PROPERTIES ( +"replication_num" = "1", +"in_memory" = "false", +"enable_persistent_index" = "false", +"replicated_storage" = "true", +"fast_schema_evolution" = "true", +"compression" = "LZ4" +); + +INSERT INTO t_generated_column_complex_rewrite_3 VALUES (1); +function: assert_explain_contains('SELECT COUNT(*) FROM t_generated_column_complex_rewrite_3 WHERE cast(id + 10 as string) IS NOT NULL', 'col') +DROP table t_generated_column_complex_rewrite_3;