diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java index 7d081a1b88ab3..cad4520382db5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java @@ -223,6 +223,12 @@ public boolean prune(OptimizerContext ctx, OptExpression queryExpression) { final List mvTables = getBaseTables(); final OperatorType queryOp = queryExpression.getOp().getOpType(); + // if a query has been applied this mv, return false directly. + List scanOperators = MvUtils.getScanOperator(queryExpression); + if (scanOperators.stream().anyMatch(op -> op.isOpAppliedMV(mv.getId()))) { + return false; + } + if (!checkOperatorCompatible(queryOp)) { return false; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java index 0ebcfd138e1be..4ebf14ccad68d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java @@ -980,7 +980,7 @@ public static LogicalOlapScanOperator createScanMvOperator(MaterializedView mv, } } final PartitionNames partitionNames = new PartitionNames(false, selectedPartitionNames); - + // MV's selected partition ids/tablet ids are necessary for MV rewrite. return LogicalOlapScanOperator.builder() .setTable(mv) .setColRefToColumnMetaMap(colRefToColumnMetaMapBuilder.build()) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java index 85e87f016a2e6..9f3b06a392912 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java @@ -128,6 +128,9 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_TRANSPARENT_REWRITE; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_UNION_REWRITE; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED; import static com.starrocks.sql.optimizer.rule.RuleType.TF_MATERIALIZED_VIEW; /** @@ -406,7 +409,7 @@ private void pruneTables(OptExpression tree, TaskContext rootTaskContext, Column */ private OptExpression transparentMVRewrite(OptExpression tree, TaskContext rootTaskContext) { ruleRewriteOnlyOnce(tree, rootTaskContext, new MaterializedViewTransparentRewriteRule()); - if (Utils.isOptHasAppliedRule(tree, Operator.OP_TRANSPARENT_MV_BIT)) { + if (Utils.isOptHasAppliedRule(tree, OP_MV_TRANSPARENT_REWRITE)) { tree = new SeparateProjectRule().rewrite(tree, rootTaskContext); } return tree; @@ -425,7 +428,12 @@ private void ruleBasedMaterializedViewRewrite(OptExpression tree, // NOTE: Since union rewrite will generate Filter -> Union -> OlapScan -> OlapScan, need to push filter below Union // and do partition predicate again. // TODO: Do it in CBO if needed later. - if (MvUtils.isAppliedMVUnionRewrite(tree)) { + boolean isNeedFurtherPartitionPrune = Utils.isOptHasAppliedRule(tree, op -> op.isOpRuleBitSet(OP_MV_UNION_REWRITE)); + if (isNeedFurtherPartitionPrune && context.getQueryMaterializationContext().hasRewrittenSuccess()) { + // reset partition prune bit to do partition prune again. + MvUtils.getScanOperator(tree).forEach(scan -> { + scan.resetOpRuleBit(OP_PARTITION_PRUNED); + }); // Do predicate push down if union rewrite successes. tree = new SeparateProjectRule().rewrite(tree, rootTaskContext); deriveLogicalProperty(tree); @@ -520,7 +528,6 @@ private OptExpression logicalRuleRewrite( ruleRewriteIterative(tree, rootTaskContext, RuleSetType.PUSH_DOWN_PREDICATE); ruleRewriteOnlyOnce(tree, rootTaskContext, SchemaTableEvaluateRule.getInstance()); - ruleRewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule()); ruleRewriteOnlyOnce(tree, rootTaskContext, RuleSetType.ELIMINATE_OP_WITH_CONSTANT); ruleRewriteOnlyOnce(tree, rootTaskContext, EliminateAggRule.getInstance()); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerTraceUtil.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerTraceUtil.java index 14f83a6048ec3..618dddbf7e758 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerTraceUtil.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerTraceUtil.java @@ -88,11 +88,11 @@ public static void logMVRewriteFailReason(String mvName, String format, Object.. public static void logMVRewrite(MaterializationContext mvContext, String format, Object... object) { Tracers.log(Tracers.Module.MV, input -> { Object[] args = new Object[] { - mvContext.getMv().getName(), mvContext.getOptimizerContext().isInMemoPhase(), + mvContext.getMv().getName(), MessageFormatter.arrayFormat(format, object).getMessage() }; - return MessageFormatter.arrayFormat("[MV TRACE] [REWRITE {}] [InMemo:{}] {}", + return MessageFormatter.arrayFormat("[MV TRACE] [REWRITE] [InMemo:{}] [{}] {}", args).getMessage(); }); } @@ -102,11 +102,11 @@ public static void logMVRewrite(MvRewriteContext mvRewriteContext, String format Tracers.log(Tracers.Module.MV, input -> { Object[] args = new Object[] { mvRewriteContext.getRule().type().name(), - mvContext.getMv().getName(), mvRewriteContext.getMaterializationContext().getOptimizerContext().isInMemoPhase(), + mvContext.getMv().getName(), MessageFormatter.arrayFormat(format, object).getMessage() }; - return MessageFormatter.arrayFormat("[MV TRACE] [REWRITE {} {}] [InMemo:{}] {}", + return MessageFormatter.arrayFormat("[MV TRACE] [REWRITE {}] [InMemo:{}] [{}] {}", args).getMessage(); }); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Utils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Utils.java index e1a814f54c04b..5dba3f893708c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Utils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Utils.java @@ -55,7 +55,6 @@ import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rewrite.ReplaceColumnRefRewriter; import com.starrocks.sql.optimizer.rewrite.ScalarOperatorRewriter; -import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils; import com.starrocks.sql.optimizer.statistics.ColumnStatistic; import com.starrocks.sql.optimizer.statistics.StatisticsCalculator; import org.apache.commons.collections4.CollectionUtils; @@ -847,45 +846,6 @@ public static OptExpression mergeProjection(OptExpression input, return input; } - /** - * Check if the operator has applied the rule - * @param op input operator to be checked - * @param ruleMask specific rule mask - * @return true if the operator has applied the rule, false otherwise - */ - public static boolean isOpAppliedRule(Operator op, int ruleMask) { - if (op == null) { - return false; - } - // TODO: support cte inline - int opRuleMask = op.getOpRuleMask(); - return (opRuleMask & ruleMask) != 0; - } - - /** - * Set the rule mask to the operator - * @param op input operator - * @param ruleMask specific rule mask - */ - public static void setOpAppliedRule(Operator op, int ruleMask) { - if (op == null) { - return; - } - op.setOpRuleMask(op.getOpRuleMask() | ruleMask); - } - - /** - * Reset the rule mask to the operator - * @param op input operator - * @param ruleMask specific rule mask - */ - public static void resetOpAppliedRule(Operator op, int ruleMask) { - if (op == null) { - return; - } - op.resetOpRuleMask(ruleMask); - } - /** * Check if the optExpression has applied the rule in recursively * @param optExpression input optExpression to be checked @@ -893,32 +853,24 @@ public static void resetOpAppliedRule(Operator op, int ruleMask) { * @return true if the optExpression or its children have applied the rule, false otherwise */ public static boolean isOptHasAppliedRule(OptExpression optExpression, int ruleMask) { + return isOptHasAppliedRule(optExpression, op -> op.isOpRuleBitSet(ruleMask)); + } + + public static boolean isOptHasAppliedRule(OptExpression optExpression, Predicate pred) { if (optExpression == null) { return false; } - if (isOpAppliedRule(optExpression.getOp(), ruleMask)) { + if (pred.test(optExpression.getOp())) { return true; } for (OptExpression child : optExpression.getInputs()) { - if (isOptHasAppliedRule(child, ruleMask)) { + if (isOptHasAppliedRule(child, pred)) { return true; } } return false; } - - public static void setOptScanOpsBit(OptExpression input, - int bit) { - List scanOps = MvUtils.getScanOperator(input); - scanOps.stream().forEach(op -> op.setOpRuleMask(op.getOpRuleMask() | bit)); - } - - public static void setOpBit(OptExpression input, - int bit) { - input.getOp().setOpRuleMask(input.getOp().getOpRuleMask() | bit); - } - @SuppressWarnings("unchecked") public static Optional downcast(T obj, Class klass) { Preconditions.checkArgument(obj != null); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/base/LogicalProperty.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/base/LogicalProperty.java index e287573bdd9ea..1c21edc29c699 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/base/LogicalProperty.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/base/LogicalProperty.java @@ -176,7 +176,8 @@ public OneTabletProperty visitLogicalViewScan(LogicalViewScanOperator node, Expr @Override public OneTabletProperty visitLogicalTableScan(LogicalScanOperator node, ExpressionContext context) { if (node instanceof LogicalOlapScanOperator) { - if (((LogicalOlapScanOperator) node).getSelectedTabletId().size() <= 1) { + LogicalOlapScanOperator olapScanOperator = (LogicalOlapScanOperator) node; + if (olapScanOperator.getSelectedTabletId() != null && olapScanOperator.getSelectedTabletId().size() <= 1) { Set distributionColumnNames = node.getTable().getDistributionColumnNames(); List bucketColumns = Lists.newArrayList(); for (Map.Entry entry : node.getColRefToColumnMetaMap().entrySet()) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/OpRuleBit.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/OpRuleBit.java new file mode 100644 index 0000000000000..26c9ed7d7b86e --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/OpRuleBit.java @@ -0,0 +1,33 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer.operator; + +public class OpRuleBit { + // Like LogicalJoinOperator#transformMask, add a mask to avoid one operator's dead-loop in one transform rule. + // eg: MV's UNION-ALL RULE: + // UNION UNION + // / \ / \ + // OP --> EXTRA-OP MV-SCAN --> UNION MV-SCAN ---> .... + // / \ + // EXTRA-OP MV-SCAN + // Operator has been union rewrite or not, if union all, no need to union again. + public static final int OP_MV_UNION_REWRITE = 0; + // Operator has been push down predicates or not, if push down predicates, no need to push down again. + public static final int OP_MV_AGG_PUSH_DOWN_REWRITE = 1; + // Operator has been transparent mv rewrite or not, if transparent mv rewrite, no need to rewrite again. + public static final int OP_MV_TRANSPARENT_REWRITE = 2; + // Operator has been partition pruned or not, if partition pruned, no need to prune again. + public static final int OP_PARTITION_PRUNED = 3; +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/Operator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/Operator.java index 73ddf70567129..9f498f77b9661 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/Operator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/Operator.java @@ -14,6 +14,7 @@ package com.starrocks.sql.optimizer.operator; +import com.google.api.client.util.Sets; import com.google.common.collect.Lists; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptExpressionVisitor; @@ -25,9 +26,11 @@ import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.property.DomainProperty; +import java.util.BitSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; public abstract class Operator { public static final long DEFAULT_LIMIT = -1; @@ -55,20 +58,10 @@ public abstract class Operator { // or self reference of groups protected long salt = 0; - protected int opRuleMask = 0; - // Like LogicalJoinOperator#transformMask, add a mask to avoid one operator's dead-loop in one transform rule. - // eg: MV's UNION-ALL RULE: - // UNION UNION - // / \ / \ - // OP --> EXTRA-OP MV-SCAN --> UNION MV-SCAN ---> .... - // / \ - // EXTRA-OP MV-SCAN - // Operator's rule mask: operator that has been union rewrite and no needs to rewrite again. - public static final int OP_UNION_ALL_BIT = 1 << 0; - // Operator's rule mask: operator that has been push down rewrite and no needs to rewrite again. - public static final int OP_PUSH_DOWN_BIT = 1 << 1; - public static final int OP_TRANSPARENT_MV_BIT = 1 << 2; - public static final int OP_PARTITION_PRUNE_BIT = 1 << 3; + // mark which rule(bit) has been applied to the operator. + protected BitSet opRuleBits = new BitSet(); + // mark which mv has been applied to the operator + protected Set opAppliedMVs = Sets.newHashSet(); // an operator logically equivalent to 'this' operator // used by view based mv rewrite @@ -159,20 +152,24 @@ public long getSalt() { return salt; } - public int getOpRuleMask() { - return opRuleMask; + public void setOpRuleBit(int bit) { + this.opRuleBits.set(bit); } - public void setOpRuleMask(int bit) { - this.opRuleMask |= bit; + public void resetOpRuleBit(int bit) { + this.opRuleBits.clear(bit); } - public void resetOpRuleMask(int bit) { - this.opRuleMask &= (~ bit); + public boolean isOpRuleBitSet(int bit) { + return opRuleBits.get(bit); } - public boolean isOpRuleMaskSet(int bit) { - return (opRuleMask & bit) != 0; + public void setOpAppliedMV(long mvId) { + this.opAppliedMVs.add(mvId); + } + + public boolean isOpAppliedMV(long mvId) { + return opAppliedMVs.contains(mvId); } public Operator getEquivalentOp() { @@ -279,8 +276,9 @@ public B withOperator(O operator) { builder.predicate = operator.predicate; builder.projection = operator.projection; builder.salt = operator.salt; - builder.opRuleMask = operator.opRuleMask; builder.equivalentOp = operator.equivalentOp; + builder.opRuleBits.or(operator.opRuleBits); + builder.opAppliedMVs.addAll(operator.opAppliedMVs); return (B) this; } @@ -326,8 +324,8 @@ public B addSalt() { return (B) this; } - public B setOpBitSet(int opRuleMask) { - builder.opRuleMask = opRuleMask; + public B setOpBitSet(BitSet opRuleMask) { + builder.opRuleBits = opRuleMask; return (B) this; } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java index 8e51174147d88..b5a1570322c80 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java @@ -18,8 +18,6 @@ import com.google.common.collect.Lists; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; -import com.starrocks.sql.optimizer.Utils; -import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; import com.starrocks.sql.optimizer.operator.pattern.Pattern; @@ -28,8 +26,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Collections; import java.util.List; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED; + public class ExternalScanPartitionPruneRule extends TransformationRule { private static final Logger LOG = LogManager.getLogger(ExternalScanPartitionPruneRule.class); @@ -56,21 +57,14 @@ public ExternalScanPartitionPruneRule(OperatorType logicalOperatorType) { super(RuleType.TF_PARTITION_PRUNE, Pattern.create(logicalOperatorType)); } - @Override - public boolean check(final OptExpression input, OptimizerContext context) { - Operator op = input.getOp(); - // if the partition id is already selected, no need to prune again - if (Utils.isOpAppliedRule(op, Operator.OP_PARTITION_PRUNE_BIT)) { - return false; - } - return true; - } - @Override public List transform(OptExpression input, OptimizerContext context) { LogicalScanOperator operator = (LogicalScanOperator) input.getOp(); + if (operator.isOpRuleBitSet(OP_PARTITION_PRUNED)) { + return Collections.emptyList(); + } OptExternalPartitionPruner.prunePartitions(context, operator); - Utils.setOpAppliedRule(operator, Operator.OP_PARTITION_PRUNE_BIT); + operator.setOpRuleBit(OP_PARTITION_PRUNED); return Lists.newArrayList(); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MaterializedViewTransparentRewriteRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MaterializedViewTransparentRewriteRule.java index 5fad4d24c465f..0ea457805ac73 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MaterializedViewTransparentRewriteRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MaterializedViewTransparentRewriteRule.java @@ -32,7 +32,6 @@ import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.QueryMaterializationContext; import com.starrocks.sql.optimizer.Utils; -import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; @@ -56,6 +55,7 @@ import java.util.stream.Collectors; import static com.starrocks.sql.optimizer.OptimizerTraceUtil.logMVRewrite; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_TRANSPARENT_REWRITE; import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.deriveLogicalProperty; public class MaterializedViewTransparentRewriteRule extends TransformationRule { @@ -67,7 +67,7 @@ public MaterializedViewTransparentRewriteRule() { public boolean check(final OptExpression input, OptimizerContext context) { // To avoid dead-loop rewrite, no rewrite when query extra predicate is not changed - if (Utils.isOpAppliedRule(input.getOp(), Operator.OP_TRANSPARENT_MV_BIT)) { + if (input.getOp().isOpRuleBitSet(OP_MV_TRANSPARENT_REWRITE)) { return false; } return true; @@ -124,7 +124,7 @@ private OptExpression doTransform(ConnectContext connectContext, public static void setOpRuleMask(OptExpression input) { List scanOps = MvUtils.getScanOperator(input); - scanOps.stream().forEach(op -> op.setOpRuleMask(Operator.OP_TRANSPARENT_MV_BIT)); + scanOps.stream().forEach(op -> op.setOpRuleBit(OP_MV_TRANSPARENT_REWRITE)); } /** diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRule.java index 64cde27734188..dc51de77495d9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRule.java @@ -18,16 +18,17 @@ import com.google.common.collect.Lists; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; -import com.starrocks.sql.optimizer.Utils; -import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.operator.pattern.Pattern; import com.starrocks.sql.optimizer.rewrite.OptOlapPartitionPruner; import com.starrocks.sql.optimizer.rule.RuleType; +import java.util.Collections; import java.util.List; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED; + /** * This class does: * 1. Prune the Olap table partition ids, Dependency predicates push down scan node @@ -57,19 +58,13 @@ public PartitionPruneRule() { super(RuleType.TF_PARTITION_PRUNE, Pattern.create(OperatorType.LOGICAL_OLAP_SCAN)); } - @Override - public boolean check(final OptExpression input, OptimizerContext context) { - Operator op = input.getOp(); - // if the partition id is already selected, no need to prune again - if (Utils.isOpAppliedRule(op, Operator.OP_PARTITION_PRUNE_BIT)) { - return false; - } - return true; - } - @Override public List transform(OptExpression input, OptimizerContext context) { LogicalOlapScanOperator logicalOlapScanOperator = (LogicalOlapScanOperator) input.getOp(); + if (logicalOlapScanOperator.isOpRuleBitSet(OP_PARTITION_PRUNED)) { + return Collections.emptyList(); + } + LogicalOlapScanOperator prunedOlapScanOperator = null; if (logicalOlapScanOperator.getSelectedPartitionId() == null) { prunedOlapScanOperator = OptOlapPartitionPruner.prunePartitions(logicalOlapScanOperator); @@ -77,7 +72,7 @@ public List transform(OptExpression input, OptimizerContext conte // do merge pruned partitions with new pruned partitions prunedOlapScanOperator = OptOlapPartitionPruner.mergePartitionPrune(logicalOlapScanOperator); } - Utils.setOpAppliedRule(prunedOlapScanOperator, Operator.OP_PARTITION_PRUNE_BIT); + prunedOlapScanOperator.setOpRuleBit(OP_PARTITION_PRUNED); return Lists.newArrayList(OptExpression.create(prunedOlapScanOperator, input.getInputs())); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/AggregatedMaterializedViewPushDownRewriter.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/AggregatedMaterializedViewPushDownRewriter.java index aba8627287909..66962d35227bc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/AggregatedMaterializedViewPushDownRewriter.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/AggregatedMaterializedViewPushDownRewriter.java @@ -22,11 +22,9 @@ import com.starrocks.sql.optimizer.MvRewriteContext; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptExpressionVisitor; -import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.operator.AggType; -import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.Projection; import com.starrocks.sql.optimizer.operator.logical.LogicalAggregationOperator; @@ -52,6 +50,7 @@ import java.util.stream.Collectors; import static com.starrocks.sql.optimizer.OptimizerTraceUtil.logMVRewrite; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_AGG_PUSH_DOWN_REWRITE; import static com.starrocks.sql.optimizer.rule.transformation.materialization.common.AggregateFunctionRollupUtils.isSupportedAggFunctionPushDown; import static com.starrocks.sql.optimizer.rule.transformation.materialization.common.AggregatePushDownUtils.doRewritePushDownAgg; import static com.starrocks.sql.optimizer.rule.transformation.materialization.common.AggregatePushDownUtils.getPushDownRollupFinalAggregateOpt; @@ -83,11 +82,11 @@ public OptExpression doRewrite(MvRewriteContext mvContext) { Optional res = rewriteInfo.getOp(); logMVRewrite(mvContext, "AggregateJoin pushdown rewrite success"); OptExpression result = res.get(); - Utils.setOptScanOpsBit(result, Operator.OP_PUSH_DOWN_BIT); + result.getOp().setOpRuleBit(OP_MV_AGG_PUSH_DOWN_REWRITE); return result; } else { logMVRewrite(mvContext, "AggregateJoin pushdown rewrite failed"); - Utils.setOpBit(input, Operator.OP_PUSH_DOWN_BIT); + input.getOp().setOpRuleBit(OP_MV_AGG_PUSH_DOWN_REWRITE); return null; } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/AggregatedTimeSeriesRewriter.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/AggregatedTimeSeriesRewriter.java index 6e70e99d4c695..1df434976aee7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/AggregatedTimeSeriesRewriter.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/AggregatedTimeSeriesRewriter.java @@ -36,10 +36,11 @@ import com.starrocks.sql.optimizer.MaterializationContext; import com.starrocks.sql.optimizer.MvRewriteContext; import com.starrocks.sql.optimizer.OptExpression; +import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.AggType; -import com.starrocks.sql.optimizer.operator.Operator; +import com.starrocks.sql.optimizer.operator.OpRuleBit; import com.starrocks.sql.optimizer.operator.Projection; import com.starrocks.sql.optimizer.operator.ScanOperatorPredicates; import com.starrocks.sql.optimizer.operator.logical.LogicalAggregationOperator; @@ -242,6 +243,17 @@ public OptExpression doRewrite(MvRewriteContext mvRewriteContext) { return result; } + @Override + public OptExpression postRewrite(OptimizerContext optimizerContext, + MvRewriteContext mvRewriteContext, + OptExpression candidate) { + OptExpression result = super.postRewrite(optimizerContext, mvRewriteContext, candidate); + MvUtils.getScanOperator(result) + .stream() + .forEach(op -> op.resetOpRuleBit(OpRuleBit.OP_PARTITION_PRUNED)); + return result; + } + private OptExpression doPushDownAggregateRewrite(OptExpression pdOptExpression, AggregatePushDownContext ctx, Pair splitPartitionPredicates) { @@ -450,7 +462,6 @@ public Pair> getPushDownQueryOptExpressio List scanOperators = MvUtils.getScanOperator(queryExpression); LogicalScanOperator logicalOlapScanOp = scanOperators.get(0); logicalOlapScanOp.setPredicate(newPredicate); - Utils.resetOpAppliedRule(logicalOlapScanOp, Operator.OP_PARTITION_PRUNE_BIT); LogicalAggregationOperator aggregateOp = (LogicalAggregationOperator) queryExpression.getOp(); Map newAggregations = Maps.newHashMap(); Map aggColRefMapping = Maps.newHashMap(); @@ -483,7 +494,7 @@ public Pair> getPushDownQueryOptExpressio .collect(Collectors.toList()); List newQueryOutputCols = duplicator.getMappedColumns(newOrigOutputColumns); - Utils.setOptScanOpsBit(queryDuplicateOptExpression, Operator.OP_UNION_ALL_BIT); + queryDuplicateOptExpression.getOp().setOpRuleBit(OpRuleBit.OP_MV_UNION_REWRITE); return Pair.create(queryDuplicateOptExpression, newQueryOutputCols); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java index 0f6c6f58dcce4..a144bad33ea27 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java @@ -38,6 +38,8 @@ import java.util.List; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED; + public class MVPartitionPruner { private final OptimizerContext optimizerContext; private final MvRewriteContext mvRewriteContext; @@ -51,21 +53,6 @@ public OptExpression prunePartition(OptExpression queryExpression) { return queryExpression.getOp().accept(new MVPartitionPrunerVisitor(), queryExpression, null); } - /** - * For input query expression, reset/clear pruned partitions and return new query expression to be pruned again. - */ - public static LogicalOlapScanOperator resetSelectedPartitions(LogicalOlapScanOperator olapScanOperator) { - final LogicalOlapScanOperator.Builder mvScanBuilder = OperatorBuilderFactory.build(olapScanOperator); - // reset original partition predicates to prune partitions/tablets again - mvScanBuilder.withOperator(olapScanOperator) - .setSelectedPartitionId(null) - .setPrunedPartitionPredicates(Lists.newArrayList()) - .setSelectedTabletId(Lists.newArrayList()); - LogicalOlapScanOperator result = mvScanBuilder.build(); - Utils.resetOpAppliedRule(result, Operator.OP_PARTITION_PRUNE_BIT); - return result; - } - private class MVPartitionPrunerVisitor extends OptExpressionVisitor { private boolean isAddMVPrunePredicate(LogicalOlapScanOperator olapScanOperator) { if (mvRewriteContext == null) { @@ -86,8 +73,8 @@ private ScalarOperator getMVPrunePredicate(LogicalOlapScanOperator scanOperator) @Override public OptExpression visitLogicalTableScan(OptExpression optExpression, Void context) { + LogicalScanOperator result = null; LogicalScanOperator scanOperator = optExpression.getOp().cast(); - if (scanOperator instanceof LogicalOlapScanOperator) { LogicalOlapScanOperator.Builder builder = new LogicalOlapScanOperator.Builder(); LogicalOlapScanOperator olapScanOperator = (LogicalOlapScanOperator) (scanOperator); @@ -99,12 +86,13 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Void con if (isAddMvPrunePredicate) { builder.setPredicate(getMVPrunePredicate(olapScanOperator)); } - LogicalOlapScanOperator newOlapScanOperator = builder.build(); + LogicalOlapScanOperator cloned = builder.build(); // prune partition List selectedPartitionIds = olapScanOperator.getSelectedPartitionId(); - if (selectedPartitionIds == null || selectedPartitionIds.isEmpty()) { - newOlapScanOperator = OptOlapPartitionPruner.prunePartitions(newOlapScanOperator); + LogicalOlapScanOperator newOlapScanOperator = cloned; + if (selectedPartitionIds == null) { + newOlapScanOperator = OptOlapPartitionPruner.prunePartitions(cloned); } // prune distribution key @@ -123,7 +111,7 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Void con } LogicalOlapScanOperator.Builder rewrittenBuilder = new LogicalOlapScanOperator.Builder(); - scanOperator = rewrittenBuilder.withOperator(newOlapScanOperator) + result = rewrittenBuilder.withOperator(newOlapScanOperator) .setPredicate(MvUtils.canonizePredicate(scanPredicate)) .setSelectedTabletId(selectedTabletIds) .build(); @@ -137,10 +125,13 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Void con Operator.Builder builder = OperatorBuilderFactory.build(scanOperator); LogicalScanOperator copiedScanOperator = (LogicalScanOperator) builder.withOperator(scanOperator).build(); - scanOperator = OptExternalPartitionPruner.prunePartitions(optimizerContext, + result = OptExternalPartitionPruner.prunePartitions(optimizerContext, copiedScanOperator); } - return OptExpression.create(scanOperator); + if (result != null) { + result.setOpRuleBit(OP_PARTITION_PRUNED); + } + return OptExpression.create(result); } public OptExpression visit(OptExpression optExpression, Void context) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java index 1f9b641dfd3f0..989ad933b6ee6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java @@ -97,7 +97,8 @@ import java.util.stream.Collectors; import static com.starrocks.sql.optimizer.OptimizerTraceUtil.logMVRewrite; -import static com.starrocks.sql.optimizer.operator.Operator.OP_UNION_ALL_BIT; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_UNION_REWRITE; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED; import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvPartitionCompensator.getMvTransparentPlan; import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.deriveLogicalProperty; @@ -1299,9 +1300,15 @@ private OptExpression buildMVScanOptExpression(RewriteContext rewriteContext, // the rewritten expression to replace query // should copy the op because the op will be modified and reused final LogicalOlapScanOperator mvScanOperator = materializationContext.getScanMvOperator(); - // reset original partition predicates to prune partitions/tablets again - final LogicalOlapScanOperator newMvScanOperator = MVPartitionPruner.resetSelectedPartitions(mvScanOperator); - OptExpression mvScanOptExpression = OptExpression.create(newMvScanOperator); + // clone mv's scan operator to avoid changing its original variables + final LogicalOlapScanOperator cloned = new LogicalOlapScanOperator.Builder() + .withOperator(mvScanOperator) + .setSelectedPartitionId(null) + .setPrunedPartitionPredicates(Lists.newArrayList()) + .setSelectedTabletId(Lists.newArrayList()) + .build(); + cloned.resetOpRuleBit(OP_PARTITION_PRUNED); + OptExpression mvScanOptExpression = OptExpression.create(cloned); // Rewrite original mv's predicates into query if needed. if (mvRewriteContext.getMvPruneConjunct() != null && !mvRewriteContext.getMvPruneConjunct().isTrue()) { @@ -1884,8 +1891,8 @@ private OptExpression doUnionRewrite(RewriteContext rewriteContext, // OP --> EXTRA-OP MV-SCAN --> UNION MV-SCAN ---> .... // / \ // EXTRA-OP MV-SCAN - queryInput.getOp().setOpRuleMask(OP_UNION_ALL_BIT); - viewInput.getOp().setOpRuleMask(OP_UNION_ALL_BIT); + queryInput.getOp().setOpRuleBit(OP_MV_UNION_REWRITE); + viewInput.getOp().setOpRuleBit(OP_MV_UNION_REWRITE); // createUnion will return the union all result of queryInput and viewInput // Union diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvPartitionCompensator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvPartitionCompensator.java index d50afdad01f4e..a8b6a11d8fc50 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvPartitionCompensator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvPartitionCompensator.java @@ -89,7 +89,7 @@ import java.util.stream.Collectors; import static com.starrocks.sql.optimizer.OptimizerTraceUtil.logMVRewrite; -import static com.starrocks.sql.optimizer.operator.Operator.OP_UNION_ALL_BIT; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_UNION_REWRITE; import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.deriveLogicalProperty; import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.mergeRanges; @@ -182,7 +182,7 @@ private static Pair> getMvScanPlan(Materi OptExpression newMvScanPlan = duplicator.duplicate(mvScanOptExpression); // output columns order by mv's columns List mvScanOutputColumns = duplicator.getMappedColumns(orgMvScanOutputColumns); - newMvScanPlan.getOp().setOpRuleMask(OP_UNION_ALL_BIT); + newMvScanPlan.getOp().setOpRuleBit(OP_MV_UNION_REWRITE); return Pair.create(newMvScanPlan, mvScanOutputColumns); } @@ -209,7 +209,7 @@ private static Pair> getMvQueryPlan(Mater deriveLogicalProperty(newMvQueryPlan); List orgMvQueryOutputColumnRefs = mvContext.getMvOutputColumnRefs(); List mvQueryOutputColumnRefs = duplicator.getMappedColumns(orgMvQueryOutputColumnRefs); - newMvQueryPlan.getOp().setOpRuleMask(OP_UNION_ALL_BIT); + newMvQueryPlan.getOp().setOpRuleBit(OP_MV_UNION_REWRITE); return Pair.create(newMvQueryPlan, mvQueryOutputColumnRefs); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java index f584dac0ad0af..231b18487c3cd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java @@ -1302,16 +1302,6 @@ public static OptExpression cloneExpression(OptExpression logicalTree) { return OptExpression.create(newOp, inputs); } - /** - * Check whether opt expression or its children have applied mv union rewrite. - * - * @param optExpression: opt expression to check - * @return : true if opt expression or its children have applied mv union rewrite, false otherwise. - */ - public static boolean isAppliedMVUnionRewrite(OptExpression optExpression) { - return Utils.isOptHasAppliedRule(optExpression, Operator.OP_UNION_ALL_BIT); - } - /** * Return mv's plan context. If mv's plan context is not in cache, optimize it. * diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/OptCompensator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/OptCompensator.java index d50b2b5a91247..e91a04ae9318d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/OptCompensator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/OptCompensator.java @@ -28,7 +28,6 @@ import com.starrocks.sql.optimizer.OptExpressionVisitor; import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.Utils; -import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.OperatorBuilderFactory; import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; @@ -41,6 +40,7 @@ import java.util.Map; import java.util.Optional; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED; import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvPartitionCompensator.SUPPORTED_PARTITION_COMPENSATE_EXTERNAL_SCAN_TYPES; import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.convertPartitionKeysToListPredicate; @@ -65,8 +65,6 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Void con LogicalScanOperator scanOperator = optExpression.getOp().cast(); Table refBaseTable = scanOperator.getTable(); - // reset the partition prune flag to be pruned again. - Utils.resetOpAppliedRule(scanOperator, Operator.OP_PARTITION_PRUNE_BIT); if (refBaseTable.isNativeTableOrMaterializedView()) { List olapTableCompensatePartitionIds = Lists.newArrayList(); if (compensations.containsKey(refBaseTable)) { @@ -76,6 +74,8 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Void con } LogicalOlapScanOperator olapScanOperator = (LogicalOlapScanOperator) scanOperator; LogicalScanOperator newScanOperator = getOlapTableCompensatePlan(olapScanOperator, olapTableCompensatePartitionIds); + // reset the partition prune flag to be pruned again. + newScanOperator.resetOpRuleBit(OP_PARTITION_PRUNED); return OptExpression.create(newScanOperator); } else if (SUPPORTED_PARTITION_COMPENSATE_EXTERNAL_SCAN_TYPES.contains(scanOperator.getOpType())) { List partitionKeys = Lists.newArrayList(); @@ -85,6 +85,8 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Void con partitionKeys = externalTableCompensation.getCompensations(); } LogicalScanOperator newScanOperator = getExternalTableCompensatePlan(scanOperator, partitionKeys); + // reset the partition prune flag to be pruned again. + newScanOperator.resetOpRuleBit(OP_PARTITION_PRUNED); return OptExpression.create(newScanOperator); } else { return optExpression; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/AggregateJoinPushDownRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/AggregateJoinPushDownRule.java index 24097fccb12fe..164e90e4fd847 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/AggregateJoinPushDownRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/AggregateJoinPushDownRule.java @@ -41,6 +41,7 @@ import java.util.stream.Collectors; import static com.starrocks.sql.optimizer.OptimizerTraceUtil.logMVRewrite; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_AGG_PUSH_DOWN_REWRITE; /** * Support to push down aggregate functions below join operator and rewrite the query by mv transparently. @@ -84,7 +85,7 @@ public boolean check(OptExpression input, OptimizerContext context) { if (!context.getSessionVariable().isEnableMaterializedViewPushDownRewrite()) { return false; } - if (Utils.isOptHasAppliedRule(input, Operator.OP_PUSH_DOWN_BIT)) { + if (Utils.isOptHasAppliedRule(input, OP_MV_AGG_PUSH_DOWN_REWRITE)) { return false; } return super.check(input, context); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java index 2ea893b048e93..0fc2b45f64074 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java @@ -25,6 +25,7 @@ import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.QueryMaterializationContext; +import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; @@ -46,9 +47,9 @@ import java.util.stream.Collectors; import static com.starrocks.sql.optimizer.OptimizerTraceUtil.logMVRewrite; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_UNION_REWRITE; import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.deriveLogicalProperty; import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.getQuerySplitPredicate; -import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.isAppliedMVUnionRewrite; public abstract class BaseMaterializedViewRewriteRule extends TransformationRule implements IMaterializedViewRewriteRule { @@ -78,7 +79,7 @@ private boolean checkOlapScanWithoutTabletOrPartitionHints(OptExpression input) @Override public boolean check(OptExpression input, OptimizerContext context) { // To avoid dead-loop rewrite, no rewrite when query extra predicate is not changed - if (isAppliedMVUnionRewrite(input)) { + if (Utils.isOptHasAppliedRule(input, OP_MV_UNION_REWRITE)) { return false; } return !context.getCandidateMvs().isEmpty() && checkOlapScanWithoutTabletOrPartitionHints(input); @@ -151,6 +152,11 @@ public List doTransform(OptExpression queryExpression, OptimizerC logMVRewrite(context, this, "too many MV candidates, truncate them to " + numCandidates); mvCandidateContexts = mvCandidateContexts.subList(0, numCandidates); } + if (mvCandidateContexts.isEmpty()) { + return Lists.newArrayList(); + } + logMVRewrite(context, this, "MV Candidates: {}", + mvCandidateContexts.stream().map(x -> x.getMv().getName()).collect(Collectors.toList())); // 3. do rewrite with associated mvs return doTransform(mvCandidateContexts, queryExpression, context); @@ -197,20 +203,20 @@ protected List doTransform(List mvCandida IMaterializedViewRewriter mvRewriter = createRewriter(context, mvRewriteContext); if (mvRewriter == null) { - logMVRewrite(context, this, "create materialized view rewriter failed"); + logMVRewrite(mvRewriteContext, "create materialized view rewriter failed"); continue; } // rewrite query OptExpression candidate = mvRewriter.doRewrite(mvRewriteContext); if (candidate == null) { - logMVRewrite(context, this, "doRewrite phase failed"); + logMVRewrite(mvRewriteContext, "doRewrite phase failed"); continue; } candidate = mvRewriter.postRewrite(context, mvRewriteContext, candidate); if (candidate == null) { - logMVRewrite(context, this, "doPostAfterRewrite phase failed"); + logMVRewrite(mvRewriteContext, "doPostAfterRewrite phase failed"); continue; } @@ -233,10 +239,15 @@ protected List doTransform(List mvCandida // Do not try to enumerate all plans, it would take a lot of time int limit = context.getSessionVariable().getCboMaterializedViewRewriteRuleOutputLimit(); if (limit > 0 && results.size() >= limit) { - logMVRewrite(context, this, "too many MV rewrite results generated, but limit to {}", limit); + logMVRewrite(mvRewriteContext, "too many MV rewrite results generated, but limit to {}", limit); break; } + // mark this mv has applied this query + MvUtils.getScanOperator(candidate) + .stream() + .forEach(op -> op.setOpAppliedMV(mvContext.getMv().getId())); + // Give up rewrite if it exceeds the optimizer timeout context.checkTimeout(); } diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java index a2ff79c8b39f1..e4f26a21ff607 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java @@ -189,7 +189,7 @@ public void testPartitionPrune_SingleTable3() throws Exception { .contains("partial_mv_6") .contains(" TABLE: test_base_part\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 10: c3 < 3000, 9: c2 < 3000, 9: c2 >= 2000\n" + + " PREDICATES: 9: c2 < 3000, 9: c2 >= 2000\n" + " partitions=5/5"); // test query delta sql("select c1, c3, c2 from test_base_part where c2 < 1000 and c3 < 1000") diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java index 75c86c533c9bd..c41251fab52ed 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java @@ -285,8 +285,7 @@ public void testJoinMvRewriteByForceRuleRewrite() throws Exception { } @Test - public void testJoinMvRewrite() throws Exception { - connectContext.getSessionVariable().setOptimizerExecuteTimeout(30000000); + public void testJoinMvRewrite1() throws Exception { createAndRefreshMv("create materialized view join_mv_1" + " distributed by hash(v1)" + " as " + @@ -340,7 +339,10 @@ public void testJoinMvRewrite() throws Exception { PlanTestBase.assertNotContains(plan6, "join_mv_1"); dropMv("test", "join_mv_1"); + } + @Test + public void testJoinMvRewrite2() throws Exception { createAndRefreshMv("create materialized view join_mv_2" + " distributed by hash(v1)" + " as " + @@ -377,7 +379,10 @@ public void testJoinMvRewrite() throws Exception { PlanTestBase.assertContains(plan10, "join_mv_2"); dropMv("test", "join_mv_2"); + } + @Test + public void testJoinMvRewrite3() throws Exception { createAndRefreshMv("create materialized view join_mv_3" + " distributed by hash(empid)" + " as" + @@ -440,7 +445,7 @@ public void testJoinMvRewrite() throws Exception { // query delta depends on join reorder String query16 = "select dependents.empid from depts join dependents on (depts.name = dependents.name)" + " join emps on (emps.deptno = depts.deptno)"; - String plan16 = getFragmentPlan(query16); + String plan16 = getFragmentPlan(query16, "MV"); PlanTestBase.assertContains(plan16, "join_mv_3"); OptExpression optExpression16 = getOptimizedPlan(query16, connectContext); List scanOperators16 = getScanOperators(optExpression16, "join_mv_3"); @@ -460,7 +465,10 @@ public void testJoinMvRewrite() throws Exception { PlanTestBase.assertContains(plan17, "join_mv_3"); dropMv("test", "join_mv_3"); + } + @Test + public void testJoinMvRewrite4() throws Exception { createAndRefreshMv("create materialized view join_mv_4" + " distributed by hash(empid)" + " as" + @@ -475,7 +483,10 @@ public void testJoinMvRewrite() throws Exception { String plan18 = getFragmentPlan(query18); PlanTestBase.assertContains(plan18, "join_mv_4"); dropMv("test", "join_mv_4"); + } + @Test + public void testJoinMvRewrite5() throws Exception { createAndRefreshMv("create materialized view join_mv_5" + " distributed by hash(empid)" + " as" + @@ -495,7 +506,10 @@ public void testJoinMvRewrite() throws Exception { dropMv("test", "join_mv_5"); dropMv("test", "join_mv_6"); + } + @Test + public void testJoinMvRewrite6() throws Exception { createAndRefreshMv("create materialized view join_mv_7" + " distributed by hash(empid)" + " as" + @@ -506,7 +520,10 @@ public void testJoinMvRewrite() throws Exception { String plan20 = getFragmentPlan(query20); PlanTestBase.assertContains(plan20, "join_mv_7"); dropMv("test", "join_mv_7"); + } + @Test + public void testJoinMvRewrite7() throws Exception { // multi relations test createAndRefreshMv("create materialized view join_mv_8" + " distributed by hash(empid)" + @@ -517,7 +534,10 @@ public void testJoinMvRewrite() throws Exception { String plan21 = getFragmentPlan(query21); PlanTestBase.assertContains(plan21, "join_mv_8"); dropMv("test", "join_mv_8"); + } + @Test + public void testJoinMvRewrite8() throws Exception { createAndRefreshMv("create materialized view join_mv_9" + " distributed by hash(empid)" + " as" + diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTimeSeriesRewriteWithOlapTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTimeSeriesRewriteWithOlapTest.java index 183a508f0d4f7..50d6a4b702b34 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTimeSeriesRewriteWithOlapTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTimeSeriesRewriteWithOlapTest.java @@ -78,10 +78,29 @@ public static void beforeClass() throws Exception { "PROPERTIES (\n" + "\"replication_num\" = \"1\"\n" + ") ;"); - cluster.runSql("test", "INSERT INTO t1 VALUES ('2020-10-22','2020-10-22 12:12:12','k3','k4',0,1,2,2,4,5,1.1," + - "1.12,2.889),\n" + - " ('2020-10-23','2020-10-23 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889),\n" + - " ('2020-10-24','2020-10-24 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889);"); + cluster.runSql("test", "INSERT INTO t1 VALUES " + + "('2020-10-22','2020-10-22 12:12:12','k3','k4',0,1,2,2,4,5,1.1,1.12,2.889),\n" + + "('2020-10-23','2020-10-23 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889),\n" + + "('2020-10-24','2020-10-24 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889);"); + + starRocksAssert.withTable("CREATE TABLE t2(\n" + + " ts datetime,\n" + + " v1 INT,\n" + + " v2 INT)\n" + + " DUPLICATE KEY(ts)\n" + + " PARTITION BY date_trunc('day', ts)\n" + + "DISTRIBUTED BY HASH(ts);"); + cluster.runSql("test", "INSERT INTO t2 VALUES \n" + + " ('2020-01-22 12:12:12', 0,1),\n" + + " ('2020-02-23 12:12:12',1,1),\n" + + " ('2020-03-24 12:12:12',1,2),\n" + + " ('2020-04-25 12:12:12',3,3),\n" + + " ('2020-05-22 12:12:12', 0,1),\n" + + " ('2020-06-23 12:12:12',1,1),\n" + + " ('2020-07-24 12:12:12',1,2),\n" + + " ('2020-08-24 12:12:12',1,2),\n" + + " ('2020-09-24 12:12:12',1,2),\n" + + " ('2020-10-25 12:12:12',3,3);"); } @AfterClass @@ -187,20 +206,21 @@ public void testAggregateTimeSeriesRollupWithNestedMV() throws Exception { ") as select date_trunc('month', dt) as dt, sum(sum_v1) as sum_v1, max(max_v2) as max_v2 " + "from test_mv1 group by date_trunc('month', dt);"); // date column should be the same with date_trunc('day', ct) - String query = "select sum(v1), max(v2) from t0 " + - "where k1 >= '2024-01-01 01:00:00'"; + String query = "select sum(v1), max(v2) from t0 where k1 >= '2024-01-01 01:00:00'"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, " TABLE: test_mv2\n" + " PREAGGREGATION: ON\n" + " PREDICATES: 36: dt >= '2024-01-01 01:00:00'\n" + " partitions=3/4"); - PlanTestBase.assertContains(plan, " TABLE: test_mv1\n" + + PlanTestBase.assertContains(plan, " 8:OlapScanNode\n" + + " TABLE: test_mv1\n" + " PREAGGREGATION: ON\n" + " PREDICATES: date_trunc('month', 45: dt) < '2024-01-01 01:00:00', 45: dt >= '2024-01-01 01:00:00'\n" + " partitions=31/63"); PlanTestBase.assertContains(plan, " TABLE: t0\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: date_trunc('day', 25: k1) < '2024-01-01 01:00:00', 25: k1 >= '2024-01-01 01:00:00'"); + " PREDICATES: date_trunc('day', 25: k1) < '2024-01-01 01:00:00', 25: k1 >= '2024-01-01 01:00:00'\n" + + " partitions=1/5"); starRocksAssert.dropMaterializedView("test_mv1"); starRocksAssert.dropMaterializedView("test_mv2"); } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTransparentUnionRewriteHiveTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTransparentUnionRewriteHiveTest.java index 2714e558c180b..d93ab3af1852a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTransparentUnionRewriteHiveTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvTransparentUnionRewriteHiveTest.java @@ -387,8 +387,8 @@ public void testTransparentRewriteWithPartitionPrune() { " PREAGGREGATION: ON\n" + " partitions=3/4", // case 1 " TABLE: lineitem_par\n" + - " PARTITION PREDICATES: (41: l_shipdate < '1998-01-02') OR " + - "(41: l_shipdate IS NULL), 41: l_shipdate >= '1998-01-01'\n" + + " PARTITION PREDICATES: 41: l_shipdate >= '1998-01-01', " + + "(41: l_shipdate < '1998-01-02') OR (41: l_shipdate IS NULL)\n" + " NON-PARTITION PREDICATES: 40: l_suppkey > 1\n" + " MIN/MAX PREDICATES: 40: l_suppkey > 1\n" + " partitions=1/6", diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtilsTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtilsTest.java index 4dc41e3fed07d..96188386bfb7d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtilsTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtilsTest.java @@ -47,6 +47,7 @@ import java.util.Set; +import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED; import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvPartitionCompensator.convertToDateRange; public class MvUtilsTest { @@ -212,12 +213,12 @@ public void testConvertToDateRange() throws AnalysisException { public void testResetOpAppliedRule() { LogicalScanOperator.Builder builder = new LogicalOlapScanOperator.Builder(); Operator op = builder.build(); - Assert.assertFalse(Utils.isOpAppliedRule(op, Operator.OP_PARTITION_PRUNE_BIT)); + Assert.assertFalse(op.isOpRuleBitSet(OP_PARTITION_PRUNED)); // set - Utils.setOpAppliedRule(op, Operator.OP_PARTITION_PRUNE_BIT); - Assert.assertTrue(Utils.isOpAppliedRule(op, Operator.OP_PARTITION_PRUNE_BIT)); + op.setOpRuleBit(OP_PARTITION_PRUNED); + Assert.assertTrue(op.isOpRuleBitSet(OP_PARTITION_PRUNED)); // reset - Utils.resetOpAppliedRule(op, Operator.OP_PARTITION_PRUNE_BIT); - Assert.assertFalse(Utils.isOpAppliedRule(op, Operator.OP_PARTITION_PRUNE_BIT)); + op.resetOpRuleBit(OP_PARTITION_PRUNED); + Assert.assertFalse(op.isOpRuleBitSet(OP_PARTITION_PRUNED)); } } diff --git a/test/sql/test_transparent_mv/R/test_transparent_mv_union_iceberg b/test/sql/test_transparent_mv/R/test_transparent_mv_union_iceberg index 5bbe0a1973394..d08860b188065 100644 --- a/test/sql/test_transparent_mv/R/test_transparent_mv_union_iceberg +++ b/test/sql/test_transparent_mv/R/test_transparent_mv_union_iceberg @@ -408,71 +408,77 @@ SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0} 2020-06-18 120 2020-06-21 168 -- !result -SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3 - +SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3; +-- result: +2020-06-21 168 +2020-06-21 168 +2020-06-24 216 +-- !result INSERT INTO mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 VALUES (1, "2020-06-15"), (4, "2020-07-25"); -- result: -E: (1064, "Getting syntax error at line 3, column 0. Detail message: Unexpected input 'INSERT', the most similar input is {, ';'}.") -- !result -function: check_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1") +function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1") -- result: -None +False -- !result -function: check_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1") +function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1") -- result: -None +False -- !result -function: check_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1", "UNION") +function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1", "UNION") -- result: -None +False -- !result -function: check_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION") +function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION") -- result: -None +False -- !result -function: check_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION") +function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION") -- result: -None +False -- !result -function: check_hit_materialized_view("SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3", "test_mv1", "UNION") +function: print_hit_materialized_view("SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3", "test_mv1", "UNION") -- result: -None +True -- !result SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt order by 1, 2 limit 3; -- result: -2020-07-02 72 -2020-07-05 120 -2020-07-08 168 +2020-07-02 78 +2020-07-05 130 +2020-07-08 182 -- !result SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3; -- result: -2020-07-02 72 -2020-07-05 120 -2020-07-08 168 +2020-07-02 78 +2020-07-05 130 +2020-07-08 182 -- !result SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt order by 1, 2 limit 3; -- result: -2020-06-21 168 -2020-06-24 216 -2020-07-02 72 +2020-06-21 182 +2020-06-24 234 +2020-07-02 78 -- !result SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3; -- result: -2020-06-21 168 -2020-06-24 216 -2020-07-02 72 +2020-06-21 182 +2020-06-24 234 +2020-07-02 78 -- !result SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3; -- result: -2020-06-15 72 -2020-06-18 120 -2020-06-21 168 +2020-06-15 78 +2020-06-18 130 +2020-06-21 182 +-- !result +SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3; +-- result: +2020-06-21 182 +2020-06-21 182 +2020-06-24 234 -- !result -SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3 - DROP MATERIALIZED VIEW test_mv1; -- result: -E: (1064, "Getting syntax error at line 3, column 0. Detail message: Unexpected input 'DROP', the most similar input is {, ';'}.") -- !result drop table mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 force; -- result: diff --git a/test/sql/test_transparent_mv/T/test_transparent_mv_union_iceberg b/test/sql/test_transparent_mv/T/test_transparent_mv_union_iceberg index 2ad1c9b2d8a38..bfa30559dafd9 100644 --- a/test/sql/test_transparent_mv/T/test_transparent_mv_union_iceberg +++ b/test/sql/test_transparent_mv/T/test_transparent_mv_union_iceberg @@ -138,22 +138,22 @@ SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0} SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt order by 1, 2 limit 3; SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3; SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3; -SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3 +SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3; INSERT INTO mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 VALUES (1, "2020-06-15"), (4, "2020-07-25"); -function: check_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1") -function: check_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1") -function: check_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1", "UNION") -function: check_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION") -function: check_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION") -function: check_hit_materialized_view("SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3", "test_mv1", "UNION") +function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1") +function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1") +function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1", "UNION") +function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION") +function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION") +function: print_hit_materialized_view("SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3", "test_mv1", "UNION") SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt order by 1, 2 limit 3; SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-07-01' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3; SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt order by 1, 2 limit 3; SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3; SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3; -SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3 +SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 join mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t2 on t1.dt and t2.dt where t2.dt > '2020-06-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3; DROP MATERIALIZED VIEW test_mv1;