From 67befe3d896c3678c849fcb5a338daa0060938ea Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 3 Dec 2024 21:01:24 +0800 Subject: [PATCH] [opt](mtmv) Support mv rewrite when mv has date_trunc but query not --- ...AbstractMaterializedViewAggregateRule.java | 4 +- .../mv/AbstractMaterializedViewJoinRule.java | 5 +- .../mv/AbstractMaterializedViewRule.java | 72 ++++++++++++++++--- .../mv/AbstractMaterializedViewScanRule.java | 4 +- .../rules/exploration/mv/Predicates.java | 39 ++++++++-- .../exploration/mv/PredicatesSplitter.java | 1 + .../doris/nereids/util/ExpressionUtils.java | 23 ++++++ 7 files changed, 129 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 5a854f085b2f773..fd2b5b4b8aaa06f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -57,6 +57,7 @@ import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -126,7 +127,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryTopPlan, materializationContext.getShuttledExprToScanExprMapping(), viewToQuerySlotMapping, - queryStructInfo.getTableBitSet()); + queryStructInfo.getTableBitSet(), + ImmutableMap.of(), cascadesContext); boolean isRewrittenQueryExpressionValid = true; if (!rewrittenQueryExpressions.isEmpty()) { List projects = new ArrayList<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 28ea4dc2c7e16f4..122688b766e3a67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -26,6 +26,8 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import com.google.common.collect.ImmutableMap; + import java.util.List; import java.util.stream.Collectors; @@ -49,7 +51,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryStructInfo.getTopPlan(), materializationContext.getShuttledExprToScanExprMapping(), targetToSourceMapping, - queryStructInfo.getTableBitSet() + queryStructInfo.getTableBitSet(), + ImmutableMap.of(), cascadesContext ); // Can not rewrite, bail out if (expressionsRewritten.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index e6f384502d620fd..9ab33416f7d9c06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -37,6 +37,8 @@ import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; @@ -45,6 +47,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; @@ -242,7 +245,9 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca // Try to rewrite compensate predicates by using mv scan List rewriteCompensatePredicates = rewriteExpression(compensatePredicates.toList(), queryPlan, materializationContext.getShuttledExprToScanExprMapping(), - viewToQuerySlotMapping, queryStructInfo.getTableBitSet()); + viewToQuerySlotMapping, queryStructInfo.getTableBitSet(), + compensatePredicates.getRangePredicateMap(), + cascadesContext); if (rewriteCompensatePredicates.isEmpty()) { materializationContext.recordFailReason(queryStructInfo, "Rewrite compensate predicate by view fail", @@ -560,7 +565,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf * then use the corresponding value of mapping to replace it */ protected List rewriteExpression(List sourceExpressionsToWrite, Plan sourcePlan, - ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, BitSet sourcePlanBitSet) { + ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, BitSet sourcePlanBitSet, + Map shuttledQueryMap, CascadesContext cascadesContext) { // Firstly, rewrite the target expression using source with inverse mapping // then try to use the target expression to represent the query. if any of source expressions // can not be represented by target expressions, return null. @@ -579,18 +585,62 @@ protected List rewriteExpression(List sourceEx rewrittenExpressions.add(expressionShuttledToRewrite); continue; } - final Set slotsToRewrite = + final Set slotsToRewrite = expressionShuttledToRewrite.collectToSet(expression -> expression instanceof Slot); final Set variants = expressionShuttledToRewrite.collectToSet(expression -> expression instanceof SlotReference - && ((SlotReference) expression).getDataType() instanceof VariantType); + && ((SlotReference) expression).getDataType() instanceof VariantType); extendMappingByVariant(variants, targetToTargetReplacementMappingQueryBased); Expression replacedExpression = ExpressionUtils.replace(expressionShuttledToRewrite, targetToTargetReplacementMappingQueryBased); - if (replacedExpression.anyMatch(slotsToRewrite::contains)) { - // if contains any slot to rewrite, which means can not be rewritten by target, bail out - return ImmutableList.of(); + Set replacedExpressionSlotQueryUsed = replacedExpression.collect(slotsToRewrite::contains); + if (!replacedExpressionSlotQueryUsed.isEmpty()) { + // if contains any slot to rewrite, which means can not be rewritten by target, + // expressionShuttledToRewrite is slot#0 > '2024-01-01' but mv plan output is date_trunc(slot#0, 'day') + // which would try to rewrite + // paramExpressionToDateTruncMap is {slot#0 : date_trunc(slot#0, 'day')} + Map paramExpressionToDateTruncMap = new HashMap<>(); + targetToTargetReplacementMappingQueryBased.keySet().forEach(expr -> { + if (expr instanceof DateTrunc) { + paramExpressionToDateTruncMap.put(expr.child(0), (DateTrunc) expr); + } + }); + Expression queryExpr = expressionShuttledToRewrite.child(0); + Map shuttledQueryParamToExpressionMap = new HashMap<>(); + // TODO: 2024/12/5 optimize performance + for (Map.Entry expressionEntry : shuttledQueryMap.entrySet()) { + Expression shuttledQueryParamExpression = ExpressionUtils.shuttleExpressionWithLineage( + expressionEntry.getKey(), sourcePlan, sourcePlanBitSet); + shuttledQueryParamToExpressionMap.put(shuttledQueryParamExpression.child(0) instanceof Literal + ? shuttledQueryParamExpression.child(1) : shuttledQueryParamExpression.child(0), + expressionEntry.getValue()); + } + + if (paramExpressionToDateTruncMap.isEmpty() || shuttledQueryMap.isEmpty() + || !shuttledQueryMap.containsKey(expressionShuttledToRewrite) + || !paramExpressionToDateTruncMap.containsKey(queryExpr)) { + // mv date_trunc expression can not offer expression for query, + // can not try to rewrite by date_trunc, bail out + return ImmutableList.of(); + } + + Map datetruncMap = new HashMap<>(); + Literal queryLiteral = shuttledQueryMap.get(expressionShuttledToRewrite); + datetruncMap.put(queryExpr, queryLiteral); + Expression replacedWithLiteral = ExpressionUtils.replace( + paramExpressionToDateTruncMap.get(queryExpr), datetruncMap); + Expression foldedExpressionWithLiteral = FoldConstantRuleOnFE.evaluate(replacedWithLiteral, + new ExpressionRewriteContext(cascadesContext)); + if (foldedExpressionWithLiteral.equals(queryLiteral)) { + // after date_trunc simplify if equals to original expression, could rewritten by mv + replacedExpression = ExpressionUtils.replace(expressionShuttledToRewrite, + targetToTargetReplacementMappingQueryBased, + paramExpressionToDateTruncMap); + } + if (replacedExpression.anyMatch(slotsToRewrite::contains)) { + return ImmutableList.of(); + } } rewrittenExpressions.add(replacedExpression); } @@ -758,7 +808,7 @@ protected SplitPredicate predicatesCompensate( viewToQuerySlotMapping, comparisonResult); // range compensate - final Set rangeCompensatePredicates = Predicates.compensateRangePredicate( + final Map rangeCompensatePredicates = Predicates.compensateRangePredicate( queryStructInfo, viewStructInfo, viewToQuerySlotMapping, @@ -775,7 +825,8 @@ protected SplitPredicate predicatesCompensate( return SplitPredicate.INVALID_INSTANCE; } if (equalCompensateConjunctions.stream().anyMatch(expr -> expr.containsType(AggregateFunction.class)) - || rangeCompensatePredicates.stream().anyMatch(expr -> expr.containsType(AggregateFunction.class)) + || rangeCompensatePredicates.keySet().stream() + .anyMatch(expr -> expr.containsType(AggregateFunction.class)) || residualCompensatePredicates.stream().anyMatch(expr -> expr.containsType(AggregateFunction.class))) { return SplitPredicate.INVALID_INSTANCE; @@ -783,7 +834,8 @@ protected SplitPredicate predicatesCompensate( return SplitPredicate.of(equalCompensateConjunctions.isEmpty() ? BooleanLiteral.TRUE : ExpressionUtils.and(equalCompensateConjunctions), rangeCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE - : ExpressionUtils.and(rangeCompensatePredicates), + : ExpressionUtils.and(rangeCompensatePredicates.keySet()), + rangeCompensatePredicates.isEmpty() ? ImmutableMap.of() : rangeCompensatePredicates, residualCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE : ExpressionUtils.and(residualCompensatePredicates)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java index 7cd49c94b09ed6d..7b2f685cf308c6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.List; @@ -50,7 +51,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryStructInfo.getTopPlan(), materializationContext.getShuttledExprToScanExprMapping(), targetToSourceMapping, - queryStructInfo.getTableBitSet() + queryStructInfo.getTableBitSet(), + ImmutableMap.of(), cascadesContext ); // Can not rewrite, bail out if (expressionsRewritten.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 20ded0415ade075..24f786f94486e10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -23,18 +23,24 @@ import org.apache.doris.nereids.rules.expression.ExpressionNormalization; import org.apache.doris.nereids.rules.expression.ExpressionOptimization; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -139,7 +145,7 @@ public static Set compensateEquivalence(StructInfo queryStructInfo, /** * compensate range predicates */ - public static Set compensateRangePredicate(StructInfo queryStructInfo, + public static Map compensateRangePredicate(StructInfo queryStructInfo, StructInfo viewStructInfo, SlotMapping viewToQuerySlotMapping, ComparisonResult comparisonResult, @@ -159,7 +165,7 @@ public static Set compensateRangePredicate(StructInfo queryStructInf Sets.difference(viewRangeQueryBasedSet, queryRangeSet).copyInto(differentExpressions); // the range predicate in query and view is same, don't need to compensate if (differentExpressions.isEmpty()) { - return differentExpressions; + return ImmutableMap.of(); } // try to normalize the different expressions Set normalizedExpressions = @@ -168,7 +174,18 @@ public static Set compensateRangePredicate(StructInfo queryStructInf // normalized expressions is not in query, can not compensate return null; } - return normalizedExpressions; + Map normalizedExpressionsWithLiteral = new HashMap<>(); + for (Expression expression : normalizedExpressions) { + Set literalSet = expression.collect(expressionTreeNode -> expressionTreeNode instanceof Literal); + if (!(expression instanceof ComparisonPredicate) + || (expression instanceof GreaterThan || expression instanceof LessThan) + || literalSet.size() != 1) { + normalizedExpressionsWithLiteral.put(expression, null); + continue; + } + normalizedExpressionsWithLiteral.put(expression, literalSet.iterator().next()); + } + return normalizedExpressionsWithLiteral; } private static Set normalizeExpression(Expression expression, CascadesContext cascadesContext) { @@ -220,14 +237,19 @@ public String toString() { */ public static final class SplitPredicate { public static final SplitPredicate INVALID_INSTANCE = - SplitPredicate.of(null, null, null); + SplitPredicate.of(null, null, null, null); private final Optional equalPredicate; private final Optional rangePredicate; + private final Optional> rangePredicateMap; private final Optional residualPredicate; - public SplitPredicate(Expression equalPredicate, Expression rangePredicate, Expression residualPredicate) { + public SplitPredicate(Expression equalPredicate, + Expression rangePredicate, + Map rangePredicateMap, + Expression residualPredicate) { this.equalPredicate = Optional.ofNullable(equalPredicate); this.rangePredicate = Optional.ofNullable(rangePredicate); + this.rangePredicateMap = Optional.ofNullable(rangePredicateMap); this.residualPredicate = Optional.ofNullable(residualPredicate); } @@ -239,6 +261,10 @@ public Expression getRangePredicate() { return rangePredicate.orElse(BooleanLiteral.TRUE); } + public Map getRangePredicateMap() { + return rangePredicateMap.orElse(ImmutableMap.of()); + } + public Expression getResidualPredicate() { return residualPredicate.orElse(BooleanLiteral.TRUE); } @@ -248,8 +274,9 @@ public Expression getResidualPredicate() { */ public static SplitPredicate of(Expression equalPredicates, Expression rangePredicates, + Map rangePredicateSet, Expression residualPredicates) { - return new SplitPredicate(equalPredicates, rangePredicates, residualPredicates); + return new SplitPredicate(equalPredicates, rangePredicates, rangePredicateSet, residualPredicates); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java index f7182eeab7386c0..e66a847cae7f01c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java @@ -104,6 +104,7 @@ public Predicates.SplitPredicate getSplitPredicate() { return Predicates.SplitPredicate.of( equalPredicates.isEmpty() ? null : ExpressionUtils.and(equalPredicates), rangePredicates.isEmpty() ? null : ExpressionUtils.and(rangePredicates), + null, residualPredicates.isEmpty() ? null : ExpressionUtils.and(residualPredicates)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 53ab8a50683ecbd..e6070007962ff91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -403,6 +403,29 @@ public static Expression replace(Expression expr, Map + * input expression: a > 1 + * replaceMap: a -> b + c + * + * output: + * b + c > 1 + * + */ + public static Expression replace(Expression expr, Map replaceMap, + Map transferMap) { + return expr.rewriteDownShortCircuit(e -> { + Expression replacedExpr = replaceMap.get(e); + if (replacedExpr != null) { + return replacedExpr; + } + replacedExpr = replaceMap.get(transferMap.get(e)); + return replacedExpr == null ? e : replacedExpr; + }); + } + public static List replace(List exprs, Map replaceMap) { ImmutableList.Builder result = ImmutableList.builderWithExpectedSize(exprs.size());