From 4c8230cca5df4929eca420470a4b5bd3cba8b07b Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 18 Dec 2024 21:03:49 +0800 Subject: [PATCH] fix --- .../NereidsSortedPartitionsCacheManager.java | 23 +++++-- .../rules/PartitionItemToRange.java | 31 +++++---- .../expression/rules/PartitionPruner.java | 66 ++++++++++++------- .../rules/SortedPartitionRanges.java | 31 +++++++-- 4 files changed, 104 insertions(+), 47 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java index 0b4ab71aaae90a4..d13b91e15fb6fcf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java @@ -28,12 +28,12 @@ import org.apache.doris.nereids.rules.expression.rules.MultiColumnBound; import org.apache.doris.nereids.rules.expression.rules.PartitionItemToRange; import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges; +import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges.PartitionItemAndId; import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges.PartitionItemAndRange; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.collect.Range; -import com.google.common.collect.TreeRangeSet; import lombok.AllArgsConstructor; import lombok.Data; import org.apache.hadoop.util.Lists; @@ -84,14 +84,23 @@ private SortedPartitionRanges loadCache(TableIdentifier key, OlapTable olapTa Map allPartitions = partitionInfo.getIdToItem(false); List> sortedList = Lists.newArrayList(allPartitions.entrySet()); List> sortedRanges = Lists.newArrayListWithCapacity(allPartitions.size()); + List> defaultPartitions = Lists.newArrayList(); for (Entry entry : sortedList) { - TreeRangeSet rangeSet = PartitionItemToRange.toRangeSets(entry.getValue()); - sortedRanges.add(new PartitionItemAndRange<>(entry.getKey(), entry.getValue(), rangeSet)); + PartitionItem partitionItem = entry.getValue(); + Long id = entry.getKey(); + if (!partitionItem.isDefaultPartition()) { + List> ranges = PartitionItemToRange.toRanges(partitionItem); + for (Range range : ranges) { + sortedRanges.add(new PartitionItemAndRange<>(id, partitionItem, range)); + } + } else { + defaultPartitions.add(new PartitionItemAndId<>(id, partitionItem)); + } } sortedRanges.sort((o1, o2) -> { - Range span1 = o1.ranges.span(); - Range span2 = o2.ranges.span(); + Range span1 = o1.range; + Range span2 = o2.range; int result = span1.lowerEndpoint().compareTo(span2.lowerEndpoint()); if (result != 0) { return result; @@ -99,7 +108,9 @@ private SortedPartitionRanges loadCache(TableIdentifier key, OlapTable olapTa result = span1.upperEndpoint().compareTo(span2.upperEndpoint()); return result; }); - SortedPartitionRanges sortedPartitionRanges = new SortedPartitionRanges(sortedRanges); + SortedPartitionRanges sortedPartitionRanges = new SortedPartitionRanges( + sortedRanges, defaultPartitions + ); PartitionCacheContext context = new PartitionCacheContext( olapTable.getId(), olapTable.getVisibleVersion(), sortedPartitionRanges); partitionCaches.put(key, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionItemToRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionItemToRange.java index bde2f3cb1a9c891..435dd700713940d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionItemToRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionItemToRange.java @@ -23,25 +23,25 @@ import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.types.DataType; import com.google.common.collect.ImmutableList; import com.google.common.collect.Range; -import com.google.common.collect.TreeRangeSet; import java.util.List; /** PartitionItemToRange */ public class PartitionItemToRange { /** toRangeSets */ - public static TreeRangeSet toRangeSets(PartitionItem partitionItem) { + public static List> toRanges(PartitionItem partitionItem) { if (partitionItem instanceof RangePartitionItem) { Range range = partitionItem.getItems(); - TreeRangeSet oneRangePartitionRanges = TreeRangeSet.create(); PartitionKey lowerKey = range.lowerEndpoint(); ImmutableList.Builder lowerBounds = ImmutableList.builderWithExpectedSize(lowerKey.getKeys().size()); for (LiteralExpr key : lowerKey.getKeys()) { - Literal literal = Literal.fromLegacyLiteral(key, key.getType()); + Literal literal = toNereidsLiteral(key); lowerBounds.add(ColumnBound.of(literal)); } @@ -53,26 +53,35 @@ public static TreeRangeSet toRangeSets(PartitionItem partition upperBounds.add(ColumnBound.of(literal)); } - oneRangePartitionRanges.add(Range.closedOpen( + return ImmutableList.of(Range.closedOpen( new MultiColumnBound(lowerBounds.build()), - new MultiColumnBound(upperBounds.build()))); - return oneRangePartitionRanges; + new MultiColumnBound(upperBounds.build()) + )); } else if (partitionItem instanceof ListPartitionItem) { - TreeRangeSet oneListPartitionRanges = TreeRangeSet.create(); List partitionKeys = partitionItem.getItems(); + ImmutableList.Builder> ranges + = ImmutableList.builderWithExpectedSize(partitionKeys.size()); for (PartitionKey partitionKey : partitionKeys) { ImmutableList.Builder bounds = ImmutableList.builderWithExpectedSize(partitionKeys.size()); for (LiteralExpr key : partitionKey.getKeys()) { - Literal literal = Literal.fromLegacyLiteral(key, key.getType()); + Literal literal = toNereidsLiteral(key); bounds.add(ColumnBound.of(literal)); } MultiColumnBound bound = new MultiColumnBound(bounds.build()); - oneListPartitionRanges.add(Range.singleton(bound)); + ranges.add(Range.singleton(bound)); } - return oneListPartitionRanges; + return ranges.build(); } else { throw new UnsupportedOperationException(partitionItem.getClass().getName()); } } + + private static Literal toNereidsLiteral(LiteralExpr partitionKeyLiteral) { + if (!partitionKeyLiteral.isMinValue()) { + return Literal.fromLegacyLiteral(partitionKeyLiteral, partitionKeyLiteral.getType()); + } else { + return new NullLiteral(DataType.fromCatalogType(partitionKeyLiteral.getType())); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java index cc3309c3c055f3d..57db28cbc83abb5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges.PartitionItemAndId; import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges.PartitionItemAndRange; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; @@ -42,12 +43,15 @@ import com.google.common.collect.Lists; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; +import com.google.common.collect.Sets; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * PartitionPruner @@ -106,7 +110,7 @@ public Expression visitComparisonPredicate(ComparisonPredicate cp, Void context) } /** prune */ - public List prune() { + public > List prune() { Builder scanPartitionIdents = ImmutableList.builder(); for (OnePartitionEvaluator partition : partitions) { if (!canBePrunedOut(partitionPredicate, partition)) { @@ -116,7 +120,7 @@ public List prune() { return scanPartitionIdents.build(); } - public static List prune(List partitionSlots, Expression partitionPredicate, + public static > List prune(List partitionSlots, Expression partitionPredicate, Map idToPartitions, CascadesContext cascadesContext, PartitionTableType partitionTableType) { return prune(partitionSlots, partitionPredicate, idToPartitions, @@ -126,7 +130,7 @@ public static List prune(List partitionSlots, Expression partitionP /** * prune partition with `idToPartitions` as parameter. */ - public static List prune(List partitionSlots, Expression partitionPredicate, + public static > List prune(List partitionSlots, Expression partitionPredicate, Map idToPartitions, CascadesContext cascadesContext, PartitionTableType partitionTableType, Optional> sortedPartitionRanges) { partitionPredicate = PartitionPruneExpressionExtractor.extract( @@ -177,61 +181,73 @@ public static OnePartitionEvaluator toPartitionEvaluator(K id, PartitionI } } - private static List binarySearchFiltering( + private static > List binarySearchFiltering( SortedPartitionRanges sortedPartitionRanges, List partitionSlots, Expression partitionPredicate, CascadesContext cascadesContext, int expandThreshold, RangeSet predicateRanges) { List> sortedPartitions = sortedPartitionRanges.sortedPartitions; - List selectedPartitions = Lists.newArrayList(); + Set selectedIdSets = Sets.newTreeSet(); int leftIndex = 0; - int midIndex = 0; for (Range predicateRange : predicateRanges.asRanges()) { int rightIndex = sortedPartitions.size(); if (leftIndex >= rightIndex) { break; } - MultiColumnBound predicateLowerBound = predicateRange.lowerEndpoint(); + + int midIndex; MultiColumnBound predicateUpperBound = predicateRange.upperEndpoint(); + MultiColumnBound predicateLowerBound = predicateRange.lowerEndpoint(); + while (leftIndex + 1 < rightIndex) { midIndex = (leftIndex + rightIndex) / 2; PartitionItemAndRange partition = sortedPartitions.get(midIndex); - Range partitionSpan = partition.ranges.span(); - MultiColumnBound partitionLowerBound = partitionSpan.lowerEndpoint(); + Range partitionSpan = partition.range; - int compare = predicateLowerBound.compareTo(partitionLowerBound); - if (compare == 0) { - break; - } else if (compare < 0) { + if (predicateUpperBound.compareTo(partitionSpan.lowerEndpoint()) < 0) { rightIndex = midIndex; - } else { + } else if (predicateLowerBound.compareTo(partitionSpan.upperEndpoint()) > 0) { leftIndex = midIndex; + } else { + break; } } - for (leftIndex = midIndex; leftIndex < sortedPartitions.size(); leftIndex++) { + for (; leftIndex < sortedPartitions.size(); leftIndex++) { PartitionItemAndRange partition = sortedPartitions.get(leftIndex); - if (leftIndex > midIndex) { - Range partitionSpan = partition.ranges.span(); - MultiColumnBound partitionLowerBound = partitionSpan.lowerEndpoint(); - if (partitionLowerBound.compareTo(predicateUpperBound) > 0) { - break; - } - } K partitionId = partition.id; + // list partition will expand to multiple PartitionItemAndRange, we should skip evaluate it again + if (selectedIdSets.contains(partitionId)) { + continue; + } + + Range partitionSpan = partition.range; + if (predicateUpperBound.compareTo(partitionSpan.lowerEndpoint()) < 0) { + break; + } + OnePartitionEvaluator partitionEvaluator = toPartitionEvaluator( partitionId, partition.partitionItem, partitionSlots, cascadesContext, expandThreshold); if (!canBePrunedOut(partitionPredicate, partitionEvaluator)) { - selectedPartitions.add(partitionId); + selectedIdSets.add(partitionId); } } } - return selectedPartitions; + for (PartitionItemAndId defaultPartition : sortedPartitionRanges.defaultPartitions) { + K partitionId = defaultPartition.id; + OnePartitionEvaluator partitionEvaluator = toPartitionEvaluator( + partitionId, defaultPartition.partitionItem, partitionSlots, cascadesContext, expandThreshold); + if (!canBePrunedOut(partitionPredicate, partitionEvaluator)) { + selectedIdSets.add(partitionId); + } + } + + return Utils.fastToImmutableList(selectedIdSets); } - private static List sequentialFiltering(Map idToPartitions, List partitionSlots, + private static > List sequentialFiltering(Map idToPartitions, List partitionSlots, Expression partitionPredicate, CascadesContext cascadesContext, int expandThreshold) { List> evaluators = Lists.newArrayListWithCapacity(idToPartitions.size()); for (Entry kv : idToPartitions.entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SortedPartitionRanges.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SortedPartitionRanges.java index a98df8d9d8dad53..fe67836405ecf9a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SortedPartitionRanges.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SortedPartitionRanges.java @@ -20,7 +20,7 @@ import org.apache.doris.catalog.PartitionItem; import org.apache.doris.nereids.util.Utils; -import com.google.common.collect.RangeSet; +import com.google.common.collect.Range; import java.util.List; import java.util.Objects; @@ -28,23 +28,44 @@ /** SortedPartitionRanges */ public class SortedPartitionRanges { public final List> sortedPartitions; + public final List> defaultPartitions; - public SortedPartitionRanges(List> sortedPartitions) { + public SortedPartitionRanges( + List> sortedPartitions, List> defaultPartitions) { this.sortedPartitions = Utils.fastToImmutableList( Objects.requireNonNull(sortedPartitions, "sortedPartitions bounds can not be null") ); + this.defaultPartitions = Utils.fastToImmutableList( + Objects.requireNonNull(defaultPartitions, "defaultPartitions bounds can not be null") + ); } /** PartitionItemAndRange */ public static class PartitionItemAndRange { public final K id; public final PartitionItem partitionItem; - public final RangeSet ranges; + public final Range range; + + public PartitionItemAndRange(K id, PartitionItem partitionItem, Range range) { + this.id = id; + this.partitionItem = partitionItem; + this.range = range; + } + + @Override + public String toString() { + return range.toString(); + } + } + + /** PartitionItemAndId */ + public static class PartitionItemAndId { + public final K id; + public final PartitionItem partitionItem; - public PartitionItemAndRange(K id, PartitionItem partitionItem, RangeSet ranges) { + public PartitionItemAndId(K id, PartitionItem partitionItem) { this.id = id; this.partitionItem = partitionItem; - this.ranges = ranges; } } }