From 5492dd3ccb1a36254b5bf37b70e6ae032989edd5 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 4 Dec 2024 19:52:23 +0800 Subject: [PATCH] [feature](function) support orthogonal_bitmap_expr_calculate & orthogonal_bitmap_expr_calculate_count for nereids (#44991) support orthogonal_bitmap_expr_calculate & orthogonal_bitmap_expr_calculate_count for nereids (cherry picked from commit d58a972601683bb58a0956b1bb3af64f7bcd751d) --- .../aggregate_function_orthogonal_bitmap.h | 2 +- .../catalog/BuiltinAggregateFunctions.java | 4 + .../implementation/AggregateStrategies.java | 12 +++ .../functions/agg/AggregateFunction.java | 4 + .../functions/agg/AggregatePhase.java | 23 +++++ .../agg/OrthogonalBitmapExprCalculate.java | 84 ++++++++++++++++++ .../OrthogonalBitmapExprCalculateCount.java | 86 +++++++++++++++++++ .../trees/plans/logical/LogicalAggregate.java | 12 +++ ...st_orthogonal_bitmap_expr_calculate.groovy | 55 ++++++++++++ 9 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregatePhase.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculate.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculateCount.java create mode 100644 regression-test/suites/nereids_function_p0/agg_function/test_orthogonal_bitmap_expr_calculate.groovy diff --git a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h index 2a7dbd44b8a4f6..8a729a515fe29c 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h +++ b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h @@ -228,7 +228,7 @@ struct AggOrthBitmapExprCalBaseData { void init_add_key(const IColumn** columns, size_t row_num, int argument_size) { if (first_init) { DCHECK(argument_size > 1); - const auto& col = assert_cast(*columns[2]); + const auto& col = assert_cast(*columns[2]); std::string expr = col.get_data_at(row_num).to_string(); bitmap_expr_cal.bitmap_calculation_init(expr); first_init = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java index 4933e83d91635e..d8f95b2f6bcc84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java @@ -53,6 +53,8 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctSum; import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctSum0; import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv; +import org.apache.doris.nereids.trees.expressions.functions.agg.OrthogonalBitmapExprCalculate; +import org.apache.doris.nereids.trees.expressions.functions.agg.OrthogonalBitmapExprCalculateCount; import org.apache.doris.nereids.trees.expressions.functions.agg.OrthogonalBitmapIntersect; import org.apache.doris.nereids.trees.expressions.functions.agg.OrthogonalBitmapIntersectCount; import org.apache.doris.nereids.trees.expressions.functions.agg.OrthogonalBitmapUnionCount; @@ -124,6 +126,8 @@ public class BuiltinAggregateFunctions implements FunctionHelper { agg(MultiDistinctSum.class, "multi_distinct_sum"), agg(MultiDistinctSum0.class, "multi_distinct_sum0"), agg(Ndv.class, "approx_count_distinct", "ndv"), + agg(OrthogonalBitmapExprCalculate.class, "orthogonal_bitmap_expr_calculate"), + agg(OrthogonalBitmapExprCalculateCount.class, "orthogonal_bitmap_expr_calculate_count"), agg(OrthogonalBitmapIntersect.class, "orthogonal_bitmap_intersect"), agg(OrthogonalBitmapIntersectCount.class, "orthogonal_bitmap_intersect_count"), agg(OrthogonalBitmapUnionCount.class, "orthogonal_bitmap_union_count"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index a0bfab4cb11551..738ddc2630f37f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -47,6 +47,7 @@ import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregatePhase; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat; import org.apache.doris.nereids.trees.expressions.functions.agg.Max; @@ -285,31 +286,37 @@ public List buildRules() { RuleType.ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT.build( basePattern .when(agg -> agg.getDistinctArguments().isEmpty()) + .when(agg -> agg.supportAggregatePhase(AggregatePhase.ONE)) .thenApplyMulti(ctx -> onePhaseAggregateWithoutDistinct(ctx.root, ctx.connectContext)) ), RuleType.TWO_PHASE_AGGREGATE_WITHOUT_DISTINCT.build( basePattern .when(agg -> agg.getDistinctArguments().isEmpty()) + .when(agg -> agg.supportAggregatePhase(AggregatePhase.TWO)) .thenApplyMulti(ctx -> twoPhaseAggregateWithoutDistinct(ctx.root, ctx.connectContext)) ), // RuleType.TWO_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI.build( // basePattern // .when(this::containsCountDistinctMultiExpr) + // .when(agg -> agg.supportAggregatePhase(AggregatePhase.TWO)) // .thenApplyMulti(ctx -> twoPhaseAggregateWithCountDistinctMulti(ctx.root, ctx.cascadesContext)) // ), RuleType.THREE_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI.build( basePattern .when(this::containsCountDistinctMultiExpr) + .when(agg -> agg.supportAggregatePhase(AggregatePhase.THREE)) .thenApplyMulti(ctx -> threePhaseAggregateWithCountDistinctMulti(ctx.root, ctx.cascadesContext)) ), RuleType.ONE_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI.build( basePattern .when(agg -> agg.getDistinctArguments().size() == 1 && couldConvertToMulti(agg)) + .when(agg -> agg.supportAggregatePhase(AggregatePhase.ONE)) .thenApplyMulti(ctx -> onePhaseAggregateWithMultiDistinct(ctx.root, ctx.connectContext)) ), RuleType.TWO_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI.build( basePattern .when(agg -> agg.getDistinctArguments().size() == 1 && couldConvertToMulti(agg)) + .when(agg -> agg.supportAggregatePhase(AggregatePhase.TWO)) .thenApplyMulti(ctx -> twoPhaseAggregateWithMultiDistinct(ctx.root, ctx.connectContext)) ), RuleType.TWO_PHASE_AGGREGATE_WITH_MULTI_DISTINCT.build( @@ -317,17 +324,20 @@ public List buildRules() { .when(agg -> agg.getDistinctArguments().size() > 1 && !containsCountDistinctMultiExpr(agg) && couldConvertToMulti(agg)) + .when(agg -> agg.supportAggregatePhase(AggregatePhase.TWO)) .thenApplyMulti(ctx -> twoPhaseAggregateWithMultiDistinct(ctx.root, ctx.connectContext)) ), // RuleType.TWO_PHASE_AGGREGATE_WITH_DISTINCT.build( // basePattern // .when(agg -> agg.getDistinctArguments().size() == 1) + // .when(agg -> agg.supportAggregatePhase(AggregatePhase.TWO)) // .thenApplyMulti(ctx -> twoPhaseAggregateWithDistinct(ctx.root, ctx.connectContext)) // ), RuleType.THREE_PHASE_AGGREGATE_WITH_DISTINCT.build( basePattern .when(agg -> agg.getDistinctArguments().size() == 1) .whenNot(agg -> agg.mustUseMultiDistinctAgg()) + .when(agg -> agg.supportAggregatePhase(AggregatePhase.THREE)) .thenApplyMulti(ctx -> threePhaseAggregateWithDistinct(ctx.root, ctx.connectContext)) ), /* @@ -352,6 +362,7 @@ && couldConvertToMulti(agg)) .when(agg -> agg.getDistinctArguments().size() == 1) .when(agg -> agg.getGroupByExpressions().isEmpty()) .whenNot(agg -> agg.mustUseMultiDistinctAgg()) + .when(agg -> agg.supportAggregatePhase(AggregatePhase.FOUR)) .thenApplyMulti(ctx -> { Function, RequireProperties> secondPhaseRequireDistinctHash = groupByAndDistinct -> RequireProperties.of( @@ -399,6 +410,7 @@ && couldConvertToMulti(agg)) } return couldConvertToMulti(agg); }) + .when(agg -> agg.supportAggregatePhase(AggregatePhase.FOUR)) .thenApplyMulti(ctx -> { Function, RequireProperties> secondPhaseRequireGroupByAndDistinctHash = groupByAndDistinct -> RequireProperties.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java index 58b9d0274ddfc5..90df2f531da3fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java @@ -131,6 +131,10 @@ public String toString() { return getName() + "(" + (distinct ? "DISTINCT " : "") + args + ")"; } + public boolean supportAggregatePhase(AggregatePhase aggregatePhase) { + return true; + } + public List getDistinctArguments() { return distinct ? getArguments() : ImmutableList.of(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregatePhase.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregatePhase.java new file mode 100644 index 00000000000000..9115e5890ad66c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregatePhase.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.agg; + +/** AggregatePhase */ +public enum AggregatePhase { + ONE, TWO, THREE, FOUR +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculate.java new file mode 100644 index 00000000000000..2575709fa318c2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculate.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.agg; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.types.BitmapType; +import org.apache.doris.nereids.types.VarcharType; +import org.apache.doris.nereids.types.coercion.CharacterType; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** OrthogonalBitmapExprCalculate */ +public class OrthogonalBitmapExprCalculate extends NotNullableAggregateFunction + implements OrthogonalBitmapFunction, ExplicitlyCastableSignature { + + static final List FUNCTION_SIGNATURES = ImmutableList.of( + FunctionSignature.ret(BitmapType.INSTANCE) + .varArgs(BitmapType.INSTANCE, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT) + ); + + /** + * constructor with 3 arguments. + */ + public OrthogonalBitmapExprCalculate( + Expression bitmap, Expression filterColumn, VarcharLiteral inputString) { + super("orthogonal_bitmap_expr_calculate", ExpressionUtils.mergeArguments(bitmap, filterColumn, inputString)); + } + + /** + * constructor with 3 arguments. + */ + public OrthogonalBitmapExprCalculate(boolean distinct, + Expression bitmap, Expression filterColumn, VarcharLiteral inputString) { + super("orthogonal_bitmap_expr_calculate", distinct, + ExpressionUtils.mergeArguments(bitmap, filterColumn, inputString)); + } + + @Override + public boolean supportAggregatePhase(AggregatePhase aggregatePhase) { + return aggregatePhase == AggregatePhase.TWO; + } + + @Override + public Expression resultForEmptyInput() { + return new BitmapEmpty(); + } + + @Override + public OrthogonalBitmapExprCalculate withDistinctAndChildren(boolean distinct, List children) { + Preconditions.checkArgument(children.size() == 3 + && children.get(2).getDataType() instanceof CharacterType + && children.get(2).getDataType() instanceof VarcharType); + return new OrthogonalBitmapExprCalculate( + distinct, children.get(0), children.get(1), (VarcharLiteral) children.get(2)); + } + + @Override + public List getSignatures() { + return FUNCTION_SIGNATURES; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculateCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculateCount.java new file mode 100644 index 00000000000000..0ae69f9fe89c28 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculateCount.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.agg; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.BitmapType; +import org.apache.doris.nereids.types.VarcharType; +import org.apache.doris.nereids.types.coercion.CharacterType; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** OrthogonalBitmapExprCalculateCount */ +public class OrthogonalBitmapExprCalculateCount extends NotNullableAggregateFunction + implements OrthogonalBitmapFunction, ExplicitlyCastableSignature { + + static final List FUNCTION_SIGNATURES = ImmutableList.of( + FunctionSignature.ret(BigIntType.INSTANCE) + .varArgs(BitmapType.INSTANCE, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT) + ); + + /** + * constructor with 3 arguments. + */ + public OrthogonalBitmapExprCalculateCount( + Expression bitmap, Expression filterColumn, VarcharLiteral inputString) { + super("orthogonal_bitmap_expr_calculate_count", + ExpressionUtils.mergeArguments(bitmap, filterColumn, inputString)); + } + + /** + * constructor with 3 arguments. + */ + public OrthogonalBitmapExprCalculateCount(boolean distinct, + Expression bitmap, Expression filterColumn, VarcharLiteral inputString) { + super("orthogonal_bitmap_expr_calculate_count", distinct, + ExpressionUtils.mergeArguments(bitmap, filterColumn, inputString)); + } + + @Override + public boolean supportAggregatePhase(AggregatePhase aggregatePhase) { + return aggregatePhase == AggregatePhase.TWO; + } + + @Override + public Expression resultForEmptyInput() { + return new BitmapEmpty(); + } + + @Override + public OrthogonalBitmapExprCalculateCount withDistinctAndChildren(boolean distinct, List children) { + Preconditions.checkArgument(children.size() == 3 + && children.get(2).getDataType() instanceof CharacterType + && children.get(2).getDataType() instanceof VarcharType); + return new OrthogonalBitmapExprCalculateCount( + distinct, children.get(0), children.get(1), (VarcharLiteral) children.get(2)); + } + + @Override + public List getSignatures() { + return FUNCTION_SIGNATURES; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 031ea8a46b5e30..c4a9b8741e9045 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -28,6 +28,8 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregatePhase; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv; import org.apache.doris.nereids.trees.plans.Plan; @@ -402,4 +404,14 @@ public ImmutableSet computeFdItems() { public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies()); } + + /** supportAggregatePhase */ + public boolean supportAggregatePhase(AggregatePhase aggregatePhase) { + for (AggregateFunction aggregateFunction : getAggregateFunctions()) { + if (!aggregateFunction.supportAggregatePhase(aggregatePhase)) { + return false; + } + } + return true; + } } diff --git a/regression-test/suites/nereids_function_p0/agg_function/test_orthogonal_bitmap_expr_calculate.groovy b/regression-test/suites/nereids_function_p0/agg_function/test_orthogonal_bitmap_expr_calculate.groovy new file mode 100644 index 00000000000000..b47315e96dbcec --- /dev/null +++ b/regression-test/suites/nereids_function_p0/agg_function/test_orthogonal_bitmap_expr_calculate.groovy @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_orthogonal_bitmap_expr_calculate") { + multi_sql """ + drop table if exists test_orthogonal_bitmap_expr_calculate; + + create table test_orthogonal_bitmap_expr_calculate( + id int, + tag int, + user_id bitmap bitmap_union + ) + aggregate key(id, tag) + distributed by hash(id) buckets 1 + properties( + 'replication_num'='1' + ); + + insert into test_orthogonal_bitmap_expr_calculate values + (1, 100, bitmap_from_string('1,2,3,4,5')), + (1, 200, bitmap_from_string('3,4,5,6,7')); + + set enable_fallback_to_original_planner=false; + """ + + test { + sql """ + select bitmap_to_string(orthogonal_bitmap_expr_calculate(user_id, tag, '(100&200)')) + from test_orthogonal_bitmap_expr_calculate + """ + result([['3,4,5']]) + } + + test { + sql """ + select orthogonal_bitmap_expr_calculate_count(user_id, tag, '(100&200)') + from test_orthogonal_bitmap_expr_calculate + """ + result([[3L]]) + } +}