diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/PruneSubfieldRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/PruneSubfieldRule.java index 5d224a8e91509..c2dbf05d86605 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/PruneSubfieldRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/PruneSubfieldRule.java @@ -56,13 +56,21 @@ public class PruneSubfieldRule extends TransformationRule { .add(FunctionSet.JSON_LENGTH) .build(); - public static final List SUPPORT_FUNCTIONS = ImmutableList.builder() + public static final List PRUNE_FUNCTIONS = ImmutableList.builder() .add(FunctionSet.MAP_KEYS, FunctionSet.MAP_SIZE) .add(FunctionSet.ARRAY_LENGTH) .add(FunctionSet.CARDINALITY) .addAll(SUPPORT_JSON_FUNCTIONS) .build(); + public static final List PUSHDOWN_FUNCTIONS = ImmutableList.builder() + .addAll(PRUNE_FUNCTIONS) + .add(FunctionSet.ARRAY_CONTAINS, FunctionSet.ARRAY_CONTAINS_ALL) + .add(FunctionSet.ARRAY_MAX, FunctionSet.ARRAY_MIN, FunctionSet.ARRAY_SUM, FunctionSet.ARRAY_AVG) + .add(FunctionSet.ARRAY_POSITION) + .add(FunctionSet.ARRAY_JOIN) + .build(); + public PruneSubfieldRule() { super(RuleType.TF_PRUNE_SUBFIELD, Pattern.create(OperatorType.LOGICAL_PROJECT, OperatorType.PATTERN_SCAN)); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/PushDownSubfieldRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/PushDownSubfieldRule.java index d54cc9f643121..82bc87db8cc4e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/PushDownSubfieldRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/PushDownSubfieldRule.java @@ -222,7 +222,7 @@ public OptExpression visitLogicalProject(OptExpression optExpression, Context co ColumnRefSet allUsedColumns = new ColumnRefSet(); context.pushDownExprUseColumns.values().forEach(allUsedColumns::union); - SubfieldExpressionCollector collector = new SubfieldExpressionCollector(); + SubfieldExpressionCollector collector = SubfieldExpressionCollector.buildPushdownCollector(); for (ScalarOperator value : lpo.getColumnRefMap().values()) { // check repeat put complex column, like that // project( columnB: structA.b.c.d ) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/SubfieldAccessPathNormalizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/SubfieldAccessPathNormalizer.java index 6d1e3fbffafe9..dae39d2fe768d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/SubfieldAccessPathNormalizer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/SubfieldAccessPathNormalizer.java @@ -192,7 +192,7 @@ public Optional visitCollectionElement(CollectionElementOperator col @Override public Optional visitCall(CallOperator call, List> childrenAccessPaths) { - if (!PruneSubfieldRule.SUPPORT_FUNCTIONS.contains(call.getFnName())) { + if (!PruneSubfieldRule.PRUNE_FUNCTIONS.contains(call.getFnName())) { return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/SubfieldExpressionCollector.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/SubfieldExpressionCollector.java index 01753be2a395b..9de1a12bb175c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/SubfieldExpressionCollector.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/prunesubfield/SubfieldExpressionCollector.java @@ -15,6 +15,7 @@ package com.starrocks.sql.optimizer.rule.tree.prunesubfield; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.starrocks.catalog.Type; import com.starrocks.sql.optimizer.operator.scalar.CallOperator; import com.starrocks.sql.optimizer.operator.scalar.CollectionElementOperator; @@ -24,13 +25,14 @@ import com.starrocks.sql.optimizer.operator.scalar.SubfieldOperator; import java.util.List; +import java.util.Set; /* * collect all complex expressions, such as: MAP_KEYS, MAP_VALUES, map['key'], struct.a.b.c ... */ public class SubfieldExpressionCollector extends ScalarOperatorVisitor { private final List complexExpressions = Lists.newArrayList(); - + private Set checkFunctions; private final boolean enableJsonCollect; public List getComplexExpressions() { @@ -43,6 +45,19 @@ public SubfieldExpressionCollector() { public SubfieldExpressionCollector(boolean enableJsonCollect) { this.enableJsonCollect = enableJsonCollect; + this.checkFunctions = Sets.newHashSet(PruneSubfieldRule.PRUNE_FUNCTIONS); + } + + public static SubfieldExpressionCollector buildPruneCollector() { + SubfieldExpressionCollector collector = new SubfieldExpressionCollector(); + collector.checkFunctions = Sets.newHashSet(PruneSubfieldRule.PRUNE_FUNCTIONS); + return collector; + } + + public static SubfieldExpressionCollector buildPushdownCollector() { + SubfieldExpressionCollector collector = new SubfieldExpressionCollector(); + collector.checkFunctions = Sets.newHashSet(PruneSubfieldRule.PUSHDOWN_FUNCTIONS); + return collector; } @Override @@ -85,7 +100,7 @@ public Void visitCall(CallOperator call, Void context) { return null; } - if (!PruneSubfieldRule.SUPPORT_FUNCTIONS.contains(call.getFnName())) { + if (!checkFunctions.contains(call.getFnName())) { return visit(call, context); } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/LowCardinalityArrayTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/LowCardinalityArrayTest.java index f9f5e779910fb..438620554a520 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/LowCardinalityArrayTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/LowCardinalityArrayTest.java @@ -260,16 +260,13 @@ public void testArrayProjectOrderLimit() throws Exception { " reverse(a1), array_slice(a2, 2, 4), cardinality(a2)\n" + "from s2 where a1[1] = 'Jiangsu' and a2[2] = 'GD' order by v1 limit 2;"; String plan = getVerboseExplain(sql); - Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + - " 23: DictDefine(22: a2, [])\n" + - " 24: DictDefine(21: a1, [])\n" + - " 25: DictDefine(21: a1, [])\n" + - " 26: DictDefine(22: a2, [])\n" + - " 27: DictDefine(21: a1, [])\n" + - " 28: DictDefine(22: a2, [])\n" + - "\n" + - " 5:Decode\n" + - " | : ")); + Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + + " 25: DictDefine(24: a2, [])\n" + + " 26: DictDefine(23: a1, [])\n" + + " 27: DictDefine(23: a1, [])\n" + + " 28: DictDefine(24: a2, [])\n" + + " 29: DictDefine(23: a1, [])\n" + + " 30: DictDefine(24: a2, [])")); } @Test @@ -292,15 +289,21 @@ public void testArrayShuffleProject() throws Exception { "from supplier_nullable xx join[shuffle] table_int t on S_NATIONKEY = id_int " + "where S_ADDRESS[0] = 'a'"; String plan = getVerboseExplain(sql); - Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + - " 19: DictDefine(18: S_ADDRESS, [])\n" + + Assert.assertTrue(plan, plan.contains("Global Dict Exprs:\n" + + " 20: DictDefine(19: S_ADDRESS, [])\n" + + " 21: DictDefine(19: S_ADDRESS, [])\n" + + " 22: DictDefine(19: S_ADDRESS, [])\n" + "\n" + - " 6:Project\n" + + " 1:Project\n" + " | output columns:\n" + - " | 11 <-> DictDecode(18: S_ADDRESS, [], array_min(18: S_ADDRESS))\n" + - " | 12 <-> DictDecode(19: expr, [])\n" + - " | 13 <-> DictDecode(18: S_ADDRESS, [], array_max(18: S_ADDRESS))\n" + - " | 14 <-> [17: array_length, INT, true]\n" + + " | 4 <-> [4: S_NATIONKEY, INT, false]\n" + + " | 18 <-> array_length[([19: S_ADDRESS, ARRAY, true]); args: INVALID_TYPE; result: INT; args " + + "nullable: true; result nullable: true]\n" + + " | 20 <-> array_distinct(array_slice(19: S_ADDRESS, 2, 4))[0]\n" + + " | 21 <-> array_max[([19: S_ADDRESS, ARRAY, true]); args: INVALID_TYPE; result: INT; args " + + "nullable: true; result nullable: true]\n" + + " | 22 <-> array_min[([19: S_ADDRESS, ARRAY, true]); args: INVALID_TYPE; result: INT; args " + + "nullable: true; result nullable: true]\n" + " | cardinality: 1")); } @@ -317,38 +320,56 @@ public void testArrayShuffleProject2() throws Exception { "from supplier_nullable xx join[shuffle] table_int t on S_NATIONKEY = id_int " + "where S_ADDRESS[0] = 'a'"; String plan = getVerboseExplain(sql); - Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + - " 24: DictDefine(23: S_ADDRESS, [])\n" + - " 25: DictDefine(23: S_ADDRESS, [])\n" + - " 26: DictDefine(23: S_ADDRESS, [])\n" + + Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + + " 32: DictDefine(26: S_ADDRESS, [])\n" + + " 27: DictDefine(26: S_ADDRESS, [])\n" + + " 28: DictDefine(26: S_ADDRESS, [])\n" + + " 29: DictDefine(26: S_ADDRESS, [])\n" + + " 30: DictDefine(26: S_ADDRESS, [])\n" + + " 31: DictDefine(26: S_ADDRESS, [])\n" + "\n" + " 1:Project\n" + " | output columns:\n" + " | 4 <-> [4: S_NATIONKEY, INT, false]\n" + - " | 23 <-> [23: S_ADDRESS, ARRAY, true]\n" + - " | 24 <-> reverse(array_distinct(reverse(23: S_ADDRESS)))[2]\n" + - " | 25 <-> array_distinct(array_slice(23: S_ADDRESS, 2, 4))[0]\n" + - " | 26 <-> array_slice(23: S_ADDRESS, 1, 2)[0]\n" + + " | 26 <-> [26: S_ADDRESS, ARRAY, true]\n" + + " | 27 <-> array_max[([35: reverse, ARRAY, true]); args: INVALID_TYPE; result: INT; args " + + "nullable: true; result nullable: true]\n" + + " | 28 <-> 35: reverse[2]\n" + + " | 29 <-> array_min[([26: S_ADDRESS, ARRAY, true]); args: INVALID_TYPE; result: INT; args " + + "nullable: true; result nullable: true]\n" + + " | 30 <-> array_distinct(array_slice(26: S_ADDRESS, 2, 4))[0]\n" + + " | 31 <-> array_slice(26: S_ADDRESS, 1, 2)[0]\n" + + " | 32 <-> array_max[([26: S_ADDRESS, ARRAY, true]); args: INVALID_TYPE; result: INT; args " + + "nullable: true; result nullable: true]\n" + + " | common expressions:\n" + + " | 33 <-> reverse[([26: S_ADDRESS, ARRAY, true]); args: INVALID_TYPE; result: ARRAY; " + + "args nullable: true; result nullable: true]\n" + + " | 34 <-> array_distinct[([33: reverse, ARRAY, true]); args: INVALID_TYPE; result: " + + "ARRAY; args nullable: true; result nullable: true]\n" + + " | 35 <-> reverse[([34: array_distinct, ARRAY, true]); args: INVALID_TYPE; result: " + + "ARRAY; args nullable: true; result nullable: true]\n" + " | cardinality: 1")); - assertContains(plan, " Global Dict Exprs:\n" + - " 24: DictDefine(23: S_ADDRESS, [])\n" + - " 25: DictDefine(23: S_ADDRESS, [])\n" + - " 26: DictDefine(23: S_ADDRESS, [])\n" + + assertContains(plan, " Global Dict Exprs:\n" + + " 32: DictDefine(26: S_ADDRESS, [])\n" + + " 27: DictDefine(26: S_ADDRESS, [])\n" + + " 28: DictDefine(26: S_ADDRESS, [])\n" + + " 29: DictDefine(26: S_ADDRESS, [])\n" + + " 30: DictDefine(26: S_ADDRESS, [])\n" + + " 31: DictDefine(26: S_ADDRESS, [])\n" + "\n" + " 6:Project\n" + " | output columns:\n" + - " | 11 <-> DictDecode(23: S_ADDRESS, [], array_min(23: S_ADDRESS))\n" + - " | 12 <-> DictDecode(25: expr, [])\n" + - " | 13 <-> DictDecode(26: expr, [hex()])\n" + - " | 14 <-> DictDecode(23: S_ADDRESS, [upper()], array_max(23: S_ADDRESS))\n" + - " | 15 <-> DictDecode(23: S_ADDRESS, [], array_distinct(array_filter(23: S_ADDRESS, " + + " | 11 <-> DictDecode(29: array_min, [])\n" + + " | 12 <-> DictDecode(30: expr, [])\n" + + " | 13 <-> DictDecode(31: expr, [hex()])\n" + + " | 14 <-> DictDecode(32: array_max, [upper()])\n" + + " | 15 <-> DictDecode(26: S_ADDRESS, [], array_distinct(array_filter(26: S_ADDRESS, " + "[TRUE,FALSE])))\n" + - " | 16 <-> DictDecode(23: S_ADDRESS, [], reverse(array_distinct(reverse(23: " + + " | 16 <-> DictDecode(26: S_ADDRESS, [], reverse(array_distinct(reverse(26: " + "S_ADDRESS))))\n" + - " | 17 <-> DictDecode(23: S_ADDRESS, [], array_max(reverse(array_distinct(reverse(23:" + - " S_ADDRESS)))))\n" + - " | 18 <-> DictDecode(24: expr, [])\n" + + " | 17 <-> DictDecode(27: array_max, [])\n" + + " | 18 <-> DictDecode(28: expr, [])\n" + " | cardinality: 1"); } @@ -373,67 +394,79 @@ public void testArrayShuffleProjectStringCountDistinct() throws Exception { "from supplier_nullable xx join[shuffle] table_int t on S_NATIONKEY = id_int " + "where S_ADDRESS[0] = 'a' ) as yyy"; String plan = getVerboseExplain(sql); - assertContains(plan, " Global Dict Exprs:\n" + - " 43: DictDefine(33: S_ADDRESS, [])\n" + - " 44: DictDefine(33: S_ADDRESS, [])\n" + + assertContains(plan, " Global Dict Exprs:\n" + + " 48: DictDefine(35: S_ADDRESS, [])\n" + + " 47: DictDefine(35: S_ADDRESS, [])\n" + "\n" + " 10:Decode\n" + - " | : \n" + - " | : \n" + + " | : \n" + + " | : \n" + " | cardinality: 1"); - Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + - " 34: DictDefine(33: S_ADDRESS, [])\n" + - " 35: DictDefine(33: S_ADDRESS, [])\n" + - " 36: DictDefine(33: S_ADDRESS, [])\n" + - " 37: DictDefine(33: S_ADDRESS, [])\n" + - " 38: DictDefine(33: S_ADDRESS, [])\n" + - " 39: DictDefine(33: S_ADDRESS, [])\n" + - " 40: DictDefine(33: S_ADDRESS, [])\n" + - " 41: DictDefine(33: S_ADDRESS, [])\n" + - " 42: DictDefine(33: S_ADDRESS, [])\n" + - " 43: DictDefine(33: S_ADDRESS, [])\n" + - " 44: DictDefine(33: S_ADDRESS, [])\n" + - " 45: DictDefine(33: S_ADDRESS, [])\n" + - " 46: DictDefine(33: S_ADDRESS, [])\n" + - " 47: DictDefine(33: S_ADDRESS, [])\n" + - " 48: DictDefine(33: S_ADDRESS, [])")); - assertContains(plan, " 6:Project\n" + + Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + + " 36: DictDefine(35: S_ADDRESS, [])\n" + + " 37: DictDefine(35: S_ADDRESS, [])\n" + + " 38: DictDefine(35: S_ADDRESS, [])\n" + + " 39: DictDefine(35: S_ADDRESS, [])\n" + + " 40: DictDefine(35: S_ADDRESS, [])\n" + + " 41: DictDefine(35: S_ADDRESS, [])\n" + + " 42: DictDefine(35: S_ADDRESS, [])\n" + + " 43: DictDefine(35: S_ADDRESS, [])\n" + + " 44: DictDefine(35: S_ADDRESS, [])\n" + + " 45: DictDefine(35: S_ADDRESS, [])\n" + + " 46: DictDefine(35: S_ADDRESS, [])\n" + + " 47: DictDefine(35: S_ADDRESS, [])\n" + + " 48: DictDefine(35: S_ADDRESS, [])\n" + + " 49: DictDefine(35: S_ADDRESS, [])\n" + + " 50: DictDefine(35: S_ADDRESS, [])\n" + + " 51: DictDefine(35: S_ADDRESS, [])\n" + + " 52: DictDefine(35: S_ADDRESS, [])\n" + + " 53: DictDefine(35: S_ADDRESS, [])")); + assertContains(plan, " 6:Project\n" + " | output columns:\n" + - " | 33 <-> [33: S_ADDRESS, ARRAY, true]\n" + - " | 36 <-> DictDefine(46: expr, [])\n" + - " | 37 <-> DictDefine(47: expr, [])\n" + - " | 39 <-> DictDefine(48: expr, [])\n" + - " | 40 <-> DictDefine(34: expr, [])\n" + - " | 42 <-> DictDefine(45: expr, [])\n" + + " | 39 <-> DictDefine(51: array_min, [])\n" + + " | 40 <-> DictDefine(52: expr, [])\n" + + " | 41 <-> DictDefine(53: expr, [])\n" + + " | 42 <-> DictDefine(36: array_max, [])\n" + + " | 43 <-> DictDefine(37: expr, [])\n" + + " | 44 <-> DictDefine(38: expr, [])\n" + + " | 45 <-> DictDefine(49: array_max, [])\n" + + " | 46 <-> DictDefine(50: expr, [])\n" + " | cardinality: 1"); - assertContains(plan, " Global Dict Exprs:\n" + - " 48: DictDefine(33: S_ADDRESS, [])\n" + - " 34: DictDefine(33: S_ADDRESS, [])\n" + - " 45: DictDefine(33: S_ADDRESS, [])\n" + - " 46: DictDefine(33: S_ADDRESS, [])\n" + - " 47: DictDefine(33: S_ADDRESS, [])\n" + - "\n" + - " 1:Project\n" + + assertContains(plan, " Global Dict Exprs:\n" + + " 49: DictDefine(35: S_ADDRESS, [])\n" + + " 50: DictDefine(35: S_ADDRESS, [])\n" + + " 51: DictDefine(35: S_ADDRESS, [])\n" + + " 52: DictDefine(35: S_ADDRESS, [])\n" + + " 36: DictDefine(35: S_ADDRESS, [])\n" + + " 53: DictDefine(35: S_ADDRESS, [])\n" + + " 37: DictDefine(35: S_ADDRESS, [])\n" + + " 38: DictDefine(35: S_ADDRESS, [])"); + assertContains(plan, " 1:Project\n" + " | output columns:\n" + " | 4 <-> [4: S_NATIONKEY, INT, false]\n" + - " | 33 <-> [33: S_ADDRESS, ARRAY, true]\n" + - " | 34 <-> 51: reverse[1]\n" + - " | 45 <-> 51: reverse[2]\n" + - " | 46 <-> array_distinct(array_slice(33: S_ADDRESS, 2, 4))[0]\n" + - " | 47 <-> array_slice(33: S_ADDRESS, 1, 2)[0]\n" + - " | 48 <-> array_distinct(array_filter(33: S_ADDRESS, [TRUE,FALSE]))[3]\n" + + " | 36 <-> array_max[([35: S_ADDRESS, ARRAY, true]); args: INVALID_TYPE; result: INT; args " + + "nullable: true; result nullable: true]\n" + + " | 37 <-> array_distinct(array_filter(35: S_ADDRESS, [TRUE,FALSE]))[3]\n" + + " | 38 <-> 56: reverse[1]\n" + + " | 49 <-> array_max[([56: reverse, ARRAY, true]); args: INVALID_TYPE; result: INT; args " + + "nullable: true; result nullable: true]\n" + + " | 50 <-> 56: reverse[2]\n" + + " | 51 <-> array_min[([35: S_ADDRESS, ARRAY, true]); args: INVALID_TYPE; result: INT; args " + + "nullable: true; result nullable: true]\n" + + " | 52 <-> array_distinct(array_slice(35: S_ADDRESS, 2, 4))[0]\n" + + " | 53 <-> array_slice(35: S_ADDRESS, 1, 2)[0]\n" + " | common expressions:\n" + - " | 49 <-> reverse[([33: S_ADDRESS, ARRAY, true]); args: INVALID_TYPE; result: ARRAY; " + + " | 54 <-> reverse[([35: S_ADDRESS, ARRAY, true]); args: INVALID_TYPE; result: ARRAY; " + "args nullable: true; result nullable: true]\n" + - " | 50 <-> array_distinct[([49: reverse, ARRAY, true]); args: INVALID_TYPE; result: " + + " | 55 <-> array_distinct[([54: reverse, ARRAY, true]); args: INVALID_TYPE; result: " + "ARRAY; args nullable: true; result nullable: true]\n" + - " | 51 <-> reverse[([50: array_distinct, ARRAY, true]); args: INVALID_TYPE; result: " + + " | 56 <-> reverse[([55: array_distinct, ARRAY, true]); args: INVALID_TYPE; result: " + "ARRAY; args nullable: true; result nullable: true]\n" + " | cardinality: 1"); - assertContains(plan, " 0:OlapScanNode\n" + + assertContains(plan, "0:OlapScanNode\n" + " table: supplier_nullable, rollup: supplier_nullable\n" + " preAggregation: on\n" + - " Predicates: DictDecode(33: S_ADDRESS, [ = 'a'], 33: S_ADDRESS[0])"); + " Predicates: DictDecode(35: S_ADDRESS, [ = 'a'], 35: S_ADDRESS[0])"); } @Test @@ -451,19 +484,20 @@ public void testArrayShuffleProjectStringProject() throws Exception { "from supplier_nullable xx join[shuffle] table_int t on S_NATIONKEY = id_int " + "where S_ADDRESS[0] = 'a' ) as yyy"; String plan = getVerboseExplain(sql); - Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + - " 26: DictDefine(25: S_ADDRESS, [])\n" + - " 27: DictDefine(25: S_ADDRESS, [])\n" + - " 28: DictDefine(25: S_ADDRESS, [])")); - - assertContains(plan, " 6:Project\n" + + Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + + " 27: DictDefine(26: S_ADDRESS, [])\n" + + " 28: DictDefine(26: S_ADDRESS, [])\n" + + " 29: DictDefine(26: S_ADDRESS, [])\n" + + " 30: DictDefine(26: S_ADDRESS, [])\n" + + " 31: DictDefine(26: S_ADDRESS, [])")); + + assertContains(plan, "6:Project\n" + " | output columns:\n" + - " | 16 <-> DictDecode(25: S_ADDRESS, [upper()], array_min(25: S_ADDRESS))\n" + - " | 17 <-> DictDecode(26: expr, [ltrim()])\n" + - " | 18 <-> DictDecode(27: expr, [if(CAST( AS BOOLEAN), 'a', 'b')])\n" + - " | 19 <-> DictDecode(25: S_ADDRESS, [lower()], array_max(25: S_ADDRESS))\n" + - " | 20 <-> DictDecode(28: expr, [concat()])\n" + - " | cardinality: 1"); + " | 16 <-> DictDecode(31: array_min, [upper()])\n" + + " | 17 <-> DictDecode(27: expr, [ltrim()])\n" + + " | 18 <-> DictDecode(28: expr, [if(CAST( AS BOOLEAN), 'a', 'b')])\n" + + " | 19 <-> DictDecode(29: array_max, [lower()])\n" + + " | 20 <-> DictDecode(30: expr, [concat()])"); } @Test @@ -656,31 +690,25 @@ public void testAggreagateOrUnique() throws Exception { " reverse(a1), array_slice(a2, 2, 4), cardinality(a2)\n" + "from s4 where a1[1] = 'Jiangsu' and a2[2] = 'GD' order by v1 limit 2;"; String plan = getVerboseExplain(sql); - Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + - " 23: DictDefine(22: a2, [])\n" + - " 24: DictDefine(21: a1, [])\n" + - " 25: DictDefine(21: a1, [])\n" + - " 26: DictDefine(22: a2, [])\n" + - " 27: DictDefine(21: a1, [])\n" + - " 28: DictDefine(22: a2, [])\n" + - "\n" + - " 5:Decode\n" + - " | : ")); + Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + + " 25: DictDefine(24: a2, [])\n" + + " 26: DictDefine(23: a1, [])\n" + + " 27: DictDefine(23: a1, [])\n" + + " 28: DictDefine(24: a2, [])\n" + + " 29: DictDefine(23: a1, [])\n" + + " 30: DictDefine(24: a2, [])")); sql = "select array_length(a1), array_max(a2), array_min(a1), array_distinct(a1), array_sort(a2),\n" + " reverse(a1), array_slice(a2, 2, 4), cardinality(a2)\n" + "from s5 where a1[1] = 'Jiangsu' and a2[2] = 'GD' order by v1 limit 2;"; plan = getVerboseExplain(sql); - Assert.assertTrue(plan, plan.contains(" Global Dict Exprs:\n" + - " 23: DictDefine(22: a2, [])\n" + - " 24: DictDefine(21: a1, [])\n" + - " 25: DictDefine(21: a1, [])\n" + - " 26: DictDefine(22: a2, [])\n" + - " 27: DictDefine(21: a1, [])\n" + - " 28: DictDefine(22: a2, [])\n" + - "\n" + - " 5:Decode\n" + - " | : ")); + Assert.assertTrue(plan, plan.contains("Global Dict Exprs:\n" + + " 25: DictDefine(24: a2, [])\n" + + " 26: DictDefine(23: a1, [])\n" + + " 27: DictDefine(23: a1, [])\n" + + " 28: DictDefine(24: a2, [])\n" + + " 29: DictDefine(23: a1, [])\n" + + " 30: DictDefine(24: a2, [])")); } @Test diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PushDownJoinProjectionTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PushDownJoinProjectionTest.java new file mode 100644 index 0000000000000..ccb7b1b436bae --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PushDownJoinProjectionTest.java @@ -0,0 +1,116 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.plan; + +import org.junit.Test; + +public class PushDownJoinProjectionTest extends PlanTestBase { + + @Test + public void testPushdown() throws Exception { + starRocksAssert.query("select tarray.v1, array_contains(tarray.v3, 123) " + + "from t0 join tarray on(t0.v1 = tarray.v1) ") + .explainContains(" 1:Project\n" + + " | : 4: v1\n" + + " | : array_contains(6: v3, 123)\n" + + " | \n" + + " 0:OlapScanNode"); + starRocksAssert.query("select t0.v1, t0.v2, tarray.v1 as c1, array_contains(tarray.v3, 123) as c2\n" + + "from t0 join tarray on(t0.v1 = tarray.v1) ") + .explainContains(" 2:Project\n" + + " | : 4: v1\n" + + " | : array_contains(6: v3, 123)\n" + + " | \n" + + " 1:OlapScanNode"); + starRocksAssert.query("select t0.v1, t0.v2, tarray.v1 as c1, 1 + array_contains(tarray.v3, 123) " + + "from t0 join tarray on(t0.v1 = tarray.v1) ") + .explainContains(" 2:Project\n" + + " | : 4: v1\n" + + " | : array_contains(6: v3, 123)\n" + + " | \n" + + " 1:OlapScanNode\n" + + " TABLE: tarray"); + starRocksAssert.query("select t0.v1, t0.v2, array_map(tarray.v3, x -> x + 1) " + + "from t0 join tarray on(t0.v1 = tarray.v1) ") + .explainContains(" 4:Project\n" + + " | : 1: v1\n" + + " | : 2: v2\n" + + " | : array_map( -> + 1, 6: v3)\n" + + " | \n" + + " 3:HASH JOIN"); + + // multiple table join + starRocksAssert.query("select t0.v1, t0.v2, " + + " a1.v1 as c1, array_contains(a1.v3, 123) as c2, " + + " t1.v4, t1.v5" + + " from t0 " + + " join tarray a1 on(t0.v1 = a1.v1) " + + " join t1 on(t0.v1 = t1.v4) ") + .explainContains(" 3:Project\n" + + " | : 4: v1\n" + + " | : array_contains(6: v3, 123)\n" + + " | \n" + + " 2:OlapScanNode\n" + + " TABLE: tarray"); + starRocksAssert.query("select t0.v1, t0.v2, " + + " a1.v1 as c1, array_contains(a1.v3, 123) as c2, " + + " a2.v1 as c3, map_size(a2.v3) as c4 " + + " from t0 " + + " join tarray a1 on(t0.v1 = a1.v1) " + + " join tmap a2 on(t0.v1 = a2.v1) ") + .explainContains(" 3:Project\n" + + " | : 4: v1\n" + + " | : array_contains(6: v3, 123)\n" + + " | \n" + + " 2:OlapScanNode\n" + + " TABLE: tarray", + " 7:Project\n" + + " | : 7: v1\n" + + " | : map_size(9: v3)\n" + + " | \n" + + " 6:OlapScanNode\n" + + " TABLE: tmap"); + starRocksAssert.query("select " + + " a1.v1 as c1, array_contains(a1.v3, 123) as c2, " + + " a2.v1 as c3, array_contains(a2.v3, 456) as c4 " + + " from t0 " + + " join tarray a1 on(t0.v1 = a1.v1) " + + " join tarray a2 on(t0.v1 = a2.v1) ") + .explainContains(" 1:Project\n" + + " | : 4: v1\n" + + " | : array_contains(6: v3, 123)\n" + + " | \n" + + " 0:OlapScanNode\n" + + " TABLE: tarray", + "7:Project\n" + + " | : 7: v1\n" + + " | : array_contains(9: v3, 456)\n" + + " | \n" + + " 6:OlapScanNode\n" + + " TABLE: tarray"); + + // simple expression: do not pushdown + starRocksAssert.query("select t0.v1, t0.v2, " + + "tarray.v1 as c1, 1 + tarray.v1 as c2\n" + + "from t0 join tarray on(t0.v1 = tarray.v1) ") + .explainContains(" UNPARTITIONED\n" + + "\n" + + " 1:OlapScanNode\n" + + " TABLE: tarray", "-2:EXCHANGE\n" + + " | \n" + + " 0:OlapScanNode\n" + + " TABLE: t0"); + } +}