From eb05917444782b41b84e62d21992cd3983905abc Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Fri, 6 Sep 2024 10:09:11 +0800 Subject: [PATCH 1/2] [BugFix] MV rewrite may generate wrong plans if query only contains constant call operators (#50757) Signed-off-by: shuming.li (cherry picked from commit 215c16499f9e873dde3fe81b3c49eea53e0f21b0) # Conflicts: # fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java --- .../materialization/EquationRewriter.java | 9 +- .../planner/MaterializedViewManualTest.java | 115 ++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/EquationRewriter.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/EquationRewriter.java index 5e6df3bb3e847..62a2610f1dfd0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/EquationRewriter.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/EquationRewriter.java @@ -148,7 +148,14 @@ public ScalarOperator visitCall(CallOperator call, Void context) { return rewritten; } } - + // If count(1)/sum(1) cannot be rewritten by mv's defined equivalents, return null directly, + // otherwise it may cause a wrong plan. + // mv : SELECT 1, count(distinct k1) from tbl1; + // query : SELECT count(1) from tbl1; + // MV should not rewrite the query. + if (call.isAggregate() && call.isConstant()) { + return null; + } return super.visitCall(call, context); } diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java index ad315f01dd655..2d3f986ec540f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java @@ -356,4 +356,119 @@ public void testRewriteWithCaseWhen() { } }); } +<<<<<<< HEAD +======= + + @Test + public void testRewriteWithEliminateJoinsBasic2() throws Exception { + starRocksAssert.withTable("CREATE TABLE `tbl1` (\n" + + " `k1` date,\n" + + " `k2` decimal64(18, 2),\n" + + " `k3` varchar(255),\n" + + " `v1` varchar(255)\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "DISTRIBUTED BY RANDOM\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW `mv1` \n" + + "DISTRIBUTED BY RANDOM\n" + + "REFRESH ASYNC\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ")\n" + + "AS SELECT k1, k2, k3, sum(v1) from tbl1 group by k1, k2, k3"); + { + String sql = "with cte as(" + + " select " + + " '2024-07-20' as date1," + + " date_add('2024-07-20', interval -1 month) AS start_date," + + " date_add('2024-07-20', interval 1 month) AS end_date," + + " 'k3' as k3" + + ") select " + + " cte.date1, cte.start_date, cte.end_date, t1.k1, t1.k2, t1.k3, sum(t1.v1) " + + " from cte join tbl1 t1 " + + " on cte.k3 = t1.k3 " + + " group by cte.date1, cte.start_date, cte.end_date, t1.k1, t1.k2, t1.k3"; + sql(sql).contains("mv1") + .contains(" 1:Project\n" + + " | : 11: k1\n" + + " | : 12: k2\n" + + " | : 13: k3\n" + + " | : '2024-07-20'\n" + + " | : '2024-06-20 00:00:00'\n" + + " | : '2024-08-20 00:00:00'\n" + + " | : 14: sum(v1)\n" + + " | \n" + + " 0:OlapScanNode\n" + + " TABLE: mv1"); + } + + { + String sql = "with cte as(" + + " select " + + " '2024-07-20' as date1," + + " date_add('2024-07-20', interval -1 month) AS start_date," + + " date_add('2024-07-20', interval 1 month) AS end_date," + + " 'k3' as k3" + + ") select " + + " cte.date1, cte.start_date, cte.end_date, t1.k1, sum(t1.v1) " + + " from cte join tbl1 t1 " + + " on cte.k3 = t1.k3 " + + " group by cte.date1, cte.start_date, cte.end_date, t1.k1"; + sql(sql).contains("mv1") + .contains("1:Project\n" + + "| : col$: k1\n" + + "| : col$: sum(v1)\n" + + "|\n" + + "0:OlapScanNode\n" + + "TABLE: mv1"); + } + starRocksAssert.dropMaterializedView("mv1"); + starRocksAssert.dropTable("tbl1"); + } + + @Test + public void testWrongMVRewrite() throws Exception { + starRocksAssert.withTable("CREATE TABLE `tbl1` (\n" + + " `k1` date,\n" + + " `k2` decimal64(18, 2),\n" + + " `k3` varchar(255),\n" + + " `v1` varchar(255)\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "DISTRIBUTED BY RANDOM\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + { + starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW `mv1` \n" + + "DISTRIBUTED BY RANDOM\n" + + "REFRESH ASYNC\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ")\n" + + "AS SELECT 1, count(distinct k1) from tbl1"); + sql("select count(distinct k1) from tbl1").contains("mv1"); + sql("select count(1) from tbl1").notContain("mv1"); + sql("select count(*) from tbl1").notContain("mv1"); + starRocksAssert.dropMaterializedView("mv1"); + } + { + starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW `mv1` \n" + + "DISTRIBUTED BY RANDOM\n" + + "REFRESH ASYNC\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ")\n" + + "AS SELECT 1, count(1) from tbl1"); + sql("select count(distinct k1) from tbl1").notContain("mv1"); + sql("select count(1) from tbl1").contains("mv1"); + sql("select count(*) from tbl1").contains("mv1"); + starRocksAssert.dropMaterializedView("mv1"); + } + starRocksAssert.dropTable("tbl1"); + } +>>>>>>> 215c16499f ([BugFix] MV rewrite may generate wrong plans if query only contains constant call operators (#50757)) } From 651ad49aa58626785fc2e8d93a92acf610108244 Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Mon, 9 Sep 2024 10:12:02 +0800 Subject: [PATCH 2/2] fix bugs Signed-off-by: shuming.li --- .../planner/MaterializedViewManualTest.java | 73 ------------------- 1 file changed, 73 deletions(-) diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java index 2d3f986ec540f..14986e02c6667 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java @@ -356,78 +356,6 @@ public void testRewriteWithCaseWhen() { } }); } -<<<<<<< HEAD -======= - - @Test - public void testRewriteWithEliminateJoinsBasic2() throws Exception { - starRocksAssert.withTable("CREATE TABLE `tbl1` (\n" + - " `k1` date,\n" + - " `k2` decimal64(18, 2),\n" + - " `k3` varchar(255),\n" + - " `v1` varchar(255)\n" + - ") ENGINE=OLAP \n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "DISTRIBUTED BY RANDOM\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ");"); - starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW `mv1` \n" + - "DISTRIBUTED BY RANDOM\n" + - "REFRESH ASYNC\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ")\n" + - "AS SELECT k1, k2, k3, sum(v1) from tbl1 group by k1, k2, k3"); - { - String sql = "with cte as(" + - " select " + - " '2024-07-20' as date1," + - " date_add('2024-07-20', interval -1 month) AS start_date," + - " date_add('2024-07-20', interval 1 month) AS end_date," + - " 'k3' as k3" + - ") select " + - " cte.date1, cte.start_date, cte.end_date, t1.k1, t1.k2, t1.k3, sum(t1.v1) " + - " from cte join tbl1 t1 " + - " on cte.k3 = t1.k3 " + - " group by cte.date1, cte.start_date, cte.end_date, t1.k1, t1.k2, t1.k3"; - sql(sql).contains("mv1") - .contains(" 1:Project\n" + - " | : 11: k1\n" + - " | : 12: k2\n" + - " | : 13: k3\n" + - " | : '2024-07-20'\n" + - " | : '2024-06-20 00:00:00'\n" + - " | : '2024-08-20 00:00:00'\n" + - " | : 14: sum(v1)\n" + - " | \n" + - " 0:OlapScanNode\n" + - " TABLE: mv1"); - } - - { - String sql = "with cte as(" + - " select " + - " '2024-07-20' as date1," + - " date_add('2024-07-20', interval -1 month) AS start_date," + - " date_add('2024-07-20', interval 1 month) AS end_date," + - " 'k3' as k3" + - ") select " + - " cte.date1, cte.start_date, cte.end_date, t1.k1, sum(t1.v1) " + - " from cte join tbl1 t1 " + - " on cte.k3 = t1.k3 " + - " group by cte.date1, cte.start_date, cte.end_date, t1.k1"; - sql(sql).contains("mv1") - .contains("1:Project\n" + - "| : col$: k1\n" + - "| : col$: sum(v1)\n" + - "|\n" + - "0:OlapScanNode\n" + - "TABLE: mv1"); - } - starRocksAssert.dropMaterializedView("mv1"); - starRocksAssert.dropTable("tbl1"); - } @Test public void testWrongMVRewrite() throws Exception { @@ -470,5 +398,4 @@ public void testWrongMVRewrite() throws Exception { } starRocksAssert.dropTable("tbl1"); } ->>>>>>> 215c16499f ([BugFix] MV rewrite may generate wrong plans if query only contains constant call operators (#50757)) }