branch-3.0: [fix](mtmv) Fix mv is deleted in nested mv causing query …

…err and fix some test #45744 (#45843) Cherry-picked from #45744 Co-authored-by: seawinde <[email protected]>
apache · Dec 24, 2024 · 819813d · 819813d
1 parent f4e5727
commit 819813d
Show file tree

Hide file tree

Showing 8 changed files with 169 additions and 14 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
@@ -29,6 +29,7 @@
 import org.apache.doris.nereids.analyzer.UnboundRelation;
 import org.apache.doris.nereids.analyzer.UnboundResultSink;
 import org.apache.doris.nereids.analyzer.UnboundTableSink;
+import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.parser.NereidsParser;
 import org.apache.doris.nereids.pattern.MatchingContext;
 import org.apache.doris.nereids.properties.PhysicalProperties;
@@ -197,7 +198,13 @@ private void collectMTMVCandidates(TableIf table, CascadesContext cascadesContex
                 try {
                     for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTables()) {
                         LOG.info("mtmv {} related base table include {}", new BaseTableInfo(mtmv), baseTableInfo);
-                        cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(), TableFrom.MTMV);
+                        try {
+                            cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(),
+                                    TableFrom.MTMV);
+                        } catch (AnalysisException exception) {
+                            LOG.warn("mtmv related base table get err, related table is "
+                                            + baseTableInfo.toList(), exception);
+                        }
                     }
                 } finally {
                     mtmv.readMvUnlock();

diff --git a/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out b/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !query_after_delete --
+1	1	o	10.50	2023-12-08	a	b	1	yy	\N
+1	1	o	9.50	2023-12-08	a	b	1	yy	1
+2	1	o	11.50	2023-12-09	a	b	1	yy	2
+3	1	o	12.50	2023-12-10	a	b	1	yy	\N
+3	1	o	33.50	2023-12-10	a	b	1	yy	3
+4	2	o	43.20	2023-12-11	c	d	2	mm	\N
+5	2	o	1.20	2023-12-12	c	d	2	mi	\N
+5	2	o	56.20	2023-12-12	c	d	2	mi	4
+
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -1929,6 +1929,9 @@ class Suite implements GroovyInterceptable {
                     for (String mv_name : mv_names) {
                         success = success && result.contains("(${mv_name})")
                     }
+                    if (!success) {
+                        logger.info("mv_rewrite_all_success fail =" + result)
+                    }
                     Assert.assertEquals(true, success)
                 }
             }
@@ -1939,7 +1942,11 @@ class Suite implements GroovyInterceptable {
             check { result ->
                 boolean success = true;
                 for (String mv_name : mv_names) {
-                    Assert.assertEquals(true, result.contains("${mv_name} chose"))
+                    def contains = result.contains("${mv_name} chose")
+                    if (!contains) {
+                        logger.info("mv_rewrite_all_success fail =" + result)
+                    }
+                    Assert.assertEquals(true, contains)
                 }
             }
         }
@@ -1966,6 +1973,9 @@ class Suite implements GroovyInterceptable {
                     for (String mv_name : mv_names) {
                         success = success || result.contains("(${mv_name})")
                     }
+                    if (!success) {
+                        logger.info("mv_rewrite_any_success fail =" + result)
+                    }
                     Assert.assertEquals(true, success)
                 }
             }
@@ -1978,6 +1988,9 @@ class Suite implements GroovyInterceptable {
                 for (String mv_name : mv_names) {
                     success = success || result.contains("${mv_name} chose")
                 }
+                if (!success) {
+                    logger.info("mv_rewrite_any_success fail =" + result)
+                }
                 Assert.assertEquals(true, success)
             }
         }
@@ -1998,6 +2011,9 @@ class Suite implements GroovyInterceptable {
                         def each_result =  splitResult.length == 2 ? splitResult[0].contains(mv_name) : false
                         success = success && (result.contains("(${mv_name})") || each_result)
                     }
+                    if (!success) {
+                        logger.info("mv_rewrite_all_success_without_check_chosen fail =" + result)
+                    }
                     Assert.assertEquals(true, success)
                 }
             }
@@ -2011,6 +2027,9 @@ class Suite implements GroovyInterceptable {
                     boolean stepSuccess = result.contains("${mv_name} chose") || result.contains("${mv_name} not chose")
                     success = success && stepSuccess
                 }
+                if (!success) {
+                    logger.info("mv_rewrite_all_success_without_check_chosen fail =" + result)
+                }
                 Assert.assertEquals(true, success)
             }
         }
@@ -2031,6 +2050,9 @@ class Suite implements GroovyInterceptable {
                         def each_result =  splitResult.length == 2 ? splitResult[0].contains(mv_name) : false
                         success = success || (result.contains("(${mv_name})") || each_result)
                     }
+                    if (!success) {
+                        logger.info("mv_rewrite_any_success_without_check_chosen fail =" + result)
+                    }
                     Assert.assertEquals(true, success)
                 }
             }
@@ -2043,6 +2065,9 @@ class Suite implements GroovyInterceptable {
                 for (String mv_name : mv_names) {
                     success = success || result.contains("${mv_name} chose") || result.contains("${mv_name} not chose")
                 }
+                if (!success) {
+                    logger.info("mv_rewrite_any_success_without_check_chosen fail =" + result)
+                }
                 Assert.assertEquals(true, success)
             }
         }
@@ -2101,6 +2126,9 @@ class Suite implements GroovyInterceptable {
                         boolean stepFail = !result.contains("(${mv_name})")
                         fail = fail && stepFail
                     }
+                    if (!fail) {
+                        logger.info("mv_rewrite_all_fail =" + result)
+                    }
                     Assert.assertEquals(true, fail)
                 }
             }
@@ -2114,6 +2142,9 @@ class Suite implements GroovyInterceptable {
                     boolean stepFail = result.contains("${mv_name} fail")
                     fail = fail && stepFail
                 }
+                if (!fail) {
+                    logger.info("mv_rewrite_all_fail =" + result)
+                }
                 Assert.assertEquals(true, fail)
             }
         }
@@ -2131,6 +2162,9 @@ class Suite implements GroovyInterceptable {
                     for (String mv_name : mv_names) {
                         fail = fail || !result.contains("(${mv_name})")
                     }
+                    if (!fail) {
+                        logger.info("mv_rewrite_any_fail =" + result)
+                    }
                     Assert.assertEquals(true, fail)
                 }
             }
@@ -2143,6 +2177,9 @@ class Suite implements GroovyInterceptable {
                 for (String mv_name : mv_names) {
                     fail = fail || result.contains("${mv_name} fail")
                 }
+                if (!fail) {
+                    logger.info("mv_rewrite_any_fail =" + result)
+                }
                 Assert.assertEquals(true, fail)
             }
         }

diff --git a/regression-test/suites/auth_p0/test_select_column_auth.groovy b/regression-test/suites/auth_p0/test_select_column_auth.groovy
@@ -130,6 +130,10 @@ suite("test_select_column_auth","p0,auth") {
     sql """grant select_priv(sum_id) on ${dbName}.${mtmv_name} to ${user}"""
     sql """grant select_priv(id) on ${dbName}.${tableName} to ${user}"""
     connect(user, "${pwd}", context.config.jdbcUrl) {
+        def show_grants = sql """show grants;"""
+        logger.info("show grants:" + show_grants.toString())
+        // If exec on fe follower, wait meta data is ready on follower
+        Thread.sleep(2000)
         sql "SET enable_materialized_view_rewrite=true"
         explain {
             sql("""select username, sum(id) from ${dbName}.${tableName} group by username""")

diff --git a/regression-test/suites/mv_p0/unique/unique_rewrite.groovy b/regression-test/suites/mv_p0/unique/unique_rewrite.groovy
@@ -96,6 +96,10 @@ suite("mv_on_unique_table") {
     AS
     ${mv1}
     """)
+
+    def desc_all_mv1 = sql """desc lineitem_2_uniq all;"""
+    logger.info("desc mv1 is: " + desc_all_mv1.toString())
+
     explain {
         sql("""${query1}""")
         check {result ->
@@ -124,6 +128,11 @@ suite("mv_on_unique_table") {
     AS
     ${mv2}
     """)
+
+    def desc_all_mv2 = sql """desc lineitem_2_uniq all;"""
+    logger.info("desc mv2 is" + desc_all_mv2)
+    // If exec on fe follower, wait meta data is ready on follower
+    Thread.sleep(2000)
     explain {
         sql("""${query2}""")
         check {result ->

diff --git a/...sion-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy b/...sion-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy
@@ -169,7 +169,7 @@ suite("mtmv_range_date_part_up_rewrite") {
     for (int i = 0; i < mv_name_list.size(); i++) {
         def job_name = getJobName(db, mv_name_list[i])
         waitingMTMVTaskFinished(job_name)
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
@@ -178,38 +178,38 @@ suite("mtmv_range_date_part_up_rewrite") {
     sql """insert into lineitem_range_date_union values 
         (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01')"""
     for (int i = 0; i < mv_name_list.size(); i++) {
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
     for (int i = 0; i < mv_name_list.size(); i++) {
         sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
     sql """insert into lineitem_range_date_union values 
         (2, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01');"""
     for (int i = 0; i < mv_name_list.size(); i++) {
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
     for (int i = 0; i < mv_name_list.size(); i++) {
         sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
     sql """ALTER TABLE lineitem_range_date_union DROP PARTITION IF EXISTS p4 FORCE"""
     for (int i = 0; i < mv_name_list.size(); i++) {
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
     for (int i = 0; i < mv_name_list.size(); i++) {
         sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 

diff --git a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
@@ -83,7 +83,9 @@ suite("is_in_debug_mode") {
         AS select * from orders where o_orderkey > 2;
         """
     } catch (Exception e) {
-        Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+        def message = e.getMessage()
+        logger.info("test_create_mv1" + message)
+        Assert.assertTrue(message.contains("because is in debug mode"))
     }
     sql """set skip_delete_sign = false;"""
 
@@ -99,7 +101,9 @@ suite("is_in_debug_mode") {
         AS select * from orders where o_orderkey > 2;
         """
     } catch (Exception e) {
-        Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+        def message = e.getMessage()
+        logger.info("test_create_mv2" + message)
+        Assert.assertTrue(message.contains("because is in debug mode"))
     }
     sql """set skip_storage_engine_merge = false;"""
 
@@ -115,7 +119,9 @@ suite("is_in_debug_mode") {
         AS select * from orders where o_orderkey > 2;
         """
     } catch (Exception e) {
-        Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+        def message = e.getMessage()
+        logger.info("test_create_mv3: " + message)
+        Assert.assertTrue(message.contains("because is in debug mode"))
     }
     sql """set skip_delete_bitmap = false;"""
 
@@ -131,7 +137,9 @@ suite("is_in_debug_mode") {
         AS select * from orders where o_orderkey > 2;
         """
     } catch (Exception e) {
-        Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+        def message = e.getMessage()
+        logger.info("test_create_mv4" + message)
+        Assert.assertTrue(message.contains("because is in debug mode"))
     }
     sql """set skip_delete_predicate = false;"""
 
@@ -147,7 +155,9 @@ suite("is_in_debug_mode") {
         AS select * from orders where o_orderkey > 2;
         """
     } catch (Exception e) {
-        Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+        def message = e.getMessage()
+        logger.info("test_create_mv5" + message)
+        Assert.assertTrue(message.contains("because is in debug mode"))
     }
     sql """set show_hidden_columns = false;"""
 

diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy
@@ -0,0 +1,77 @@
+package mv.nested
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("nested_mv_delete") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    sql "set runtime_filter_mode=OFF";
+    sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+
+    sql """
+    drop table if exists orders_1
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS orders_1  (
+      o_orderkey       INTEGER NOT NULL,
+      o_custkey        INTEGER NOT NULL,
+      o_orderstatus    CHAR(1) NOT NULL,
+      o_totalprice     DECIMALV3(15,2) NOT NULL,
+      o_orderdate      DATE NOT NULL,
+      o_orderpriority  CHAR(15) NOT NULL,  
+      o_clerk          CHAR(15) NOT NULL, 
+      o_shippriority   INTEGER NOT NULL,
+      o_comment        VARCHAR(79) NOT NULL,
+      public_col       INT NULL
+    )
+    DUPLICATE KEY(o_orderkey, o_custkey)
+    DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+    PROPERTIES (
+      "replication_num" = "1"
+    );
+    """
+
+    sql """
+    insert into orders_1 values
+    (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy', 1),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy', null),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy', 2),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy', null),
+    (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy', 3),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm', null),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi', 4),
+    (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi', null);  
+    """
+
+    sql """alter table orders_1 modify column o_comment set stats ('row_count'='8');"""
+
+
+    create_async_mv(db, "mv_level_1", """
+    select * from orders_1;
+    """)
+
+    create_async_mv(db, "mv_level_2", """
+    select * from mv_level_1;
+    """)
+
+    sql """drop materialized view mv_level_1;"""
+
+    order_qt_query_after_delete "select * from mv_level_2"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv_level_2"""
+}