diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java index 9c6e3adbe74e1b..01adc549e3686d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.pattern.MatchingContext; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -197,7 +198,13 @@ private void collectMTMVCandidates(TableIf table, CascadesContext cascadesContex try { for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTables()) { LOG.info("mtmv {} related base table include {}", new BaseTableInfo(mtmv), baseTableInfo); - cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(), TableFrom.MTMV); + try { + cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(), + TableFrom.MTMV); + } catch (AnalysisException exception) { + LOG.warn("mtmv related base table get err, related table is " + + baseTableInfo.toList(), exception); + } } } finally { mtmv.readMvUnlock(); diff --git a/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out b/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out new file mode 100644 index 00000000000000..65b48e2b8c1fce --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query_after_delete -- +1 1 o 10.50 2023-12-08 a b 1 yy \N +1 1 o 9.50 2023-12-08 a b 1 yy 1 +2 1 o 11.50 2023-12-09 a b 1 yy 2 +3 1 o 12.50 2023-12-10 a b 1 yy \N +3 1 o 33.50 2023-12-10 a b 1 yy 3 +4 2 o 43.20 2023-12-11 c d 2 mm \N +5 2 o 1.20 2023-12-12 c d 2 mi \N +5 2 o 56.20 2023-12-12 c d 2 mi 4 + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index cb220bffb903e9..ee9a87251fdaed 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -1929,6 +1929,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { success = success && result.contains("(${mv_name})") } + if (!success) { + logger.info("mv_rewrite_all_success fail =" + result) + } Assert.assertEquals(true, success) } } @@ -1939,7 +1942,11 @@ class Suite implements GroovyInterceptable { check { result -> boolean success = true; for (String mv_name : mv_names) { - Assert.assertEquals(true, result.contains("${mv_name} chose")) + def contains = result.contains("${mv_name} chose") + if (!contains) { + logger.info("mv_rewrite_all_success fail =" + result) + } + Assert.assertEquals(true, contains) } } } @@ -1966,6 +1973,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { success = success || result.contains("(${mv_name})") } + if (!success) { + logger.info("mv_rewrite_any_success fail =" + result) + } Assert.assertEquals(true, success) } } @@ -1978,6 +1988,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { success = success || result.contains("${mv_name} chose") } + if (!success) { + logger.info("mv_rewrite_any_success fail =" + result) + } Assert.assertEquals(true, success) } } @@ -1998,6 +2011,9 @@ class Suite implements GroovyInterceptable { def each_result = splitResult.length == 2 ? splitResult[0].contains(mv_name) : false success = success && (result.contains("(${mv_name})") || each_result) } + if (!success) { + logger.info("mv_rewrite_all_success_without_check_chosen fail =" + result) + } Assert.assertEquals(true, success) } } @@ -2011,6 +2027,9 @@ class Suite implements GroovyInterceptable { boolean stepSuccess = result.contains("${mv_name} chose") || result.contains("${mv_name} not chose") success = success && stepSuccess } + if (!success) { + logger.info("mv_rewrite_all_success_without_check_chosen fail =" + result) + } Assert.assertEquals(true, success) } } @@ -2031,6 +2050,9 @@ class Suite implements GroovyInterceptable { def each_result = splitResult.length == 2 ? splitResult[0].contains(mv_name) : false success = success || (result.contains("(${mv_name})") || each_result) } + if (!success) { + logger.info("mv_rewrite_any_success_without_check_chosen fail =" + result) + } Assert.assertEquals(true, success) } } @@ -2043,6 +2065,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { success = success || result.contains("${mv_name} chose") || result.contains("${mv_name} not chose") } + if (!success) { + logger.info("mv_rewrite_any_success_without_check_chosen fail =" + result) + } Assert.assertEquals(true, success) } } @@ -2101,6 +2126,9 @@ class Suite implements GroovyInterceptable { boolean stepFail = !result.contains("(${mv_name})") fail = fail && stepFail } + if (!fail) { + logger.info("mv_rewrite_all_fail =" + result) + } Assert.assertEquals(true, fail) } } @@ -2114,6 +2142,9 @@ class Suite implements GroovyInterceptable { boolean stepFail = result.contains("${mv_name} fail") fail = fail && stepFail } + if (!fail) { + logger.info("mv_rewrite_all_fail =" + result) + } Assert.assertEquals(true, fail) } } @@ -2131,6 +2162,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { fail = fail || !result.contains("(${mv_name})") } + if (!fail) { + logger.info("mv_rewrite_any_fail =" + result) + } Assert.assertEquals(true, fail) } } @@ -2143,6 +2177,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { fail = fail || result.contains("${mv_name} fail") } + if (!fail) { + logger.info("mv_rewrite_any_fail =" + result) + } Assert.assertEquals(true, fail) } } diff --git a/regression-test/suites/auth_p0/test_select_column_auth.groovy b/regression-test/suites/auth_p0/test_select_column_auth.groovy index 52f1dc02697dd4..36cc2a0a09cf1c 100644 --- a/regression-test/suites/auth_p0/test_select_column_auth.groovy +++ b/regression-test/suites/auth_p0/test_select_column_auth.groovy @@ -130,6 +130,10 @@ suite("test_select_column_auth","p0,auth") { sql """grant select_priv(sum_id) on ${dbName}.${mtmv_name} to ${user}""" sql """grant select_priv(id) on ${dbName}.${tableName} to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { + def show_grants = sql """show grants;""" + logger.info("show grants:" + show_grants.toString()) + // If exec on fe follower, wait meta data is ready on follower + Thread.sleep(2000) sql "SET enable_materialized_view_rewrite=true" explain { sql("""select username, sum(id) from ${dbName}.${tableName} group by username""") diff --git a/regression-test/suites/mv_p0/unique/unique_rewrite.groovy b/regression-test/suites/mv_p0/unique/unique_rewrite.groovy index e8c3dd05f80c92..1e8a37c70919ba 100644 --- a/regression-test/suites/mv_p0/unique/unique_rewrite.groovy +++ b/regression-test/suites/mv_p0/unique/unique_rewrite.groovy @@ -96,6 +96,10 @@ suite("mv_on_unique_table") { AS ${mv1} """) + + def desc_all_mv1 = sql """desc lineitem_2_uniq all;""" + logger.info("desc mv1 is: " + desc_all_mv1.toString()) + explain { sql("""${query1}""") check {result -> @@ -124,6 +128,11 @@ suite("mv_on_unique_table") { AS ${mv2} """) + + def desc_all_mv2 = sql """desc lineitem_2_uniq all;""" + logger.info("desc mv2 is" + desc_all_mv2) + // If exec on fe follower, wait meta data is ready on follower + Thread.sleep(2000) explain { sql("""${query2}""") check {result -> diff --git a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy index 88d8ad6ea2d84d..35d4a60e6255bb 100644 --- a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy @@ -169,7 +169,7 @@ suite("mtmv_range_date_part_up_rewrite") { for (int i = 0; i < mv_name_list.size(); i++) { def job_name = getJobName(db, mv_name_list[i]) waitingMTMVTaskFinished(job_name) - mv_rewrite_success(query_stmt_list[i], mv_name_list[i]) + mv_rewrite_any_success(query_stmt_list[i], mv_name_list) compare_res(query_stmt_list[i] + " order by 1,2,3") } @@ -178,38 +178,38 @@ suite("mtmv_range_date_part_up_rewrite") { sql """insert into lineitem_range_date_union values (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01')""" for (int i = 0; i < mv_name_list.size(); i++) { - mv_rewrite_success(query_stmt_list[i], mv_name_list[i]) + mv_rewrite_any_success(query_stmt_list[i], mv_name_list) compare_res(query_stmt_list[i] + " order by 1,2,3") } for (int i = 0; i < mv_name_list.size(); i++) { sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;""" - mv_rewrite_success(query_stmt_list[i], mv_name_list[i]) + mv_rewrite_any_success(query_stmt_list[i], mv_name_list) compare_res(query_stmt_list[i] + " order by 1,2,3") } sql """insert into lineitem_range_date_union values (2, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01');""" for (int i = 0; i < mv_name_list.size(); i++) { - mv_rewrite_success(query_stmt_list[i], mv_name_list[i]) + mv_rewrite_any_success(query_stmt_list[i], mv_name_list) compare_res(query_stmt_list[i] + " order by 1,2,3") } for (int i = 0; i < mv_name_list.size(); i++) { sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;""" - mv_rewrite_success(query_stmt_list[i], mv_name_list[i]) + mv_rewrite_any_success(query_stmt_list[i], mv_name_list) compare_res(query_stmt_list[i] + " order by 1,2,3") } sql """ALTER TABLE lineitem_range_date_union DROP PARTITION IF EXISTS p4 FORCE""" for (int i = 0; i < mv_name_list.size(); i++) { - mv_rewrite_success(query_stmt_list[i], mv_name_list[i]) + mv_rewrite_any_success(query_stmt_list[i], mv_name_list) compare_res(query_stmt_list[i] + " order by 1,2,3") } for (int i = 0; i < mv_name_list.size(); i++) { sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;""" - mv_rewrite_success(query_stmt_list[i], mv_name_list[i]) + mv_rewrite_any_success(query_stmt_list[i], mv_name_list) compare_res(query_stmt_list[i] + " order by 1,2,3") } diff --git a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy index 15d93e32f65dc2..f973d031adeee0 100644 --- a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy @@ -83,7 +83,9 @@ suite("is_in_debug_mode") { AS select * from orders where o_orderkey > 2; """ } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + def message = e.getMessage() + logger.info("test_create_mv1" + message) + Assert.assertTrue(message.contains("because is in debug mode")) } sql """set skip_delete_sign = false;""" @@ -99,7 +101,9 @@ suite("is_in_debug_mode") { AS select * from orders where o_orderkey > 2; """ } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + def message = e.getMessage() + logger.info("test_create_mv2" + message) + Assert.assertTrue(message.contains("because is in debug mode")) } sql """set skip_storage_engine_merge = false;""" @@ -115,7 +119,9 @@ suite("is_in_debug_mode") { AS select * from orders where o_orderkey > 2; """ } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + def message = e.getMessage() + logger.info("test_create_mv3: " + message) + Assert.assertTrue(message.contains("because is in debug mode")) } sql """set skip_delete_bitmap = false;""" @@ -131,7 +137,9 @@ suite("is_in_debug_mode") { AS select * from orders where o_orderkey > 2; """ } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + def message = e.getMessage() + logger.info("test_create_mv4" + message) + Assert.assertTrue(message.contains("because is in debug mode")) } sql """set skip_delete_predicate = false;""" @@ -147,7 +155,9 @@ suite("is_in_debug_mode") { AS select * from orders where o_orderkey > 2; """ } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + def message = e.getMessage() + logger.info("test_create_mv5" + message) + Assert.assertTrue(message.contains("because is in debug mode")) } sql """set show_hidden_columns = false;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy new file mode 100644 index 00000000000000..e7556094e71c10 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy @@ -0,0 +1,77 @@ +package mv.nested +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("nested_mv_delete") { + + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders_1 + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders_1 ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL, + public_col INT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into orders_1 values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy', 1), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy', null), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy', 2), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy', null), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy', 3), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm', null), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi', 4), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi', null); + """ + + sql """alter table orders_1 modify column o_comment set stats ('row_count'='8');""" + + + create_async_mv(db, "mv_level_1", """ + select * from orders_1; + """) + + create_async_mv(db, "mv_level_2", """ + select * from mv_level_1; + """) + + sql """drop materialized view mv_level_1;""" + + order_qt_query_after_delete "select * from mv_level_2" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv_level_2""" +}