diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java index bf8c53259ce6ad..67edbe4f4ced09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.pattern.MatchingContext; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -196,7 +197,13 @@ private void collectMTMVCandidates(TableIf table, CascadesContext cascadesContex try { for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTables()) { LOG.info("mtmv {} related base table include {}", new BaseTableInfo(mtmv), baseTableInfo); - cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(), TableFrom.MTMV); + try { + cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(), + TableFrom.MTMV); + } catch (AnalysisException exception) { + LOG.warn("mtmv related base table get err, related table is " + + baseTableInfo.toList(), exception); + } } } finally { mtmv.readMvUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java index 43ad573bf795ef..9d234ae9e69f33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java @@ -179,7 +179,10 @@ public RowBatch getNext(Status status) throws TException { status.updateStatus(TStatusCode.TIMEOUT, e.getMessage()); } else { status.updateStatus(TStatusCode.THRIFT_RPC_ERROR, e.getMessage()); - SimpleScheduler.addToBlacklist(backendId, e.getMessage()); + // Shutdown maybe called by other request, should ignore this case. + if (!e.getMessage().contains("shutdown")) { + SimpleScheduler.addToBlacklist(backendId, e.getMessage()); + } } } catch (TimeoutException e) { LOG.warn("fetch result timeout, finstId={}", DebugUtil.printId(finstId), e); diff --git a/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out b/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out new file mode 100644 index 00000000000000..65b48e2b8c1fce --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query_after_delete -- +1 1 o 10.50 2023-12-08 a b 1 yy \N +1 1 o 9.50 2023-12-08 a b 1 yy 1 +2 1 o 11.50 2023-12-09 a b 1 yy 2 +3 1 o 12.50 2023-12-10 a b 1 yy \N +3 1 o 33.50 2023-12-10 a b 1 yy 3 +4 2 o 43.20 2023-12-11 c d 2 mm \N +5 2 o 1.20 2023-12-12 c d 2 mi \N +5 2 o 56.20 2023-12-12 c d 2 mi 4 + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index e13e3eca4a37c6..e9367e2bcc8633 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -1609,6 +1609,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { success = success && result.contains("(${mv_name})") } + if (!success) { + logger.info("mv_rewrite_all_success fail =" + result) + } Assert.assertEquals(true, success) } } @@ -1619,7 +1622,11 @@ class Suite implements GroovyInterceptable { check { result -> boolean success = true; for (String mv_name : mv_names) { - Assert.assertEquals(true, result.contains("${mv_name} chose")) + def contains = result.contains("${mv_name} chose") + if (!contains) { + logger.info("mv_rewrite_all_success fail =" + result) + } + Assert.assertEquals(true, contains) } } } @@ -1646,6 +1653,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { success = success || result.contains("(${mv_name})") } + if (!success) { + logger.info("mv_rewrite_any_success fail =" + result) + } Assert.assertEquals(true, success) } } @@ -1658,6 +1668,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { success = success || result.contains("${mv_name} chose") } + if (!success) { + logger.info("mv_rewrite_any_success fail =" + result) + } Assert.assertEquals(true, success) } } @@ -1678,6 +1691,9 @@ class Suite implements GroovyInterceptable { def each_result = splitResult.length == 2 ? splitResult[0].contains(mv_name) : false success = success && (result.contains("(${mv_name})") || each_result) } + if (!success) { + logger.info("mv_rewrite_all_success_without_check_chosen fail =" + result) + } Assert.assertEquals(true, success) } } @@ -1691,6 +1707,9 @@ class Suite implements GroovyInterceptable { boolean stepSuccess = result.contains("${mv_name} chose") || result.contains("${mv_name} not chose") success = success && stepSuccess } + if (!success) { + logger.info("mv_rewrite_all_success_without_check_chosen fail =" + result) + } Assert.assertEquals(true, success) } } @@ -1711,6 +1730,9 @@ class Suite implements GroovyInterceptable { def each_result = splitResult.length == 2 ? splitResult[0].contains(mv_name) : false success = success || (result.contains("(${mv_name})") || each_result) } + if (!success) { + logger.info("mv_rewrite_any_success_without_check_chosen fail =" + result) + } Assert.assertEquals(true, success) } } @@ -1723,6 +1745,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { success = success || result.contains("${mv_name} chose") || result.contains("${mv_name} not chose") } + if (!success) { + logger.info("mv_rewrite_any_success_without_check_chosen fail =" + result) + } Assert.assertEquals(true, success) } } @@ -1781,6 +1806,9 @@ class Suite implements GroovyInterceptable { boolean stepFail = !result.contains("(${mv_name})") fail = fail && stepFail } + if (!fail) { + logger.info("mv_rewrite_all_fail =" + result) + } Assert.assertEquals(true, fail) } } @@ -1794,6 +1822,9 @@ class Suite implements GroovyInterceptable { boolean stepFail = result.contains("${mv_name} fail") fail = fail && stepFail } + if (!fail) { + logger.info("mv_rewrite_all_fail =" + result) + } Assert.assertEquals(true, fail) } } @@ -1811,6 +1842,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { fail = fail || !result.contains("(${mv_name})") } + if (!fail) { + logger.info("mv_rewrite_any_fail =" + result) + } Assert.assertEquals(true, fail) } } @@ -1823,6 +1857,9 @@ class Suite implements GroovyInterceptable { for (String mv_name : mv_names) { fail = fail || result.contains("${mv_name} fail") } + if (!fail) { + logger.info("mv_rewrite_any_fail =" + result) + } Assert.assertEquals(true, fail) } } diff --git a/regression-test/suites/auth_p0/test_select_column_auth.groovy b/regression-test/suites/auth_p0/test_select_column_auth.groovy index 04b426830a6fdc..afb19ceb0b77f1 100644 --- a/regression-test/suites/auth_p0/test_select_column_auth.groovy +++ b/regression-test/suites/auth_p0/test_select_column_auth.groovy @@ -119,6 +119,10 @@ suite("test_select_column_auth","p0,auth") { sql """grant select_priv(sum_id) on ${dbName}.${mtmv_name} to ${user}""" sql """grant select_priv(id) on ${dbName}.${tableName} to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { + def show_grants = sql """show grants;""" + logger.info("show grants:" + show_grants.toString()) + // If exec on fe follower, wait meta data is ready on follower + Thread.sleep(2000) sql "SET enable_materialized_view_rewrite=true" explain { sql("""select username, sum(id) from ${dbName}.${tableName} group by username""") diff --git a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy index 15d93e32f65dc2..f973d031adeee0 100644 --- a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy @@ -83,7 +83,9 @@ suite("is_in_debug_mode") { AS select * from orders where o_orderkey > 2; """ } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + def message = e.getMessage() + logger.info("test_create_mv1" + message) + Assert.assertTrue(message.contains("because is in debug mode")) } sql """set skip_delete_sign = false;""" @@ -99,7 +101,9 @@ suite("is_in_debug_mode") { AS select * from orders where o_orderkey > 2; """ } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + def message = e.getMessage() + logger.info("test_create_mv2" + message) + Assert.assertTrue(message.contains("because is in debug mode")) } sql """set skip_storage_engine_merge = false;""" @@ -115,7 +119,9 @@ suite("is_in_debug_mode") { AS select * from orders where o_orderkey > 2; """ } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + def message = e.getMessage() + logger.info("test_create_mv3: " + message) + Assert.assertTrue(message.contains("because is in debug mode")) } sql """set skip_delete_bitmap = false;""" @@ -131,7 +137,9 @@ suite("is_in_debug_mode") { AS select * from orders where o_orderkey > 2; """ } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + def message = e.getMessage() + logger.info("test_create_mv4" + message) + Assert.assertTrue(message.contains("because is in debug mode")) } sql """set skip_delete_predicate = false;""" @@ -147,7 +155,9 @@ suite("is_in_debug_mode") { AS select * from orders where o_orderkey > 2; """ } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + def message = e.getMessage() + logger.info("test_create_mv5" + message) + Assert.assertTrue(message.contains("because is in debug mode")) } sql """set show_hidden_columns = false;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy new file mode 100644 index 00000000000000..e7556094e71c10 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy @@ -0,0 +1,77 @@ +package mv.nested +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("nested_mv_delete") { + + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders_1 + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders_1 ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL, + public_col INT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into orders_1 values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy', 1), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy', null), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy', 2), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy', null), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy', 3), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm', null), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi', 4), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi', null); + """ + + sql """alter table orders_1 modify column o_comment set stats ('row_count'='8');""" + + + create_async_mv(db, "mv_level_1", """ + select * from orders_1; + """) + + create_async_mv(db, "mv_level_2", """ + select * from mv_level_1; + """) + + sql """drop materialized view mv_level_1;""" + + order_qt_query_after_delete "select * from mv_level_2" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv_level_2""" +} diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy index 4554947d5ea8bf..e9afd352727e1c 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy @@ -16,6 +16,7 @@ // under the License. suite("int_as_date_partition_col") { + sql "set ENABLE_FOLD_CONSTANT_BY_BE=false" sql "drop table if exists partition_int" sql """CREATE TABLE partition_int(a int, dt int) PARTITION BY range(dt) ( partition p20240101 values less than ("20240101"), diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy index 4e04fed0cc4d39..a52f2577477435 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy @@ -16,6 +16,7 @@ // under the License. suite("one_col_range_partition") { + sql "set ENABLE_FOLD_CONSTANT_BY_BE=false" sql "drop table if exists one_col_range_partition_date" sql """ create table one_col_range_partition_date(a int, dt datetime, d date, c varchar(100)) duplicate key(a) diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy index 0cfe4963f27b2a..a5b3a4375c12bc 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy @@ -16,6 +16,7 @@ // under the License. suite("varchar_as_date_partition_col") { + sql "set ENABLE_FOLD_CONSTANT_BY_BE=false" sql"""drop table if exists partition_varchar;""" sql """CREATE TABLE partition_varchar(a int, dt varchar(10), rdt datetime) PARTITION BY list(dt) ( partition p20240101 values in ("20240101","20240102"),