From c10777bc8c1dc319b54c4f007eb9dddd8fc4762b Mon Sep 17 00:00:00 2001 From: Abhishek Ravi Date: Thu, 27 Dec 2018 16:55:44 -0800 Subject: [PATCH] Fix failures in JPPD tests (#526) * Fix plan verification failures due to Semi Join - In case of nested join scenario, multiple RTFs arrives at scan on probe side. This resulted in plan showing multiple RuntimeFilters. This has been fixed and hence changes are required. - Semi Join is enable by default and the plan has to factor in the changes. - Disable Semi Join where Agg is required before Join. * Disable failing queries with a reference to the bug --- .../tpch_sf100/parquet/blocking_op_build_side.sql | 4 +++- .../parquet/blocking_op_build_side.sql.explain | 4 +++- ...roadcast_hash_01.sql => broadcast_hash_01.sql.fail} | 1 + .../tpch_sf100/parquet/broadcast_hash_02.plan | 4 +--- ...roadcast_hash_02.sql => broadcast_hash_02.sql.fail} | 1 + .../tpch_sf100/parquet/hash_join_date.plan | 10 ++++------ .../{hash_join_date.sql => hash_join_date.sql.fail} | 1 + .../tpch_sf100/parquet/hash_join_double.plan | 10 ++++------ ...{hash_join_double.sql => hash_join_double.sql.fail} | 1 + .../tpch_sf100/parquet/nested_broadcast.plan | 4 +--- ...{nested_broadcast.sql => nested_broadcast.sql.fail} | 1 + 11 files changed, 21 insertions(+), 20 deletions(-) rename framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/{broadcast_hash_01.sql => broadcast_hash_01.sql.fail} (96%) rename framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/{broadcast_hash_02.sql => broadcast_hash_02.sql.fail} (97%) rename framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/{hash_join_date.sql => hash_join_date.sql.fail} (92%) rename framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/{hash_join_double.sql => hash_join_double.sql.fail} (92%) rename framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/{nested_broadcast.sql => nested_broadcast.sql.fail} (91%) diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/blocking_op_build_side.sql b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/blocking_op_build_side.sql index 8d6983808..403a5187f 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/blocking_op_build_side.sql +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/blocking_op_build_side.sql @@ -1,6 +1,7 @@ set `exec.hashjoin.enable.runtime_filter` = true; set `exec.hashjoin.runtime_filter.max.waiting.time` = 10000; set `planner.enable_broadcast_join` = false; +set `planner.enable_semijoin` = false; select l.l_shipdate, @@ -23,4 +24,5 @@ limit 10; reset `exec.hashjoin.enable.runtime_filter`; reset `exec.hashjoin.runtime_filter.max.waiting.time`; -reset `planner.enable_broadcast_join`; \ No newline at end of file +reset `planner.enable_broadcast_join`; +reset `planner.enable_semijoin`; diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/blocking_op_build_side.sql.explain b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/blocking_op_build_side.sql.explain index 3f727b391..e8e18b3a3 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/blocking_op_build_side.sql.explain +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/blocking_op_build_side.sql.explain @@ -1,6 +1,7 @@ set `exec.hashjoin.enable.runtime_filter` = true; set `exec.hashjoin.runtime_filter.max.waiting.time` = 10000; set `planner.enable_broadcast_join` = false; +set `planner.enable_semijoin` = false; explain plan for select @@ -24,4 +25,5 @@ limit 10; reset `exec.hashjoin.enable.runtime_filter`; reset `exec.hashjoin.runtime_filter.max.waiting.time`; -reset `planner.enable_broadcast_join`; \ No newline at end of file +reset `planner.enable_broadcast_join`; +reset `planner.enable_semijoin`; \ No newline at end of file diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_01.sql b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_01.sql.fail similarity index 96% rename from framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_01.sql rename to framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_01.sql.fail index faf995162..e1f5d9b46 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_01.sql +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_01.sql.fail @@ -1,3 +1,4 @@ +-- Enable when DRILL-6914 is fixed set `exec.hashjoin.enable.runtime_filter` = true; set `exec.hashjoin.runtime_filter.max.waiting.time` = 10000; set `planner.enable_broadcast_join` = true; diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_02.plan b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_02.plan index ba081c2ba..ed98bc42b 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_02.plan +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_02.plan @@ -17,9 +17,7 @@ 04-03\s+SelectionVectorRemover 04-04\s+Filter.* 04-05\s+RuntimeFilter -04-06\s+SelectionVectorRemover -04-07\s+RuntimeFilter -04-08\s+Scan.*lineitem.*columns=\[`l_suppkey`, `l_orderkey`, `l_shipdate`, `l_extendedprice`, `l_discount`\]\]\]\) +04-06\s+Scan.*lineitem.*columns=\[`l_suppkey`, `l_orderkey`, `l_shipdate`, `l_extendedprice`, `l_discount`\]\]\]\) 04-02\s+BroadcastExchange 06-01\s+HashJoin.* 06-03\s+Scan.*supplier.*columns=\[`s_suppkey`, `s_nationkey`\]\]\]\) diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_02.sql b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_02.sql.fail similarity index 97% rename from framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_02.sql rename to framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_02.sql.fail index 73f81c0d8..6688a213e 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_02.sql +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/broadcast_hash_02.sql.fail @@ -1,3 +1,4 @@ +-- Enable when DRILL-6914 is fixed set `exec.hashjoin.enable.runtime_filter` = true; set `exec.hashjoin.runtime_filter.max.waiting.time` = 10000; set `planner.enable_broadcast_join` = true; diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_date.plan b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_date.plan index c112462d4..3da486e06 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_date.plan +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_date.plan @@ -6,11 +6,9 @@ 04-02\s+SelectionVectorRemover 04-03\s+RuntimeFilter 04-04\s+Scan.*lineitem.*columns=\[`l_shipdate`\]\]\]\) -01-04\s+HashAgg.* -01-06\s+Project.* -01-08\s+HashToRandomExchange.* +01-04\s+Project.* +01-06\s+HashToRandomExchange.* 03-01\s+UnorderedMuxExchange 05-01\s+Project.* -05-02\s+HashAgg.* -05-03\s+Project\(EXPR\$0=\[CAST\(\$0\)\:\DATE\]\) -05-04\s+Scan.*lineitem.*columns=\[`l_shipdate`\]\]\]\) \ No newline at end of file +05-02\s+Project\(EXPR\$0=\[CAST\(\$0\)\:\DATE\]\) +05-03\s+Scan.*lineitem.*columns=\[`l_shipdate`\]\]\]\) \ No newline at end of file diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_date.sql b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_date.sql.fail similarity index 92% rename from framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_date.sql rename to framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_date.sql.fail index b6c015647..d75ce2cc2 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_date.sql +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_date.sql.fail @@ -1,3 +1,4 @@ +-- Enable when DRILL-6914 is fixed set `exec.hashjoin.enable.runtime_filter` = true; set `exec.hashjoin.runtime_filter.max.waiting.time` = 10000; set `planner.enable_broadcast_join` = false; diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_double.plan b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_double.plan index cdf4227ad..39e793048 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_double.plan +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_double.plan @@ -6,11 +6,9 @@ 04-02\s+SelectionVectorRemover 04-03\s+RuntimeFilter 04-04\s+Scan.*lineitem.*columns=\[`l_discount`\]\]\]\) -01-04\s+HashAgg.* -01-06\s+Project.* -01-08\s+HashToRandomExchange.* +01-04\s+Project.* +01-06\s+HashToRandomExchange.* 03-01\s+UnorderedMuxExchange 05-01\s+Project.* -05-02\s+HashAgg.* -05-03\s+Project\(EXPR\$0=\[CAST\(\$0\)\:DOUBLE\]\) -05-04\s+Scan.*lineitem.*columns=\[`l_discount`\]\]\]\) \ No newline at end of file +05-02\s+Project\(EXPR\$0=\[CAST\(\$0\)\:DOUBLE\]\) +05-03\s+Scan.*lineitem.*columns=\[`l_discount`\]\]\]\) \ No newline at end of file diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_double.sql b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_double.sql.fail similarity index 92% rename from framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_double.sql rename to framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_double.sql.fail index 07bdfe363..d1cde5341 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_double.sql +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/hash_join_double.sql.fail @@ -1,3 +1,4 @@ +-- Enable when DRILL-6914 is fixed set `exec.hashjoin.enable.runtime_filter` = true; set `exec.hashjoin.runtime_filter.max.waiting.time` = 10000; set `planner.enable_broadcast_join` = false; diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/nested_broadcast.plan b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/nested_broadcast.plan index 8aa78f87c..392c1b501 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/nested_broadcast.plan +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/nested_broadcast.plan @@ -2,9 +2,7 @@ 03-06\s+HashJoin.* 03-08\s+SelectionVectorRemover 03-09\s+RuntimeFilter -03-10\s+SelectionVectorRemover -03-11\s+RuntimeFilter -03-12\s+Scan.*partsupp.*columns=\[`ps_suppkey`, `ps_partkey`\]\]\]\) +03-10\s+Scan.*partsupp.*columns=\[`ps_suppkey`, `ps_partkey`\]\]\]\) 03-07\s+BroadcastExchange 04-01\s+HashJoin.* 04-03\s+SelectionVectorRemover diff --git a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/nested_broadcast.sql b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/nested_broadcast.sql.fail similarity index 91% rename from framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/nested_broadcast.sql rename to framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/nested_broadcast.sql.fail index 577b80308..8110c9369 100644 --- a/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/nested_broadcast.sql +++ b/framework/resources/Advanced/runtimefilter/tpch_sf100/parquet/nested_broadcast.sql.fail @@ -1,3 +1,4 @@ +-- Enable when DRILL-6792 is fixed set `exec.hashjoin.enable.runtime_filter` = true; set `exec.hashjoin.runtime_filter.max.waiting.time` = 10000; set `planner.enable_broadcast_join` = true;