add query instrumentation and regress case with profile

amorynan · Aug 10, 2024 · 2374ae3 · 2374ae3
1 parent 214d536
commit 2374ae3
Show file tree

Hide file tree

Showing 5 changed files with 222 additions and 2 deletions.
diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h
@@ -172,6 +172,21 @@ class FunctionArrayIndex : public IFunction {
         }
 
         *bitmap = *roaring;
+        // here debug for check array_contains function really filter rows by inverted index correctly
+        DBUG_EXECUTE_IF("array_func.array_contains", {
+            auto result_bitmap = DebugPoints::instance()->get_debug_param_or_default<uint64_t>(
+                    "array_func.array_contains", "result_bitmap", 0);
+            if (result_bitmap < 0) {
+                return Status::Error<ErrorCode::INTERNAL_ERROR>(
+                        "result_bitmap count cannot be negative");
+            }
+            if (bitmap->cardinality() != result_bitmap) {
+                return Status::Error<ErrorCode::INTERNAL_ERROR>(
+                        "array_contains really filtered {} by inverted index not equal to expected "
+                        "{}",
+                        bitmap->cardinality(), result_bitmap);
+            }
+        })
         return Status::OK();
     }
 

diff --git a/regression-test/data/inverted_index_p0/test_arrayInvertedIdx_profile.out b/regression-test/data/inverted_index_p0/test_arrayInvertedIdx_profile.out
@@ -0,0 +1,87 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+7
+
+-- !sql --
+2019-01-01	0974e7a82e30d1af83205e474fadd0a2	["w"]
+
+-- !sql --
+2019-01-01	0974e7a82e30d1af83205e474fadd0a2	["w"]
+
+-- !sql --
+
+-- !sql --
+
+-- !sql --
+2019-01-01	a648a447b8f71522f11632eba4b4adde	["p", "q", "r", "s", "t"]
+
+-- !sql --
+2017-01-01	021603e7dcfe65d44af0efd0e5aee154	["n"]
+2017-01-01	48a33ec3453a28bce84b8f96fe161956	["m"]
+2017-01-01	6afef581285b6608bf80d5a4e46cf839	["a", "b", "c"]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a3	\N
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a4	\N
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a5	[]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a6	[null, null, null]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a7	[null, null, null]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a8	[]
+2017-01-01	9fcb57ae675f0af4d613d9e6c0e8a2a2	["o"]
+2017-01-01	d93d942d985a8fb7547c72dada8d332d	["d", "e", "f", "g", "h", "i", "j", "k", "l"]
+2019-01-01	a648a447b8f71522f11632eba4b4adde	["p", "q", "r", "s", "t"]
+
+-- !sql --
+2017-01-01	021603e7dcfe65d44af0efd0e5aee154	["n"]
+2017-01-01	48a33ec3453a28bce84b8f96fe161956	["m"]
+2017-01-01	6afef581285b6608bf80d5a4e46cf839	["a", "b", "c"]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a5	[]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a6	[null, null, null]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a7	[null, null, null]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a8	[]
+2017-01-01	9fcb57ae675f0af4d613d9e6c0e8a2a2	["o"]
+2017-01-01	d93d942d985a8fb7547c72dada8d332d	["d", "e", "f", "g", "h", "i", "j", "k", "l"]
+2019-01-01	0974e7a82e30d1af83205e474fadd0a2	["w"]
+2019-01-01	26823b3995ee38bd145ddd910b2f6300	["x"]
+2019-01-01	a9fb5c985c90bf05f3bee5ca3ae95260	["u", "v"]
+2019-01-01	ee27ee1da291e46403c408e220bed6e1	["y"]
+
+-- !sql --
+2017-01-01	021603e7dcfe65d44af0efd0e5aee154	["n"]
+2017-01-01	48a33ec3453a28bce84b8f96fe161956	["m"]
+2017-01-01	6afef581285b6608bf80d5a4e46cf839	["a", "b", "c"]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a5	[]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a6	[null, null, null]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a7	[null, null, null]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a8	[]
+2017-01-01	9fcb57ae675f0af4d613d9e6c0e8a2a2	["o"]
+2017-01-01	d93d942d985a8fb7547c72dada8d332d	["d", "e", "f", "g", "h", "i", "j", "k", "l"]
+
+-- !sql --
+2019-01-01	0974e7a82e30d1af83205e474fadd0a2	["w"]
+2019-01-01	26823b3995ee38bd145ddd910b2f6300	["x"]
+2019-01-01	a9fb5c985c90bf05f3bee5ca3ae95260	["u", "v"]
+2019-01-01	ee27ee1da291e46403c408e220bed6e1	["y"]
+
+-- !sql --
+2017-01-01	021603e7dcfe65d44af0efd0e5aee154	["n"]
+2017-01-01	48a33ec3453a28bce84b8f96fe161956	["m"]
+2017-01-01	6afef581285b6608bf80d5a4e46cf839	["a", "b", "c"]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a3	\N
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a4	\N
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a5	[]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a6	[null, null, null]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a7	[null, null, null]
+2017-01-01	8fcb57ae675f0af4d613d9e6c0e8a2a8	[]
+2017-01-01	9fcb57ae675f0af4d613d9e6c0e8a2a2	["o"]
+2017-01-01	d93d942d985a8fb7547c72dada8d332d	["d", "e", "f", "g", "h", "i", "j", "k", "l"]
+2019-01-01	0974e7a82e30d1af83205e474fadd0a2	["w"]
+2019-01-01	26823b3995ee38bd145ddd910b2f6300	["x"]
+2019-01-01	a9fb5c985c90bf05f3bee5ca3ae95260	["u", "v"]
+2019-01-01	ee27ee1da291e46403c408e220bed6e1	["y"]
+
+-- !sql --
+2019-01-01	0974e7a82e30d1af83205e474fadd0a2	["w"]
+2019-01-01	26823b3995ee38bd145ddd910b2f6300	["x"]
+2019-01-01	a648a447b8f71522f11632eba4b4adde	["p", "q", "r", "s", "t"]
+2019-01-01	a9fb5c985c90bf05f3bee5ca3ae95260	["u", "v"]
+2019-01-01	ee27ee1da291e46403c408e220bed6e1	["y"]
+
diff --git a/regression-test/suites/inverted_index_p0/test_arrayInvertedIdx_profile.groovy b/regression-test/suites/inverted_index_p0/test_arrayInvertedIdx_profile.groovy
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import groovy.json.JsonSlurper
+
+suite("test_arrayInvertedIdx_profile", "nonConcurrent"){
+    // prepare test table
+    def indexTblName = "taip"
+    def httpGet = { url ->
+        def dst = 'http://' + context.config.feHttpAddress
+        def conn = new URL(dst + url).openConnection()
+        conn.setRequestMethod("GET")
+        def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
+                (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
+        conn.setRequestProperty("Authorization", "Basic ${encoding}")
+        return conn.getInputStream().getText()
+    }
+
+    def checkRowsInvertedIndexFilter = { sql, expectedRowsInvertedIndexFiltered ->
+        order_qt_sql sql
+        def profileUrl = '/rest/v1/query_profile/'
+        def profiles = httpGet(profileUrl)
+        log.debug("profiles:{}", profiles);
+        profiles = new JsonSlurper().parseText(profiles)
+        assertEquals(0, profiles.code)
+
+        def profileId = null;
+        for (def profile in profiles["data"]["rows"]) {
+            if (profile["Sql Statement"].contains(sql)) {
+                profileId = profile["Profile ID"]
+                break;
+            }
+        }
+        log.info("profileId:{}", profileId);
+        def profileDetail = httpGet("/rest/v1/query_profile/" + profileId)
+        assertTrue(profileDetail.contains("RowsInvertedIndexFiltered:&nbsp;&nbsp;" + expectedRowsInvertedIndexFiltered))
+    }
+
+    // If we use common expr pass to inverted index , we should set enable_common_expr_pushdown = true
+    sql """ set enable_common_expr_pushdown = true; """
+    sql """ set enable_common_expr_pushdown_for_inverted_index = true; """
+    sql """ set enable_pipeline_x_engine = true;"""
+    sql """ set enable_profile = true;"""
+
+    sql "DROP TABLE IF EXISTS ${indexTblName}"
+    // create 1 replica table
+    sql """
+	CREATE TABLE IF NOT EXISTS `${indexTblName}` (
+      `apply_date` date NULL COMMENT '',
+      `id` varchar(60) NOT NULL COMMENT '',
+      `inventors` array<text> NULL COMMENT '',
+      INDEX index_inverted_inventors(inventors) USING INVERTED  COMMENT ''
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`apply_date`, `id`)
+    COMMENT 'OLAP'
+    DISTRIBUTED BY HASH(`id`) BUCKETS 1
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1",
+    "is_being_synced" = "false",
+    "storage_format" = "V2",
+    "light_schema_change" = "true",
+    "disable_auto_compaction" = "false",
+    "enable_single_replica_compaction" = "false"
+    );
+    """
+
+    sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES
+        ('2017-01-01', '6afef581285b6608bf80d5a4e46cf839', '[\"a\", \"b\", \"c\"]'),
+        ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a3', []),
+        ('2017-01-01', 'd93d942d985a8fb7547c72dada8d332d', '[\"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\"]'),
+        ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a4', NULL),
+         ('2017-01-01', 'd93d942d985a8fb7547c72dada8d332e', '[\"m\", \"n\", \"o\", \"p\", \"q\", \"r\", \"s\", \"t\", \"u\"]'),
+         ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a6', '[null,null,null]'),
+         ('2019-01-01', 'd93d942d985a8fb7547c72dada8d332f', '[\"v\", \"w\", \"x\", \"y\", \"z\"]'); """
+
+
+    qt_sql """ select count() from ${indexTblName}"""
+    def checkpoints_name = "array_func.array_contains"
+    try {
+        GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [result_bitmap: 1])
+        order_qt_sql "select * from tai where array_contains(inventors, 'w') order by id;"
+    } finally {
+        GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+    }
+
+    checkRowsInvertedIndexFilter.call("select * from tai where array_contains(inventors, 'w') order by id;", 1)
+
+    try {
+        GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [result_bitmap: 1])
+         order_qt_sql """ select * from tai where array_contains(inventors, 's') and apply_date = '2017-01-01' order by id; """
+    } finally {
+        GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+    }
+    checkRowsInvertedIndexFilter.call("select * from tai where array_contains(inventors, 's') and apply_date = '2017-01-01' order by id", 1)
+
+    order_qt_sql """ select * from tai where array_contains(inventors, 's') and apply_date = '2019-01-01' order by id; """
+
+    order_qt_sql """ select * from tai where array_contains(inventors, 's') or apply_date = '2017-01-01' order by id; """
+    order_qt_sql """ select * from tai where !array_contains(inventors, 's') order by id; """
+    order_qt_sql """ select * from tai where !array_contains(inventors, 's') and apply_date = '2017-01-01' order by id; """
+    order_qt_sql """ select * from tai where !array_contains(inventors, 's') and apply_date = '2019-01-01' order by id; """
+    order_qt_sql """ select * from tai where !array_contains(inventors, 's') or apply_date = '2017-01-01' order by id; """
+    order_qt_sql """ select * from tai where (array_contains(inventors, 's') and apply_date = '2017-01-01') or apply_date = '2019-01-01' order by id; """
+}
diff --git a/...on-test/suites/inverted_index_p0/test_array_inverted_index/test_array_index_parser.groovy b/...on-test/suites/inverted_index_p0/test_array_inverted_index/test_array_index_parser.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_array_index_parser"){
+suite("test_array_index_parser", "nonConcurrent"){
 
     def create_array_index_table = {testTablex, parser ->
         sql "DROP TABLE IF EXISTS " + testTablex + ";"

diff --git a/...ion-test/suites/inverted_index_p0/test_array_inverted_index/test_array_index_write.groovy b/...ion-test/suites/inverted_index_p0/test_array_inverted_index/test_array_index_write.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_array_index_write"){
+suite("test_array_index_write", "nonConcurrent"){
 
     def create_array_index_table = {testTablex, parser ->
         def stmt = "CREATE TABLE IF NOT EXISTS " + testTablex + "(\n" +