edit

be/src/cloud/cloud_storage_engine.cpp

@@ -832,17 +832,11 @@ void CloudStorageEngine::_check_tablet_delete_bitmap_score_callback() {
             return;
         }
         uint64_t max_delete_bitmap_score = 0;
-        uint64_t max_base_rowset_delete_bitmap_score = 0;
         std::vector<CloudTabletSPtr> tablets;
-        tablet_mgr().get_max_tablet_delete_bitmap_score(&max_delete_bitmap_score,
-                                                        &max_base_rowset_delete_bitmap_score);
+        tablet_mgr().get_topn_tablet_delete_bitmap_score(&max_delete_bitmap_score);
         if (max_delete_bitmap_score > 0) {
             _tablet_max_delete_bitmap_score_metrics->set_value(max_delete_bitmap_score);
         }
-        if (max_base_rowset_delete_bitmap_score > 0) {
-            _tablet_max_base_rowset_delete_bitmap_score_metrics->set_value(
-                    max_base_rowset_delete_bitmap_score);
-        }
     }
 }
 

be/src/cloud/cloud_tablet_mgr.cpp

@@ -419,12 +419,13 @@ void CloudTabletMgr::get_tablet_info(int64_t num_tablets, std::vector<TabletInfo
     }
 }
 
-void CloudTabletMgr::get_max_tablet_delete_bitmap_score(
-        uint64_t* max_delete_bitmap_score, uint64_t* max_base_rowset_delete_bitmap_score) {
+void CloudTabletMgr::get_topn_tablet_delete_bitmap_score(uint64_t* max_delete_bitmap_score) {
     int64_t max_delete_bitmap_score_tablet_id = 0;
-    int64_t max_base_rowset_delete_bitmap_score_tablet_id = 0;
     OlapStopWatch watch;
     uint64_t total_delete_map_count = 0;
+    int n = config::check_tablet_delete_bitmap_score_top_n;
+    std::vector<std::pair<int64_t, int64_t>> buf;
+    buf.reserve(n + 1);
     auto handler = [&](const std::weak_ptr<CloudTablet>& tablet_wk) {
         auto t = tablet_wk.lock();
         if (!t) return;
@@ -435,19 +436,25 @@ void CloudTabletMgr::get_max_tablet_delete_bitmap_score(
             max_delete_bitmap_score_tablet_id = t->tablet_id();
             *max_delete_bitmap_score = delete_bitmap_count;
         }
-        t->get_base_rowset_delete_bitmap_count(max_base_rowset_delete_bitmap_score,
-                                               &max_base_rowset_delete_bitmap_score_tablet_id);
+        buf.emplace_back(t.get()->tablet_id(), delete_bitmap_count);
+        std::sort(buf.begin(), buf.end(), [](auto& a, auto& b) { return a.second > b.second; });
+        if (buf.size() > n) {
+            buf.pop_back();
+        }
     };
     auto weak_tablets = get_weak_tablets();
     std::for_each(weak_tablets.begin(), weak_tablets.end(), handler);
-    LOG(INFO) << "tablet size=" << weak_tablets.size()
+    std::stringstream ss;
+    for (auto& i : buf) {
+        ss << i.first << ":" << i.second << ",";
+    }
+    LOG(INFO) << "get_topn_tablet_delete_bitmap_score, n=" << n
+              << ",tablet size=" << weak_tablets.size()
               << ",total_delete_map_count=" << total_delete_map_count
               << ",cost(us)=" << watch.get_elapse_time_us()
               << ",max_delete_bitmap_score=" << *max_delete_bitmap_score
               << ",max_delete_bitmap_score_tablet_id=" << max_delete_bitmap_score_tablet_id
-              << ",max_base_rowset_delete_bitmap_score=" << *max_base_rowset_delete_bitmap_score
-              << ",max_base_rowset_delete_bitmap_score_tablet_id="
-              << max_base_rowset_delete_bitmap_score_tablet_id;
+              << ",tablets=[" << ss.str() << "]";
 }
 
 } // namespace doris

be/src/cloud/cloud_tablet_mgr.h

@@ -82,8 +82,7 @@ class CloudTabletMgr {
 
     void get_tablet_info(int64_t num_tablets, std::vector<TabletInfo>* tablets_info);
 
-    void get_max_tablet_delete_bitmap_score(uint64_t* max_delete_bitmap_score,
-                                            uint64_t* max_base_rowset_delete_bitmap_score);
+    void get_topn_tablet_delete_bitmap_score(uint64_t* max_delete_bitmap_score);
 
 private:
     CloudStorageEngine& _engine;

be/src/common/config.cpp

@@ -1407,7 +1407,8 @@ DEFINE_mBool(enable_sleep_between_delete_cumu_compaction, "false");
 
 DEFINE_mInt32(compaction_num_per_round, "1");
 
-DEFINE_mInt32(check_tablet_delete_bitmap_interval_seconds, "30");
+DEFINE_mInt32(check_tablet_delete_bitmap_interval_seconds, "60");
+DEFINE_mInt32(check_tablet_delete_bitmap_score_top_n, "10");
 DEFINE_mBool(enable_check_tablet_delete_bitmap_score, "true");
 
 // clang-format off

be/src/common/config.h

@@ -1495,6 +1495,7 @@ DECLARE_mBool(enable_sleep_between_delete_cumu_compaction);
 DECLARE_mInt32(compaction_num_per_round);
 
 DECLARE_mInt32(check_tablet_delete_bitmap_interval_seconds);
+DECLARE_mInt32(check_tablet_delete_bitmap_score_top_n);
 DECLARE_mBool(enable_check_tablet_delete_bitmap_score);
 
 #ifdef BE_TEST

be/src/olap/olap_server.cpp

@@ -1656,17 +1656,11 @@ void StorageEngine::_check_tablet_delete_bitmap_score_callback() {
             return;
         }
         uint64_t max_delete_bitmap_score = 0;
-        uint64_t max_base_rowset_delete_bitmap_score = 0;
         std::vector<CloudTabletSPtr> tablets;
-        _tablet_manager.get()->get_max_tablet_delete_bitmap_score(
-                &max_delete_bitmap_score, &max_base_rowset_delete_bitmap_score);
+        _tablet_manager.get()->get_topn_tablet_delete_bitmap_score(&max_delete_bitmap_score);
         if (max_delete_bitmap_score > 0) {
             _tablet_max_delete_bitmap_score_metrics->set_value(max_delete_bitmap_score);
         }
-        if (max_base_rowset_delete_bitmap_score > 0) {
-            _tablet_max_base_rowset_delete_bitmap_score_metrics->set_value(
-                    max_base_rowset_delete_bitmap_score);
-        }
     }
 }
 

be/src/olap/storage_engine.cpp

@@ -105,8 +105,6 @@ BaseStorageEngine::BaseStorageEngine(Type type, const UniqueId& backend_uid)
             static_cast<int64_t>(MemInfo::soft_mem_limit() * config::schema_change_mem_limit_frac);
     _tablet_max_delete_bitmap_score_metrics =
             std::make_shared<bvar::Status<size_t>>("tablet_max", "delete_bitmap_score", 0);
-    _tablet_max_base_rowset_delete_bitmap_score_metrics = std::make_shared<bvar::Status<size_t>>(
-            "tablet_max_base_rowset", "delete_bitmap_score", 0);
 }
 
 BaseStorageEngine::~BaseStorageEngine() = default;

be/src/olap/storage_engine.h

@@ -171,7 +171,6 @@ class BaseStorageEngine {
     std::shared_ptr<StreamLoadRecorder> _stream_load_recorder;
 
     std::shared_ptr<bvar::Status<size_t>> _tablet_max_delete_bitmap_score_metrics;
-    std::shared_ptr<bvar::Status<size_t>> _tablet_max_base_rowset_delete_bitmap_score_metrics;
 };
 
 class CompactionSubmitRegistry {

be/src/olap/tablet_manager.cpp

@@ -1764,12 +1764,13 @@ bool TabletManager::update_tablet_partition_id(::doris::TPartitionId partition_i
     return true;
 }
 
-void TabletManager::get_max_tablet_delete_bitmap_score(
-        uint64_t* max_delete_bitmap_score, uint64_t* max_base_rowset_delete_bitmap_score) {
+void TabletManager::get_topn_tablet_delete_bitmap_score(uint64_t* max_delete_bitmap_score) {
     int64_t max_delete_bitmap_score_tablet_id = 0;
-    int64_t max_base_rowset_delete_bitmap_score_tablet_id = 0;
     OlapStopWatch watch;
     uint64_t total_delete_map_count = 0;
+    int n = config::check_tablet_delete_bitmap_score_top_n;
+    std::vector<std::pair<int64_t, int64_t>> buf;
+    buf.reserve(n + 1);
     auto handler = [&](const TabletSharedPtr& tablet) {
         uint64_t delete_bitmap_count =
                 tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count();
@@ -1778,18 +1779,24 @@ void TabletManager::get_max_tablet_delete_bitmap_score(
             max_delete_bitmap_score_tablet_id = tablet->tablet_id();
             *max_delete_bitmap_score = delete_bitmap_count;
         }
-        tablet->get_base_rowset_delete_bitmap_count(max_base_rowset_delete_bitmap_score,
-                                                    &max_base_rowset_delete_bitmap_score_tablet_id);
+        buf.emplace_back(tablet->tablet_id(), delete_bitmap_count);
+        std::sort(buf.begin(), buf.end(), [](auto& a, auto& b) { return a.second > b.second; });
+        if (buf.size() > n) {
+            buf.pop_back();
+        }
     };
     for_each_tablet(handler, filter_all_tablets);
-    LOG(INFO) << "tablet size=" << _tablets_shards.size()
+    std::stringstream ss;
+    for (auto& i : buf) {
+        ss << i.first << ":" << i.second << ",";
+    }
+    LOG(INFO) << "get_topn_tablet_delete_bitmap_score, n=" << n
+              << ",tablet size=" << _tablets_shards.size()
               << ",total_delete_map_count=" << total_delete_map_count
               << ",cost(us)=" << watch.get_elapse_time_us()
-              << "max_delete_bitmap_score=" << *max_delete_bitmap_score
+              << ",max_delete_bitmap_score=" << *max_delete_bitmap_score
               << ",max_delete_bitmap_score_tablet_id=" << max_delete_bitmap_score_tablet_id
-              << ",max_base_rowset_delete_bitmap_score=" << *max_base_rowset_delete_bitmap_score
-              << ",max_base_rowset_delete_bitmap_score_tablet_id="
-              << max_base_rowset_delete_bitmap_score_tablet_id;
+              << ",tablets=[" << ss.str() << "]";
 }
 
 } // end namespace doris

be/src/olap/tablet_manager.h

@@ -178,8 +178,7 @@ class TabletManager {
     bool update_tablet_partition_id(::doris::TPartitionId partition_id,
                                     ::doris::TTabletId tablet_id);
 
-    void get_max_tablet_delete_bitmap_score(uint64_t* max_delete_bitmap_score,
-                                            uint64_t* max_base_rowset_delete_bitmap_score);
+    void get_topn_tablet_delete_bitmap_score(uint64_t* max_delete_bitmap_score);
 
 private:
     // Add a tablet pointer to StorageEngine

regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy

@@ -143,90 +143,80 @@ suite("test_delete_bitmap_metrics", "p0") {
     sql testTableDDL
     sql "sync"
 
-    // store the original value
-    get_be_param("check_tablet_delete_bitmap_score_enable")
-
-    try {
-        set_be_param("check_tablet_delete_bitmap_score_enable", "true")
-        sql "sync"
-        sql """ INSERT INTO ${testTable} VALUES (0,0,'1'),(1,1,'1'); """
-        sql """ INSERT INTO ${testTable} VALUES (0,0,'2'),(2,2,'2'); """
-        sql """ INSERT INTO ${testTable} VALUES (0,0,'3'),(3,3,'3'); """
-        sql """ INSERT INTO ${testTable} VALUES (0,0,'4'),(4,4,'4'); """
-        sql """ INSERT INTO ${testTable} VALUES (0,0,'5'),(5,5,'5'); """
-        sql """ INSERT INTO ${testTable} VALUES (0,0,'6'),(6,6,'6'); """
-        sql """ INSERT INTO ${testTable} VALUES (0,0,'7'),(7,7,'7'); """
-        sql """ INSERT INTO ${testTable} VALUES (0,0,'8'),(8,8,'8'); """
-
-        qt_sql "select * from ${testTable} order by plan_id"
-
-
-        def tablets = sql_return_maparray """ show tablets from ${testTable}; """
-        logger.info("tablets: " + tablets)
-        def local_delete_bitmap_count = 0
-        def ms_delete_bitmap_count = 0
-        def local_delete_bitmap_cardinality = 0;
-        def ms_delete_bitmap_cardinality = 0;
-        for (def tablet in tablets) {
-            String tablet_id = tablet.TabletId
-            def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """
-            logger.info("tablet: " + tablet_info)
-            String trigger_backend_id = tablet.BackendId
-
-            // before compaction, delete_bitmap_count is (rowsets num - 1)
-            local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-            local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
-            logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
-            logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality)
-            assertTrue(local_delete_bitmap_count == 7)
-            assertTrue(local_delete_bitmap_cardinality == 7)
-
-            if (isCloudMode()) {
-                ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-                ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
-                logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
-                logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
-                assertTrue(ms_delete_bitmap_count == 7)
-                assertTrue(ms_delete_bitmap_cardinality == 7)
-            }
-            def tablet_delete_bitmap_count = 0;
-            def base_rowset_delete_bitmap_count = 0;
-            int retry_time = 0;
-            while (retry_time < 10) {
-                log.info("retry_time: ${retry_time}")
-                getMetricsMethod.call() {
-                    respCode, body ->
-                        logger.info("test get delete bitmap count resp Code {}", "${respCode}".toString())
-                        assertEquals("${respCode}".toString(), "200")
-                        String out = "${body}".toString()
-                        def strs = out.split('\n')
-                        for (String line in strs) {
-                            if (line.startsWith("tablet_max_delete_bitmap_score")) {
-                                logger.info("find: {}", line)
-                                tablet_delete_bitmap_count = line.replaceAll("tablet_max_delete_bitmap_score ", "").toInteger()
-                                break
-                            }
+    sql "sync"
+    sql """ INSERT INTO ${testTable} VALUES (0,0,'1'),(1,1,'1'); """
+    sql """ INSERT INTO ${testTable} VALUES (0,0,'2'),(2,2,'2'); """
+    sql """ INSERT INTO ${testTable} VALUES (0,0,'3'),(3,3,'3'); """
+    sql """ INSERT INTO ${testTable} VALUES (0,0,'4'),(4,4,'4'); """
+    sql """ INSERT INTO ${testTable} VALUES (0,0,'5'),(5,5,'5'); """
+    sql """ INSERT INTO ${testTable} VALUES (0,0,'6'),(6,6,'6'); """
+    sql """ INSERT INTO ${testTable} VALUES (0,0,'7'),(7,7,'7'); """
+    sql """ INSERT INTO ${testTable} VALUES (0,0,'8'),(8,8,'8'); """
+
+    qt_sql "select * from ${testTable} order by plan_id"
+
+
+    def tablets = sql_return_maparray """ show tablets from ${testTable}; """
+    logger.info("tablets: " + tablets)
+    def local_delete_bitmap_count = 0
+    def ms_delete_bitmap_count = 0
+    def local_delete_bitmap_cardinality = 0;
+    def ms_delete_bitmap_cardinality = 0;
+    for (def tablet in tablets) {
+        String tablet_id = tablet.TabletId
+        def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """
+        logger.info("tablet: " + tablet_info)
+        String trigger_backend_id = tablet.BackendId
+
+        // before compaction, delete_bitmap_count is (rowsets num - 1)
+        local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+        local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+        logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
+        logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality)
+        assertTrue(local_delete_bitmap_count == 7)
+        assertTrue(local_delete_bitmap_cardinality == 7)
+
+        if (isCloudMode()) {
+            ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+            ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+            logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
+            logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
+            assertTrue(ms_delete_bitmap_count == 7)
+            assertTrue(ms_delete_bitmap_cardinality == 7)
+        }
+        def tablet_delete_bitmap_count = 0;
+        int retry_time = 0;
+        while (retry_time < 10) {
+            log.info("retry_time: ${retry_time}")
+            getMetricsMethod.call() {
+                respCode, body ->
+                    logger.info("test get delete bitmap count resp Code {}", "${respCode}".toString())
+                    assertEquals("${respCode}".toString(), "200")
+                    String out = "${body}".toString()
+                    def strs = out.split('\n')
+                    for (String line in strs) {
+                        if (line.startsWith("tablet_max_delete_bitmap_score")) {
+                            logger.info("find: {}", line)
+                            tablet_delete_bitmap_count = line.replaceAll("tablet_max_delete_bitmap_score ", "").toInteger()
+                            break
                         }
-                        for (String line in strs) {
-                            if (line.startsWith("tablet_max_base_rowset_delete_bitmap_score")) {
-                                logger.info("find: {}", line)
-                                base_rowset_delete_bitmap_count = line.replaceAll("tablet_max_base_rowset_delete_bitmap_score ", "").toInteger()
-                                break
-                            }
+                    }
+                    for (String line in strs) {
+                        if (line.startsWith("tablet_max_base_rowset_delete_bitmap_score")) {
+                            logger.info("find: {}", line)
+                            base_rowset_delete_bitmap_count = line.replaceAll("tablet_max_base_rowset_delete_bitmap_score ", "").toInteger()
+                            break
                         }
-                }
-                if (tablet_delete_bitmap_count > 0 && base_rowset_delete_bitmap_count > 0) {
-                    break;
-                } else {
-                    Thread.sleep(10000)
-                    retry_time++;
-                }
+                    }
+            }
+            if (tablet_delete_bitmap_count > 0) {
+                break;
+            } else {
+                Thread.sleep(20000)
+                retry_time++;
             }
-            assertTrue(tablet_delete_bitmap_count > 0)
-            assertTrue(base_rowset_delete_bitmap_count > 0)
         }
-    } finally {
-        reset_be_param("check_tablet_delete_bitmap_score_enable")
+        assertTrue(tablet_delete_bitmap_count > 0)
     }
 
 }