From 1c114e42f3daa806d762b10cb6c746974d416249 Mon Sep 17 00:00:00 2001
From: Murphy <96611012+murphyatwork@users.noreply.github.com>
Date: Thu, 5 Sep 2024 07:50:17 +0800
Subject: [PATCH] [Enhancement] reduce lock granular of TabletStatMgr (#50668)

Signed-off-by: Murphy <mofei@starrocks.com>
(cherry picked from commit 0b796e0ba0c108ec898423ebff5a8e99a949b7b4)
Signed-off-by: Murphy <mofei@starrocks.com>

# Conflicts:
#	fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java
---
 .../com/starrocks/catalog/TabletStatMgr.java  | 49 ++++++++++++++-----
 1 file changed, 37 insertions(+), 12 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java
index 8c1f4399a4581..04bd582ae810b 100644
--- a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java
@@ -36,6 +36,7 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.starrocks.catalog.MaterializedIndex.IndexExtState;
 import com.starrocks.common.ClientPool;
 import com.starrocks.common.Config;
@@ -102,14 +103,16 @@ protected void runAfterCatalogReady() {
             if (db == null) {
                 continue;
             }
-            db.writeLock();
-            try {
-                for (Table table : db.getTables()) {
-                    long totalRowCount = 0L;
-                    if (!table.isNativeTableOrMaterializedView()) {
-                        continue;
-                    }
+            for (Table table : db.getTables()) {
+                long totalRowCount = 0L;
+                if (!table.isNativeTableOrMaterializedView()) {
+                    continue;
+                }
 
+                // NOTE: calculate the row first with read lock, then update the stats with write lock
+                db.readLock();
+                Map<Long, Long> indexRowCountMap = Maps.newHashMap();
+                try {
                     OlapTable olapTable = (OlapTable) table;
                     for (Partition partition : olapTable.getAllPartitions()) {
                         for (PhysicalPartition physicalPartition : partition.getSubPartitions()) {
@@ -117,10 +120,11 @@ protected void runAfterCatalogReady() {
                             for (MaterializedIndex index : physicalPartition.getMaterializedIndices(
                                     IndexExtState.VISIBLE)) {
                                 long indexRowCount = 0L;
+                                // NOTE: can take a rather long time to iterate lots of tablets
                                 for (Tablet tablet : index.getTablets()) {
                                     indexRowCount += tablet.getRowCount(version);
                                 } // end for tablets
-                                index.setRowCount(indexRowCount);
+                                indexRowCountMap.put(index.getId(), indexRowCount);
                                 if (!olapTable.isTempPartition(partition.getId())) {
                                     totalRowCount += indexRowCount;
                                 }
@@ -129,10 +133,29 @@ protected void runAfterCatalogReady() {
                     } // end for partitions
                     LOG.debug("finished to set row num for table: {} in database: {}",
                             table.getName(), db.getFullName());
+                } finally {
+                    db.readUnlock();
+                }
+
+                // update
+                db.writeLock();
+                try {
+                    OlapTable olapTable = (OlapTable) table;
+                    for (Partition partition : olapTable.getAllPartitions()) {
+                        for (PhysicalPartition physicalPartition : partition.getSubPartitions()) {
+                            for (MaterializedIndex index :
+                                    physicalPartition.getMaterializedIndices(IndexExtState.VISIBLE)) {
+                                Long indexRowCount = indexRowCountMap.get(index.getId());
+                                if (indexRowCount != null) {
+                                    index.setRowCount(indexRowCount);
+                                }
+                            }
+                        }
+                    }
                     adjustStatUpdateRows(table.getId(), totalRowCount);
+                } finally {
+                    db.writeUnlock();
                 }
-            } finally {
-                db.writeUnlock();
             }
         }
         LOG.info("finished to update index row num of all databases. cost: {} ms",
@@ -144,7 +167,8 @@ private void updateLocalTabletStat() {
         if (!RunMode.isSharedNothingMode()) {
             return;
         }
-        ImmutableMap<Long, Backend> backends = GlobalStateMgr.getCurrentSystemInfo().getIdToBackend();
+        ImmutableMap<Long, Backend> backends =
+                GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getIdToBackend();
 
         long start = System.currentTimeMillis();
         for (Backend backend : backends.values()) {
@@ -357,7 +381,8 @@ private void sendTasks() {
                     LOG.debug("Sent tablet stat collection task to node {} for partition {} of version {}. tablet count={}",
                                 node.getHost(), debugName(), version, entry.getValue().size());
                 } catch (Throwable e) {
-                    LOG.warn("Fail to send tablet stat task to host {} for partition {}: {}", node.getHost(), debugName(),
+                    LOG.warn("Fail to send tablet stat task to host {} for partition {}: {}", node.getHost(),
+                            debugName(),
                             e.getMessage());
                 }
             }