diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java index 2f7f4f7f7fc11..2bf9a61df211c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java @@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.starrocks.catalog.MaterializedIndex.IndexExtState; import com.starrocks.common.Config; import com.starrocks.common.util.FrontendDaemon; @@ -107,6 +108,7 @@ protected void runAfterCatalogReady() { continue; } Locker locker = new Locker(); +<<<<<<< HEAD locker.lockDatabase(db, LockType.WRITE); try { for (Table table : db.getTables()) { @@ -114,7 +116,18 @@ protected void runAfterCatalogReady() { if (!table.isNativeTableOrMaterializedView()) { continue; } +======= + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { + long totalRowCount = 0L; + if (!table.isNativeTableOrMaterializedView()) { + continue; + } +>>>>>>> 0b796e0ba0 ([Enhancement] reduce lock granular of TabletStatMgr (#50668)) + // NOTE: calculate the row first with read lock, then update the stats with write lock + locker.lockTableWithIntensiveDbLock(db, table.getId(), LockType.READ); + Map indexRowCountMap = Maps.newHashMap(); + try { OlapTable olapTable = (OlapTable) table; for (Partition partition : olapTable.getAllPartitions()) { for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { @@ -122,10 +135,11 @@ protected void runAfterCatalogReady() { for (MaterializedIndex index : physicalPartition.getMaterializedIndices( IndexExtState.VISIBLE)) { long indexRowCount = 0L; + // NOTE: can take a rather long time to iterate lots of tablets for (Tablet tablet : index.getTablets()) { indexRowCount += tablet.getRowCount(version); } // end for tablets - index.setRowCount(indexRowCount); + indexRowCountMap.put(index.getId(), indexRowCount); if (!olapTable.isTempPartition(partition.getId())) { totalRowCount += indexRowCount; } @@ -134,10 +148,29 @@ protected void runAfterCatalogReady() { } // end for partitions LOG.debug("finished to set row num for table: {} in database: {}", table.getName(), db.getFullName()); + } finally { + locker.unLockTableWithIntensiveDbLock(db, table, LockType.READ); + } + + // update + locker.lockTableWithIntensiveDbLock(db, table.getId(), LockType.WRITE); + try { + OlapTable olapTable = (OlapTable) table; + for (Partition partition : olapTable.getAllPartitions()) { + for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { + for (MaterializedIndex index : + physicalPartition.getMaterializedIndices(IndexExtState.VISIBLE)) { + Long indexRowCount = indexRowCountMap.get(index.getId()); + if (indexRowCount != null) { + index.setRowCount(indexRowCount); + } + } + } + } adjustStatUpdateRows(table.getId(), totalRowCount); + } finally { + locker.unLockTableWithIntensiveDbLock(db, table, LockType.WRITE); } - } finally { - locker.unLockDatabase(db, LockType.WRITE); } } LOG.info("finished to update index row num of all databases. cost: {} ms", @@ -149,7 +182,8 @@ private void updateLocalTabletStat() { if (!RunMode.isSharedNothingMode()) { return; } - ImmutableMap backends = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getIdToBackend(); + ImmutableMap backends = + GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getIdToBackend(); long start = System.currentTimeMillis(); for (Backend backend : backends.values()) { @@ -354,10 +388,13 @@ private void sendTasks() { LakeService lakeService = BrpcProxy.getLakeService(node.getHost(), node.getBrpcPort()); Future responseFuture = lakeService.getTabletStats(request); responseList.add(responseFuture); - LOG.debug("Sent tablet stat collection task to node {} for partition {} of version {}. tablet count={}", + LOG.debug( + "Sent tablet stat collection task to node {} for partition {} of version {}. tablet " + + "count={}", node.getHost(), debugName(), version, entry.getValue().size()); } catch (Throwable e) { - LOG.warn("Fail to send tablet stat task to host {} for partition {}: {}", node.getHost(), debugName(), + LOG.warn("Fail to send tablet stat task to host {} for partition {}: {}", node.getHost(), + debugName(), e.getMessage()); } }