perf: 优化多个大文件同时索引的问题

jamebal · Jun 10, 2024 · d26a3f3 · d26a3f3
1 parent 8ea6f9c
commit d26a3f3
Showing 1 changed file with 27 additions and 6 deletions.
diff --git a/src/main/java/com/jmal/clouddisk/lucene/LuceneService.java b/src/main/java/com/jmal/clouddisk/lucene/LuceneService.java
@@ -78,6 +78,10 @@ public class LuceneService {
      * 更新索引内容线程池
      */
     private ExecutorService executorUpdateContentIndexService;
+    /**
+     * 更新大文件索引内容线程池
+     */
+    private ExecutorService executorUpdateBigContentIndexService;
 
     /**
      * 定时任务线程池
@@ -129,6 +133,9 @@ public void init() {
             log.info("updateContentIndexTask 线程数: {}, maxProcessors: {}", processors, maxProcessors);
             executorUpdateContentIndexService = ThreadUtil.newFixedExecutor(processors, 1, "updateContentIndexTask", true);
         }
+        if (executorUpdateBigContentIndexService == null) {
+            executorUpdateBigContentIndexService = ThreadUtil.newFixedExecutor(2, 100, "updateBigContentIndexTask", true);
+        }
     }
 
     /**
@@ -665,17 +672,28 @@ public void processFilesToBeIndexed() throws IOException {
                 if (fileIntroVO != null) {
                     // 处理待索引文件
                     updateIndexStatus(fileIntroVO, IndexStatus.INDEXING);
-                    long size = fileIntroVO.getSize();
-                    if (RebuildIndexTaskService.isSyncFile() || size > 20 * 1024 * 1024) {
-                        updateIndex(true, fileIntroVO);
-                    } else {
-                        executorUpdateContentIndexService.execute(() -> updateIndex(true, fileIntroVO));
-                    }
+                    processFileThreaded(fileIntroVO);
                 }
             }
         }
     }
 
+    private void processFileThreaded(FileIntroVO fileIntroVO) {
+        long size = fileIntroVO.getSize();
+        if (RebuildIndexTaskService.isSyncFile()) {
+            // 单线程处理
+            updateIndex(true, fileIntroVO);
+        } else {
+            if (RebuildIndexTaskService.isSyncFile() || size > 20 * 1024 * 1024) {
+                // 大文件线程
+                executorUpdateBigContentIndexService.execute(() -> updateIndex(true, fileIntroVO));
+            } else {
+                // 小文件线程
+                executorUpdateContentIndexService.execute(() -> updateIndex(true, fileIntroVO));
+            }
+        }
+    }
+
     private void startProcessFilesToBeIndexed() {
         executorCreateIndexService.execute(() -> {
             if (!toBeIndexedLock.tryLock()) {
@@ -701,6 +719,9 @@ public void destroy() {
         if (executorUpdateContentIndexService != null) {
             executorUpdateContentIndexService.shutdown();
         }
+        if (executorUpdateBigContentIndexService != null) {
+            executorUpdateBigContentIndexService.shutdown();
+        }
         if (scheduler != null) {
             scheduler.shutdown();
         }