feat: run all offline checks in a single job

immich-app · Dec 2, 2024 · 80aa615 · 80aa615
1 parent ba71fd4
commit 80aa615
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 33 deletions.
diff --git a/server/src/interfaces/job.interface.ts b/server/src/interfaces/job.interface.ts
@@ -85,7 +85,7 @@ export enum JobName {
   LIBRARY_QUEUE_SYNC_FILES = 'library-queue-sync-files',
   LIBRARY_QUEUE_SYNC_ASSETS = 'library-queue-sync-assets',
   LIBRARY_SYNC_FILE = 'library-sync-file',
-  LIBRARY_SYNC_ASSET = 'library-sync-asset',
+  LIBRARY_SYNC_ASSETS = 'library-sync-assets',
   LIBRARY_DELETE = 'library-delete',
   LIBRARY_QUEUE_SYNC_ALL = 'library-queue-sync-all',
   LIBRARY_QUEUE_CLEANUP = 'library-queue-cleanup',
@@ -148,15 +148,15 @@ export interface ILibraryFileJob extends IEntityJob {
   assetPath: string;
 }
 
-export interface ILibraryAssetJob extends IEntityJob {
-  importPaths: string[];
-  exclusionPatterns: string[];
-}
-
 export interface IBulkEntityJob extends IBaseJob {
   ids: string[];
 }
 
+export interface ILibraryAssetsJob extends IBulkEntityJob {
+  importPaths: string[];
+  exclusionPatterns: string[];
+}
+
 export interface IDeleteFilesJob extends IBaseJob {
   files: Array<string | null | undefined>;
 }
@@ -287,7 +287,7 @@ export type JobItem =
   | { name: JobName.LIBRARY_SYNC_FILE; data: ILibraryFileJob }
   | { name: JobName.LIBRARY_QUEUE_SYNC_FILES; data: IEntityJob }
   | { name: JobName.LIBRARY_QUEUE_SYNC_ASSETS; data: IEntityJob }
-  | { name: JobName.LIBRARY_SYNC_ASSET; data: ILibraryAssetJob }
+  | { name: JobName.LIBRARY_SYNC_ASSETS; data: ILibraryAssetsJob }
   | { name: JobName.LIBRARY_DELETE; data: IEntityJob }
   | { name: JobName.LIBRARY_QUEUE_SYNC_ALL; data?: IBaseJob }
   | { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob }

diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts
@@ -10,7 +10,7 @@ import { ICronRepository } from 'src/interfaces/cron.interface';
 import { IDatabaseRepository } from 'src/interfaces/database.interface';
 import {
   IJobRepository,
-  ILibraryAssetJob,
+  ILibraryAssetsJob,
   ILibraryFileJob,
   JobName,
   JOBS_LIBRARY_PAGINATION_SIZE,
@@ -250,7 +250,7 @@ describe(LibraryService.name, () => {
 
   describe('handleSyncAsset', () => {
     it('should skip missing assets', async () => {
-      const mockAssetJob: ILibraryAssetJob = {
+      const mockAssetJob: ILibraryAssetsJob = {
         id: assetStub.external.id,
         importPaths: ['/'],
         exclusionPatterns: [],
@@ -264,7 +264,7 @@ describe(LibraryService.name, () => {
     });
 
     it('should offline assets no longer on disk', async () => {
-      const mockAssetJob: ILibraryAssetJob = {
+      const mockAssetJob: ILibraryAssetsJob = {
         id: assetStub.external.id,
         importPaths: ['/'],
         exclusionPatterns: [],
@@ -282,7 +282,7 @@ describe(LibraryService.name, () => {
     });
 
     it('should offline assets matching an exclusion pattern', async () => {
-      const mockAssetJob: ILibraryAssetJob = {
+      const mockAssetJob: ILibraryAssetsJob = {
         id: assetStub.external.id,
         importPaths: ['/'],
         exclusionPatterns: ['**/user1/**'],
@@ -298,7 +298,7 @@ describe(LibraryService.name, () => {
     });
 
     it('should set assets outside of import paths as offline', async () => {
-      const mockAssetJob: ILibraryAssetJob = {
+      const mockAssetJob: ILibraryAssetsJob = {
         id: assetStub.external.id,
         importPaths: ['/data/user2'],
         exclusionPatterns: [],
@@ -316,7 +316,7 @@ describe(LibraryService.name, () => {
     });
 
     it('should do nothing with online assets', async () => {
-      const mockAssetJob: ILibraryAssetJob = {
+      const mockAssetJob: ILibraryAssetsJob = {
         id: assetStub.external.id,
         importPaths: ['/'],
         exclusionPatterns: [],
@@ -331,7 +331,7 @@ describe(LibraryService.name, () => {
     });
 
     it('should un-trash an asset previously marked as offline', async () => {
-      const mockAssetJob: ILibraryAssetJob = {
+      const mockAssetJob: ILibraryAssetsJob = {
         id: assetStub.external.id,
         importPaths: ['/'],
         exclusionPatterns: [],
@@ -353,7 +353,7 @@ describe(LibraryService.name, () => {
   });
 
   it('should update file when mtime has changed', async () => {
-    const mockAssetJob: ILibraryAssetJob = {
+    const mockAssetJob: ILibraryAssetsJob = {
       id: assetStub.external.id,
       importPaths: ['/'],
       exclusionPatterns: [],

diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts
@@ -242,12 +242,10 @@ export class LibraryService extends BaseService {
   }
 
   private async syncAssets({ importPaths, exclusionPatterns }: LibraryEntity, assetIds: string[]) {
-    await this.jobRepository.queueAll(
-      assetIds.map((assetId) => ({
-        name: JobName.LIBRARY_SYNC_ASSET,
-        data: { id: assetId, importPaths, exclusionPatterns },
-      })),
-    );
+    await this.jobRepository.queue({
+      name: JobName.LIBRARY_SYNC_ASSETS,
+      data: { ids: assetIds, importPaths, exclusionPatterns },
+    });
   }
 
   private async validateImportPath(importPath: string): Promise<ValidateLibraryImportPathResponseDto> {
@@ -472,27 +470,35 @@ export class LibraryService extends BaseService {
     return JobStatus.SUCCESS;
   }
 
-  @OnJob({ name: JobName.LIBRARY_SYNC_ASSET, queue: QueueName.LIBRARY })
-  async handleSyncAsset(job: JobOf<JobName.LIBRARY_SYNC_ASSET>): Promise<JobStatus> {
-    const asset = await this.assetRepository.getById(job.id);
+  @OnJob({ name: JobName.LIBRARY_SYNC_ASSETS, queue: QueueName.LIBRARY })
+  async handleSyncAssets(job: JobOf<JobName.LIBRARY_SYNC_ASSETS>): Promise<JobStatus> {
+    for (const id of job.ids) {
+      await this.handleSyncAsset(id, job.importPaths, job.exclusionPatterns);
+    }
+
+    return JobStatus.SUCCESS;
+  }
+
+  async handleSyncAsset(id: string, importPaths: string[], exclusionPatterns: string[]): Promise<JobStatus> {
+    const asset = await this.assetRepository.getById(id);
     if (!asset) {
       return JobStatus.SKIPPED;
     }
 
     const markOffline = async (explanation: string) => {
       if (!asset.isOffline) {
-        this.logger.debug(`${explanation}, removing: ${asset.originalPath}`);
+        this.logger.debug(`${explanation}, moving to trash: ${asset.originalPath}`);
         await this.assetRepository.updateAll([asset.id], { isOffline: true, deletedAt: new Date() });
       }
     };
 
-    const isInPath = job.importPaths.find((path) => asset.originalPath.startsWith(path));
+    const isInPath = importPaths.find((path) => asset.originalPath.startsWith(path));
     if (!isInPath) {
       await markOffline('Asset is no longer in an import path');
       return JobStatus.SUCCESS;
     }
 
-    const isExcluded = job.exclusionPatterns.some((pattern) => picomatch.isMatch(asset.originalPath, pattern));
+    const isExcluded = exclusionPatterns.some((pattern) => picomatch.isMatch(asset.originalPath, pattern));
     if (isExcluded) {
       await markOffline('Asset is covered by an exclusion pattern');
       return JobStatus.SUCCESS;
@@ -597,12 +603,14 @@ export class LibraryService extends BaseService {
     for await (const assets of onlineAssets) {
       assetCount += assets.length;
       this.logger.debug(`Discovered ${assetCount} asset(s) in library ${library.id}...`);
-      await this.jobRepository.queueAll(
-        assets.map((asset) => ({
-          name: JobName.LIBRARY_SYNC_ASSET,
-          data: { id: asset.id, importPaths: library.importPaths, exclusionPatterns: library.exclusionPatterns },
-        })),
-      );
+      await this.jobRepository.queue({
+        name: JobName.LIBRARY_SYNC_ASSETS,
+        data: {
+          ids: assets.map((asset) => asset.id),
+          importPaths: library.importPaths,
+          exclusionPatterns: library.exclusionPatterns,
+        },
+      });
       this.logger.debug(`Queued check of ${assets.length} asset(s) in library ${library.id}...`);
     }