wip move back to typeorm brackets

immich-app · Dec 10, 2024 · a3be620 · a3be620
1 parent 02c5765
commit a3be620
Show file tree

Hide file tree

Showing 9 changed files with 327 additions and 101 deletions.
diff --git a/e2e/src/api/specs/library.e2e-spec.ts b/e2e/src/api/specs/library.e2e-spec.ts
@@ -421,7 +421,7 @@ describe('/libraries', () => {
       const { status } = await request(app)
         .post(`/libraries/${library.id}/scan`)
         .set('Authorization', `Bearer ${admin.accessToken}`)
-        .send({ refreshModifiedFiles: true });
+        .send();
       expect(status).toBe(204);
 
       await utils.waitForQueueFinish(admin.accessToken, 'library');
@@ -453,7 +453,7 @@ describe('/libraries', () => {
       const { status } = await request(app)
         .post(`/libraries/${library.id}/scan`)
         .set('Authorization', `Bearer ${admin.accessToken}`)
-        .send({ refreshModifiedFiles: true });
+        .send();
       expect(status).toBe(204);
 
       await utils.waitForQueueFinish(admin.accessToken, 'library');
@@ -577,7 +577,7 @@ describe('/libraries', () => {
       ]);
     });
 
-    it('should not trash an online asset', async () => {
+    it('should not set an asset offline if its file exists, is in an import path, and not covered by an exclusion pattern', async () => {
       const library = await utils.createLibrary(admin.accessToken, {
         ownerId: admin.userId,
         importPaths: [`${testAssetDirInternal}/temp`],
@@ -601,6 +601,195 @@ describe('/libraries', () => {
 
       expect(assets).toEqual(assetsBefore);
     });
+
+    it('should set an offline asset to online if its file exists, is in an import path, and not covered by an exclusion pattern', async () => {
+      utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);
+
+      const library = await utils.createLibrary(admin.accessToken, {
+        ownerId: admin.userId,
+        importPaths: [`${testAssetDirInternal}/temp/offline`],
+      });
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
+
+      utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`);
+
+      {
+        const { status } = await request(app)
+          .post(`/libraries/${library.id}/scan`)
+          .set('Authorization', `Bearer ${admin.accessToken}`)
+          .send();
+        expect(status).toBe(204);
+      }
+
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
+      expect(offlineAsset.isTrashed).toBe(true);
+      expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
+      expect(offlineAsset.isOffline).toBe(true);
+
+      {
+        const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
+        expect(assets.count).toBe(1);
+      }
+
+      utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`);
+
+      {
+        const { status } = await request(app)
+          .post(`/libraries/${library.id}/scan`)
+          .set('Authorization', `Bearer ${admin.accessToken}`)
+          .send();
+        expect(status).toBe(204);
+      }
+
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      const backOnlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
+
+      expect(backOnlineAsset.isTrashed).toBe(false);
+      expect(backOnlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
+      expect(backOnlineAsset.isOffline).toBe(false);
+
+      {
+        const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
+        expect(assets.count).toBe(1);
+      }
+    });
+
+    it('should not set an offline asset to online if its file exists, is not covered by an exclusion pattern, but is outside of all import paths', async () => {
+      utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);
+
+      const library = await utils.createLibrary(admin.accessToken, {
+        ownerId: admin.userId,
+        importPaths: [`${testAssetDirInternal}/temp/offline`],
+      });
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
+
+      utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`);
+
+      {
+        const { status } = await request(app)
+          .post(`/libraries/${library.id}/scan`)
+          .set('Authorization', `Bearer ${admin.accessToken}`)
+          .send();
+        expect(status).toBe(204);
+      }
+
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      {
+        const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
+        expect(assets.count).toBe(1);
+      }
+
+      const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
+
+      expect(offlineAsset.isTrashed).toBe(true);
+      expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
+      expect(offlineAsset.isOffline).toBe(true);
+
+      utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`);
+
+      utils.createDirectory(`${testAssetDir}/temp/another-path/`);
+
+      await utils.updateLibrary(admin.accessToken, library.id, {
+        importPaths: [`${testAssetDirInternal}/temp/another-path`],
+      });
+
+      {
+        const { status } = await request(app)
+          .post(`/libraries/${library.id}/scan`)
+          .set('Authorization', `Bearer ${admin.accessToken}`)
+          .send();
+        expect(status).toBe(204);
+      }
+
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      const stillOfflineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
+
+      expect(stillOfflineAsset.isTrashed).toBe(true);
+      expect(stillOfflineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
+      expect(stillOfflineAsset.isOffline).toBe(true);
+
+      {
+        const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
+        expect(assets.count).toBe(1);
+      }
+
+      utils.removeDirectory(`${testAssetDir}/temp/another-path/`);
+    });
+
+    it('should not set an offline asset to online if its file exists, is in an import path, but is covered by an exclusion pattern', async () => {
+      utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);
+
+      const library = await utils.createLibrary(admin.accessToken, {
+        ownerId: admin.userId,
+        importPaths: [`${testAssetDirInternal}/temp/offline`],
+      });
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
+
+      utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`);
+
+      {
+        const { status } = await request(app)
+          .post(`/libraries/${library.id}/scan`)
+          .set('Authorization', `Bearer ${admin.accessToken}`)
+          .send();
+        expect(status).toBe(204);
+      }
+
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      {
+        const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
+        expect(assets.count).toBe(1);
+      }
+
+      const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
+
+      expect(offlineAsset.isTrashed).toBe(true);
+      expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
+      expect(offlineAsset.isOffline).toBe(true);
+
+      utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`);
+
+      await utils.updateLibrary(admin.accessToken, library.id, { exclusionPatterns: ['**/offline/**'] });
+
+      {
+        const { status } = await request(app)
+          .post(`/libraries/${library.id}/scan`)
+          .set('Authorization', `Bearer ${admin.accessToken}`)
+          .send();
+        expect(status).toBe(204);
+      }
+
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      const stillOfflineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
+
+      expect(stillOfflineAsset.isTrashed).toBe(true);
+      expect(stillOfflineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
+      expect(stillOfflineAsset.isOffline).toBe(true);
+
+      {
+        const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
+        expect(assets.count).toBe(1);
+      }
+    });
   });
 
   describe('POST /libraries/:id/validate', () => {

diff --git a/e2e/src/utils.ts b/e2e/src/utils.ts
@@ -10,6 +10,7 @@ import {
   Permission,
   PersonCreateDto,
   SharedLinkCreateDto,
+  UpdateLibraryDto,
   UserAdminCreateDto,
   UserPreferencesUpdateDto,
   ValidateLibraryDto,
@@ -35,14 +36,15 @@ import {
   updateAlbumUser,
   updateAssets,
   updateConfig,
+  updateLibrary,
   updateMyPreferences,
   upsertTags,
   validate,
 } from '@immich/sdk';
 import { BrowserContext } from '@playwright/test';
 import { exec, spawn } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path, { dirname } from 'node:path';
 import { setTimeout as setAsyncTimeout } from 'node:timers/promises';
@@ -392,6 +394,14 @@ export const utils = {
     rmSync(path);
   },
 
+  renameImageFile: (oldPath: string, newPath: string) => {
+    if (!existsSync(oldPath)) {
+      return;
+    }
+
+    renameSync(oldPath, newPath);
+  },
+
   removeDirectory: (path: string) => {
     if (!existsSync(path)) {
       return;
@@ -444,6 +454,9 @@ export const utils = {
   createLibrary: (accessToken: string, dto: CreateLibraryDto) =>
     createLibrary({ createLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),
 
+  updateLibrary: (accessToken: string, id: string, dto: UpdateLibraryDto) =>
+    updateLibrary({ id, updateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),
+
   validateLibrary: (accessToken: string, id: string, dto: ValidateLibraryDto) =>
     validate({ id, validateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),
 

diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts
@@ -1,7 +1,6 @@
 import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
 import { ExifEntity } from 'src/entities/exif.entity';
-import { LibraryEntity } from 'src/entities/library.entity';
 import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';
 import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface';
 import { Paginated, PaginationOptions } from 'src/utils/pagination';
@@ -193,5 +192,5 @@ export interface IAssetRepository {
   getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
   upsertFile(file: UpsertFileOptions): Promise<void>;
   upsertFiles(files: UpsertFileOptions[]): Promise<void>;
-  updateOffline(pagination: PaginationOptions, library: LibraryEntity): Paginated<AssetEntity>;
+  updateOffline(importPaths: string[], exclusionPatterns: string[]): Promise<UpdateResult>;
 }
diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts
@@ -1,12 +1,10 @@
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
-import picomatch from 'picomatch';
 import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
 import { AssetFileEntity } from 'src/entities/asset-files.entity';
 import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
 import { ExifEntity } from 'src/entities/exif.entity';
-import { LibraryEntity } from 'src/entities/library.entity';
 import { AssetFileType, AssetOrder, AssetStatus, AssetType, PaginationMode } from 'src/enum';
 import {
   AssetBuilderOptions,
@@ -716,39 +714,26 @@ export class AssetRepository implements IAssetRepository {
     await this.fileRepository.upsert(files, { conflictPaths: ['assetId', 'type'] });
   }
 
-  updateOffline(pagination: PaginationOptions, library: LibraryEntity): Paginated<AssetEntity> {
-    return this.dataSource.manager.transaction(async (transactionalEntityManager) =>
-      transactionalEntityManager.query(
-        `
-        WITH updated_rows AS (
-            UPDATE assets
-            SET "isOffline" = $1, "deletedAt" = $2
-            WHERE "isOffline" = $3
-            AND (
-                "originalPath" NOT SIMILAR TO $4
-                OR "originalPath" SIMILAR TO $5
-            )
-            RETURNING id
-        )
-        SELECT * 
-        FROM assets
-        WHERE id NOT IN (SELECT id FROM updated_rows)
-        AND "libraryId" = $6
-        AND ($7 OR "deletedAt" IS NULL)
-        LIMIT $8 OFFSET $9;
-        `,
-        [
-          true, // $1 - is_offline = true
-          new Date(), // $2 - deleted_at = current timestamp
-          false, // $3 - is_offline = false
-          library.importPaths.map((importPath) => `${importPath}%`).join('|'), // $4 - importPartMatcher pattern
-          library.exclusionPatterns.map(globToSqlPattern).join('|'), // $5 - exclusionPatternMatcher pattern
-          library.id, // $6 - libraryId matches job.id
-          true, // $7 - withDeleted flag
-          pagination.take, // $8 - LIMIT
-          pagination.skip, // $9 - OFFSET
-        ],
-      ),
-    );
+  updateOffline(importPaths: string[], exclusionPatterns: string[]): Promise<UpdateResult> {
+    const paths = importPaths.map((importPath) => `${importPath}%`).join('|');
+    const exclusions = exclusionPatterns.map((pattern) => globToSqlPattern(pattern)).join('|');
+    return this.repository
+      .createQueryBuilder()
+      .update()
+      .set({
+        isOffline: true,
+        deletedAt: new Date(),
+      })
+      .where({ isOffline: false })
+      .andWhere(
+        new Brackets((qb) => {
+          qb.where('originalPath NOT SIMILAR TO :paths', {
+            paths,
+          }).orWhere('originalPath SIMILAR TO :exclusions', {
+            exclusions,
+          });
+        }),
+      )
+      .execute();
   }
 }