diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index 35bb41aac068df..70ef5f7b81f2e2 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -178,9 +178,9 @@ export interface IAssetRepository { getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated; getRandom(userIds: string[], count: number): Promise; getLastUpdatedAssetForAlbumId(albumId: string): Promise; - getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise; deleteAll(ownerId: string): Promise; getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated; + getAllInLibrary(pagination: PaginationOptions, libraryId: string): Paginated; getAllByDeviceId(userId: string, deviceId: string): Promise; getLivePhotoCount(motionId: string): Promise; updateAll(ids: string[], options: Partial): Promise; diff --git a/server/src/interfaces/library.interface.ts b/server/src/interfaces/library.interface.ts index 803cf1bc4ed310..341d7dd6472d55 100644 --- a/server/src/interfaces/library.interface.ts +++ b/server/src/interfaces/library.interface.ts @@ -19,4 +19,5 @@ export interface ILibraryRepository { softDelete(id: string): Promise; update(library: Partial): Promise; getStatistics(id: string): Promise; + getAssetCount(id: string, withDeleted: boolean): Promise; } diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index da619f0d356b23..683953edc2e2a7 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -231,6 +231,20 @@ export class AssetRepository implements IAssetRepository { }); } + getAllInLibrary(pagination: PaginationOptions, libraryId: string): Paginated { + const builder = this.repository + .createQueryBuilder('asset') + .select('asset.id') + .where('asset.libraryId = :libraryId', { libraryId }) + .withDeleted(); + + return paginatedBuilder(builder, { + mode: PaginationMode.SKIP_TAKE, + skip: pagination.skip, + take: pagination.take, + }); + } + /** * Get assets by device's Id on the database * @param ownerId diff --git a/server/src/repositories/library.repository.ts b/server/src/repositories/library.repository.ts index 1446395854aabf..b461a7d5fcd0eb 100644 --- a/server/src/repositories/library.repository.ts +++ b/server/src/repositories/library.repository.ts @@ -67,6 +67,22 @@ export class LibraryRepository implements ILibraryRepository { return this.save(library); } + @GenerateSql({ params: [DummyValue.UUID] }) + getAssetCount(id: string, withDeleted = false): Promise { + const deletedAt = withDeleted + ? 'assets.deletedAt IS NULL OR assets.deletedAt IS NOT NULL' + : 'assets.deletedAt IS NULL'; + + return this.repository + .createQueryBuilder('libraries') + .select('COUNT(assets.id)', 'count') + .leftJoin('libraries.assets', 'assets') + .where('libraries.id = :id', { id }) + .andWhere(deletedAt) + .getRawOne() + .then((result) => Number(result.count)); + } + @GenerateSql({ params: [DummyValue.UUID] }) async getStatistics(id: string): Promise { const stats = await this.repository @@ -80,6 +96,19 @@ export class LibraryRepository implements ILibraryRepository { .where('libraries.id = :id', { id }) .getRawOne(); + const sql = this.repository + .createQueryBuilder('libraries') + .addSelect(`COUNT(assets.id) FILTER (WHERE assets.type = 'IMAGE' AND assets.isVisible)`, 'photos') + .addSelect(`COUNT(assets.id) FILTER (WHERE assets.type = 'VIDEO' AND assets.isVisible)`, 'videos') + .addSelect('COALESCE(SUM(exif.fileSizeInByte), 0)', 'usage') + .leftJoin('libraries.assets', 'assets') + .leftJoin('assets.exifInfo', 'exif') + .groupBy('libraries.id') + .where('libraries.id = :id', { id }) + .getSql(); + + console.warn(sql); + if (!stats) { return; } diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index dc391d241d58e4..6685b025e84e60 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -388,15 +388,14 @@ export class LibraryService extends BaseService { this.logger.log(`Starting to scan library ${id}`); - await this.jobRepository.queueAll([ - { - name: JobName.LIBRARY_QUEUE_SYNC_FILES, - data: { - id, - }, + await this.jobRepository.queue({ + name: JobName.LIBRARY_QUEUE_SYNC_FILES, + data: { + id, }, - { name: JobName.LIBRARY_QUEUE_SYNC_ASSETS, data: { id } }, - ]); + }); + + await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SYNC_ASSETS, data: { id } }); } @OnJob({ name: JobName.LIBRARY_QUEUE_SYNC_ALL, queue: QueueName.LIBRARY }) @@ -434,6 +433,8 @@ export class LibraryService extends BaseService { const assetIdsToOffline: string[] = []; const assetIdsToUpdate: string[] = []; + this.logger.debug(`Checking batch of ${assets.length} existing asset(s) in library ${job.libraryId}`); + for (const asset of assets) { const action = await this.handleSyncAsset(asset); switch (action) { @@ -447,30 +448,28 @@ export class LibraryService extends BaseService { } if (assetIdsToOffline.length) { - await this.assetRepository.updateAll(assetIdsToOffline, { isOffline: true, deletedAt: new Date() }); - this.logger.log( - `Originals are missing for ${assetIdsToOffline.length} asset(s) in library ${job.libraryId}, marked offline`, - ); + await this.assetRepository.updateAll(assetIdsToOffline, { + isOffline: true, + status: AssetStatus.TRASHED, + deletedAt: new Date(), + }); } if (assetIdsToUpdate.length) { //TODO: When we have asset status, we need to leave deletedAt as is when status is trashed await this.assetRepository.updateAll(assetIdsToUpdate, { isOffline: false, + status: AssetStatus.ACTIVE, deletedAt: null, }); - - this.logger.log( - `Found ${assetIdsToOffline.length} asset(s) with modified files for library ${job.libraryId}, queuing refresh...`, - ); - await this.queuePostSyncJobs(assetIdsToUpdate); } const remainingCount = assets.length - assetIdsToOffline.length - assetIdsToUpdate.length; - if (remainingCount > 0) { - this.logger.log(`${remainingCount} asset(s) are unchanged in library ${job.libraryId}, no action required`); - } + + this.logger.log( + `Checked existing asset(s): ${assetIdsToOffline.length} offlined, ${assetIdsToUpdate.length} updated, ${remainingCount} unchanged of batch of ${assets.length} in library ${job.libraryId}.`, + ); return JobStatus.SUCCESS; } @@ -621,11 +620,17 @@ export class LibraryService extends BaseService { return JobStatus.SKIPPED; } - const assetCount = (await this.getStatistics(library.id)).total; + const assetCount = await this.libraryRepository.getAssetCount(library.id, true); + + if (!assetCount) { + this.logger.log(`Library ${library.id} is empty, no need to check assets`); + return JobStatus.SUCCESS; + } this.logger.log( - `Scanning library ${library.id} for assets outside of import paths and/or matching an exclusion pattern...`, + `${assetCount} asset(s) in library ${library.id} will be checked against import paths and exclusion patterns...`, ); + const offlineResult = await this.assetRepository.updateOffline(library); const affectedAssetCount = offlineResult.affected; @@ -638,6 +643,8 @@ export class LibraryService extends BaseService { this.logger.log( `All ${assetCount} asset(s) in ${library.id} are outside of import paths and/or match an exclusion pattern, marked as offline`, ); + + return JobStatus.SUCCESS; } else if (affectedAssetCount !== assetCount && affectedAssetCount > 0) { this.logger.log( `${offlineResult.affected} asset(s) out of ${assetCount} were marked offline due to import paths and/or exclusion patterns for library ${library.id}`, @@ -651,7 +658,7 @@ export class LibraryService extends BaseService { this.logger.log(`Scanning library ${library.id} for assets missing from disk...`); const existingAssets = usePagination(JOBS_LIBRARY_PAGINATION_SIZE, (pagination) => - this.assetRepository.getAll(pagination, { libraryId: job.id, withDeleted: true }), + this.assetRepository.getAllInLibrary(pagination, job.id), ); let currentAssetCount = 0; @@ -667,12 +674,12 @@ export class LibraryService extends BaseService { }); this.logger.log( - `Queued check of ${assets.length} existing asset(s) in library ${library.id}, ${currentAssetCount} of ${assetCount} queued in total`, + `Queued check of ${currentAssetCount} of ${assetCount} existing asset(s) so far in library ${library.id}`, ); } if (currentAssetCount) { - this.logger.log(`Finished queuing ${currentAssetCount} file checks for library ${library.id}`); + this.logger.log(`Finished queuing ${currentAssetCount} asset check(s) for library ${library.id}`); } return JobStatus.SUCCESS;