Skip to content

Commit

Permalink
Merge branch 'feat/inline-offline-check' of https://github.com/immich…
Browse files Browse the repository at this point in the history 
…-app/immich into feat/inline-offline-check
  • Loading branch information
@etnoy
etnoy committed Dec 18, 2024
2 parents 3deeaad + 26ffde6 commit d800c70
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 39 deletions.
3 changes: 2 additions & 1 deletion server/src/interfaces/asset.interface.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,9 +178,9 @@ export interface IAssetRepository {
getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>;
getRandom(userIds: string[], count: number): Promise<AssetEntity[]>;
getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
deleteAll(ownerId: string): Promise<void>;
getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
getAllInLibrary(pagination: PaginationOptions, libraryId: string): Paginated<AssetEntity>;
getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
getLivePhotoCount(motionId: string): Promise<number>;
updateAll(ids: string[], options: Partial<AssetUpdateAllOptions>): Promise<void>;
Expand All @@ -201,4 +201,5 @@ export interface IAssetRepository {
upsertFiles(files: UpsertFileOptions[]): Promise<void>;
updateOffline(library: LibraryEntity): Promise<UpdateResult>;
getNewPaths(libraryId: string, paths: string[]): Promise<string[]>;
getAssetCount(id: string, options: AssetSearchOptions): Promise<number | undefined>;
}
28 changes: 20 additions & 8 deletions server/src/repositories/asset.repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,14 +197,6 @@ export class AssetRepository implements IAssetRepository {
return this.getAll(pagination, { ...options, userIds: [userId] });
}

@GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] })
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null> {
return this.repository.findOne({
where: { library: { id: libraryId }, originalPath },
withDeleted: true,
});
}

@GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING]] })
@ChunkedArray({ paramIndex: 1 })
async getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise<string[]> {
Expand All @@ -231,6 +223,20 @@ export class AssetRepository implements IAssetRepository {
});
}

getAllInLibrary(pagination: PaginationOptions, libraryId: string): Paginated<AssetEntity> {
const builder = this.repository
.createQueryBuilder('asset')
.select('asset.id')
.where('asset.libraryId = :libraryId', { libraryId })
.withDeleted();

return paginatedBuilder<AssetEntity>(builder, {
mode: PaginationMode.SKIP_TAKE,
skip: pagination.skip,
take: pagination.take,
});
}

/**
* Get assets by device's Id on the database
* @param ownerId
Expand Down Expand Up @@ -779,4 +785,10 @@ export class AssetRepository implements IAssetRepository {
.query(rawSql, [paths, libraryId])
.then((result) => result.map((row: { path: string }) => row.path));
}

async getAssetCount(id: string, options: AssetSearchOptions = {}): Promise<number | undefined> {
let builder = this.repository.createQueryBuilder('asset').leftJoinAndSelect('asset.files', 'files');
builder = searchAssetBuilder(builder, options);
return builder.getCount();
}
}
12 changes: 8 additions & 4 deletions server/src/repositories/trash.repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ export class TrashRepository implements ITrashRepository {
.createQueryBuilder('asset')
.select('asset.id')
.where({ status: AssetStatus.DELETED })
.orWhere({ isOffline: true })
.withDeleted(),
pagination,
);
Expand All @@ -34,10 +35,13 @@ export class TrashRepository implements ITrashRepository {
}

async empty(userId: string): Promise<number> {
const result = await this.assetRepository.update(
{ ownerId: userId, status: AssetStatus.TRASHED },
{ status: AssetStatus.DELETED },
);
const result = await this.assetRepository
.createQueryBuilder()
.update(AssetEntity)
.set({ status: AssetStatus.DELETED })
.where({ ownerId: userId, status: AssetStatus.TRASHED })
.orWhere({ ownerId: userId, isOffline: true })
.execute();

return result.affected || 0;
}
Expand Down
59 changes: 33 additions & 26 deletions server/src/services/library.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,15 +388,14 @@ export class LibraryService extends BaseService {

this.logger.log(`Starting to scan library ${id}`);

await this.jobRepository.queueAll([
{
name: JobName.LIBRARY_QUEUE_SYNC_FILES,
data: {
id,
},
await this.jobRepository.queue({
name: JobName.LIBRARY_QUEUE_SYNC_FILES,
data: {
id,
},
{ name: JobName.LIBRARY_QUEUE_SYNC_ASSETS, data: { id } },
]);
});

await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SYNC_ASSETS, data: { id } });
}

@OnJob({ name: JobName.LIBRARY_QUEUE_SYNC_ALL, queue: QueueName.LIBRARY })
Expand Down Expand Up @@ -434,6 +433,8 @@ export class LibraryService extends BaseService {
const assetIdsToOffline: string[] = [];
const assetIdsToUpdate: string[] = [];

this.logger.debug(`Checking batch of ${assets.length} existing asset(s) in library ${job.libraryId}`);

for (const asset of assets) {
const action = await this.handleSyncAsset(asset);
switch (action) {
Expand All @@ -447,30 +448,28 @@ export class LibraryService extends BaseService {
}

if (assetIdsToOffline.length) {

Check failure on line 450 in server/src/services/library.service.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server 

Use `.length > 0` when checking length is not zero
await this.assetRepository.updateAll(assetIdsToOffline, { isOffline: true, deletedAt: new Date() });
this.logger.log(
`Originals are missing for ${assetIdsToOffline.length} asset(s) in library ${job.libraryId}, marked offline`,
);
await this.assetRepository.updateAll(assetIdsToOffline, {
isOffline: true,
status: AssetStatus.TRASHED,
deletedAt: new Date(),
});
}

if (assetIdsToUpdate.length) {

Check failure on line 458 in server/src/services/library.service.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server 

Use `.length > 0` when checking length is not zero
//TODO: When we have asset status, we need to leave deletedAt as is when status is trashed
await this.assetRepository.updateAll(assetIdsToUpdate, {
isOffline: false,
status: AssetStatus.ACTIVE,
deletedAt: null,
});

this.logger.log(
`Found ${assetIdsToOffline.length} asset(s) with modified files for library ${job.libraryId}, queuing refresh...`,
);

await this.queuePostSyncJobs(assetIdsToUpdate);
}

const remainingCount = assets.length - assetIdsToOffline.length - assetIdsToUpdate.length;
if (remainingCount > 0) {
this.logger.log(`${remainingCount} asset(s) are unchanged in library ${job.libraryId}, no action required`);
}

this.logger.log(
`Checked existing asset(s): ${assetIdsToOffline.length} offlined, ${assetIdsToUpdate.length} updated, ${remainingCount} unchanged of batch of ${assets.length} in library ${job.libraryId}.`,
);

return JobStatus.SUCCESS;
}
Expand Down Expand Up @@ -606,7 +605,7 @@ export class LibraryService extends BaseService {
`Finished crawling ${crawlCount} file(s) of which ${importCount} file(s) are queued for import for library ${library.id}`,
);
} else {
this.logger.log(`All ${crawlCount} file(s) on disk are already in ${library.id}`);
this.logger.log(`All ${crawlCount} file(s) on disk are already in library ${library.id}`);
}

await this.libraryRepository.update({ id: job.id, refreshedAt: new Date() });
Expand All @@ -621,11 +620,17 @@ export class LibraryService extends BaseService {
return JobStatus.SKIPPED;
}

const assetCount = (await this.getStatistics(library.id)).total;
const assetCount = await this.assetRepository.getAssetCount(library.id, { withDeleted: true });

Check failure on line 623 in server/src/services/library.service.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/library.service.spec.ts > LibraryService > handleQueueRemoveDeleted > should queue online check of existing assets 

TypeError: this.assetRepository.getAssetCount is not a function
 ❯ LibraryService.handleQueueSyncAssets src/services/library.service.ts:623:51
 ❯ src/services/library.service.spec.ts:230:7

if (!assetCount) {
this.logger.log(`Library ${library.id} is empty, no need to check assets`);
return JobStatus.SUCCESS;
}

this.logger.log(
`Scanning library ${library.id} for assets outside of import paths and/or matching an exclusion pattern...`,
`${assetCount} asset(s) in library ${library.id} will be checked against import paths and exclusion patterns...`,
);

const offlineResult = await this.assetRepository.updateOffline(library);

const affectedAssetCount = offlineResult.affected;
Expand All @@ -638,6 +643,8 @@ export class LibraryService extends BaseService {
this.logger.log(
`All ${assetCount} asset(s) in ${library.id} are outside of import paths and/or match an exclusion pattern, marked as offline`,
);

return JobStatus.SUCCESS;
} else if (affectedAssetCount !== assetCount && affectedAssetCount > 0) {
this.logger.log(
`${offlineResult.affected} asset(s) out of ${assetCount} were marked offline due to import paths and/or exclusion patterns for library ${library.id}`,
Expand All @@ -651,7 +658,7 @@ export class LibraryService extends BaseService {
this.logger.log(`Scanning library ${library.id} for assets missing from disk...`);

const existingAssets = usePagination(JOBS_LIBRARY_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getAll(pagination, { libraryId: job.id, withDeleted: true }),
this.assetRepository.getAllInLibrary(pagination, job.id),
);

let currentAssetCount = 0;
Expand All @@ -667,12 +674,12 @@ export class LibraryService extends BaseService {
});

this.logger.log(
`Queued check of ${assets.length} existing asset(s) in library ${library.id}, ${currentAssetCount} of ${assetCount} queued in total`,
`Queued check of ${currentAssetCount} of ${assetCount} existing asset(s) so far in library ${library.id}`,
);
}

if (currentAssetCount) {
this.logger.log(`Finished queuing ${currentAssetCount} file checks for library ${library.id}`);
this.logger.log(`Finished queuing ${currentAssetCount} asset check(s) for library ${library.id}`);
}

return JobStatus.SUCCESS;
Expand Down

0 comments on commit d800c70

Please sign in to comment.