Skip to content

Commit

Permalink
do e2e instead
Browse files Browse the repository at this point in the history
  • Loading branch information
@etnoy
etnoy committed Feb 5, 2025
1 parent e8c4d27 commit dca3f96
Show file tree
Hide file tree
Showing 4 changed files with 192 additions and 43 deletions.
1 change: 0 additions & 1 deletion e2e/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ services:
- IMMICH_ENV=testing
- IMMICH_PORT=2285
- IMMICH_IGNORE_MOUNT_CHECK_ERRORS=true
- IMMICH_LOG_LEVEL=verbose
volumes:
- ./test-assets:/test-assets
extra_hosts:
Expand Down
217 changes: 185 additions & 32 deletions e2e/src/api/specs/jobs.e2e-spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { JobCommand, JobName, LoginResponseDto, updateConfig } from '@immich/sdk';
import { cpSync } from 'node:fs';
import { cpSync, rmSync } from 'node:fs';
import { readFile, writeFile } from 'node:fs/promises';
import { basename } from 'node:path';
import { errorDto } from 'src/responses';
Expand Down Expand Up @@ -32,6 +32,16 @@ describe('/jobs', () => {
force: false,
});

await utils.jobCommand(admin.accessToken, JobName.SmartSearch, {
command: JobCommand.Resume,
force: false,
});

await utils.jobCommand(admin.accessToken, JobName.DuplicateDetection, {
command: JobCommand.Resume,
force: false,
});

const config = await utils.getSystemConfig(admin.accessToken);
config.machineLearning.duplicateDetection.enabled = false;
config.machineLearning.enabled = false;
Expand All @@ -47,22 +57,15 @@ describe('/jobs', () => {
});

it('should queue metadata extraction for missing assets', async () => {
const path1 = `${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`;
const path2 = `${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`;

await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(path1), filename: basename(path1) },
});

await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
const path = `${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`;

await utils.jobCommand(admin.accessToken, JobName.MetadataExtraction, {
command: JobCommand.Pause,
force: false,
});

const { id } = await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(path2), filename: basename(path2) },
assetData: { bytes: await readFile(path), filename: basename(path) },
});

await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
Expand Down Expand Up @@ -101,36 +104,62 @@ describe('/jobs', () => {
}
});

it('should queue thumbnail extraction for missing assets', async () => {
const path1 = `${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`;
const path2 = `${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`;
it('should not re-extract metadata for existing assets', async () => {
const path = `${testAssetDir}/temp/metadata/asset.jpg`;

await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(path1), filename: basename(path1) },
cpSync(`${testAssetDir}/formats/raw/Nikon/D700/philadelphia.nef`, path);

const { id } = await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(path), filename: basename(path) },
});

await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');

{
const asset = await utils.getAssetInfo(admin.accessToken, id);

expect(asset.exifInfo).toBeDefined();
expect(asset.exifInfo?.model).toBe('NIKON D700');
}

cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, path);

await utils.jobCommand(admin.accessToken, JobName.MetadataExtraction, {
command: JobCommand.Start,
force: false,
});

await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');

{
const asset = await utils.getAssetInfo(admin.accessToken, id);

expect(asset.exifInfo).toBeDefined();
expect(asset.exifInfo?.model).toBe('NIKON D700');
}

rmSync(path);
});

it('should queue thumbnail extraction for assets missing thumbs', async () => {
const path = `${testAssetDir}/albums/nature/tanners_ridge.jpg`;

await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
command: JobCommand.Pause,
force: false,
});

const { id } = await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(path2), filename: basename(path2) },
assetData: { bytes: await readFile(path), filename: basename(path) },
});

await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);

{
const asset = await utils.getAssetInfo(admin.accessToken, id);
const assetBefore = await utils.getAssetInfo(admin.accessToken, id);
expect(assetBefore.thumbhash).toBeNull();

expect(asset.thumbhash).toBeNull();
}

await utils.jobCommand(admin.accessToken, JobName.MetadataExtraction, {
await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
command: JobCommand.Empty,
force: false,
});
Expand All @@ -151,11 +180,46 @@ describe('/jobs', () => {
await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);

{
const asset = await utils.getAssetInfo(admin.accessToken, id);
const assetAfter = await utils.getAssetInfo(admin.accessToken, id);
expect(assetAfter.thumbhash).not.toBeNull();
});

expect(asset.thumbhash).not.toBeNull();
}
it('should not reload existing thumbnail when running thumb job for missing assets', async () => {
const path = `${testAssetDir}/temp/thumbs/asset1.jpg`;

cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, path);

const { id } = await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(path), filename: basename(path) },
});

await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);

const assetBefore = await utils.getAssetInfo(admin.accessToken, id);

cpSync(`${testAssetDir}/albums/nature/notocactus_minimus.jpg`, path);

await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
command: JobCommand.Resume,
force: false,
});

// This runs the missing thumbnail job
await utils.jobCommand(admin.accessToken, JobName.ThumbnailGeneration, {
command: JobCommand.Start,
force: false,
});

await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);

const assetAfter = await utils.getAssetInfo(admin.accessToken, id);

// Asset 1 thumbnail should be untouched since its thumb should not have been reloaded, even though the file was changed
expect(assetAfter.thumbhash).toEqual(assetBefore.thumbhash);

rmSync(path);
});

it('should queue duplicate detection for missing duplicates', async () => {
Expand Down Expand Up @@ -235,20 +299,22 @@ describe('/jobs', () => {

expect(asset1.duplicateId).toEqual(asset2.duplicateId);
}

rmSync(`${testAssetDir}/temp/dupes/asset1.jpg`);
rmSync(`${testAssetDir}/temp/dupes/asset2.jpg`);
}, 120_000);

it('should queue smart search for missing assets', async () => {
{
const config = await utils.getSystemConfig(admin.accessToken);
config.machineLearning.duplicateDetection.enabled = false;
config.machineLearning.enabled = false;
config.machineLearning.clip.enabled = false;
await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) });
}

const path = `${testAssetDir}/albums/nature/prairie_falcon.jpg`;

const { id: id1 } = await utils.createAsset(admin.accessToken, {
await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(path), filename: basename(path) },
});

Expand Down Expand Up @@ -281,7 +347,49 @@ describe('/jobs', () => {
const results = await utils.searchSmart(admin.accessToken, { query: 'bird' });
expect(results.assets.count).toBeGreaterThanOrEqual(1);
}
}, 120_000);
}, 60_000);

it('should not re-do smart search for already-indexed assets', async () => {
{
const config = await utils.getSystemConfig(admin.accessToken);
config.machineLearning.enabled = true;
config.machineLearning.clip.enabled = true;
await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) });
}

const path = `${testAssetDir}/temp/smart/asset.jpg`;

cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, path);

await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(path), filename: basename(path) },
});

await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
await utils.waitForQueueFinish(admin.accessToken, JobName.SmartSearch);

{
const results = await utils.searchSmart(admin.accessToken, { query: 'bird' });
expect(results.assets.count).toBeGreaterThanOrEqual(1);
}

cpSync(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, path);

await utils.jobCommand(admin.accessToken, JobName.SmartSearch, {
command: JobCommand.Start,
force: false,
});

await utils.waitForQueueFinish(admin.accessToken, JobName.SmartSearch, 60_000);

{
const results = await utils.searchSmart(admin.accessToken, { query: 'bird' });
expect(results.assets.count).toBeGreaterThanOrEqual(1);
}

rmSync(path);
}, 60_000);

it('should queue face detection for missing faces', async () => {
const path = `${testAssetDir}/metadata/faces/solvay.jpg`;
Expand Down Expand Up @@ -350,6 +458,51 @@ describe('/jobs', () => {
expect(asset.people).toEqual([]);
expect(asset.unassignedFaces?.length).toBeGreaterThan(10);
}
}, 120_000);
}, 60_000);

it('should not rerun face detection for existing faces', async () => {
const config = await utils.getSystemConfig(admin.accessToken);
config.metadata.faces.import = true;
config.machineLearning.enabled = true;
await updateConfig({ systemConfigDto: config }, { headers: asBearerAuth(admin.accessToken) });

const path = `${testAssetDir}/temp/faces/asset.jpg`;

cpSync(`${testAssetDir}/metadata/faces/solvay.jpg`, path);

const { id } = await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(path), filename: basename(path) },
});

await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
await utils.waitForQueueFinish(admin.accessToken, JobName.FaceDetection);

{
const asset = await utils.getAssetInfo(admin.accessToken, id);
expect(asset.people).toEqual([]);
expect(asset.unassignedFaces?.length).toBeGreaterThan(10);
}

cpSync(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, path);

await utils.jobCommand(admin.accessToken, JobName.FaceDetection, {
command: JobCommand.Start,
force: false,
});

await utils.waitForQueueFinish(admin.accessToken, JobName.MetadataExtraction);
await utils.waitForQueueFinish(admin.accessToken, JobName.ThumbnailGeneration);
await utils.waitForQueueFinish(admin.accessToken, JobName.FaceDetection, 60_000);

{
const asset = await utils.getAssetInfo(admin.accessToken, id);

expect(asset.people).toEqual([]);
expect(asset.unassignedFaces?.length).toBeGreaterThan(10);
}

rmSync(path);
}, 60_000);
});
});
2 changes: 0 additions & 2 deletions e2e/src/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import {
PersonCreateDto,
SharedLinkCreateDto,
SmartSearchDto,
SystemConfigDto,
UpdateLibraryDto,
UserAdminCreateDto,
UserPreferencesUpdateDto,
Expand Down Expand Up @@ -61,7 +60,6 @@ import { io, type Socket } from 'socket.io-client';
import { loginDto, signupDto } from 'src/fixtures';
import { makeRandomImage } from 'src/generators';
import request from 'supertest';
import { a } from 'vitest/dist/chunks/suite.B2jumIFP.js';

type CommandResponse = { stdout: string; stderr: string; exitCode: number | null };
type EventType = 'assetUpload' | 'assetUpdate' | 'assetDelete' | 'userDelete' | 'assetHidden' | 'configUpdate';
Expand Down
15 changes: 7 additions & 8 deletions server/src/repositories/asset.repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -454,8 +454,8 @@ export class AssetRepository implements IAssetRepository {
.selectAll('assets')
.$if(property === WithoutProperty.DUPLICATE, (qb) =>
qb
.leftJoin('asset_job_status as job_status', 'assets.id', 'job_status.assetId')
.where((eb) => eb.or([eb('job_status.duplicatesDetectedAt', 'is', null), eb('assetId', 'is', null)]))
.innerJoin('asset_job_status as job_status', 'assets.id', 'job_status.assetId')
.where('job_status.duplicatesDetectedAt', 'is', null)
.where('job_status.previewAt', 'is not', null)
.where((eb) => eb.exists(eb.selectFrom('smart_search').where('assetId', '=', eb.ref('assets.id'))))
.where('assets.isVisible', '=', true),
Expand All @@ -473,9 +473,9 @@ export class AssetRepository implements IAssetRepository {
)
.$if(property === WithoutProperty.FACES, (qb) =>
qb
.leftJoin('asset_job_status as job_status', 'assetId', 'assets.id')
.where((eb) => eb.or([eb('job_status.facesRecognizedAt', 'is', null), eb('assetId', 'is', null)]))
.innerJoin('asset_job_status as job_status', 'assetId', 'assets.id')
.where('job_status.previewAt', 'is not', null)
.where('job_status.facesRecognizedAt', 'is', null)
.where('assets.isVisible', '=', true),
)
.$if(property === WithoutProperty.SIDECAR, (qb) =>
Expand All @@ -485,20 +485,19 @@ export class AssetRepository implements IAssetRepository {
)
.$if(property === WithoutProperty.SMART_SEARCH, (qb) =>
qb
.leftJoin('asset_job_status as job_status', 'assetId', 'assets.id')
.where((eb) => eb.or([eb('job_status.previewAt', 'is not', null), eb('assetId', 'is', null)]))
.innerJoin('asset_job_status as job_status', 'assetId', 'assets.id')
.where('job_status.previewAt', 'is not', null)
.where('assets.isVisible', '=', true)
.where((eb) =>
eb.not((eb) => eb.exists(eb.selectFrom('smart_search').whereRef('assetId', '=', 'assets.id'))),
),
)
.$if(property === WithoutProperty.THUMBNAIL, (qb) =>
qb
.leftJoin('asset_job_status as job_status', 'assetId', 'assets.id')
.innerJoin('asset_job_status as job_status', 'assetId', 'assets.id')
.where('assets.isVisible', '=', true)
.where((eb) =>
eb.or([
eb('assetId', 'is', null),
eb('job_status.previewAt', 'is', null),
eb('job_status.thumbnailAt', 'is', null),
eb('assets.thumbhash', 'is', null),
Expand Down

0 comments on commit dca3f96

Please sign in to comment.