From 5301fee0f79b5d832d22c387402450ce7000105e Mon Sep 17 00:00:00 2001 From: sergey-misuk-valor <168101676+sergey-misuk-valor@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:02:26 +0300 Subject: [PATCH] Add possibility to ignore filename pairs (#90) --- src/hope_dedup_engine/apps/api/const.py | 7 +- .../apps/api/deduplication/process.py | 42 +++++-- ...keypair_ignoredreferencepkpair_and_more.py | 31 +++++ .../migrations/0008_ignoredfilenamepair.py | 40 +++++++ .../apps/api/models/deduplication.py | 37 ++++-- src/hope_dedup_engine/apps/api/serializers.py | 31 +++-- src/hope_dedup_engine/apps/api/urls.py | 13 +- src/hope_dedup_engine/apps/api/views.py | 46 ++++++-- tests/api/api_const.py | 6 +- tests/api/conftest.py | 8 +- tests/api/test_auth.py | 6 +- tests/api/test_find_duplicates.py | 29 ++++- tests/api/test_ignored_filename_create.py | 111 ++++++++++++++++++ tests/api/test_ignored_filename_list.py | 35 ++++++ tests/api/test_ignored_keys_list.py | 31 ----- ...py => test_ignored_reference_pk_create.py} | 69 ++++++----- tests/api/test_ignored_reference_pk_list.py | 35 ++++++ tests/extras/testutils/factories/api.py | 20 +++- 18 files changed, 477 insertions(+), 120 deletions(-) create mode 100644 src/hope_dedup_engine/apps/api/migrations/0007_rename_ignoredkeypair_ignoredreferencepkpair_and_more.py create mode 100644 src/hope_dedup_engine/apps/api/migrations/0008_ignoredfilenamepair.py create mode 100644 tests/api/test_ignored_filename_create.py create mode 100644 tests/api/test_ignored_filename_list.py delete mode 100644 tests/api/test_ignored_keys_list.py rename tests/api/{test_ignored_keys_create.py => test_ignored_reference_pk_create.py} (52%) create mode 100644 tests/api/test_ignored_reference_pk_list.py diff --git a/src/hope_dedup_engine/apps/api/const.py b/src/hope_dedup_engine/apps/api/const.py index 13f4c475..573cb593 100644 --- a/src/hope_dedup_engine/apps/api/const.py +++ b/src/hope_dedup_engine/apps/api/const.py @@ -15,5 +15,8 @@ DUPLICATE = "duplicate" DUPLICATE_LIST = f"{DUPLICATE}s" -IGNORED_KEYS = "ignored_key" -IGNORED_KEYS_LIST = f"{IGNORED_KEYS}s" +IGNORED = "ignored" +REFERENCE_PK = "reference_pk" +FILENAME = "filename" +IGNORED_REFERENCE_PK_LIST = f"{IGNORED}/{REFERENCE_PK}s" +IGNORED_FILENAME_LIST = f"{IGNORED}/{FILENAME}s" diff --git a/src/hope_dedup_engine/apps/api/deduplication/process.py b/src/hope_dedup_engine/apps/api/deduplication/process.py index 88388c39..c6a69737 100644 --- a/src/hope_dedup_engine/apps/api/deduplication/process.py +++ b/src/hope_dedup_engine/apps/api/deduplication/process.py @@ -18,12 +18,40 @@ def _sort_keys(pair: DuplicateKeyPair) -> DuplicateKeyPair: def _save_duplicates( finder: DuplicateFinder, deduplication_set: DeduplicationSet, - ignored_key_pairs: frozenset[tuple[str, str]], lock_enabled: bool, lock: DeduplicationSetLock, ) -> None: + reference_pk_to_filename_mapping = dict( + deduplication_set.image_set.values_list("reference_pk", "filename") + ) + ignored_filename_pairs = frozenset( + map( + tuple, + map( + sorted, + deduplication_set.ignoredfilenamepair_set.values_list( + "first", "second" + ), + ), + ) + ) + + ignored_reference_pk_pairs = frozenset( + deduplication_set.ignoredreferencepkpair_set.values_list("first", "second") + ) + for first, second, score in map(_sort_keys, finder.run()): - if (first, second) not in ignored_key_pairs: + first_filename, second_filename = sorted( + ( + reference_pk_to_filename_mapping[first], + reference_pk_to_filename_mapping[second], + ) + ) + ignored = (first, second) in ignored_reference_pk_pairs or ( + first_filename, + second_filename, + ) in ignored_filename_pairs + if not ignored: duplicate, _ = Duplicate.objects.get_or_create( deduplication_set=deduplication_set, first_reference_pk=first, @@ -54,17 +82,9 @@ def find_duplicates(deduplication_set_id: str, serialized_lock: str) -> None: # clean results Duplicate.objects.filter(deduplication_set=deduplication_set).delete() - ignored_key_pairs = frozenset( - deduplication_set.ignoredkeypair_set.values_list( - "first_reference_pk", "second_reference_pk" - ) - ) - weight_total = 0 for finder in get_finders(deduplication_set): - _save_duplicates( - finder, deduplication_set, ignored_key_pairs, lock_enabled, lock - ) + _save_duplicates(finder, deduplication_set, lock_enabled, lock) weight_total += finder.weight for duplicate in deduplication_set.duplicate_set.all(): diff --git a/src/hope_dedup_engine/apps/api/migrations/0007_rename_ignoredkeypair_ignoredreferencepkpair_and_more.py b/src/hope_dedup_engine/apps/api/migrations/0007_rename_ignoredkeypair_ignoredreferencepkpair_and_more.py new file mode 100644 index 00000000..908b7ce1 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/migrations/0007_rename_ignoredkeypair_ignoredreferencepkpair_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 5.0.7 on 2024-09-25 10:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0006_alter_deduplicationset_state_and_more"), + ] + + operations = [ + migrations.RenameModel( + old_name="IgnoredKeyPair", + new_name="IgnoredReferencePkPair", + ), + migrations.RenameField( + model_name="ignoredreferencepkpair", + old_name="first_reference_pk", + new_name="first", + ), + migrations.RenameField( + model_name="ignoredreferencepkpair", + old_name="second_reference_pk", + new_name="second", + ), + migrations.AlterUniqueTogether( + name="ignoredreferencepkpair", + unique_together={("deduplication_set", "first", "second")}, + ), + ] diff --git a/src/hope_dedup_engine/apps/api/migrations/0008_ignoredfilenamepair.py b/src/hope_dedup_engine/apps/api/migrations/0008_ignoredfilenamepair.py new file mode 100644 index 00000000..052cd42a --- /dev/null +++ b/src/hope_dedup_engine/apps/api/migrations/0008_ignoredfilenamepair.py @@ -0,0 +1,40 @@ +# Generated by Django 5.0.7 on 2024-09-25 11:24 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0007_rename_ignoredkeypair_ignoredreferencepkpair_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="IgnoredFilenamePair", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("first", models.CharField(max_length=100)), + ("second", models.CharField(max_length=100)), + ( + "deduplication_set", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="api.deduplicationset", + ), + ), + ], + options={ + "unique_together": {("deduplication_set", "first", "second")}, + }, + ), + ] diff --git a/src/hope_dedup_engine/apps/api/models/deduplication.py b/src/hope_dedup_engine/apps/api/models/deduplication.py index 19abd75d..61414194 100644 --- a/src/hope_dedup_engine/apps/api/models/deduplication.py +++ b/src/hope_dedup_engine/apps/api/models/deduplication.py @@ -115,21 +115,36 @@ class Duplicate(models.Model): score = models.FloatField(default=0) -class IgnoredKeyPair(models.Model): +class IgnoredPair(models.Model): deduplication_set = models.ForeignKey(DeduplicationSet, on_delete=models.CASCADE) - first_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) - second_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) class Meta: - unique_together = ( - "deduplication_set", - "first_reference_pk", - "second_reference_pk", - ) + abstract = True @override def save(self, **kwargs: Any) -> None: - self.first_reference_pk, self.second_reference_pk = sorted( - (self.first_reference_pk, self.second_reference_pk) - ) + self.first, self.second = sorted((self.first, self.second)) super().save(**kwargs) + + +UNIQUE_FOR_IGNORED_PAIR = ( + "deduplication_set", + "first", + "second", +) + + +class IgnoredReferencePkPair(IgnoredPair): + first = models.CharField(max_length=REFERENCE_PK_LENGTH) + second = models.CharField(max_length=REFERENCE_PK_LENGTH) + + class Meta: + unique_together = UNIQUE_FOR_IGNORED_PAIR + + +class IgnoredFilenamePair(IgnoredPair): + first = models.CharField(max_length=REFERENCE_PK_LENGTH) + second = models.CharField(max_length=REFERENCE_PK_LENGTH) + + class Meta: + unique_together = UNIQUE_FOR_IGNORED_PAIR diff --git a/src/hope_dedup_engine/apps/api/serializers.py b/src/hope_dedup_engine/apps/api/serializers.py index 0e763acd..c178c7f1 100644 --- a/src/hope_dedup_engine/apps/api/serializers.py +++ b/src/hope_dedup_engine/apps/api/serializers.py @@ -6,7 +6,8 @@ from hope_dedup_engine.apps.api.models.deduplication import ( Config, Duplicate, - IgnoredKeyPair, + IgnoredFilenamePair, + IgnoredReferencePkPair, Image, ) @@ -96,16 +97,32 @@ class Meta: fields = "first", "second", "score" -class IgnoredKeyPairSerializer(serializers.ModelSerializer): +CREATE_PAIR_FIELDS = "first", "second" +PAIR_FIELDS = ("id", "deduplication_set") + CREATE_PAIR_FIELDS + + +class IgnoredReferencePkPairSerializer(serializers.ModelSerializer): + class Meta: + model = IgnoredReferencePkPair + fields = PAIR_FIELDS + + +class CreateIgnoredReferencePkPairSerializer(serializers.ModelSerializer): + class Meta: + model = IgnoredReferencePkPair + fields = CREATE_PAIR_FIELDS + + +class IgnoredFilenamePairSerializer(serializers.ModelSerializer): class Meta: - model = IgnoredKeyPair - fields = "__all__" + model = IgnoredFilenamePair + fields = PAIR_FIELDS -class CreateIgnoredKeyPairSerializer(serializers.ModelSerializer): +class CreateIgnoredFilenamePairSerializer(serializers.ModelSerializer): class Meta: - model = IgnoredKeyPair - fields = ("first_reference_pk", "second_reference_pk") + model = IgnoredFilenamePair + fields = CREATE_PAIR_FIELDS class EmptySerializer(serializers.Serializer): diff --git a/src/hope_dedup_engine/apps/api/urls.py b/src/hope_dedup_engine/apps/api/urls.py index 12a25e78..5414a839 100644 --- a/src/hope_dedup_engine/apps/api/urls.py +++ b/src/hope_dedup_engine/apps/api/urls.py @@ -13,14 +13,16 @@ DEDUPLICATION_SET, DEDUPLICATION_SET_LIST, DUPLICATE_LIST, - IGNORED_KEYS_LIST, + IGNORED_FILENAME_LIST, + IGNORED_REFERENCE_PK_LIST, IMAGE_LIST, ) from hope_dedup_engine.apps.api.views import ( BulkImageViewSet, DeduplicationSetViewSet, DuplicateViewSet, - IgnoredKeyPairViewSet, + IgnoredFilenamePairViewSet, + IgnoredReferencePkPairViewSet, ImageViewSet, ) @@ -40,7 +42,12 @@ DUPLICATE_LIST, DuplicateViewSet, basename=DUPLICATE_LIST ) deduplication_sets_router.register( - IGNORED_KEYS_LIST, IgnoredKeyPairViewSet, basename=IGNORED_KEYS_LIST + IGNORED_FILENAME_LIST, IgnoredFilenamePairViewSet, basename=IGNORED_FILENAME_LIST +) +deduplication_sets_router.register( + IGNORED_REFERENCE_PK_LIST, + IgnoredReferencePkPairViewSet, + basename=IGNORED_REFERENCE_PK_LIST, ) urlpatterns = [ diff --git a/src/hope_dedup_engine/apps/api/views.py b/src/hope_dedup_engine/apps/api/views.py index bc8bf879..75baab22 100644 --- a/src/hope_dedup_engine/apps/api/views.py +++ b/src/hope_dedup_engine/apps/api/views.py @@ -27,17 +27,20 @@ from hope_dedup_engine.apps.api.models import DeduplicationSet from hope_dedup_engine.apps.api.models.deduplication import ( Duplicate, - IgnoredKeyPair, + IgnoredFilenamePair, + IgnoredReferencePkPair, Image, ) from hope_dedup_engine.apps.api.serializers import ( CreateDeduplicationSetSerializer, - CreateIgnoredKeyPairSerializer, + CreateIgnoredFilenamePairSerializer, + CreateIgnoredReferencePkPairSerializer, CreateImageSerializer, DeduplicationSetSerializer, DuplicateSerializer, EmptySerializer, - IgnoredKeyPairSerializer, + IgnoredFilenamePairSerializer, + IgnoredReferencePkPairSerializer, ImageSerializer, ) from hope_dedup_engine.apps.api.utils import delete_model_data, start_processing @@ -272,8 +275,8 @@ def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: return super().list(request, *args, **kwargs) -class IgnoredKeyPairViewSet( - nested_viewsets.NestedViewSetMixin[IgnoredKeyPair], +class IgnoredPairViewSet[T]( + nested_viewsets.NestedViewSetMixin[T], mixins.ListModelMixin, mixins.CreateModelMixin, viewsets.GenericViewSet, @@ -284,8 +287,6 @@ class IgnoredKeyPairViewSet( AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem, ) - serializer_class = IgnoredKeyPairSerializer - queryset = IgnoredKeyPair.objects.all() parent_lookup_kwargs = { DEDUPLICATION_SET_PARAM: DEDUPLICATION_SET_FILTER, } @@ -297,13 +298,38 @@ def perform_create(self, serializer: Serializer) -> None: deduplication_set.updated_by = self.request.user deduplication_set.save() - @extend_schema(description="List all ignored key pairs for the deduplication set") + +class IgnoredFilenamePairViewSet(IgnoredPairViewSet[IgnoredFilenamePair]): + serializer_class = IgnoredFilenamePairSerializer + queryset = IgnoredFilenamePair.objects.all() + + @extend_schema( + description="List all ignored filename pairs for the deduplication set" + ) + def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().list(request, *args, **kwargs) + + @extend_schema( + request=CreateIgnoredFilenamePairSerializer, + description="Add ignored filename pair for the deduplication set", + ) + def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: + return super().create(request, *args, **kwargs) + + +class IgnoredReferencePkPairViewSet(IgnoredPairViewSet[IgnoredReferencePkPair]): + serializer_class = IgnoredReferencePkPairSerializer + queryset = IgnoredReferencePkPair.objects.all() + + @extend_schema( + description="List all ignored reference pk pairs for the deduplication set" + ) def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: return super().list(request, *args, **kwargs) @extend_schema( - request=CreateIgnoredKeyPairSerializer, - description="Add ignored key pair for the deduplication set", + request=CreateIgnoredReferencePkPairSerializer, + description="Add ignored reference pk pair for the deduplication set", ) def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: return super().create(request, *args, **kwargs) diff --git a/tests/api/api_const.py b/tests/api/api_const.py index 1c2cbb9e..2accb963 100644 --- a/tests/api/api_const.py +++ b/tests/api/api_const.py @@ -2,7 +2,8 @@ BULK_IMAGE_LIST, DEDUPLICATION_SET_LIST, DUPLICATE_LIST, - IGNORED_KEYS_LIST, + IGNORED_FILENAME_LIST, + IGNORED_REFERENCE_PK_LIST, IMAGE_LIST, ) @@ -17,4 +18,5 @@ BULK_IMAGE_LIST_VIEW = f"{BULK_IMAGE_LIST}-{LIST}" BULK_IMAGE_CLEAR_VIEW = f"{BULK_IMAGE_LIST}-clear" DUPLICATE_LIST_VIEW = f"{DUPLICATE_LIST}-{LIST}" -IGNORED_KEYS_LIST_VIEW = f"{IGNORED_KEYS_LIST}-{LIST}" +IGNORED_REFERENCE_PK_LIST_VIEW = f"{IGNORED_REFERENCE_PK_LIST}-{LIST}" +IGNORED_FILENAME_LIST_VIEW = f"{IGNORED_FILENAME_LIST}-{LIST}" diff --git a/tests/api/conftest.py b/tests/api/conftest.py index e772d59c..f77b1ed6 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -14,7 +14,8 @@ ConfigFactory, DeduplicationSetFactory, DuplicateFactory, - IgnoredKeyPairFactory, + IgnoredFilenamePairFactory, + IgnoredReferencePkPairFactory, ImageFactory, TokenFactory, ) @@ -34,7 +35,10 @@ deduplication_Set=LazyFixture("deduplication_set"), ) register(DuplicateFactory, deduplication_set=LazyFixture("deduplication_set")) -register(IgnoredKeyPairFactory, deduplication_set=LazyFixture("deduplication_set")) +register(IgnoredFilenamePairFactory, deduplication_set=LazyFixture("deduplication_set")) +register( + IgnoredReferencePkPairFactory, deduplication_set=LazyFixture("deduplication_set") +) register(ConfigFactory) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 1ba3ed40..1d716cf7 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -7,7 +7,7 @@ BULK_IMAGE_LIST_VIEW, DEDUPLICATION_SET_DETAIL_VIEW, DEDUPLICATION_SET_LIST_VIEW, - IGNORED_KEYS_LIST_VIEW, + IGNORED_REFERENCE_PK_LIST_VIEW, IMAGE_DETAIL_VIEW, IMAGE_LIST_VIEW, JSON, @@ -33,8 +33,8 @@ (BULK_IMAGE_LIST_VIEW, HTTPMethod.POST, (PK,)), (IMAGE_DETAIL_VIEW, HTTPMethod.DELETE, (PK, PK)), (BULK_IMAGE_CLEAR_VIEW, HTTPMethod.DELETE, (PK,)), - (IGNORED_KEYS_LIST_VIEW, HTTPMethod.GET, (PK,)), - (IGNORED_KEYS_LIST_VIEW, HTTPMethod.POST, (PK,)), + (IGNORED_REFERENCE_PK_LIST_VIEW, HTTPMethod.GET, (PK,)), + (IGNORED_REFERENCE_PK_LIST_VIEW, HTTPMethod.POST, (PK,)), ) diff --git a/tests/api/test_find_duplicates.py b/tests/api/test_find_duplicates.py index a46c4068..48f4574b 100644 --- a/tests/api/test_find_duplicates.py +++ b/tests/api/test_find_duplicates.py @@ -38,7 +38,7 @@ def test_duplicates_are_stored( assert deduplication_set.duplicate_set.count() -def test_ignored_key_pairs( +def test_ignored_reference_pk_pairs( deduplication_set: DeduplicationSet, image: Image, second_image: Image, @@ -46,15 +46,34 @@ def test_ignored_key_pairs( requests_get_mock: MagicMock, ) -> None: assert not deduplication_set.duplicate_set.count() - ignored_key_pair = deduplication_set.ignoredkeypair_set.create( - first_reference_pk=image.reference_pk, - second_reference_pk=second_image.reference_pk, + ignored_reference_pk_pair = deduplication_set.ignoredreferencepkpair_set.create( + first=image.reference_pk, + second=second_image.reference_pk, ) find_duplicates( str(deduplication_set.pk), str(DeduplicationSetLock.for_deduplication_set(deduplication_set)), ) - ignored_key_pair.delete() + ignored_reference_pk_pair.delete() + assert not deduplication_set.duplicate_set.count() + + +def test_ignored_filename_pairs( + deduplication_set: DeduplicationSet, + image: Image, + second_image: Image, + all_duplicates_finder: DuplicateFinder, +) -> None: + assert not deduplication_set.duplicate_set.count() + ignored_filename_pair = deduplication_set.ignoredfilenamepair_set.create( + first=image.filename, + second=second_image.filename, + ) + find_duplicates( + str(deduplication_set.pk), + str(DeduplicationSetLock.for_deduplication_set(deduplication_set)), + ) + ignored_filename_pair.delete() assert not deduplication_set.duplicate_set.count() diff --git a/tests/api/test_ignored_filename_create.py b/tests/api/test_ignored_filename_create.py new file mode 100644 index 00000000..80b3ee1f --- /dev/null +++ b/tests/api/test_ignored_filename_create.py @@ -0,0 +1,111 @@ +from api_const import IGNORED_FILENAME_LIST_VIEW, JSON +from pytest import mark +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient +from testutils.factories.api import IgnoredFilenamePairFactory + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import IgnoredFilenamePair +from hope_dedup_engine.apps.api.serializers import IgnoredFilenamePairSerializer +from hope_dedup_engine.apps.security.models import User + + +def test_can_create_ignored_filename_pair( + api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + previous_amount = IgnoredFilenamePair.objects.filter( + deduplication_set=deduplication_set + ).count() + data = IgnoredFilenamePairSerializer(IgnoredFilenamePairFactory.build()).data + + response = api_client.post( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_201_CREATED + assert ( + IgnoredFilenamePair.objects.filter(deduplication_set=deduplication_set).count() + == previous_amount + 1 + ) + + +def test_cannot_create_ignored_filename_pair_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + previous_amount = IgnoredFilenamePair.objects.filter( + deduplication_set=deduplication_set + ).count() + data = IgnoredFilenamePairSerializer(IgnoredFilenamePairFactory.build()).data + + response = another_system_api_client.post( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + assert ( + IgnoredFilenamePair.objects.filter(deduplication_set=deduplication_set).count() + == previous_amount + ) + + +INVALID_FILENAME_VALUES = "", None + + +@mark.parametrize("first_filename", INVALID_FILENAME_VALUES) +@mark.parametrize("second_filename", INVALID_FILENAME_VALUES) +def test_invalid_values_handling( + api_client: APIClient, + deduplication_set: DeduplicationSet, + first_filename: str | None, + second_filename: str | None, +) -> None: + data = IgnoredFilenamePairSerializer(IgnoredFilenamePairFactory.build()).data + data["first"] = first_filename + data["second"] = second_filename + response = api_client.post( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert len(errors) == 2 + assert "first" in errors + assert "second" in errors + + +def test_missing_filename_handling( + api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + data = IgnoredFilenamePairSerializer(IgnoredFilenamePairFactory.build()).data + del data["first"], data["second"] + + response = api_client.post( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert "first" in errors + assert "second" in errors + + +def test_deduplication_set_is_updated( + api_client: APIClient, user: User, deduplication_set: DeduplicationSet +) -> None: + assert deduplication_set.updated_by is None + + data = IgnoredFilenamePairSerializer(IgnoredFilenamePairFactory.build()).data + response = api_client.post( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, + ) + + assert response.status_code == status.HTTP_201_CREATED + deduplication_set.refresh_from_db() + assert deduplication_set.updated_by == user diff --git a/tests/api/test_ignored_filename_list.py b/tests/api/test_ignored_filename_list.py new file mode 100644 index 00000000..f07ce386 --- /dev/null +++ b/tests/api/test_ignored_filename_list.py @@ -0,0 +1,35 @@ +from api_const import IGNORED_FILENAME_LIST_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import IgnoredFilenamePair + + +def test_can_list_ignored_filename_pairs( + api_client: APIClient, + deduplication_set: DeduplicationSet, + ignored_filename_pair: IgnoredFilenamePair, +) -> None: + response = api_client.get( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_200_OK + ignored_filename_pairs = response.json() + assert len(ignored_filename_pairs) + assert ( + len(ignored_filename_pairs) + == IgnoredFilenamePair.objects.filter( + deduplication_set=deduplication_set + ).count() + ) + + +def test_cannot_list_ignored_filename_pairs_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + response = another_system_api_client.get( + reverse(IGNORED_FILENAME_LIST_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/tests/api/test_ignored_keys_list.py b/tests/api/test_ignored_keys_list.py deleted file mode 100644 index 30c73537..00000000 --- a/tests/api/test_ignored_keys_list.py +++ /dev/null @@ -1,31 +0,0 @@ -from api_const import IGNORED_KEYS_LIST_VIEW -from rest_framework import status -from rest_framework.reverse import reverse -from rest_framework.test import APIClient - -from hope_dedup_engine.apps.api.models import DeduplicationSet -from hope_dedup_engine.apps.api.models.deduplication import IgnoredKeyPair - - -def test_can_list_ignored_key_pairs( - api_client: APIClient, - deduplication_set: DeduplicationSet, - ignored_key_pair: IgnoredKeyPair, -) -> None: - response = api_client.get(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,))) - assert response.status_code == status.HTTP_200_OK - ignored_key_pairs = response.json() - assert len(ignored_key_pairs) - assert ( - len(ignored_key_pairs) - == IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() - ) - - -def test_cannot_list_ignored_key_pairs_between_systems( - another_system_api_client: APIClient, deduplication_set: DeduplicationSet -) -> None: - response = another_system_api_client.get( - reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)) - ) - assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/tests/api/test_ignored_keys_create.py b/tests/api/test_ignored_reference_pk_create.py similarity index 52% rename from tests/api/test_ignored_keys_create.py rename to tests/api/test_ignored_reference_pk_create.py index 1f45f42a..996d4329 100644 --- a/tests/api/test_ignored_keys_create.py +++ b/tests/api/test_ignored_reference_pk_create.py @@ -1,52 +1,59 @@ from unittest.mock import MagicMock - -from api_const import IGNORED_KEYS_LIST_VIEW, JSON +from api_const import IGNORED_REFERENCE_PK_LIST_VIEW, JSON from pytest import mark from rest_framework import status from rest_framework.reverse import reverse from rest_framework.test import APIClient -from testutils.factories.api import IgnoredKeyPairFactory +from testutils.factories.api import IgnoredReferencePkPairFactory from hope_dedup_engine.apps.api.models import DeduplicationSet -from hope_dedup_engine.apps.api.models.deduplication import IgnoredKeyPair -from hope_dedup_engine.apps.api.serializers import IgnoredKeyPairSerializer +from hope_dedup_engine.apps.api.models.deduplication import IgnoredReferencePkPair +from hope_dedup_engine.apps.api.serializers import IgnoredReferencePkPairSerializer from hope_dedup_engine.apps.security.models import User -def test_can_create_ignored_key_pair( +def test_can_create_ignored_reference_pk_pair( api_client: APIClient, deduplication_set: DeduplicationSet, requests_get_mock: MagicMock, ) -> None: - previous_amount = IgnoredKeyPair.objects.filter( + previous_amount = IgnoredReferencePkPair.objects.filter( deduplication_set=deduplication_set ).count() - data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data + data = IgnoredReferencePkPairSerializer(IgnoredReferencePkPairFactory.build()).data response = api_client.post( - reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, ) assert response.status_code == status.HTTP_201_CREATED assert ( - IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() + IgnoredReferencePkPair.objects.filter( + deduplication_set=deduplication_set + ).count() == previous_amount + 1 ) -def test_cannot_create_ignored_key_pair_between_systems( +def test_cannot_create_ignored_reference_pk_pair_between_systems( another_system_api_client: APIClient, deduplication_set: DeduplicationSet ) -> None: - previous_amount = IgnoredKeyPair.objects.filter( + previous_amount = IgnoredReferencePkPair.objects.filter( deduplication_set=deduplication_set ).count() - data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data + data = IgnoredReferencePkPairSerializer(IgnoredReferencePkPairFactory.build()).data response = another_system_api_client.post( - reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, ) assert response.status_code == status.HTTP_403_FORBIDDEN assert ( - IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() + IgnoredReferencePkPair.objects.filter( + deduplication_set=deduplication_set + ).count() == previous_amount ) @@ -62,32 +69,36 @@ def test_invalid_values_handling( first_pk: str | None, second_pk: str | None, ) -> None: - data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data - data["first_reference_pk"] = first_pk - data["second_reference_pk"] = second_pk + data = IgnoredReferencePkPairSerializer(IgnoredReferencePkPairFactory.build()).data + data["first"] = first_pk + data["second"] = second_pk response = api_client.post( - reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, ) assert response.status_code == status.HTTP_400_BAD_REQUEST errors = response.json() assert len(errors) == 2 - assert "first_reference_pk" in errors - assert "second_reference_pk" in errors + assert "first" in errors + assert "second" in errors def test_missing_pk_handling( api_client: APIClient, deduplication_set: DeduplicationSet ) -> None: - data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data - del data["first_reference_pk"], data["second_reference_pk"] + data = IgnoredReferencePkPairSerializer(IgnoredReferencePkPairFactory.build()).data + del data["first"], data["second"] response = api_client.post( - reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, ) assert response.status_code == status.HTTP_400_BAD_REQUEST errors = response.json() - assert "first_reference_pk" in errors - assert "second_reference_pk" in errors + assert "first" in errors + assert "second" in errors def test_deduplication_set_is_updated( @@ -98,9 +109,11 @@ def test_deduplication_set_is_updated( ) -> None: assert deduplication_set.updated_by is None - data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data + data = IgnoredReferencePkPairSerializer(IgnoredReferencePkPairFactory.build()).data response = api_client.post( - reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)), + data=data, + format=JSON, ) assert response.status_code == status.HTTP_201_CREATED diff --git a/tests/api/test_ignored_reference_pk_list.py b/tests/api/test_ignored_reference_pk_list.py new file mode 100644 index 00000000..acf40291 --- /dev/null +++ b/tests/api/test_ignored_reference_pk_list.py @@ -0,0 +1,35 @@ +from api_const import IGNORED_REFERENCE_PK_LIST_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import IgnoredReferencePkPair + + +def test_can_list_ignored_reference_pk_pairs( + api_client: APIClient, + deduplication_set: DeduplicationSet, + ignored_reference_pk_pair: IgnoredReferencePkPair, +) -> None: + response = api_client.get( + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_200_OK + ignored_reference_pk_pairs = response.json() + assert len(ignored_reference_pk_pairs) + assert ( + len(ignored_reference_pk_pairs) + == IgnoredReferencePkPair.objects.filter( + deduplication_set=deduplication_set + ).count() + ) + + +def test_cannot_list_ignored_reference_pk_pairs_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + response = another_system_api_client.get( + reverse(IGNORED_REFERENCE_PK_LIST_VIEW, (deduplication_set.pk,)) + ) + assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/tests/extras/testutils/factories/api.py b/tests/extras/testutils/factories/api.py index 5f189b3c..a30ef1e0 100644 --- a/tests/extras/testutils/factories/api.py +++ b/tests/extras/testutils/factories/api.py @@ -6,7 +6,8 @@ from hope_dedup_engine.apps.api.models.deduplication import ( Config, Duplicate, - IgnoredKeyPair, + IgnoredFilenamePair, + IgnoredReferencePkPair, Image, ) @@ -55,10 +56,19 @@ class Meta: model = Duplicate -class IgnoredKeyPairFactory(DjangoModelFactory): +class IgnoredFilenamePairFactory(DjangoModelFactory): deduplication_set = SubFactory(DeduplicationSetFactory) - first_reference_pk = fuzzy.FuzzyText() - second_reference_pk = fuzzy.FuzzyText() + first = fuzzy.FuzzyText() + second = fuzzy.FuzzyText() + + class Meta: + model = IgnoredFilenamePair + + +class IgnoredReferencePkPairFactory(DjangoModelFactory): + deduplication_set = SubFactory(DeduplicationSetFactory) + first = fuzzy.FuzzyText() + second = fuzzy.FuzzyText() class Meta: - model = IgnoredKeyPair + model = IgnoredReferencePkPair