From eed30eb40447095d1edd5ab1020e6392152284b4 Mon Sep 17 00:00:00 2001 From: sergey-misuk-valor <168101676+sergey-misuk-valor@users.noreply.github.com> Date: Tue, 11 Jun 2024 18:33:42 +0400 Subject: [PATCH 1/2] Add ignored key pair API (#26) --- src/hope_dedup_engine/apps/api/const.py | 3 + .../apps/api/migrations/0001_initial.py | 17 ++++- .../apps/api/models/deduplication.py | 15 ++++ src/hope_dedup_engine/apps/api/serializers.py | 8 +- src/hope_dedup_engine/apps/api/urls.py | 10 ++- src/hope_dedup_engine/apps/api/views.py | 28 ++++++- tests/api/api_const.py | 23 ++++-- tests/api/conftest.py | 9 ++- tests/api/test_auth.py | 3 + tests/api/test_ignored_keys_create.py | 74 +++++++++++++++++++ tests/api/test_ignored_keys_list.py | 24 ++++++ tests/extras/testutils/factories/api.py | 11 ++- 12 files changed, 211 insertions(+), 14 deletions(-) create mode 100644 tests/api/test_ignored_keys_create.py create mode 100644 tests/api/test_ignored_keys_list.py diff --git a/src/hope_dedup_engine/apps/api/const.py b/src/hope_dedup_engine/apps/api/const.py index cf066e62..a5c8aeb1 100644 --- a/src/hope_dedup_engine/apps/api/const.py +++ b/src/hope_dedup_engine/apps/api/const.py @@ -14,3 +14,6 @@ DUPLICATE = "duplicate" DUPLICATE_LIST = f"{DUPLICATE}s" + +IGNORED_KEYS = "ignore" +IGNORED_KEYS_LIST = f"{IGNORED_KEYS}s" diff --git a/src/hope_dedup_engine/apps/api/migrations/0001_initial.py b/src/hope_dedup_engine/apps/api/migrations/0001_initial.py index 37ad2d18..07000d24 100644 --- a/src/hope_dedup_engine/apps/api/migrations/0001_initial.py +++ b/src/hope_dedup_engine/apps/api/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.6 on 2024-06-04 13:30 +# Generated by Django 5.0.6 on 2024-06-11 09:14 import django.db.models.deletion import uuid @@ -117,4 +117,19 @@ class Migration(migrations.Migration): ), ], ), + migrations.CreateModel( + name="IgnoredKeyPair", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("first_reference_pk", models.CharField(max_length=100)), + ("second_reference_pk", models.CharField(max_length=100)), + ( + "deduplication_set", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="api.deduplicationset"), + ), + ], + options={ + "unique_together": {("deduplication_set", "first_reference_pk", "second_reference_pk")}, + }, + ), ] diff --git a/src/hope_dedup_engine/apps/api/models/deduplication.py b/src/hope_dedup_engine/apps/api/models/deduplication.py index 77da90e4..4bce2d0e 100644 --- a/src/hope_dedup_engine/apps/api/models/deduplication.py +++ b/src/hope_dedup_engine/apps/api/models/deduplication.py @@ -1,3 +1,4 @@ +from typing import Any, override from uuid import uuid4 from django.conf import settings @@ -54,3 +55,17 @@ class Duplicate(models.Model): second_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) second_filename = models.CharField(max_length=255) score = models.FloatField() + + +class IgnoredKeyPair(models.Model): + deduplication_set = models.ForeignKey(DeduplicationSet, on_delete=models.CASCADE) + first_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) + second_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH) + + class Meta: + unique_together = "deduplication_set", "first_reference_pk", "second_reference_pk" + + @override + def save(self, **kwargs: Any) -> None: + self.first_reference_pk, self.second_reference_pk = sorted((self.first_reference_pk, self.second_reference_pk)) + super().save(**kwargs) diff --git a/src/hope_dedup_engine/apps/api/serializers.py b/src/hope_dedup_engine/apps/api/serializers.py index ba5ac220..2227e72a 100644 --- a/src/hope_dedup_engine/apps/api/serializers.py +++ b/src/hope_dedup_engine/apps/api/serializers.py @@ -1,7 +1,7 @@ from rest_framework import serializers from hope_dedup_engine.apps.api.models import DeduplicationSet -from hope_dedup_engine.apps.api.models.deduplication import Duplicate, Image +from hope_dedup_engine.apps.api.models.deduplication import Duplicate, IgnoredKeyPair, Image class DeduplicationSetSerializer(serializers.ModelSerializer): @@ -38,3 +38,9 @@ def get_filename(self, duplicate: Duplicate) -> str: class DuplicateSerializer(serializers.Serializer): first = EntrySerializer(prefix="first", source="*") second = EntrySerializer(prefix="second", source="*") + + +class IgnoredKeyPairSerializer(serializers.ModelSerializer): + class Meta: + model = IgnoredKeyPair + fields = "__all__" diff --git a/src/hope_dedup_engine/apps/api/urls.py b/src/hope_dedup_engine/apps/api/urls.py index 78b1ff4f..fb83633a 100644 --- a/src/hope_dedup_engine/apps/api/urls.py +++ b/src/hope_dedup_engine/apps/api/urls.py @@ -8,9 +8,16 @@ DEDUPLICATION_SET, DEDUPLICATION_SET_LIST, DUPLICATE_LIST, + IGNORED_KEYS_LIST, IMAGE_LIST, ) -from hope_dedup_engine.apps.api.views import BulkImageViewSet, DeduplicationSetViewSet, DuplicateViewSet, ImageViewSet +from hope_dedup_engine.apps.api.views import ( + BulkImageViewSet, + DeduplicationSetViewSet, + DuplicateViewSet, + IgnoredKeyPairViewSet, + ImageViewSet, +) router = routers.SimpleRouter() router.register(DEDUPLICATION_SET_LIST, DeduplicationSetViewSet, basename=DEDUPLICATION_SET_LIST) @@ -19,5 +26,6 @@ deduplication_sets_router.register(IMAGE_LIST, ImageViewSet, basename=IMAGE_LIST) deduplication_sets_router.register(BULK_IMAGE_LIST, BulkImageViewSet, basename=BULK_IMAGE_LIST) deduplication_sets_router.register(DUPLICATE_LIST, DuplicateViewSet, basename=DUPLICATE_LIST) +deduplication_sets_router.register(IGNORED_KEYS_LIST, IgnoredKeyPairViewSet, basename=IGNORED_KEYS_LIST) urlpatterns = [path("", include(router.urls)), path("", include(deduplication_sets_router.urls))] diff --git a/src/hope_dedup_engine/apps/api/views.py b/src/hope_dedup_engine/apps/api/views.py index 9afb7f70..eb07413e 100644 --- a/src/hope_dedup_engine/apps/api/views.py +++ b/src/hope_dedup_engine/apps/api/views.py @@ -20,8 +20,13 @@ ) from hope_dedup_engine.apps.api.const import DEDUPLICATION_SET_FILTER, DEDUPLICATION_SET_PARAM from hope_dedup_engine.apps.api.models import DeduplicationSet -from hope_dedup_engine.apps.api.models.deduplication import Duplicate, Image -from hope_dedup_engine.apps.api.serializers import DeduplicationSetSerializer, DuplicateSerializer, ImageSerializer +from hope_dedup_engine.apps.api.models.deduplication import Duplicate, IgnoredKeyPair, Image +from hope_dedup_engine.apps.api.serializers import ( + DeduplicationSetSerializer, + DuplicateSerializer, + IgnoredKeyPairSerializer, + ImageSerializer, +) from hope_dedup_engine.apps.api.utils import delete_model_data, start_processing MESSAGE = "message" @@ -164,3 +169,22 @@ class DuplicateViewSet(nested_viewsets.NestedViewSetMixin, mixins.ListModelMixin parent_lookup_kwargs = { DEDUPLICATION_SET_PARAM: DEDUPLICATION_SET_FILTER, } + + +class IgnoredKeyPairViewSet( + nested_viewsets.NestedViewSetMixin, mixins.ListModelMixin, mixins.CreateModelMixin, viewsets.GenericViewSet +): + authentication_classes = (HDETokenAuthentication,) + permission_classes = IsAuthenticated, AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem + serializer_class = IgnoredKeyPairSerializer + queryset = IgnoredKeyPair.objects.all() + parent_lookup_kwargs = { + DEDUPLICATION_SET_PARAM: DEDUPLICATION_SET_FILTER, + } + + def perform_create(self, serializer: Serializer) -> None: + super().perform_create(serializer) + deduplication_set = serializer.instance.deduplication_set + deduplication_set.state = DeduplicationSet.State.DIRTY + deduplication_set.updated_by = self.request.user + deduplication_set.save() diff --git a/tests/api/api_const.py b/tests/api/api_const.py index a191afe2..1c2cbb9e 100644 --- a/tests/api/api_const.py +++ b/tests/api/api_const.py @@ -1,11 +1,20 @@ -from hope_dedup_engine.apps.api.const import BULK_IMAGE_LIST, DEDUPLICATION_SET_LIST, DUPLICATE_LIST, IMAGE_LIST +from hope_dedup_engine.apps.api.const import ( + BULK_IMAGE_LIST, + DEDUPLICATION_SET_LIST, + DUPLICATE_LIST, + IGNORED_KEYS_LIST, + IMAGE_LIST, +) JSON = "json" -DEDUPLICATION_SET_LIST_VIEW = f"{DEDUPLICATION_SET_LIST}-list" -DEDUPLICATION_SET_DETAIL_VIEW = f"{DEDUPLICATION_SET_LIST}-detail" +LIST = "list" +DETAIL = "detail" +DEDUPLICATION_SET_LIST_VIEW = f"{DEDUPLICATION_SET_LIST}-{LIST}" +DEDUPLICATION_SET_DETAIL_VIEW = f"{DEDUPLICATION_SET_LIST}-{DETAIL}" DEDUPLICATION_SET_PROCESS_VIEW = f"{DEDUPLICATION_SET_LIST}-process" -IMAGE_LIST_VIEW = f"{IMAGE_LIST}-list" -IMAGE_DETAIL_VIEW = f"{IMAGE_LIST}-detail" -BULK_IMAGE_LIST_VIEW = f"{BULK_IMAGE_LIST}-list" +IMAGE_LIST_VIEW = f"{IMAGE_LIST}-{LIST}" +IMAGE_DETAIL_VIEW = f"{IMAGE_LIST}-{DETAIL}" +BULK_IMAGE_LIST_VIEW = f"{BULK_IMAGE_LIST}-{LIST}" BULK_IMAGE_CLEAR_VIEW = f"{BULK_IMAGE_LIST}-clear" -DUPLICATE_LIST_VIEW = f"{DUPLICATE_LIST}-list" +DUPLICATE_LIST_VIEW = f"{DUPLICATE_LIST}-{LIST}" +IGNORED_KEYS_LIST_VIEW = f"{IGNORED_KEYS_LIST}-{LIST}" diff --git a/tests/api/conftest.py b/tests/api/conftest.py index c9fafa5b..8fec4a87 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -5,7 +5,13 @@ from pytest_factoryboy import LazyFixture, register from pytest_mock import MockerFixture from rest_framework.test import APIClient -from testutils.factories.api import DeduplicationSetFactory, DuplicateFactory, ImageFactory, TokenFactory +from testutils.factories.api import ( + DeduplicationSetFactory, + DuplicateFactory, + IgnoredKeyPairFactory, + ImageFactory, + TokenFactory, +) from testutils.factories.user import ExternalSystemFactory, UserFactory from hope_dedup_engine.apps.api.models import HDEToken @@ -16,6 +22,7 @@ register(DeduplicationSetFactory, external_system=LazyFixture("external_system")) register(ImageFactory, deduplication_Set=LazyFixture("deduplication_set")) register(DuplicateFactory, deduplication_set=LazyFixture("deduplication_set")) +register(IgnoredKeyPairFactory, deduplication_set=LazyFixture("deduplication_set")) @fixture diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 7e1ecc43..75636dad 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -7,6 +7,7 @@ BULK_IMAGE_LIST_VIEW, DEDUPLICATION_SET_DETAIL_VIEW, DEDUPLICATION_SET_LIST_VIEW, + IGNORED_KEYS_LIST_VIEW, IMAGE_DETAIL_VIEW, IMAGE_LIST_VIEW, JSON, @@ -32,6 +33,8 @@ (BULK_IMAGE_LIST_VIEW, HTTPMethod.POST, (PK,)), (IMAGE_DETAIL_VIEW, HTTPMethod.DELETE, (PK, PK)), (BULK_IMAGE_CLEAR_VIEW, HTTPMethod.DELETE, (PK,)), + (IGNORED_KEYS_LIST_VIEW, HTTPMethod.GET, (PK,)), + (IGNORED_KEYS_LIST_VIEW, HTTPMethod.POST, (PK,)), ) diff --git a/tests/api/test_ignored_keys_create.py b/tests/api/test_ignored_keys_create.py new file mode 100644 index 00000000..317e39c0 --- /dev/null +++ b/tests/api/test_ignored_keys_create.py @@ -0,0 +1,74 @@ +from api_const import IGNORED_KEYS_LIST_VIEW, JSON +from pytest import mark +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient +from testutils.factories.api import IgnoredKeyPairFactory + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import IgnoredKeyPair +from hope_dedup_engine.apps.api.serializers import IgnoredKeyPairSerializer +from hope_dedup_engine.apps.security.models import User + + +def test_can_create_ignored_key_pair(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: + previous_amount = IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() + data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data + + response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + assert response.status_code == status.HTTP_201_CREATED + assert IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() == previous_amount + 1 + + +def test_cannot_create_ignored_key_pair_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + previous_amount = IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() + data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data + + response = another_system_api_client.post( + reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + assert IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() == previous_amount + + +INVALID_PK_VALUES = "", None + + +@mark.parametrize("first_pk", INVALID_PK_VALUES) +@mark.parametrize("second_pk", INVALID_PK_VALUES) +def test_invalid_values_handling( + api_client: APIClient, deduplication_set: DeduplicationSet, first_pk: str | None, second_pk: str | None +) -> None: + data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data + data["first_reference_pk"] = first_pk + data["second_reference_pk"] = second_pk + response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert len(errors) == 2 + assert "first_reference_pk" in errors + assert "second_reference_pk" in errors + + +def test_missing_pk_handling(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: + data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data + del data["first_reference_pk"], data["second_reference_pk"] + + response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert "first_reference_pk" in errors + assert "second_reference_pk" in errors + + +def test_deduplication_set_is_updated(api_client: APIClient, user: User, deduplication_set: DeduplicationSet) -> None: + assert deduplication_set.updated_by is None + + data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data + response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + + assert response.status_code == status.HTTP_201_CREATED + deduplication_set.refresh_from_db() + assert deduplication_set.updated_by == user diff --git a/tests/api/test_ignored_keys_list.py b/tests/api/test_ignored_keys_list.py new file mode 100644 index 00000000..8affedf4 --- /dev/null +++ b/tests/api/test_ignored_keys_list.py @@ -0,0 +1,24 @@ +from api_const import IGNORED_KEYS_LIST_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import IgnoredKeyPair + + +def test_can_list_ignored_key_pairs( + api_client: APIClient, deduplication_set: DeduplicationSet, ignored_key_pair: IgnoredKeyPair +) -> None: + response = api_client.get(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_200_OK + ignored_key_pairs = response.json() + assert len(ignored_key_pairs) + assert len(ignored_key_pairs) == IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() + + +def test_cannot_list_ignored_key_pairs_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + response = another_system_api_client.get(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/tests/extras/testutils/factories/api.py b/tests/extras/testutils/factories/api.py index a795e755..851e9c41 100644 --- a/tests/extras/testutils/factories/api.py +++ b/tests/extras/testutils/factories/api.py @@ -3,7 +3,7 @@ from testutils.factories import ExternalSystemFactory, UserFactory from hope_dedup_engine.apps.api.models import DeduplicationSet, HDEToken -from hope_dedup_engine.apps.api.models.deduplication import Duplicate, Image +from hope_dedup_engine.apps.api.models.deduplication import Duplicate, IgnoredKeyPair, Image class TokenFactory(DjangoModelFactory): @@ -43,3 +43,12 @@ class DuplicateFactory(DjangoModelFactory): class Meta: model = Duplicate + + +class IgnoredKeyPairFactory(DjangoModelFactory): + deduplication_set = SubFactory(DeduplicationSetFactory) + first_reference_pk = fuzzy.FuzzyText() + second_reference_pk = fuzzy.FuzzyText() + + class Meta: + model = IgnoredKeyPair From a0fa9dd9c79afd48998a492e3dabf2c3cb6b44d8 Mon Sep 17 00:00:00 2001 From: Stefano Apostolico Date: Tue, 11 Jun 2024 16:39:43 +0200 Subject: [PATCH 2/2] Update README.md --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f85b4790..a5ac95a0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -ABOUT HOPE Payment Gateway -========================== +ABOUT HOPE Deduplication Engine +=============================== ## Contributing @@ -14,7 +14,7 @@ ABOUT HOPE Payment Gateway **WARNING** -> Hope Payment Gateway implements **security first** policy. It means that configuration default values are "almost" production compliant. +> Hope Deduplication Engine implements **security first** policy. It means that configuration default values are "almost" production compliant. > > Es. `DEBUG=False` or `SECURE_SSL_REDIRECT=True`. > @@ -39,6 +39,3 @@ Uses `./manage.py env` to check required (and optional) variables to put ### 3. Run upgrade to run migrations and initial setup ./manage.py upgrade - -# Troubleshoot -> Exception are logged in Sentry: https://sentry.io/unicef-jk/