From 347b9a2c9a769a5a2d9058221afab7860fa44d44 Mon Sep 17 00:00:00 2001 From: Sergey Misuk Date: Mon, 16 Sep 2024 14:01:14 +0300 Subject: [PATCH] Make lock optional --- .../apps/api/deduplication/lock.py | 1 + .../apps/api/deduplication/process.py | 22 ++++++++++++++----- src/hope_dedup_engine/apps/api/utils.py | 9 +++++++- src/hope_dedup_engine/config/settings.py | 1 + 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/hope_dedup_engine/apps/api/deduplication/lock.py b/src/hope_dedup_engine/apps/api/deduplication/lock.py index ab19b40a..9262b33a 100644 --- a/src/hope_dedup_engine/apps/api/deduplication/lock.py +++ b/src/hope_dedup_engine/apps/api/deduplication/lock.py @@ -12,6 +12,7 @@ from hope_dedup_engine.apps.api.models import DeduplicationSet DELIMITER: Final[str] = "|" +LOCK_IS_NOT_ENABLED = "LOCK_IS_NOT_ENABLED" class DeduplicationSetLock: diff --git a/src/hope_dedup_engine/apps/api/deduplication/process.py b/src/hope_dedup_engine/apps/api/deduplication/process.py index dfa68a18..bef9e030 100644 --- a/src/hope_dedup_engine/apps/api/deduplication/process.py +++ b/src/hope_dedup_engine/apps/api/deduplication/process.py @@ -8,6 +8,7 @@ ) from hope_dedup_engine.apps.api.models import DeduplicationSet, Duplicate from hope_dedup_engine.apps.api.utils import send_notification +from hope_dedup_engine.config import settings def _sort_keys(pair: DuplicateKeyPair) -> DuplicateKeyPair: @@ -19,6 +20,7 @@ def _save_duplicates( finder: DuplicateFinder, deduplication_set: DeduplicationSet, ignored_key_pairs: frozenset[tuple[str, str]], + lock_enabled: bool, lock: DeduplicationSetLock, ) -> None: for first, second, score in map(_sort_keys, finder.run()): @@ -30,7 +32,8 @@ def _save_duplicates( ) duplicate.score += score * finder.weight duplicate.save() - lock.refresh() + if lock_enabled: + lock.refresh() HOUR = 60 * 60 @@ -40,10 +43,14 @@ def _save_duplicates( def find_duplicates(deduplication_set_id: str, serialized_lock: str) -> None: deduplication_set = DeduplicationSet.objects.get(pk=deduplication_set_id) try: - lock = DeduplicationSetLock.from_string(serialized_lock) + lock_enabled = settings.DEDUPLICATION_SET_LOCK_ENABLED + lock = ( + DeduplicationSetLock.from_string(serialized_lock) if lock_enabled else None + ) - # refresh lock in case we spent much time waiting in queue - lock.refresh() + if lock_enabled: + # refresh lock in case we spent much time waiting in queue + lock.refresh() # clean results Duplicate.objects.filter(deduplication_set=deduplication_set).delete() @@ -56,7 +63,9 @@ def find_duplicates(deduplication_set_id: str, serialized_lock: str) -> None: weight_total = 0 for finder in get_finders(deduplication_set): - _save_duplicates(finder, deduplication_set, ignored_key_pairs, lock) + _save_duplicates( + finder, deduplication_set, ignored_key_pairs, lock_enabled, lock + ) weight_total += finder.weight for duplicate in deduplication_set.duplicate_set.all(): @@ -66,7 +75,8 @@ def find_duplicates(deduplication_set_id: str, serialized_lock: str) -> None: deduplication_set.state = deduplication_set.State.CLEAN deduplication_set.save() - lock.release() + if lock_enabled: + lock.release() except Exception as e: deduplication_set.state = DeduplicationSet.State.ERROR diff --git a/src/hope_dedup_engine/apps/api/utils.py b/src/hope_dedup_engine/apps/api/utils.py index 93cb04db..e19d1e7f 100644 --- a/src/hope_dedup_engine/apps/api/utils.py +++ b/src/hope_dedup_engine/apps/api/utils.py @@ -1,7 +1,10 @@ +from django.conf import settings + import requests from rest_framework import status from rest_framework.exceptions import APIException +from hope_dedup_engine.apps.api.deduplication.lock import LOCK_IS_NOT_ENABLED from hope_dedup_engine.apps.api.models import DeduplicationSet @@ -16,7 +19,11 @@ def start_processing(deduplication_set: DeduplicationSet) -> None: from hope_dedup_engine.apps.api.deduplication.process import find_duplicates try: - lock = DeduplicationSetLock.for_deduplication_set(deduplication_set) + lock = ( + DeduplicationSetLock.for_deduplication_set(deduplication_set) + if settings.DEDUPLICATION_SET_LOCK_ENABLED + else LOCK_IS_NOT_ENABLED + ) deduplication_set.state = DeduplicationSet.State.PROCESSING deduplication_set.save() find_duplicates.delay(str(deduplication_set.pk), str(lock)) diff --git a/src/hope_dedup_engine/config/settings.py b/src/hope_dedup_engine/config/settings.py index 4878df4a..86335cd9 100644 --- a/src/hope_dedup_engine/config/settings.py +++ b/src/hope_dedup_engine/config/settings.py @@ -193,6 +193,7 @@ EMAIL_PORT = env("EMAIL_PORT", default=25) EMAIL_USE_TLS = env("EMAIL_USE_TLS", default=False) EMAIL_USE_SSL = env("EMAIL_USE_SSL", default=False) +DEDUPLICATION_SET_LOCK_ENABLED = env("DEDUPLICATION_SET_LOCK_ENABLED", default=False) DEDUPLICATION_SET_LAST_ACTION_TIMEOUT = env( "DEDUPLICATION_SET_LAST_ACTION_TIMEOUT", default=60 )