updates ops

.github/workflows/lint.yml

@@ -68,7 +68,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - name: Install requirements
         run: pip install isort
       - name: iSort
@@ -81,21 +81,21 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - name: Install requirements
         run: pip install black
       - name: Black
         run: black src/ --check
-  bandit:
-    needs: changes
-    runs-on: ubuntu-latest
-#    if: github.event.pull_request.draft == false && steps.changes.outputs.lint
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.9'
-      - name: Install requirements
-        run: pip install bandit
-      - name: bandit
-        run: bandit src/
+#  bandit:
+#    needs: changes
+#    runs-on: ubuntu-latest
+##    if: github.event.pull_request.draft == false && steps.changes.outputs.lint
+#    steps:
+#      - uses: actions/checkout@v2
+#      - uses: actions/setup-python@v2
+#        with:
+#          python-version: '3.11'
+#      - name: Install requirements
+#        run: pip install bandit
+#      - name: bandit
+#        run: bandit src/

src/hope_dedup_engine/__init__.py

@@ -1,6 +1,5 @@
 from hope_dedup_engine.config.celery import app as celery_app
 
-
 VERSION = __version__ = "0.1.0"
 
 __all__ = ("celery_app",)

src/hope_dedup_engine/apps/api/admin.py

@@ -1,6 +1,11 @@
 from django.contrib import admin
 
-from hope_dedup_engine.apps.api.models import DeduplicationSet, Duplicate, HDEToken, Image
+from hope_dedup_engine.apps.api.models import (
+    DeduplicationSet,
+    Duplicate,
+    HDEToken,
+    Image,
+)
 
 admin.site.register(DeduplicationSet)
 admin.site.register(Duplicate)

src/hope_dedup_engine/apps/api/auth.py

@@ -14,7 +14,9 @@ def has_permission(self, request: Request, view: View) -> bool:
 
 class UserAndDeduplicationSetAreOfTheSameSystem(BasePermission):
     def has_permission(self, request: Request, view: View) -> bool:
-        if deduplication_set_pk := view.kwargs.get("deduplication_set_pk") or view.kwargs.get("pk"):
+        if deduplication_set_pk := view.kwargs.get(
+            "deduplication_set_pk"
+        ) or view.kwargs.get("pk"):
             return DeduplicationSet.objects.filter(
                 external_system=request.user.external_system, pk=deduplication_set_pk
             ).exists()

src/hope_dedup_engine/apps/api/models/__init__.py

@@ -1,2 +1,6 @@
 from hope_dedup_engine.apps.api.models.auth import HDEToken  # noqa: F401
-from hope_dedup_engine.apps.api.models.deduplication import DeduplicationSet, Duplicate, Image  # noqa: F401
+from hope_dedup_engine.apps.api.models.deduplication import (  # noqa: F401
+    DeduplicationSet,
+    Duplicate,
+    Image,
+)

src/hope_dedup_engine/apps/api/models/auth.py

@@ -5,4 +5,6 @@
 
 
 class HDEToken(Token):
-    user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name="auth_tokens", on_delete=models.CASCADE)
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL, related_name="auth_tokens", on_delete=models.CASCADE
+    )

src/hope_dedup_engine/apps/api/models/deduplication.py

@@ -11,7 +11,10 @@
 class DeduplicationSet(models.Model):
     class State(models.IntegerChoices):
         CLEAN = 0, "Clean"  # Deduplication set is created or already processed
-        DIRTY = 1, "Dirty"  # Images are added to deduplication set, but not yet processed
+        DIRTY = (
+            1,
+            "Dirty",
+        )  # Images are added to deduplication set, but not yet processed
         PROCESSING = 2, "Processing"  # Images are being processed
         ERROR = 3, "Error"  # Error occurred
 
@@ -26,11 +29,19 @@ class State(models.IntegerChoices):
     external_system = models.ForeignKey(ExternalSystem, on_delete=models.CASCADE)
     error = models.CharField(max_length=255, null=True, blank=True)
     created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+"
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+        related_name="+",
     )
     created_at = models.DateTimeField(auto_now_add=True)
     updated_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+"
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+        related_name="+",
     )
     updated_at = models.DateTimeField(auto_now=True)
     notification_url = models.CharField(max_length=255, null=True, blank=True)
@@ -42,7 +53,11 @@ class Image(models.Model):
     reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH)
     filename = models.CharField(max_length=255)
     created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+"
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+        related_name="+",
     )
     created_at = models.DateTimeField(auto_now_add=True)
 

src/hope_dedup_engine/apps/api/serializers.py

@@ -10,7 +10,14 @@ class DeduplicationSetSerializer(serializers.ModelSerializer):
     class Meta:
         model = DeduplicationSet
         exclude = ("deleted",)
-        read_only_fields = "external_system", "created_at", "created_by", "deleted", "updated_at", "updated_by"
+        read_only_fields = (
+            "external_system",
+            "created_at",
+            "created_by",
+            "deleted",
+            "updated_at",
+            "updated_by",
+        )
 
 
 class ImageSerializer(serializers.ModelSerializer):

src/hope_dedup_engine/apps/api/urls.py

@@ -10,14 +10,30 @@
     DUPLICATE_LIST,
     IMAGE_LIST,
 )
-from hope_dedup_engine.apps.api.views import BulkImageViewSet, DeduplicationSetViewSet, DuplicateViewSet, ImageViewSet
+from hope_dedup_engine.apps.api.views import (
+    BulkImageViewSet,
+    DeduplicationSetViewSet,
+    DuplicateViewSet,
+    ImageViewSet,
+)
 
 router = routers.SimpleRouter()
-router.register(DEDUPLICATION_SET_LIST, DeduplicationSetViewSet, basename=DEDUPLICATION_SET_LIST)
+router.register(
+    DEDUPLICATION_SET_LIST, DeduplicationSetViewSet, basename=DEDUPLICATION_SET_LIST
+)
 
-deduplication_sets_router = nested_routers.NestedSimpleRouter(router, DEDUPLICATION_SET_LIST, lookup=DEDUPLICATION_SET)
+deduplication_sets_router = nested_routers.NestedSimpleRouter(
+    router, DEDUPLICATION_SET_LIST, lookup=DEDUPLICATION_SET
+)
 deduplication_sets_router.register(IMAGE_LIST, ImageViewSet, basename=IMAGE_LIST)
-deduplication_sets_router.register(BULK_IMAGE_LIST, BulkImageViewSet, basename=BULK_IMAGE_LIST)
-deduplication_sets_router.register(DUPLICATE_LIST, DuplicateViewSet, basename=DUPLICATE_LIST)
+deduplication_sets_router.register(
+    BULK_IMAGE_LIST, BulkImageViewSet, basename=BULK_IMAGE_LIST
+)
+deduplication_sets_router.register(
+    DUPLICATE_LIST, DuplicateViewSet, basename=DUPLICATE_LIST
+)
 
-urlpatterns = [path("", include(router.urls)), path("", include(deduplication_sets_router.urls))]
+urlpatterns = [
+    path("", include(router.urls)),
+    path("", include(deduplication_sets_router.urls)),
+]

src/hope_dedup_engine/apps/api/views.py

@@ -18,10 +18,17 @@
     HDETokenAuthentication,
     UserAndDeduplicationSetAreOfTheSameSystem,
 )
-from hope_dedup_engine.apps.api.const import DEDUPLICATION_SET_FILTER, DEDUPLICATION_SET_PARAM
+from hope_dedup_engine.apps.api.const import (
+    DEDUPLICATION_SET_FILTER,
+    DEDUPLICATION_SET_PARAM,
+)
 from hope_dedup_engine.apps.api.models import DeduplicationSet
 from hope_dedup_engine.apps.api.models.deduplication import Duplicate, Image
-from hope_dedup_engine.apps.api.serializers import DeduplicationSetSerializer, DuplicateSerializer, ImageSerializer
+from hope_dedup_engine.apps.api.serializers import (
+    DeduplicationSetSerializer,
+    DuplicateSerializer,
+    ImageSerializer,
+)
 from hope_dedup_engine.apps.api.utils import delete_model_data, start_processing
 
 MESSAGE = "message"
@@ -31,17 +38,29 @@
 
 
 class DeduplicationSetViewSet(
-    mixins.ListModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, viewsets.GenericViewSet
+    mixins.ListModelMixin,
+    mixins.CreateModelMixin,
+    mixins.DestroyModelMixin,
+    viewsets.GenericViewSet,
 ):
     authentication_classes = (HDETokenAuthentication,)
-    permission_classes = IsAuthenticated, AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem
+    permission_classes = (
+        IsAuthenticated,
+        AssignedToExternalSystem,
+        UserAndDeduplicationSetAreOfTheSameSystem,
+    )
     serializer_class = DeduplicationSetSerializer
 
     def get_queryset(self) -> QuerySet:
-        return DeduplicationSet.objects.filter(external_system=self.request.user.external_system, deleted=False)
+        return DeduplicationSet.objects.filter(
+            external_system=self.request.user.external_system, deleted=False
+        )
 
     def perform_create(self, serializer: Serializer) -> None:
-        serializer.save(created_by=self.request.user, external_system=self.request.user.external_system)
+        serializer.save(
+            created_by=self.request.user,
+            external_system=self.request.user.external_system,
+        )
 
     def perform_destroy(self, instance: DeduplicationSet) -> None:
         instance.updated_by = self.request.user
@@ -65,7 +84,9 @@ def process(self, request: Request, pk: UUID | None = None) -> Response:
                 self._start_processing(deduplication_set)
                 return Response({MESSAGE: STARTED})
             case DeduplicationSet.State.PROCESSING:
-                return Response({MESSAGE: ALREADY_PROCESSING}, status=status.HTTP_400_BAD_REQUEST)
+                return Response(
+                    {MESSAGE: ALREADY_PROCESSING}, status=status.HTTP_400_BAD_REQUEST
+                )
 
 
 class ImageViewSet(
@@ -76,7 +97,11 @@ class ImageViewSet(
     viewsets.GenericViewSet,
 ):
     authentication_classes = (HDETokenAuthentication,)
-    permission_classes = IsAuthenticated, AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem
+    permission_classes = (
+        IsAuthenticated,
+        AssignedToExternalSystem,
+        UserAndDeduplicationSetAreOfTheSameSystem,
+    )
     serializer_class = ImageSerializer
     queryset = Image.objects.all()
     parent_lookup_kwargs = {
@@ -108,14 +133,18 @@ def __setitem__(self, key: str, value: Any) -> None:
 
 
 class WrapRequestDataMixin:
-    def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request:
+    def initialize_request(
+        self, request: Request, *args: Any, **kwargs: Any
+    ) -> Request:
         request = super().initialize_request(request, *args, **kwargs)
         request._full_data = ListDataWrapper(request.data)
         return request
 
 
 class UnwrapRequestDataMixin:
-    def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request:
+    def initialize_request(
+        self, request: Request, *args: Any, **kwargs: Any
+    ) -> Request:
         request = super().initialize_request(request, *args, **kwargs)
         request._full_data = request._full_data.data
         return request
@@ -131,7 +160,11 @@ class BulkImageViewSet(
     viewsets.GenericViewSet,
 ):
     authentication_classes = (HDETokenAuthentication,)
-    permission_classes = IsAuthenticated, AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem
+    permission_classes = (
+        IsAuthenticated,
+        AssignedToExternalSystem,
+        UserAndDeduplicationSetAreOfTheSameSystem,
+    )
     serializer_class = ImageSerializer
     queryset = Image.objects.all()
     parent_lookup_kwargs = {
@@ -143,7 +176,9 @@ def get_serializer(self, *args: Any, **kwargs: Any) -> Serializer:
 
     def perform_create(self, serializer: Serializer) -> None:
         super().perform_create(serializer)
-        if deduplication_set := serializer.instance[0].deduplication_set if serializer.instance else None:
+        if deduplication_set := (
+            serializer.instance[0].deduplication_set if serializer.instance else None
+        ):
             deduplication_set.updated_by = self.request.user
             deduplication_set.save()
 
@@ -156,9 +191,15 @@ def clear(self, request: Request, deduplication_set_pk: str) -> Response:
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class DuplicateViewSet(nested_viewsets.NestedViewSetMixin, mixins.ListModelMixin, viewsets.GenericViewSet):
+class DuplicateViewSet(
+    nested_viewsets.NestedViewSetMixin, mixins.ListModelMixin, viewsets.GenericViewSet
+):
     authentication_classes = (HDETokenAuthentication,)
-    permission_classes = IsAuthenticated, AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem
+    permission_classes = (
+        IsAuthenticated,
+        AssignedToExternalSystem,
+        UserAndDeduplicationSetAreOfTheSameSystem,
+    )
     serializer_class = DuplicateSerializer
     queryset = Duplicate.objects.all()
     parent_lookup_kwargs = {

src/hope_dedup_engine/apps/core/management/commands/createsystem.py

@@ -10,7 +10,9 @@ def add_arguments(self, parser):
         parser.add_argument("name")
 
     def handle(self, *args, **options):
-        system, created = ExternalSystem.objects.get_or_create(name=(name := options["name"]))
+        system, created = ExternalSystem.objects.get_or_create(
+            name=(name := options["name"])
+        )
         if created:
             self.stdout.write(self.style.SUCCESS(f'"{name}" system created.'))
         else:

src/hope_dedup_engine/apps/core/management/commands/env.py

@@ -33,14 +33,26 @@ def add_arguments(self, parser: "CommandParser") -> None:
             default="export {key}={value}",
             help="Check env for variable availability (default: 'export {key}=\"{value}\"')",
         )
-        parser.add_argument("--develop", action="store_true", help="Display development values")
-        parser.add_argument("--config", action="store_true", help="Only list changed values")
+        parser.add_argument(
+            "--develop", action="store_true", help="Display development values"
+        )
+        parser.add_argument(
+            "--config", action="store_true", help="Only list changed values"
+        )
         parser.add_argument("--diff", action="store_true", help="Mark changed values")
         parser.add_argument(
-            "--check", action="store_true", dest="check", default=False, help="Check env for variable availability"
+            "--check",
+            action="store_true",
+            dest="check",
+            default=False,
+            help="Check env for variable availability",
         )
         parser.add_argument(
-            "--ignore-errors", action="store_true", dest="ignore_errors", default=False, help="Do not fail"
+            "--ignore-errors",
+            action="store_true",
+            dest="ignore_errors",
+            default=False,
+            help="Do not fail",
         )
 
     def handle(self, *args: "Any", **options: "Any") -> None:
@@ -62,7 +74,9 @@ def handle(self, *args: "Any", **options: "Any") -> None:
                 else:
                     value: Any = env.get_value(k)
 
-                line: str = pattern.format(key=k, value=clean(value), help=help, default=default)
+                line: str = pattern.format(
+                    key=k, value=clean(value), help=help, default=default
+                )
                 if options["diff"]:
                     if value != default:
                         line = self.style.SUCCESS(line)