Merge pull request #151 from ParkenDD/handle-issues-at-duplicates-mat…

…ching-bugfixes

fix: handle issues at duplicate matching, migration bugfix

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## Version 0.4.2
+
+Released 2024-05-28
+
+### Fixes
+
+* Fixes issue in database migration downgrade leading to database not matching the model
+* Catches and logs an issue with invalid data at duplicate distance calculation
+
+
 ## Version 0.4.1
 
 Released 2024-05-16

README.md

@@ -102,7 +102,7 @@ If you want to create new data sources, please have a look at
 ### Using the push command line interface
 
 In order to test push tasks or to upload files you got per e-mail, there is an upload script included in this
-repository. It's located at `/push-client/push-client.py`. You need python requests for this, please have a look at
+repository. It's located at `/scripts/push-client.py`. You need python requests for this, please have a look at
 "Prepare scripts environment" for preparations. You can use the script using:
 
 ```bash

migrations/versions/2024-05-14-06-46-51_9ad8cd0f3b0d_photo_url_duplicate_id_new_types_tags.py

@@ -107,11 +107,6 @@ def downgrade():
     sa.Enum(*new_parking_site_types, name='_parkingsitetype').drop(op.get_bind())
 
     with op.batch_alter_table('parking_site', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('is_supervised', sa.Boolean(), autoincrement=False, nullable=True))
-        batch_op.drop_column('related_location')
-        batch_op.drop_column('supervision_type')
-        batch_op.drop_column('is_covered')
-        batch_op.drop_column('purpose')
         batch_op.alter_column(
             'type',
             existing_type=sa.Enum(*new_parking_site_types, name='parkingsitetype'),

webapp/common/logging/models.py

@@ -15,6 +15,7 @@ class LogMessageType(Enum):
     EXCEPTION = 'exception'
     FAILED_SOURCE_HANDLING = 'failed-source-handling'
     FAILED_PARKING_SITE_HANDLING = 'failed-parking-site-handling'
+    DUPLICATE_HANDLING = 'duplicate-handling'
     MISC = 'misc'
 
 

webapp/services/matching_service/matching_service.py

@@ -7,6 +7,7 @@
 from decimal import Decimal
 from math import acos, cos, sin
 
+from webapp.common.logging.models import LogMessageType
 from webapp.models import ParkingSite
 from webapp.repositories import ParkingSiteRepository
 from webapp.repositories.parking_site_repository import ParkingSiteLocation
@@ -52,7 +53,15 @@ def generate_duplicates(self, existing_matches: list[tuple[int, int]]) -> list[D
                     continue
 
                 # If distance is over match radius: ignore possible match
-                if self.distance(parking_site_locations[i], parking_site_locations[j]) > match_radius:
+                try:
+                    if self.distance(parking_site_locations[i], parking_site_locations[j]) > match_radius:
+                        continue
+                # Ignore (and log) invalid data at distance calculations (eg 'math domain error')
+                except ValueError:
+                    self.logger.warning(
+                        LogMessageType.DUPLICATE_HANDLING,
+                        f'Cannot calculate distance between {parking_site_locations[i]} and {parking_site_locations[j]}',
+                    )
                     continue
 
                 matches.append((parking_site_locations[i], parking_site_locations[j]))