From 353ca8b18943670ebda1a2b759a9e4686055c66c Mon Sep 17 00:00:00 2001 From: Justine Fricou Date: Thu, 14 Nov 2024 15:07:31 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20[BUG]=20Fix=20schema=5Frandonnee?= =?UTF-8?q?=20parser=20url=20update=20issues=20(#4022)=20-=20fix=20url=20b?= =?UTF-8?q?eing=20duplicated=20in=20description=20after=20new=20import=20i?= =?UTF-8?q?f=20description=20is=20null=20or=20was=20not=20updated=20-=20de?= =?UTF-8?q?scription=20and=20url=20are=20now=20parsed=20in=20a=20filter=5F?= =?UTF-8?q?description=20method=20rather=20than=20in=20the=20end=20method?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/changelog.rst | 1 + geotrek/trekking/parsers.py | 29 ++++++------- .../description_and_url.geojson | 24 +++++++++++ .../description_no_url.geojson | 24 +++++++++++ .../find_url_in_description.geojson | 40 ++++++++++++++++++ ....geojson => no_description_no_url.geojson} | 3 +- .../update_url_after.geojson | 41 +++++++++++++++++++ .../update_url_before.geojson | 41 +++++++++++++++++++ .../url_no_description.geojson | 24 +++++++++++ geotrek/trekking/tests/test_parsers.py | 30 +++++++++++++- 10 files changed, 240 insertions(+), 17 deletions(-) create mode 100644 geotrek/trekking/tests/data/schema_randonnee_parser/description_and_url.geojson create mode 100644 geotrek/trekking/tests/data/schema_randonnee_parser/description_no_url.geojson create mode 100644 geotrek/trekking/tests/data/schema_randonnee_parser/find_url_in_description.geojson rename geotrek/trekking/tests/data/schema_randonnee_parser/{no_url.geojson => no_description_no_url.geojson} (91%) create mode 100644 geotrek/trekking/tests/data/schema_randonnee_parser/update_url_after.geojson create mode 100644 geotrek/trekking/tests/data/schema_randonnee_parser/update_url_before.geojson create mode 100644 geotrek/trekking/tests/data/schema_randonnee_parser/url_no_description.geojson diff --git a/docs/changelog.rst b/docs/changelog.rst index 2b0fc2c2a5..3b77ed494d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ CHANGELOG **Bug fixes** - Fix missing Dockerfile path on make build scripts +- Fix SchemaRandonneeParser url update when description is null or was not updated (#4022) **Documentation** diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index e3eed00c19..92825d2101 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -1338,7 +1338,7 @@ class SchemaRandonneeParser(AttachmentParserMixin, Parser): 'arrival': 'arrivee', 'duration': 'duree', 'difficulty': 'difficulte', - 'description': 'instructions', + 'description': ('instructions', 'url'), 'ambiance': 'presentation', 'description_teaser': 'presentation_courte', 'advice': 'recommandations', @@ -1391,14 +1391,12 @@ class SchemaRandonneeParser(AttachmentParserMixin, Parser): 'attachments': 'medias', 'id_local': 'id_local', 'itineraire_parent': 'itineraire_parent', - 'url': 'url', } def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.related_treks_mapping = defaultdict(list) self.id_local_to_pk_mapping = {} - self.trek_urls = {} if self.url: response = self.request_or_retry(self.url) self.root = response.json() @@ -1453,6 +1451,19 @@ def filter_parking_location(self, src, val): except ValueError: self.add_warning(_("Bad value for parking geometry: should be a Point")) + def filter_description(self, src, val): + instructions, url = val + if not instructions and not url: + return None + description = "" + if instructions: + description += instructions + if instructions and url: + description += "\n\n" + if url: + description += f'{url}' + return description + def filter_attachments(self, src, val): """Handles images only""" if val is None: @@ -1548,10 +1559,6 @@ def get_or_create_license(self, license_label): self.add_warning(_("License '{val}' does not exist in Geotrek-Admin. Please add it").format(val=license_label)) return license - def save_url(self, src, val): - if val: - self.trek_urls[self.obj.pk] = val - def save_id_local(self, src, val): if val: self.id_local_to_pk_mapping[val] = self.obj.pk @@ -1562,14 +1569,8 @@ def save_itineraire_parent(self, src, val): def end(self): """ - After all treks have been created: - - concatenate the `url` field into the description - - add children + After all treks have been created, add parent/children relationships """ - for trek_pk, trek_url in self.trek_urls.items(): - trek = Trek.objects.get(pk=trek_pk) - trek.description = f'{trek.description}\n\n{trek_url}' - trek.save() for parent_id_local, child_pks in self.related_treks_mapping.items(): parent_pk = self.id_local_to_pk_mapping.get(parent_id_local) try: diff --git a/geotrek/trekking/tests/data/schema_randonnee_parser/description_and_url.geojson b/geotrek/trekking/tests/data/schema_randonnee_parser/description_and_url.geojson new file mode 100644 index 0000000000..b1c460ea3f --- /dev/null +++ b/geotrek/trekking/tests/data/schema_randonnee_parser/description_and_url.geojson @@ -0,0 +1,24 @@ +{ + "type": "FeatureCollection", + "name": "sql_statement", + "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, + "features": [ + { + "type": "Feature", + "properties": { + "id_local": "1", + "producteur": "Producer 1", + "nom_itineraire": "Trek 1", + "pratique": "pédestre", + "depart": "Departure 1", + "arrivee": "Arrival 1", + "instructions": "Instructions 1", + "url": "https://test.com" + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ 6.449592517966364, 44.733424655086957 ], [ 6.449539623508488, 44.733394939411369 ] ] + } + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/schema_randonnee_parser/description_no_url.geojson b/geotrek/trekking/tests/data/schema_randonnee_parser/description_no_url.geojson new file mode 100644 index 0000000000..f87f9b7f04 --- /dev/null +++ b/geotrek/trekking/tests/data/schema_randonnee_parser/description_no_url.geojson @@ -0,0 +1,24 @@ +{ + "type": "FeatureCollection", + "name": "sql_statement", + "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, + "features": [ + { + "type": "Feature", + "properties": { + "id_local": "1", + "producteur": "Producer 1", + "nom_itineraire": "Trek 1", + "pratique": "pédestre", + "depart": "Departure 1", + "arrivee": "Arrival 1", + "instructions": "Instructions 1", + "url": null + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ 6.449592517966364, 44.733424655086957 ], [ 6.449539623508488, 44.733394939411369 ] ] + } + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/schema_randonnee_parser/find_url_in_description.geojson b/geotrek/trekking/tests/data/schema_randonnee_parser/find_url_in_description.geojson new file mode 100644 index 0000000000..bc934850df --- /dev/null +++ b/geotrek/trekking/tests/data/schema_randonnee_parser/find_url_in_description.geojson @@ -0,0 +1,40 @@ +{ + "type": "FeatureCollection", + "name": "sql_statement", + "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, + "features": [ + { + "type": "Feature", + "properties": { + "id_local": "1", + "producteur": "Producer 1", + "url": "https://test.com", + "nom_itineraire": "Trek 1", + "pratique": "pédestre", + "depart": "Departure 1", + "arrivee": "Arrival 1", + "instructions": "Instructions 1" + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ 6.449592517966364, 44.733424655086957 ], [ 6.449539623508488, 44.733394939411369 ] ] + } + }, + { + "type": "Feature", + "properties": { + "id_local": "2", + "producteur": "Producer 1", + "nom_itineraire": "Trek 1", + "pratique": "pédestre", + "depart": "Departure 1", + "arrivee": "Arrival 1", + "instructions": "Instructions 2" + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ 6.449592517966364, 44.733424655086957 ], [ 6.449539623508488, 44.733394939411369 ] ] + } + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/schema_randonnee_parser/no_url.geojson b/geotrek/trekking/tests/data/schema_randonnee_parser/no_description_no_url.geojson similarity index 91% rename from geotrek/trekking/tests/data/schema_randonnee_parser/no_url.geojson rename to geotrek/trekking/tests/data/schema_randonnee_parser/no_description_no_url.geojson index 8d2d4a69fc..250dde234f 100644 --- a/geotrek/trekking/tests/data/schema_randonnee_parser/no_url.geojson +++ b/geotrek/trekking/tests/data/schema_randonnee_parser/no_description_no_url.geojson @@ -12,7 +12,8 @@ "pratique": "pédestre", "depart": "Departure 1", "arrivee": "Arrival 1", - "instructions": "Instructions 1" + "instructions": null, + "url": null }, "geometry": { "type": "LineString", diff --git a/geotrek/trekking/tests/data/schema_randonnee_parser/update_url_after.geojson b/geotrek/trekking/tests/data/schema_randonnee_parser/update_url_after.geojson new file mode 100644 index 0000000000..11faa1780a --- /dev/null +++ b/geotrek/trekking/tests/data/schema_randonnee_parser/update_url_after.geojson @@ -0,0 +1,41 @@ +{ + "type": "FeatureCollection", + "name": "sql_statement", + "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, + "features": [ + { + "type": "Feature", + "properties": { + "id_local": "1", + "producteur": "Producer 1", + "url": "https://test.com", + "nom_itineraire": "Trek 1", + "pratique": "pédestre", + "depart": "Departure 1", + "arrivee": "Arrival 1", + "instructions": null + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ 6.449592517966364, 44.733424655086957 ], [ 6.449539623508488, 44.733394939411369 ] ] + } + }, + { + "type": "Feature", + "properties": { + "id_local": "2", + "producteur": "Producer 1", + "url": "https://test2.com", + "nom_itineraire": "Trek 2", + "pratique": "pédestre", + "depart": "Departure 1", + "arrivee": "Arrival 1", + "instructions": "Instructions 2" + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ 6.449592517966364, 44.733424655086957 ], [ 6.449539623508488, 44.733394939411369 ] ] + } + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/schema_randonnee_parser/update_url_before.geojson b/geotrek/trekking/tests/data/schema_randonnee_parser/update_url_before.geojson new file mode 100644 index 0000000000..5b232be4ff --- /dev/null +++ b/geotrek/trekking/tests/data/schema_randonnee_parser/update_url_before.geojson @@ -0,0 +1,41 @@ +{ + "type": "FeatureCollection", + "name": "sql_statement", + "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, + "features": [ + { + "type": "Feature", + "properties": { + "id_local": "1", + "producteur": "Producer 1", + "url": "https://test.com", + "nom_itineraire": "Trek 1", + "pratique": "pédestre", + "depart": "Departure 1", + "arrivee": "Arrival 1", + "instructions": "Instructions 1" + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ 6.449592517966364, 44.733424655086957 ], [ 6.449539623508488, 44.733394939411369 ] ] + } + }, + { + "type": "Feature", + "properties": { + "id_local": "2", + "producteur": "Producer 1", + "url": "https://test1.com", + "nom_itineraire": "Trek 2", + "pratique": "pédestre", + "depart": "Departure 1", + "arrivee": "Arrival 1", + "instructions": "Instructions 2" + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ 6.449592517966364, 44.733424655086957 ], [ 6.449539623508488, 44.733394939411369 ] ] + } + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/schema_randonnee_parser/url_no_description.geojson b/geotrek/trekking/tests/data/schema_randonnee_parser/url_no_description.geojson new file mode 100644 index 0000000000..b4d6978506 --- /dev/null +++ b/geotrek/trekking/tests/data/schema_randonnee_parser/url_no_description.geojson @@ -0,0 +1,24 @@ +{ + "type": "FeatureCollection", + "name": "sql_statement", + "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, + "features": [ + { + "type": "Feature", + "properties": { + "id_local": "1", + "producteur": "Producer 1", + "nom_itineraire": "Trek 1", + "pratique": "pédestre", + "depart": "Departure 1", + "arrivee": "Arrival 1", + "instructions": null, + "url": "https://test.com" + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ 6.449592517966364, 44.733424655086957 ], [ 6.449539623508488, 44.733394939411369 ] ] + } + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index a214f588c1..be5e62b70b 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -2030,8 +2030,34 @@ def test_incorrect_parking_location(self): self.assertIsNone(trek.parking_location) self.assertIn("Bad value for parking geometry: should be a Point", output.getvalue()) - def test_no_url(self): - self.call_import_command_with_file('no_url.geojson') + def test_description_and_url(self): + self.call_import_command_with_file('description_and_url.geojson') + self.assertEqual(Trek.objects.count(), 1) + trek = Trek.objects.get() + self.assertEqual(trek.description, 'Instructions 1\n\nhttps://test.com') + + def test_description_no_url(self): + self.call_import_command_with_file('description_no_url.geojson') self.assertEqual(Trek.objects.count(), 1) trek = Trek.objects.get() self.assertEqual(trek.description, 'Instructions 1') + + def test_url_no_description(self): + self.call_import_command_with_file('url_no_description.geojson') + self.assertEqual(Trek.objects.count(), 1) + trek = Trek.objects.get() + self.assertEqual(trek.description, 'https://test.com') + + def test_no_description_no_url(self): + self.call_import_command_with_file('no_description_no_url.geojson') + self.assertEqual(Trek.objects.count(), 1) + trek = Trek.objects.get() + self.assertEqual(trek.description, '') + + def test_update_url(self): + self.call_import_command_with_file('update_url_before.geojson') + self.call_import_command_with_file('update_url_after.geojson') + trek1 = Trek.objects.get(eid="1") + self.assertEqual(trek1.description, "https://test.com") + trek2 = Trek.objects.get(eid="2") + self.assertEqual(trek2.description, "Instructions 2\n\nhttps://test2.com")