diff --git a/docs/changelog.rst b/docs/changelog.rst index 4ecc5145f8..3a6b820008 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,20 +5,26 @@ CHANGELOG 2.101.5+dev (XXXX-XX-XX) ------------------------ +**Documentation** + +- Improve performance in spatial intersection (zoning district and zoning city) for sql views (#3600) + +**New features** + +- Add UUIDs of parent and children ``Courses`` and ``Sites`` in APIv2 (#3569) + **Improvements** +- Add missing translations for fields on ``Courses`` and ``Sites`` in APIv2 (#3569) - Allow Apidae Trek parser to handle traces not in utf-8 -**Documentation** - -- Improve performance in spatial intersection (zoning district and zoning city) for sql views (#3600) 2.101.5 (2024-01-11) -------------------- **New features** --Land: Add ``CirculationEdge`` model to manage circulation types and authorization types in the land module (#3578) +- Land: Add ``CirculationEdge`` model to manage circulation types and authorization types in the land module (#3578) - Generalize``AccessMean`` model and add field ``access`` to ``Intervention`` (#3819) **Improvements** diff --git a/geotrek/api/tests/test_v2.py b/geotrek/api/tests/test_v2.py index 976177bb9e..bad2747d2b 100644 --- a/geotrek/api/tests/test_v2.py +++ b/geotrek/api/tests/test_v2.py @@ -158,10 +158,10 @@ RESERVATION_SYSTEM_PROPERTIES_JSON_STRUCTURE = sorted(['name', 'id']) SITE_PROPERTIES_JSON_STRUCTURE = sorted([ - 'accessibility', 'advice', 'ambiance', 'attachments', 'children', 'cities', 'courses', 'description', 'description_teaser', 'districts', 'eid', - 'geometry', 'id', 'information_desks', 'labels', 'managers', 'name', 'orientation', 'parent', 'period', 'portal', - 'practice', 'provider', 'pdf', 'ratings', 'sector', 'source', 'structure', 'themes', 'type', 'url', 'uuid', - 'view_points', 'wind', 'web_links' + 'accessibility', 'advice', 'ambiance', 'attachments', 'children', 'children_uuids', 'cities', 'courses', 'courses_uuids', 'description', + 'description_teaser', 'districts', 'eid', 'geometry', 'id', 'information_desks', 'labels', 'managers', 'name', 'orientation', 'parent', + 'parent_uuid', 'period', 'portal', 'practice', 'provider', 'pdf', 'ratings', 'sector', 'source', 'structure', 'themes', 'type', 'url', 'uuid', + 'view_points', 'published', 'wind', 'web_links' ]) OUTDOORPRACTICE_PROPERTIES_JSON_STRUCTURE = sorted(['id', 'name', 'sector', 'pictogram']) @@ -186,9 +186,9 @@ COURSE_PROPERTIES_JSON_STRUCTURE = sorted([ 'accessibility', 'advice', 'cities', 'description', 'districts', 'eid', 'equipment', 'geometry', 'height', 'id', - 'length', 'name', 'ratings', 'ratings_description', 'sites', 'structure', - 'type', 'url', 'attachments', 'max_elevation', 'min_elevation', 'parents', 'provider', - 'pdf', 'points_reference', 'children', 'duration', 'gear', 'uuid' + 'length', 'name', 'ratings', 'ratings_description', 'sites', 'sites_uuids', 'structure', + 'type', 'url', 'attachments', 'max_elevation', 'min_elevation', 'parents', 'parents_uuids', 'provider', + 'pdf', 'points_reference', 'published', 'children', 'children_uuids', 'duration', 'gear', 'uuid' ]) COURSETYPE_PROPERTIES_JSON_STRUCTURE = sorted(['id', 'name', 'practice']) @@ -4429,6 +4429,11 @@ def test_site_children_published_serializing(self): self.assertIn(self.site_leaf_published.pk, children) self.assertIn(self.site_leaf_published_2.pk, children) self.assertNotIn(self.site_leaf_unpublished.pk, children) + children_uuids = response.json()['children_uuids'] + self.assertEqual(2, len(children_uuids)) + self.assertIn(str(self.site_leaf_published.uuid), children_uuids) + self.assertIn(str(self.site_leaf_published_2.uuid), children_uuids) + self.assertNotIn(str(self.site_leaf_unpublished.uuid), children_uuids) def test_site_parent_unpublished_serializing(self): response = self.get_site_detail(self.site_node_parent_unpublished.pk) @@ -4448,8 +4453,15 @@ def test_site_parent_and_children_serializing_by_lang(self): self.assertIn(self.site_leaf_published_fr.pk, children) self.assertNotIn(self.site_leaf_published_not_fr.pk, children) self.assertNotIn(self.site_leaf_unpublished_fr.pk, children) + children_uuids = site_published_fr['children_uuids'] + self.assertEqual(1, len(children_uuids)) + self.assertIn(str(self.site_leaf_published_fr.uuid), children_uuids) + self.assertNotIn(str(self.site_leaf_published_not_fr.uuid), children_uuids) + self.assertNotIn(str(self.site_leaf_unpublished_fr.uuid), children_uuids) parent = site_published_fr['parent'] + parent_uuid = site_published_fr['parent_uuid'] self.assertEqual(parent, self.site_root_fr.pk) + self.assertEqual(parent_uuid, str(self.site_root_fr.uuid)) class OutdoorFilterByPracticesTestCase(BaseApiTest): @@ -4509,6 +4521,11 @@ def setUpTestData(cls): cls.course = outdoor_factory.CourseFactory() cls.course.parent_sites.set([cls.site.pk]) cls.course2 = outdoor_factory.CourseFactory() + cls.course3 = outdoor_factory.CourseFactory() + cls.course4 = outdoor_factory.CourseFactory() + outdoor_models.OrderedCourseChild.objects.create(parent=cls.course, child=cls.course2) + outdoor_models.OrderedCourseChild.objects.create(parent=cls.course, child=cls.course4, order=1) + outdoor_models.OrderedCourseChild.objects.create(parent=cls.course3, child=cls.course) cls.information_desk = tourism_factory.InformationDeskFactory() cls.site.information_desks.set([cls.information_desk]) @@ -4522,6 +4539,44 @@ def test_filter_courses_by_portal(self): self.assertIn(self.course.pk, all_ids) self.assertNotIn(self.course2.pk, all_ids) + def test_course_serialized_parent_site_and_related_courses(self): + response = self.get_course_detail(self.course.pk) + self.assertEqual(response.status_code, 200) + parent_sites_pk = response.json()['sites'] + parent_sites_uuid = response.json()['sites_uuids'] + parent_courses_uuid = response.json()['parents_uuids'] + parent_courses_pk = response.json()['parents'] + children_courses_uuid = response.json()['children_uuids'] + children_courses_pk = response.json()['children'] + self.assertEqual(parent_sites_pk, [self.site.pk]) + self.assertEqual(parent_sites_uuid, [str(self.site.uuid)]) + self.assertEqual(parent_courses_pk, [self.course3.pk]) + self.assertEqual(parent_courses_uuid, [str(self.course3.uuid)]) + self.assertEqual(children_courses_pk, [self.course2.pk, self.course4.pk]) + self.assertEqual(children_courses_uuid, [str(self.course2.uuid), str(self.course4.uuid)]) + response = self.get_course_detail(self.course.pk, params={'language': 'en'}) + self.assertEqual(response.status_code, 200) + parent_sites_pk = response.json()['sites'] + parent_sites_uuid = response.json()['sites_uuids'] + parent_courses_uuid = response.json()['parents_uuids'] + parent_courses_pk = response.json()['parents'] + children_courses_uuid = response.json()['children_uuids'] + children_courses_pk = response.json()['children'] + self.assertEqual(parent_sites_pk, [self.site.pk]) + self.assertEqual(parent_sites_uuid, [str(self.site.uuid)]) + self.assertEqual(parent_courses_pk, [self.course3.pk]) + self.assertEqual(parent_courses_uuid, [str(self.course3.uuid)]) + self.assertEqual(children_courses_pk, [self.course2.pk, self.course4.pk]) + self.assertEqual(children_courses_uuid, [str(self.course2.uuid), str(self.course4.uuid)]) + + def test_site_serialized_children_course(self): + response = self.get_site_detail(self.site.pk) + self.assertEqual(response.status_code, 200) + child_course_pk = response.json()['courses'] + child_course_uuid = response.json()['courses_uuids'] + self.assertEqual(child_course_pk, [self.course.pk]) + self.assertEqual(child_course_uuid, [str(self.course.uuid)]) + def test_filter_courses_by_themes(self): response = self.get_course_list({'themes': self.theme.pk}) self.assertEqual(response.status_code, 200) diff --git a/geotrek/api/v2/mixins.py b/geotrek/api/v2/mixins.py index 42867f99a0..6334fb67fb 100644 --- a/geotrek/api/v2/mixins.py +++ b/geotrek/api/v2/mixins.py @@ -1,6 +1,7 @@ from django.conf import settings from django.shortcuts import get_object_or_404 from django.urls import reverse +from modeltranslation.utils import build_localized_fieldname from geotrek.common import models as common_models @@ -32,3 +33,38 @@ def get_pdf_url(self, obj): for language in settings.MODELTRANSLATION_LANGUAGES: data[language] = self._get_pdf_url_lang(obj, language, portal) return data + + +class PublishedRelatedObjectsSerializerMixin: + + def get_values_on_published_related_objects(self, related_queryset, field): + """ + Retrieve values for `field` on objects from `related_queryset` only if they are published according to requested language + """ + request = self.context['request'] + language = request.GET.get('language') + if language: + published_by_lang = build_localized_fieldname('published', language) + return list(related_queryset.filter(**{published_by_lang: True}).values_list(field, flat=True)) + else: + all_values = [] + for item in related_queryset: + if getattr(item, "any_published"): + all_values.append(getattr(item, field)) + return all_values + + def get_value_on_published_related_object(self, related_object, field): + """ + Retrieve value for `field` on instance `related_object` only if it is published according to requested language + """ + value = None + request = self.context['request'] + language = request.GET.get('language') + if related_object: + if language: + if getattr(related_object, build_localized_fieldname('published', language)): + value = getattr(related_object, field) + else: + if related_object.published: + value = getattr(related_object, field) + return value diff --git a/geotrek/api/v2/serializers.py b/geotrek/api/v2/serializers.py index 1d52de42bb..93b6518c56 100644 --- a/geotrek/api/v2/serializers.py +++ b/geotrek/api/v2/serializers.py @@ -1,6 +1,6 @@ -from bs4 import BeautifulSoup import json +from bs4 import BeautifulSoup from django.conf import settings from django.contrib.gis.db.models.functions import Transform from django.contrib.gis.geos import MultiLineString, Point @@ -13,13 +13,14 @@ from easy_thumbnails.engine import NoSourceGenerator from easy_thumbnails.exceptions import InvalidImageFormatError from easy_thumbnails.files import get_thumbnailer +from modeltranslation.utils import build_localized_fieldname from PIL.Image import DecompressionBombError from rest_framework import serializers from rest_framework.relations import HyperlinkedIdentityField from rest_framework_gis import serializers as geo_serializers from geotrek.api.v2.functions import Length3D -from geotrek.api.v2.mixins import PDFSerializerMixin +from geotrek.api.v2.mixins import PDFSerializerMixin, PublishedRelatedObjectsSerializerMixin from geotrek.api.v2.utils import build_url, get_translation_or_dict from geotrek.authent import models as authent_models from geotrek.common import models as common_models @@ -1138,14 +1139,25 @@ class Meta: model = outdoor_models.Practice fields = ('id', 'name') - class SiteSerializer(PDFSerializerMixin, DynamicFieldsMixin, serializers.ModelSerializer): + class SiteSerializer(PDFSerializerMixin, DynamicFieldsMixin, PublishedRelatedObjectsSerializerMixin, serializers.ModelSerializer): + name = serializers.SerializerMethodField() + accessibility = serializers.SerializerMethodField() + ambiance = serializers.SerializerMethodField() + advice = serializers.SerializerMethodField() + description = serializers.SerializerMethodField() + description_teaser = serializers.SerializerMethodField() + published = serializers.SerializerMethodField() + period = serializers.SerializerMethodField() url = HyperlinkedIdentityField(view_name='apiv2:site-detail') geometry = geo_serializers.GeometryField(read_only=True, source="geom_transformed", precision=7) attachments = AttachmentSerializer(many=True) sector = serializers.SerializerMethodField() courses = serializers.SerializerMethodField() + courses_uuids = serializers.SerializerMethodField() children = serializers.SerializerMethodField() + children_uuids = serializers.SerializerMethodField() parent = serializers.SerializerMethodField() + parent_uuid = serializers.SerializerMethodField() pdf = serializers.SerializerMethodField('get_pdf_url') cities = serializers.SerializerMethodField() districts = serializers.SerializerMethodField() @@ -1153,6 +1165,27 @@ class SiteSerializer(PDFSerializerMixin, DynamicFieldsMixin, serializers.ModelSe web_links = WebLinkSerializer(many=True) view_points = HDViewPointSerializer(many=True) + def get_name(self, obj): + return get_translation_or_dict('name', self, obj) + + def get_accessibility(self, obj): + return get_translation_or_dict('accessibility', self, obj) + + def get_advice(self, obj): + return get_translation_or_dict('advice', self, obj) + + def get_ambiance(self, obj): + return get_translation_or_dict('ambiance', self, obj) + + def get_description(self, obj): + return get_translation_or_dict('description', self, obj) + + def get_description_teaser(self, obj): + return get_translation_or_dict('description_teaser', self, obj) + + def get_period(self, obj): + return get_translation_or_dict('period', self, obj) + def get_cities(self, obj): return [city.code for city in obj.published_cities] @@ -1162,45 +1195,26 @@ def get_districts(self, obj): def get_labels(self, obj): return [label.pk for label in obj.published_labels] + def get_published(self, obj): + return get_translation_or_dict('published', self, obj) + def get_courses(self, obj): - courses = [] - request = self.context['request'] - language = request.GET.get('language') - for course in obj.children_courses.all(): - if language: - if getattr(course, f"published_{language}"): - courses.append(course.pk) - else: - if course.published: - courses.append(course.pk) - return courses + return self.get_values_on_published_related_objects(obj.children_courses.all(), 'pk') + + def get_courses_uuids(self, obj): + return self.get_values_on_published_related_objects(obj.children_courses.all(), 'uuid') def get_parent(self, obj): - parent = None - request = self.context['request'] - language = request.GET.get('language') - if obj.parent: - if language: - if getattr(obj.parent, f"published_{language}"): - parent = obj.parent.pk - else: - if obj.parent.published: - parent = obj.parent.pk - return parent + return self.get_value_on_published_related_object(obj.parent, 'pk') + + def get_parent_uuid(self, obj): + return self.get_value_on_published_related_object(obj.parent, 'uuid') def get_children(self, obj): - children = [] - request = self.context['request'] - language = request.GET.get('language') - if language: - for site in obj.get_children(): - if getattr(site, f"published_{language}"): - children.append(site.pk) - else: - for site in obj.get_children(): - if site.published: - children.append(site.pk) - return children + return self.get_values_on_published_related_objects(obj.get_children(), 'pk') + + def get_children_uuids(self, obj): + return self.get_values_on_published_related_objects(obj.get_children(), 'uuid') def get_sector(self, obj): if obj.practice and obj.practice.sector: @@ -1210,29 +1224,45 @@ def get_sector(self, obj): class Meta: model = outdoor_models.Site fields = ( - 'id', 'accessibility', 'advice', 'ambiance', 'attachments', 'cities', 'children', 'description', + 'id', 'accessibility', 'advice', 'ambiance', 'attachments', 'cities', 'children', 'children_uuids', 'description', 'description_teaser', 'districts', 'eid', 'geometry', 'information_desks', 'labels', 'managers', - 'name', 'orientation', 'pdf', 'period', 'parent', 'portal', 'practice', 'provider', - 'ratings', 'sector', 'source', 'structure', 'themes', 'view_points', - 'type', 'url', 'uuid', 'courses', 'web_links', 'wind', + 'name', 'orientation', 'pdf', 'period', 'parent', 'parent_uuid', 'portal', 'practice', 'provider', + 'ratings', 'sector', 'source', 'structure', 'themes', 'view_points', 'published', + 'type', 'url', 'uuid', 'courses', 'courses_uuids', 'web_links', 'wind', ) - class CourseSerializer(PDFSerializerMixin, DynamicFieldsMixin, serializers.ModelSerializer): + class CourseSerializer(PDFSerializerMixin, DynamicFieldsMixin, PublishedRelatedObjectsSerializerMixin, serializers.ModelSerializer): + name = serializers.SerializerMethodField() + advice = serializers.SerializerMethodField() + description = serializers.SerializerMethodField() url = HyperlinkedIdentityField(view_name='apiv2:course-detail') geometry = geo_serializers.GeometryField(read_only=True, source="geom_transformed", precision=7) - children = serializers.ReadOnlyField(source='children_id') - parents = serializers.ReadOnlyField(source='parents_id') + children = serializers.SerializerMethodField() + parents = serializers.SerializerMethodField() + parents_uuids = serializers.SerializerMethodField() + published = serializers.SerializerMethodField() + children_uuids = serializers.SerializerMethodField() accessibility = serializers.SerializerMethodField() attachments = AttachmentSerializer(many=True, source='sorted_attachments') equipment = serializers.SerializerMethodField() gear = serializers.SerializerMethodField() ratings_description = serializers.SerializerMethodField() sites = serializers.SerializerMethodField() + sites_uuids = serializers.SerializerMethodField() points_reference = serializers.SerializerMethodField() pdf = serializers.SerializerMethodField('get_pdf_url') cities = serializers.SerializerMethodField() districts = serializers.SerializerMethodField() + def get_name(self, obj): + return get_translation_or_dict('name', self, obj) + + def get_advice(self, obj): + return get_translation_or_dict('advice', self, obj) + + def get_description(self, obj): + return get_translation_or_dict('description', self, obj) + def get_accessibility(self, obj): return get_translation_or_dict('accessibility', self, obj) @@ -1245,25 +1275,50 @@ def get_districts(self, obj): def get_equipment(self, obj): return get_translation_or_dict('equipment', self, obj) + def get_published(self, obj): + return get_translation_or_dict('published', self, obj) + def get_gear(self, obj): return get_translation_or_dict('gear', self, obj) def get_ratings_description(self, obj): return get_translation_or_dict('ratings_description', self, obj) - def get_sites(self, obj): - sites = [] + def get_values_on_published_related_ordered_course(self, ordered_course_queryset, related_course, field): + """ + Retrieve dict of values for `field` on objects from `ordered_course_queryset` only if they are published according to requested language + """ request = self.context['request'] language = request.GET.get('language') if language: - for site in obj.parent_sites.all(): - if getattr(site, f"published_{language}"): - sites.append(site.pk) + published_by_lang = f"{related_course}__{build_localized_fieldname('published', language)}" + all_values = ordered_course_queryset.filter(**{published_by_lang: True}).values_list(f"{related_course}__{field}", flat=True) + return list(all_values) else: - for site in obj.parent_sites.all(): - if getattr(site, "published"): - sites.append(site.pk) - return sites + all_values = [] + for item in ordered_course_queryset: + related_object = getattr(item, related_course) + if getattr(related_object, "any_published"): + all_values.append(getattr(related_object, field)) + return all_values + + def get_sites(self, obj): + return self.get_values_on_published_related_objects(obj.parent_sites.all(), 'pk') + + def get_children(self, obj): + return self.get_values_on_published_related_ordered_course(obj.course_children.order_by('order'), 'child', 'pk') + + def get_parents(self, obj): + return self.get_values_on_published_related_ordered_course(obj.course_parents.order_by('order'), 'parent', 'pk') + + def get_sites_uuids(self, obj): + return self.get_values_on_published_related_objects(obj.parent_sites.all(), 'uuid') + + def get_children_uuids(self, obj): + return self.get_values_on_published_related_ordered_course(obj.course_children.order_by('order'), 'child', 'uuid') + + def get_parents_uuids(self, obj): + return self.get_values_on_published_related_ordered_course(obj.course_parents.order_by('order'), 'parent', 'uuid') def get_points_reference(self, obj): if not obj.points_reference: @@ -1274,10 +1329,10 @@ def get_points_reference(self, obj): class Meta: model = outdoor_models.Course fields = ( - 'id', 'accessibility', 'advice', 'attachments', 'children', 'cities', 'description', 'districts', 'duration', 'eid', - 'equipment', 'gear', 'geometry', 'height', 'length', 'max_elevation', - 'min_elevation', 'name', 'parents', 'pdf', 'points_reference', 'provider', 'ratings', 'ratings_description', - 'sites', 'structure', 'type', 'url', 'uuid' + 'id', 'accessibility', 'advice', 'attachments', 'children', 'children_uuids', 'cities', 'description', 'districts', 'duration', 'eid', + 'equipment', 'gear', 'geometry', 'height', 'length', 'max_elevation', 'min_elevation', 'name', 'parents', + 'parents_uuids', 'pdf', 'points_reference', 'published', 'provider', 'ratings', 'ratings_description', + 'sites', 'sites_uuids', 'structure', 'type', 'url', 'uuid' ) if 'geotrek.feedback' in settings.INSTALLED_APPS: diff --git a/geotrek/api/v2/views/outdoor.py b/geotrek/api/v2/views/outdoor.py index 37bd380c5f..0ce360926a 100644 --- a/geotrek/api/v2/views/outdoor.py +++ b/geotrek/api/v2/views/outdoor.py @@ -23,10 +23,12 @@ def get_queryset(self): activate(self.request.GET.get('language')) return outdoor_models.Site.objects \ .annotate(geom_transformed=Transform(F('geom'), settings.API_SRID)) \ + .select_related('parent', 'practice', 'type') \ .prefetch_related(Prefetch('attachments', queryset=Attachment.objects.select_related('license', 'filetype', 'filetype__structure')), Prefetch('view_points', - queryset=HDViewPoint.objects.select_related('content_type', 'license').annotate(geom_transformed=Transform(F('geom'), settings.API_SRID)))) \ + queryset=HDViewPoint.objects.select_related('content_type', 'license').annotate(geom_transformed=Transform(F('geom'), settings.API_SRID))), + 'information_desks', 'labels', 'managers', 'pois_excluded', 'portal', 'ratings', 'source', 'themes', 'web_links') \ .order_by('name') # Required for reliable pagination @@ -93,6 +95,10 @@ def get_queryset(self): activate(self.request.GET.get('language')) return outdoor_models.Course.objects \ .annotate(geom_transformed=Transform(F('geom'), settings.API_SRID)) \ + .select_related('type') \ .prefetch_related(Prefetch('attachments', - queryset=Attachment.objects.select_related('license', 'filetype', 'filetype__structure'))) \ + queryset=Attachment.objects.select_related('license', 'filetype', 'filetype__structure')), + Prefetch('course_children', queryset=outdoor_models.OrderedCourseChild.objects.select_related('parent', 'child')), + Prefetch('course_parents', queryset=outdoor_models.OrderedCourseChild.objects.select_related('parent', 'child')), + 'parent_sites', 'pois_excluded', 'ratings') \ .order_by('name') # Required for reliable pagination