MVT correct field names (#928)

* update test

* initial solution

* refactor to separate function

* further refactor

* optimize check for camelcase

Co-authored-by: Diederik van der Boor <[email protected]>

* only annotate when zoom is high enough

---------

Co-authored-by: Diederik van der Boor <[email protected]>

src/dso_api/dynamic_api/views/mvt.py

@@ -4,10 +4,13 @@
 import time
 
 from django.core.exceptions import PermissionDenied
+from django.db.models import F
 from django.http import Http404
 from django.urls.base import reverse
 from django.views.generic import TemplateView
 from schematools.contrib.django.models import Dataset
+from schematools.naming import toCamelCase
+from schematools.permissions import UserScopes
 from schematools.types import DatasetTableSchema
 from vectortiles import VectorLayer
 from vectortiles.backends import BaseVectorLayerMixin
@@ -114,27 +117,12 @@ def setup(self, request, *args, **kwargs):
         self.model = model
         self.check_permissions(request, [self.model])
 
-    def get_layers(self):
+    def get_layers(self) -> list[BaseVectorLayerMixin]:
         """Provide all layer definitions for this rendering."""
-        schema: DatasetTableSchema = self.model.table_schema()
         layer: BaseVectorLayerMixin = VectorLayer()
         layer.id = "default"
         layer.model = self.model
-        layer.queryset = self.model.objects.all()
-        layer.geom_field = schema.main_geometry_field.python_name
-
-        # If we are zoomed far out (low z), only fetch the geometry field for a smaller payload.
-        # The cutoff is arbitrary. Play around with
-        # https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#14/4.92/52.37
-        # to get a feel for the MVT zoom levels and how much detail a single tile should contain.
-        if self.z >= 15:
-            user_scopes = self.request.user_scopes
-            layer.tile_fields = tuple(
-                f.name
-                for f in self.model._meta.get_fields()
-                if f.name != layer.geom_field
-                and user_scopes.has_field_access(self.model.get_field_schema(f))
-            )
+        self._schemafy_layer(layer)
         return [layer]
 
     def get(self, request, *args, **kwargs):
@@ -152,6 +140,38 @@ def get(self, request, *args, **kwargs):
         )
         return result
 
+    def _schemafy_layer(self, layer: BaseVectorLayerMixin) -> None:
+        schema: DatasetTableSchema = self.model.table_schema()
+        user_scopes: UserScopes = self.request.user_scopes
+
+        layer.geom_field = schema.main_geometry_field.python_name
+
+        queryset = self.model.objects.all()
+        tile_fields = ()
+
+        for field in schema.fields:
+            field_name = field.name
+            if not user_scopes.has_field_access(field):
+                # 403
+                continue
+            if field.is_relation:
+                # Here we have to use the db_name, because that usually has a suffix not
+                # available on field.name.
+                field_name = toCamelCase(field.db_name)
+            if self.z >= 15 and field.db_name != layer.geom_field and field.name != "schema":
+                # If we are zoomed far out (low z), only fetch the geometry field for a
+                # smaller payload. The cutoff is arbitrary. Play around with
+                # https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/
+                # to get a feel for the MVT zoom levels and how much detail a single tile
+                # should contain. We exclude the main geometry and `schema` fields.
+                tile_fields += (field_name,)
+
+                if field_name != field.db_name and field_name.lower() != field_name:
+                    # Annotate camelCased field names so they can be found.
+                    queryset = queryset.annotate(**{field_name: F(field.db_name)})
+        layer.queryset = queryset
+        layer.tile_fields = tile_fields
+
     def check_permissions(self, request, models) -> None:
         """Override CheckPermissionsMixin to add extra checks"""
         super().check_permissions(request, models)

src/tests/test_dynamic_api/views/test_mvt.py

@@ -11,7 +11,12 @@
 @pytest.mark.django_db
 class TestDatasetMVTIndexView:
     def test_mvt_index(
-        self, api_client, afval_dataset, fietspaaltjes_dataset, filled_router, drf_request
+        self,
+        api_client,
+        afval_dataset,
+        fietspaaltjes_dataset,
+        filled_router,
+        drf_request,
     ):
         """Prove that the MVT index view works."""
         response = api_client.get("/v1/mvt/")
@@ -108,7 +113,9 @@ def test_mvt_single(api_client, afval_container, filled_router):
 
 
 @pytest.mark.django_db
-def test_mvt_content(api_client, afval_container_model, afval_cluster, filled_router):
+def test_mvt_content(
+    api_client, afval_dataset, filled_router, afval_container_model, afval_cluster
+):
     """Prove that the MVT view produces vector tiles."""
 
     # Coordinates below have been calculated using https://oms.wff.ch/calc.htm
@@ -141,15 +148,12 @@ def test_mvt_content(api_client, afval_container_model, afval_cluster, filled_ro
                 {
                     "geometry": {"type": "Point", "coordinates": [1928, 2558]},
                     "properties": {
-                        # TODO These are snake-cased database field names.
-                        # We should return schema field names instead.
-                        # Also, what happens in the case of relations?
                         "id": 1,
-                        "cluster_id": "c1",
+                        "clusterId": "c1",  # relation
                         "serienummer": "foobar-123",
-                        "datum_creatie": "2021-01-03",
-                        "eigenaar_naam": "Dataservices",
-                        "datum_leegmaken": "2021-01-03 12:13:14+01",
+                        "datumCreatie": "2021-01-03",
+                        "eigenaarNaam": "Dataservices",
+                        "datumLeegmaken": "2021-01-03 12:13:14+01",
                     },
                     "id": 0,
                     "type": "Feature",
@@ -199,7 +203,6 @@ def test_mvt_forbidden(api_client, geometry_auth_thing, fetch_auth_token, filled
 @pytest.mark.django_db
 def test_mvt_model_auth(api_client, geometry_auth_model, fetch_auth_token, filled_router):
     """Prove that unauthorized fields are excluded from vector tiles"""
-
     # See test_mvt_content for how to compute the coordinates.
     geometry_auth_model.objects.create(
         id=1,