From a51f9ac2ef95e8e8e1482bd269b6aca85b7bbfeb Mon Sep 17 00:00:00 2001 From: Andrey Rusakov Date: Mon, 16 Dec 2024 12:37:01 +0100 Subject: [PATCH 1/6] Add basic many-to-many quota table --- api/admin.py | 13 ++++++++++--- api/models.py | 25 ++++++++++++++++++------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/api/admin.py b/api/admin.py index 4dd29c7a..d604dd3b 100644 --- a/api/admin.py +++ b/api/admin.py @@ -24,7 +24,8 @@ OrderItem, Pricing, Product, - ProductFormat) + ProductFormat, + ProductQuota) UserModel = get_user_model() @@ -188,17 +189,22 @@ def response_change(self, request, obj): return HttpResponseRedirect(redirect_url) return super().response_change(request, obj) +class ProductQuotaAdmin(CustomGeoModelAdmin): + pass + +class ProductQuotaInline(admin.TabularInline): + model = ProductQuota + extra = 1 class ProductAdmin(CustomGeoModelAdmin): save_as = True - inlines = [ProductFormatInline] + inlines = [ProductFormatInline, ProductQuotaInline] raw_id_fields = ('metadata', 'group') exclude = ('ts',) search_fields = ['label'] list_filter = ('product_status',) readonly_fields = ('thumbnail_tag',) - class AbstractIdentityAdmin(CustomModelAdmin): list_display = ['last_name', 'first_name', 'company_name', 'email'] search_fields = ['first_name', 'last_name', 'company_name', 'email'] @@ -281,3 +287,4 @@ def response_change(self, request, obj): admin.site.register(Pricing, PricingAdmin) admin.site.register(Product, ProductAdmin) admin.site.register(ProductFormat) +admin.site.register(ProductQuota, ProductQuotaAdmin) \ No newline at end of file diff --git a/api/models.py b/api/models.py index cca725ee..d437aaea 100644 --- a/api/models.py +++ b/api/models.py @@ -6,6 +6,7 @@ from django.contrib.gis.db import models from django.contrib.gis.geos import MultiPolygon, Polygon from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group from django.contrib.postgres.search import SearchVectorField from django.contrib.postgres.indexes import GinIndex, BTreeIndex from django.utils import timezone @@ -412,7 +413,6 @@ class Meta: def __str__(self): return self.name - class Product(models.Model): """ A product is mostly a table or a raster. It can also be a group of products. @@ -482,12 +482,7 @@ class ProductStatus(models.TextChoices): default=settings.DEFAULT_PRODUCT_THUMBNAIL_URL, ) ts = SearchVectorField(null=True) - bbox = settings.DEFAULT_EXTENT - geom = models.MultiPolygonField( - _("geom"), - srid=settings.DEFAULT_SRID, - default=MultiPolygon(Polygon.from_bbox(bbox)), - ) + class Meta: db_table = "product" @@ -510,6 +505,22 @@ def thumbnail_tag(self): thumbnail_tag.short_description = _("thumbnail") +class ProductQuota(models.Model): + user_group = models.ForeignKey( + Group, models.CASCADE, verbose_name=_("user_group") + ) + product = models.ForeignKey( + Product, models.CASCADE, verbose_name=_("product"), default=1 + ) + bbox = settings.DEFAULT_EXTENT + geom = models.MultiPolygonField( + _("geom"), + srid=settings.DEFAULT_SRID, + default=MultiPolygon(Polygon.from_bbox(bbox))) + quota = models.FloatField(_("quota")) + + def __str__(self): + return f'Quota for "{self.user_group}" in "{self.product}" is "{self.quota}"' class Order(models.Model): """ From 8950d2fe38e0ac13bc12219102ecbdc1674bec8e Mon Sep 17 00:00:00 2001 From: Andrey Rusakov Date: Wed, 18 Dec 2024 16:32:58 +0100 Subject: [PATCH 2/6] Renamed quota to ownership --- api/admin.py | 12 +- api/management/commands/seed.py | 385 +++++++++++++++++++------------- api/models.py | 15 +- api/serializers.py | 31 ++- 4 files changed, 266 insertions(+), 177 deletions(-) diff --git a/api/admin.py b/api/admin.py index d604dd3b..e27ce5c2 100644 --- a/api/admin.py +++ b/api/admin.py @@ -25,7 +25,7 @@ Pricing, Product, ProductFormat, - ProductQuota) + ProductOwnership) UserModel = get_user_model() @@ -189,16 +189,16 @@ def response_change(self, request, obj): return HttpResponseRedirect(redirect_url) return super().response_change(request, obj) -class ProductQuotaAdmin(CustomGeoModelAdmin): +class ProductOwnershipAdmin(CustomGeoModelAdmin): pass -class ProductQuotaInline(admin.TabularInline): - model = ProductQuota +class ProductOwnershipInline(admin.TabularInline): + model = ProductOwnership extra = 1 class ProductAdmin(CustomGeoModelAdmin): save_as = True - inlines = [ProductFormatInline, ProductQuotaInline] + inlines = [ProductFormatInline, ProductOwnershipInline] raw_id_fields = ('metadata', 'group') exclude = ('ts',) search_fields = ['label'] @@ -287,4 +287,4 @@ def response_change(self, request, obj): admin.site.register(Pricing, PricingAdmin) admin.site.register(Product, ProductAdmin) admin.site.register(ProductFormat) -admin.site.register(ProductQuota, ProductQuotaAdmin) \ No newline at end of file +admin.site.register(ProductOwnership, ProductOwnershipAdmin) \ No newline at end of file diff --git a/api/management/commands/seed.py b/api/management/commands/seed.py index 5b39a6e7..0b8100c6 100644 --- a/api/management/commands/seed.py +++ b/api/management/commands/seed.py @@ -8,13 +8,25 @@ from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model from django.contrib.auth.models import User -from api.models import Contact, Order, OrderItem, OrderType, Product, DataFormat, Identity, Metadata, Pricing, Money +from api.models import ( + Contact, + Group, + Order, + OrderItem, + OrderType, + Product, + DataFormat, + Identity, + Metadata, + Pricing, + Money, +) from django.contrib.gis.geos import Polygon from api.helpers import _zip_them_all from typing import TypeVar, Generic, Type from collections.abc import MutableMapping -T = TypeVar('T', bound=models.Model) +T = TypeVar("T", bound=models.Model) UserModel = get_user_model() @@ -35,17 +47,26 @@ def success(self, msg: str): def error(self, msg: str): self.stdout.write(self.style.ERROR(msg)) - def getOrCreate(self, model: Type[T], defaults: MutableMapping[str, any] | None = ..., **kwargs) -> T: + def getOrCreate( + self, model: Type[T], defaults: MutableMapping[str, any] | None = ..., **kwargs + ) -> T: (value, created) = model.objects.get_or_create(defaults, **kwargs) typeName = model.__name__ if created: - self.success(f"Created {typeName} \"{value}\"") + self.success(f'Created {typeName} "{value}"') else: - self.notice(f"Not creating {typeName}: \"{value}\" - already exists") + self.notice(f'Not creating {typeName}: "{value}" - already exists') return value def addUser(self, username: str, email: str, password: str) -> User: - return self.getOrCreate(UserModel, username=username, defaults={"email": email, "password": password}) + return self.getOrCreate( + UserModel, + username=username, + defaults={"email": email, "password": password}, + ) + + def addGroup(self, name: str) -> Group: + return self.getOrCreate(Group, name=name, defaults={}) def setIdentity(self, user: User, params: dict[str, any]) -> Identity: for k, v in params.items(): @@ -55,218 +76,251 @@ def setIdentity(self, user: User, params: dict[str, any]) -> Identity: return user.identity def addProduct(self, user: User, label: str) -> OrderType: - return self.getOrCreate(Product, label=label, defaults={ - "metadata": self.getOrCreate(Metadata, id_name="metadata", name="metadata", defaults={"modified_user_id": user.id}), - "pricing": self.getOrCreate(Pricing, name="Free", defaults={"pricing_type": Pricing.PricingType.FREE}), - "provider": user - }) + return self.getOrCreate( + Product, + label=label, + defaults={ + "metadata": self.getOrCreate( + Metadata, + id_name="metadata", + name="metadata", + defaults={"modified_user_id": user.id}, + ), + "pricing": self.getOrCreate( + Pricing, + name="Free", + defaults={"pricing_type": Pricing.PricingType.FREE}, + ), + "provider": user, + }, + ) def seed(self): # Create users - rincevent = self.addUser('rincevent', 'rincevent@mail.com', 'rincevent') - self.setIdentity(rincevent, { - "email": os.environ.get('EMAIL_TEST_TO', 'admin@admin.com'), - "first_name": 'Jean', - "last_name": 'Michoud', - "street": 'Rue de Tivoli 22', - "postcode": '2000', - "city": 'Neuchâtel', - "country": 'Suisse', - "company_name": 'Service du Registre Foncier et de la Géomatique - SITN', - "phone": '+41 32 000 00 00'}) + extractUser = self.addUser("extract", "extract@mail.com", "extract") + extractGroup = self.addGroup("extract") + extractGroup.user_set.add(extractUser) + extractGroup.save() + + rincevent = self.addUser("rincevent", "rincevent@mail.com", "rincevent") + self.setIdentity( + rincevent, + { + "email": os.environ.get("EMAIL_TEST_TO", "admin@admin.com"), + "first_name": "Jean", + "last_name": "Michoud", + "street": "Rue de Tivoli 22", + "postcode": "2000", + "city": "Neuchâtel", + "country": "Suisse", + "company_name": "Service du Registre Foncier et de la Géomatique - SITN", + "phone": "+41 32 000 00 00", + }, + ) mmi = self.addUser("mmi", "mmi@mmi.com", "mmi") - self.setIdentity(mmi, { - "email": os.environ.get('EMAIL_TEST_TO_ARXIT', 'admin@admin.com'), - "first_name": 'Jeanne', - "last_name": 'Paschoud', - "street": 'Rue de Tivoli 22', - "postcode": '2000', - "city": 'Neuchâtel', - "country": 'Suisse', - "company_name": 'Service du Registre Foncier et de la Géomatique - SITN', - "phone": '+41 32 000 00 00'}) + self.setIdentity( + mmi, + { + "email": os.environ.get("EMAIL_TEST_TO_ARXIT", "admin@admin.com"), + "first_name": "Jeanne", + "last_name": "Paschoud", + "street": "Rue de Tivoli 22", + "postcode": "2000", + "city": "Neuchâtel", + "country": "Suisse", + "company_name": "Service du Registre Foncier et de la Géomatique - SITN", + "phone": "+41 32 000 00 00", + }, + ) mma = self.addUser("mma", "mma@mma.com", "mma") - self.setIdentity(mma, { - "email": 'mma-email@admin.com', - "first_name": 'Jean-René', - "last_name": 'Humbert-Droz L\'Authentique', - "street": 'Rue de Tivoli 22', - "postcode": '2000', - "city": 'Neuchâtel', - "country": 'Suisse', - "company_name": 'Service du Registre Foncier et de la Géomatique - SITN', - "phone": '+41 32 000 00 00'}) + self.setIdentity( + mma, + { + "email": "mma-email@admin.com", + "first_name": "Jean-René", + "last_name": "Humbert-Droz L'Authentique", + "street": "Rue de Tivoli 22", + "postcode": "2000", + "city": "Neuchâtel", + "country": "Suisse", + "company_name": "Service du Registre Foncier et de la Géomatique - SITN", + "phone": "+41 32 000 00 00", + }, + ) mka2 = self.addUser("mka", "mka@mka.com", "mka") - self.setIdentity(mka2, { - "email": 'mka2-email@ne.ch', - "first_name": 'Michaël', - "last_name": 'Kalbermatten', - "street": 'Rue de Tivoli 22', - "postcode": '2000', - "city": 'Neuchâtel', - "country": 'Suisse', - "company_name": 'Service du Registre Foncier et de la Géomatique - SITN', - "phone": '+41 32 000 00 00', - "subscribed": True}) + self.setIdentity( + mka2, + { + "email": "mka2-email@ne.ch", + "first_name": "Michaël", + "last_name": "Kalbermatten", + "street": "Rue de Tivoli 22", + "postcode": "2000", + "city": "Neuchâtel", + "country": "Suisse", + "company_name": "Service du Registre Foncier et de la Géomatique - SITN", + "phone": "+41 32 000 00 00", + "subscribed": True, + }, + ) # contacts contact1 = Contact.objects.create( - first_name='Marc', - last_name='Riedo', - email='test@admin.com', + first_name="Marc", + last_name="Riedo", + email="test@admin.com", postcode=2000, - city='Neuchâtel', - country='Suisse', - company_name='SITN', - phone='+41 00 787 45 15', - belongs_to=mmi + city="Neuchâtel", + country="Suisse", + company_name="SITN", + phone="+41 00 787 45 15", + belongs_to=mmi, ) contact1.save() contact2 = Contact.objects.create( - first_name='Marcelle', - last_name='Rieda', - email='test2@admin.com', + first_name="Marcelle", + last_name="Rieda", + email="test2@admin.com", postcode=2000, - city='Neuchâtel', - country='Suisse', - company_name='SITN', - phone='+41 00 787 45 16', - belongs_to=mmi + city="Neuchâtel", + country="Suisse", + company_name="SITN", + phone="+41 00 787 45 16", + belongs_to=mmi, ) contact2.save() contact3 = Contact.objects.create( - first_name='Jean', - last_name='Doe', - email='test3@admin.com', + first_name="Jean", + last_name="Doe", + email="test3@admin.com", postcode=2000, - city='Lausanne', - country='Suisse', - company_name='Marine de Colombier', - phone='+41 00 787 29 16', - belongs_to=mmi + city="Lausanne", + country="Suisse", + company_name="Marine de Colombier", + phone="+41 00 787 29 16", + belongs_to=mmi, ) contact3.save() contact_mka2 = Contact.objects.create( - first_name='Jean', - last_name='Doe', - email='test3@admin.com', + first_name="Jean", + last_name="Doe", + email="test3@admin.com", postcode=2000, - city='Lausanne', - country='Suisse', - company_name='Marine de Colombier', - phone='+41 00 787 29 16', - belongs_to=mka2 + city="Lausanne", + country="Suisse", + company_name="Marine de Colombier", + phone="+41 00 787 29 16", + belongs_to=mka2, ) contact_mka2.save() - order_geom = Polygon(( + order_geom = Polygon( ( - 2528577.8382161376, - 1193422.4003930448 - ), - ( - 2542482.6542869355, - 1193422.4329014618 - ), - ( - 2542482.568523701, - 1199018.36469272 - ), - ( - 2528577.807487005, - 1199018.324372703 - ), - ( - 2528577.8382161376, - 1193422.4003930448 + (2528577.8382161376, 1193422.4003930448), + (2542482.6542869355, 1193422.4329014618), + (2542482.568523701, 1199018.36469272), + (2528577.807487005, 1199018.324372703), + (2528577.8382161376, 1193422.4003930448), ) - )) + ) - order_type_prive = self.getOrCreate(OrderType, name='Privé', defaults={}) - public = self.getOrCreate(OrderType, name='Public', defaults={}) + order_type_prive = self.getOrCreate(OrderType, name="Privé", defaults={}) + public = self.getOrCreate(OrderType, name="Public", defaults={}) # Create orders order1 = Order.objects.create( - title='Plan de situation pour enquête', - description='C\'est un test', + title="Plan de situation pour enquête", + description="C'est un test", order_type=order_type_prive, client=rincevent, geom=order_geom, - invoice_reference='Dossier n°545454', - date_ordered=timezone.now()) + invoice_reference="Dossier n°545454", + date_ordered=timezone.now(), + ) order1.save() order2 = Order.objects.create( - title='Plan de situation pour enquête', - description='C\'est un test', + title="Plan de situation pour enquête", + description="C'est un test", order_type=order_type_prive, client=rincevent, geom=order_geom, - invoice_reference='Dossier n°545454', - date_ordered=timezone.now()) + invoice_reference="Dossier n°545454", + date_ordered=timezone.now(), + ) order2.save() order3 = Order.objects.create( - title='Plan de situation pour enquête', - description='C\'est un test', + title="Plan de situation pour enquête", + description="C'est un test", order_type=order_type_prive, client=rincevent, geom=order_geom, - invoice_reference='Dossier n°545454', - date_ordered=timezone.now()) + invoice_reference="Dossier n°545454", + date_ordered=timezone.now(), + ) order3.save() order4 = Order.objects.create( - title='Plan de situation pour enquête', - description='C\'est un test', + title="Plan de situation pour enquête", + description="C'est un test", order_type=order_type_prive, client=mma, geom=order_geom, - invoice_reference='Dossier n°545454', - date_ordered=timezone.now()) + invoice_reference="Dossier n°545454", + date_ordered=timezone.now(), + ) order4.save() order_mka2 = Order.objects.create( - title='Plan de situation pour enquête', - description='C\'est un test', + title="Plan de situation pour enquête", + description="C'est un test", order_type=order_type_prive, client=mka2, geom=order_geom, - invoice_reference='Dossier n°545454', - date_ordered=timezone.now()) + invoice_reference="Dossier n°545454", + date_ordered=timezone.now(), + ) order_mka2.save() order_download = Order.objects.create( - title='Commande prête à être téléchargée', - description='C\'est un test', + title="Commande prête à être téléchargée", + description="C'est un test", order_type=order_type_prive, client=mmi, geom=order_geom, - invoice_reference='Dossier 8', - date_ordered=timezone.now()) + invoice_reference="Dossier 8", + date_ordered=timezone.now(), + ) order_download.save() order_quoted = Order.objects.create( - title='Commande devisée pour test', - description='C\'est un test', + title="Commande devisée pour test", + description="C'est un test", order_type=order_type_prive, client=mmi, geom=order_geom, - invoice_reference='Dossier n°545454', - date_ordered=timezone.now()) + invoice_reference="Dossier n°545454", + date_ordered=timezone.now(), + ) order_quoted.save() # Products - product1 = self.addProduct(mma, 'MO - Cadastre complet') - product2 = self.addProduct(mma, 'Maquette 3D') - product_deprecated = self.addProduct(mma, - 'MO07 - Objets divers et éléments linéaires - linéaires') + product1 = self.addProduct(mma, "MO - Cadastre complet") + product2 = self.addProduct(mma, "Maquette 3D") + product_deprecated = self.addProduct( + mma, "MO07 - Objets divers et éléments linéaires - linéaires" + ) - data_format = self.getOrCreate(DataFormat, name='Geobat NE complet (DXF)', defaults={}) - data_format_maquette = self.getOrCreate(DataFormat, name='3dm (Fichier Rhino)', defaults={}) + data_format = self.getOrCreate( + DataFormat, name="Geobat NE complet (DXF)", defaults={} + ) + data_format_maquette = self.getOrCreate( + DataFormat, name="3dm (Fichier Rhino)", defaults={} + ) for order_item in [ OrderItem.objects.create(order=order1, product=product1), @@ -274,31 +328,37 @@ def seed(self): OrderItem.objects.create(order=order_download, product=product1), OrderItem.objects.create(order=order2, product=product1), OrderItem.objects.create( - order=order3, product=product1, data_format=data_format), + order=order3, product=product1, data_format=data_format + ), OrderItem.objects.create(order=order4, product=product2), OrderItem.objects.create( - order=order_mka2, product=product1, data_format=data_format) + order=order_mka2, product=product1, data_format=data_format + ), ]: order_item.set_price() order_item.save() order_item_deprecated = OrderItem.objects.create( - order=order_mka2, product=product_deprecated, data_format=data_format) - order_item_deprecated.set_price(price=Money( - 400, 'CHF'), base_fee=Money(150, 'CHF')) + order=order_mka2, product=product_deprecated, data_format=data_format + ) + order_item_deprecated.set_price( + price=Money(400, "CHF"), base_fee=Money(150, "CHF") + ) order_item_deprecated.price_status = OrderItem.PricingStatus.CALCULATED order_item_deprecated.save() order_item_download = OrderItem.objects.create( - order=order_download, product=product2, data_format=data_format_maquette) - order_item_download.set_price(price=Money( - 400, 'CHF'), base_fee=Money(150, 'CHF')) + order=order_download, product=product2, data_format=data_format_maquette + ) + order_item_download.set_price( + price=Money(400, "CHF"), base_fee=Money(150, "CHF") + ) order_item_download.price_status = OrderItem.PricingStatus.CALCULATED order_item_download.save() order_item_quoted = OrderItem.objects.create( - order=order_quoted, product=product2, data_format=data_format_maquette) - order_item_quoted.set_price(price=Money( - 400, 'CHF'), base_fee=Money(150, 'CHF')) + order=order_quoted, product=product2, data_format=data_format_maquette + ) + order_item_quoted.set_price(price=Money(400, "CHF"), base_fee=Money(150, "CHF")) order_item_quoted.price_status = OrderItem.PricingStatus.CALCULATED order_item_quoted.save() @@ -315,14 +375,16 @@ def seed(self): order_mka2.invoice_contact = contact_mka2 order_mka2.set_price() order_mka2.date_ordered = datetime.datetime( - 2018, 12, 1, 8, 20, 3, 0, tzinfo=datetime.timezone.utc) + 2018, 12, 1, 8, 20, 3, 0, tzinfo=datetime.timezone.utc + ) order_mka2.order_status = Order.OrderStatus.ARCHIVED order_mka2.save() order_download.set_price() - empty_zip_data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + empty_zip_data = b"PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" extract_file = SimpleUploadedFile( - "result.zip", empty_zip_data, content_type="multipart/form-data") + "result.zip", empty_zip_data, content_type="multipart/form-data" + ) for order_item in order_download.items.all(): order_item.extract_result = extract_file order_item.status = OrderItem.OrderItemStatus.PROCESSED @@ -330,13 +392,16 @@ def seed(self): order_download.order_status = Order.OrderStatus.PROCESSED # Creating zip with all zips without background process unsupported by manage.py - zip_list_path = list(order_download.items.all( - ).values_list('extract_result', flat=True)) + zip_list_path = list( + order_download.items.all().values_list("extract_result", flat=True) + ) today = timezone.now() zip_path = Path( - 'extract', - str(today.year), str(today.month), - "{}{}.zip".format('0a2ebb0a-', str(order_download.id))) + "extract", + str(today.year), + str(today.month), + "{}{}.zip".format("0a2ebb0a-", str(order_download.id)), + ) order_download.extract_result.name = zip_path.as_posix() full_zip_path = Path(settings.MEDIA_ROOT, zip_path) _zip_them_all(full_zip_path, zip_list_path) diff --git a/api/models.py b/api/models.py index d437aaea..faf90d4b 100644 --- a/api/models.py +++ b/api/models.py @@ -413,6 +413,7 @@ class Meta: def __str__(self): return self.name + class Product(models.Model): """ A product is mostly a table or a raster. It can also be a group of products. @@ -482,7 +483,12 @@ class ProductStatus(models.TextChoices): default=settings.DEFAULT_PRODUCT_THUMBNAIL_URL, ) ts = SearchVectorField(null=True) - + bbox = settings.DEFAULT_EXTENT + geom = models.MultiPolygonField( + _("geom"), + srid=settings.DEFAULT_SRID, + default=MultiPolygon(Polygon.from_bbox(bbox)), + ) class Meta: db_table = "product" @@ -505,7 +511,7 @@ def thumbnail_tag(self): thumbnail_tag.short_description = _("thumbnail") -class ProductQuota(models.Model): +class ProductOwnership(models.Model): user_group = models.ForeignKey( Group, models.CASCADE, verbose_name=_("user_group") ) @@ -517,10 +523,9 @@ class ProductQuota(models.Model): _("geom"), srid=settings.DEFAULT_SRID, default=MultiPolygon(Polygon.from_bbox(bbox))) - quota = models.FloatField(_("quota")) def __str__(self): - return f'Quota for "{self.user_group}" in "{self.product}" is "{self.quota}"' + return f'Product ownership for "{self.user_group}" in "{self.product}"' class Order(models.Model): """ @@ -582,6 +587,8 @@ class OrderStatus(models.TextChoices): null=True, ) geom = models.PolygonField(_("geom"), srid=settings.DEFAULT_SRID) + actualGeom = models.PolygonField(_("actualGeom"), srid=settings.DEFAULT_SRID, null=True) + client = models.ForeignKey( UserModel, models.PROTECT, verbose_name=_("client"), blank=True ) diff --git a/api/serializers.py b/api/serializers.py index c63d8501..7abb3d2f 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -9,7 +9,7 @@ from django.contrib.auth.forms import PasswordResetForm, SetPasswordForm from django.contrib.auth.tokens import default_token_generator from django.contrib.gis.gdal import GDALException -from django.contrib.gis.geos import Polygon, GEOSException, GEOSGeometry, WKTWriter +from django.contrib.gis.geos import Polygon, MultiPolygon, GEOSException, GEOSGeometry, WKTWriter from django.utils.translation import gettext_lazy as _ from django.utils.encoding import force_str from django.utils.http import urlsafe_base64_decode @@ -22,9 +22,9 @@ from .helpers import send_geoshop_email, zip_all_orderitems from .models import ( - Copyright, Contact, Document, DataFormat, Identity, + Copyright, Contact, Document, DataFormat, Group, Identity, Metadata, MetadataCategoryEch, MetadataContact, Order, OrderItem, OrderType, - Pricing, Product, ProductFormat, UserChange) + Pricing, Product, ProductFormat, ProductOwnership, UserChange) from typing import List, Dict @@ -301,14 +301,31 @@ def validate(self, attrs): self._errors = {} if 'geom' not in attrs: return attrs - geom = attrs['geom'] - area = geom.area - if settings.MAX_ORDER_AREA > 0 and area > settings.MAX_ORDER_AREA: + requestedGeom = Polygon( + [xy[0:2] for xy in list(attrs['geom'].coords[0])], + srid=settings.DEFAULT_SRID + ) + relevantOwnedAreas = ProductOwnership.objects.filter( + product__in=[ item['product'] for item in attrs['items']], + user_group__in=Group.objects.filter(user=self.context.get('request').user) + ).all() + + ownedAreas = MultiPolygon(srid=settings.DEFAULT_SRID) + for area in relevantOwnedAreas: + ownedAreas = ownedAreas.union(area.geom) + + ownedRequestedGeom = requestedGeom.intersection(ownedAreas) + if (round(ownedRequestedGeom.area) == 0 and settings.MAX_ORDER_AREA > 0 + and requestedGeom.area > settings.MAX_ORDER_AREA): raise ValidationError({ 'message': _(f'Order area is too large'), 'expected': settings.MAX_ORDER_AREA, - 'actual': area + 'actual': requestedGeom.area }) + + attrs['actualGeom'] = ownedRequestedGeom + attrs['geom'] = requestedGeom + return attrs class Meta: From 841c442a0f9ca69418526d40f3c88bef63a10519 Mon Sep 17 00:00:00 2001 From: Andrey Rusakov Date: Fri, 20 Dec 2024 15:46:04 +0100 Subject: [PATCH 3/6] Add tests for various cases --- api/admin.py | 1 + api/models.py | 3 +- api/serializers.py | 8 +--- api/tests/factories.py | 33 +++++++++++++++- api/tests/test_order.py | 85 ++++++++++++++++++++++++++++++++++++++++- 5 files changed, 118 insertions(+), 12 deletions(-) diff --git a/api/admin.py b/api/admin.py index e27ce5c2..0541925e 100644 --- a/api/admin.py +++ b/api/admin.py @@ -10,6 +10,7 @@ from django.http import HttpResponseRedirect from django.utils.translation import gettext_lazy as _ from django_extended_ol.forms.widgets import WMTSWidget +from django.contrib.auth.models import Group from .helpers import send_geoshop_email from .models import ( diff --git a/api/models.py b/api/models.py index faf90d4b..ab5e88b4 100644 --- a/api/models.py +++ b/api/models.py @@ -518,11 +518,10 @@ class ProductOwnership(models.Model): product = models.ForeignKey( Product, models.CASCADE, verbose_name=_("product"), default=1 ) - bbox = settings.DEFAULT_EXTENT geom = models.MultiPolygonField( _("geom"), srid=settings.DEFAULT_SRID, - default=MultiPolygon(Polygon.from_bbox(bbox))) + default=MultiPolygon(Polygon.from_bbox(settings.DEFAULT_EXTENT))) def __str__(self): return f'Product ownership for "{self.user_group}" in "{self.product}"' diff --git a/api/serializers.py b/api/serializers.py index 7abb3d2f..cc01c8bf 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -1,7 +1,5 @@ import json import copy -import shapely -import shapely.ops as ops from shapely.geometry.polygon import Polygon from django.conf import settings @@ -313,8 +311,9 @@ def validate(self, attrs): ownedAreas = MultiPolygon(srid=settings.DEFAULT_SRID) for area in relevantOwnedAreas: ownedAreas = ownedAreas.union(area.geom) - ownedRequestedGeom = requestedGeom.intersection(ownedAreas) + attrs['actualGeom'] = ownedRequestedGeom + if (round(ownedRequestedGeom.area) == 0 and settings.MAX_ORDER_AREA > 0 and requestedGeom.area > settings.MAX_ORDER_AREA): raise ValidationError({ @@ -323,9 +322,6 @@ def validate(self, attrs): 'actual': requestedGeom.area }) - attrs['actualGeom'] = ownedRequestedGeom - attrs['geom'] = requestedGeom - return attrs class Meta: diff --git a/api/tests/factories.py b/api/tests/factories.py index 17b36cbc..13b97b40 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -1,12 +1,12 @@ import os from django.contrib.auth import get_user_model -from django.contrib.gis.geos import Polygon +from django.contrib.gis.geos import Polygon, MultiPolygon from django.core import management from djmoney.money import Money from django.utils import timezone from django.urls import reverse -from api.models import Contact, DataFormat, Metadata, Order, OrderType, Pricing, Product, ProductFormat +from api.models import Contact, DataFormat, Metadata, Order, OrderType, Pricing, Product, ProductFormat, Group, ProductOwnership UserModel = get_user_model() TOKEN_URL = reverse('token_obtain_pair') @@ -194,6 +194,35 @@ def __init__(self, webclient=None): date_ordered=timezone.now() ) + self.zurichDataOwner = ProductOwnership.objects.create( + user_group=Group.objects.create(name="zurich_data_owner"), + product=self.products['free'], + geom=MultiPolygon([Polygon([ + [8.472347, 47.364641], [8.472347, 47.404275], + [8.576202, 47.404275], [8.576202, 47.364641], + [8.472347, 47.364641] + ])], srid=4326)) + self.lausanneDataOwner = ProductOwnership.objects.create( + user_group=Group.objects.create(name="lausanne_data_owner"), + product=self.products['free'], + geom=MultiPolygon([Polygon([ + [6.59008, 46.500283], [6.59008, 46.551542], + [6.694794, 46.551542], [6.694794, 46.500283], + [6.59008, 46.500283] + ])], srid=4326)) + self.switzerlandDataOwner = ProductOwnership.objects.create( + user_group=Group.objects.create(name="switzerland_data_owner"), + product=self.products['free'], + geom=MultiPolygon([Polygon([ + [5.50415, 45.713851], [5.50415, 47.857403], + [10.667725, 47.857403], [10.667725, 45.713851], + [5.50415, 45.713851] + ])], srid=4326)) + + zurichOwners = self.zurichDataOwner.user_group + zurichOwners.user_set.add(self.user_private) + zurichOwners.save() + class ExtractFactory: password = os.environ['EXTRACT_USER_PASSWORD'] diff --git a/api/tests/test_order.py b/api/tests/test_order.py index 0f9da7ba..d40ce489 100644 --- a/api/tests/test_order.py +++ b/api/tests/test_order.py @@ -4,13 +4,33 @@ from django.core import mail from django.test import override_settings +from math import isclose +from django.contrib.gis.geos import Polygon + from djmoney.money import Money from rest_framework import status from rest_framework.test import APITestCase -from api.models import Contact, OrderItem, Order, Metadata, Product, ProductFormat +from api.models import ( + Contact, + OrderItem, + Order, + Metadata, + Product, + ProductFormat, +) from api.tests.factories import BaseObjectsFactory +def areasEqual(geomA, geomB, srid: int = 2056) -> bool: + """We can consider polygons equal if """ + polyA = Polygon(geomA['coordinates'][0], srid=srid) + polyB = Polygon(geomB['coordinates'][0], srid=srid) + intAB = polyA.intersection(polyB).area + uniAB = polyA.union(polyB).area + return (isclose(intAB, polyA.area) and isclose(intAB, polyB.area) and + isclose(uniAB, polyA.area) and isclose(uniAB, polyB.area) and + isclose(polyA.difference(polyB).area, 0)) + class OrderTests(APITestCase): """ Test Orders @@ -466,4 +486,65 @@ def test_order_geom_is_fine(self): [2545488, 1203070]] ]} response = self.client.post(url, self.order_data, format='json') - self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.content) \ No newline at end of file + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.content) + + @override_settings(MAX_ORDER_AREA = 100) + def test_order_owned_contains(self): + url = reverse('order-list') + self.client.credentials(HTTP_AUTHORIZATION='Bearer ' + self.config.client_token) + self.order_data['items'] = [{ + 'product': 'Produit gratuit' + }] + self.order_data['geom'] = { + 'type': 'Polygon', + 'coordinates': [ + [[2682192.2803059844, 1246970.4157564922], + [2682178.2106039342, 1247984.965345809], + [2683720.8073948864, 1248006.558970477], + [2683735.1414241255, 1246992.0130589735], + [2682192.2803059844, 1246970.4157564922]] + ]} + response = self.client.post(url, self.order_data, format='json') + order = json.loads(response.content) + + self.assertTrue(areasEqual(order["geom"], order["actualGeom"])) + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.content) + + @override_settings(MAX_ORDER_AREA = 1000) + def test_order_owned_intersects(self): + url = reverse('order-list') + self.client.credentials(HTTP_AUTHORIZATION='Bearer ' + self.config.client_token) + self.order_data['items'] = [{ + 'product': 'Produit gratuit' + }] + self.order_data['geom'] = { + 'type': 'Polygon', + 'coordinates': [ + [[2651783.430446268, 1248297.3690953483], + [2651756.3479182185, 1251397.9173197772], + [2717461.37168784, 1252336.6990602014], + [2717522.8814288364, 1249236.639547884], + [2651783.430446268, 1248297.3690953483]] + ]} + response = self.client.post(url, self.order_data, format='json') + + order = json.loads(response.content) + self.assertFalse(areasEqual(order["geom"], order["actualGeom"])) + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.content) + + @override_settings(MAX_ORDER_AREA = 100) + def test_order_unowned_limited(self): + url = reverse('order-list') + self.client.credentials(HTTP_AUTHORIZATION='Bearer ' + self.config.client_token) + self.order_data['items'] = [{ + 'product': 'Produit gratuit' + }] + self.order_data['geom'] = { + 'type': 'Polygon', + 'coordinates': [ + [[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]] + ]} + response = self.client.post(url, self.order_data, format='json') + order = json.loads(response.content) + self.assertEqual(len(order["actualGeom"]["coordinates"]), 0) + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.content) From 10e2e9ad7e5c96c7b631e53e74b15d8da23b1353 Mon Sep 17 00:00:00 2001 From: Andrey Rusakov Date: Fri, 20 Dec 2024 16:19:40 +0100 Subject: [PATCH 4/6] Do not check for limits at all if there is no area limit --- api/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index cc01c8bf..02f77edd 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -297,7 +297,7 @@ class OrderSerializer(serializers.ModelSerializer): def validate(self, attrs): super().validate(attrs) self._errors = {} - if 'geom' not in attrs: + if ('geom' not in attrs) or (settings.MAX_ORDER_AREA == 0): return attrs requestedGeom = Polygon( [xy[0:2] for xy in list(attrs['geom'].coords[0])], @@ -314,8 +314,8 @@ def validate(self, attrs): ownedRequestedGeom = requestedGeom.intersection(ownedAreas) attrs['actualGeom'] = ownedRequestedGeom - if (round(ownedRequestedGeom.area) == 0 and settings.MAX_ORDER_AREA > 0 - and requestedGeom.area > settings.MAX_ORDER_AREA): + if (round(ownedRequestedGeom.area) == 0 and + requestedGeom.area > settings.MAX_ORDER_AREA): raise ValidationError({ 'message': _(f'Order area is too large'), 'expected': settings.MAX_ORDER_AREA, From f16cf8234c973a04b2ad109317685de08615b13c Mon Sep 17 00:00:00 2001 From: Andrey Rusakov Date: Fri, 20 Dec 2024 16:21:06 +0100 Subject: [PATCH 5/6] Add migrations --- ..._alter_orderitem_download_guid_and_more.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 api/migrations/0050_order_actualgeom_alter_orderitem_download_guid_and_more.py diff --git a/api/migrations/0050_order_actualgeom_alter_orderitem_download_guid_and_more.py b/api/migrations/0050_order_actualgeom_alter_orderitem_download_guid_and_more.py new file mode 100644 index 00000000..7a828ad9 --- /dev/null +++ b/api/migrations/0050_order_actualgeom_alter_orderitem_download_guid_and_more.py @@ -0,0 +1,37 @@ +# Generated by Django 5.1.3 on 2024-12-20 15:20 + +import django.contrib.gis.db.models.fields +import django.contrib.gis.geos.collections +import django.contrib.gis.geos.polygon +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0049_orderitem_download_guid_alter_order_order_status_and_more'), + ('auth', '0012_alter_user_first_name_max_length'), + ] + + operations = [ + migrations.AddField( + model_name='order', + name='actualGeom', + field=django.contrib.gis.db.models.fields.PolygonField(null=True, srid=2056, verbose_name='actualGeom'), + ), + migrations.AlterField( + model_name='orderitem', + name='download_guid', + field=models.UUIDField(blank=True, null=True, unique=True, verbose_name='download_guid'), + ), + migrations.CreateModel( + name='ProductOwnership', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('geom', django.contrib.gis.db.models.fields.MultiPolygonField(default=django.contrib.gis.geos.collections.MultiPolygon(django.contrib.gis.geos.polygon.Polygon(((2828694.200665463, 1075126.8548189853), (2828694.200665463, 1299777.3195268118), (2484749.5514877755, 1299777.3195268118), (2484749.5514877755, 1075126.8548189853), (2828694.200665463, 1075126.8548189853)))), srid=2056, verbose_name='geom')), + ('product', models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, to='api.product', verbose_name='product')), + ('user_group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.group', verbose_name='user_group')), + ], + ), + ] From 05f3d6e119067800b5709ae9dbbd76e5294a0db7 Mon Sep 17 00:00:00 2001 From: Andrey Rusakov Date: Mon, 23 Dec 2024 17:47:19 +0100 Subject: [PATCH 6/6] Changed way how extra area is calculated --- ...alter_orderitem_download_guid_and_more.py} | 6 ++-- api/models.py | 2 +- api/serializers.py | 11 ++++--- api/tests/factories.py | 30 +++++++++++-------- api/tests/test_order.py | 25 +++++++++------- 5 files changed, 42 insertions(+), 32 deletions(-) rename api/migrations/{0050_order_actualgeom_alter_orderitem_download_guid_and_more.py => 0050_order_excludedgeom_alter_orderitem_download_guid_and_more.py} (92%) diff --git a/api/migrations/0050_order_actualgeom_alter_orderitem_download_guid_and_more.py b/api/migrations/0050_order_excludedgeom_alter_orderitem_download_guid_and_more.py similarity index 92% rename from api/migrations/0050_order_actualgeom_alter_orderitem_download_guid_and_more.py rename to api/migrations/0050_order_excludedgeom_alter_orderitem_download_guid_and_more.py index 7a828ad9..6012efba 100644 --- a/api/migrations/0050_order_actualgeom_alter_orderitem_download_guid_and_more.py +++ b/api/migrations/0050_order_excludedgeom_alter_orderitem_download_guid_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.3 on 2024-12-20 15:20 +# Generated by Django 5.1.3 on 2024-12-23 11:39 import django.contrib.gis.db.models.fields import django.contrib.gis.geos.collections @@ -17,8 +17,8 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( model_name='order', - name='actualGeom', - field=django.contrib.gis.db.models.fields.PolygonField(null=True, srid=2056, verbose_name='actualGeom'), + name='excludedGeom', + field=django.contrib.gis.db.models.fields.PolygonField(null=True, srid=2056, verbose_name='excludedGeom'), ), migrations.AlterField( model_name='orderitem', diff --git a/api/models.py b/api/models.py index ab5e88b4..1198d54d 100644 --- a/api/models.py +++ b/api/models.py @@ -586,7 +586,7 @@ class OrderStatus(models.TextChoices): null=True, ) geom = models.PolygonField(_("geom"), srid=settings.DEFAULT_SRID) - actualGeom = models.PolygonField(_("actualGeom"), srid=settings.DEFAULT_SRID, null=True) + excludedGeom = models.PolygonField(_("excludedGeom"), srid=settings.DEFAULT_SRID, null=True) client = models.ForeignKey( UserModel, models.PROTECT, verbose_name=_("client"), blank=True diff --git a/api/serializers.py b/api/serializers.py index 02f77edd..188c52eb 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -311,15 +311,14 @@ def validate(self, attrs): ownedAreas = MultiPolygon(srid=settings.DEFAULT_SRID) for area in relevantOwnedAreas: ownedAreas = ownedAreas.union(area.geom) - ownedRequestedGeom = requestedGeom.intersection(ownedAreas) - attrs['actualGeom'] = ownedRequestedGeom - - if (round(ownedRequestedGeom.area) == 0 and - requestedGeom.area > settings.MAX_ORDER_AREA): + unownedAreas = requestedGeom.difference(ownedAreas) + attrs['excludedGeom'] = unownedAreas + if (unownedAreas.area > settings.MAX_ORDER_AREA): raise ValidationError({ 'message': _(f'Order area is too large'), 'expected': settings.MAX_ORDER_AREA, - 'actual': requestedGeom.area + 'actual': requestedGeom.area, + 'excluded': unownedAreas.area }) return attrs diff --git a/api/tests/factories.py b/api/tests/factories.py index 13b97b40..4f657341 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -198,26 +198,32 @@ def __init__(self, webclient=None): user_group=Group.objects.create(name="zurich_data_owner"), product=self.products['free'], geom=MultiPolygon([Polygon([ - [8.472347, 47.364641], [8.472347, 47.404275], - [8.576202, 47.404275], [8.576202, 47.364641], - [8.472347, 47.364641] - ])], srid=4326)) + [2678084.641714959, 1246491.459194262], + [2678026.5975109423, 1250897.6804795086], + [2685865.173103665, 1251006.1324528102], + [2685929.053569935, 1246599.9914154143], + [2678084.641714959, 1246491.459194262] + ])])) self.lausanneDataOwner = ProductOwnership.objects.create( user_group=Group.objects.create(name="lausanne_data_owner"), product=self.products['free'], geom=MultiPolygon([Polygon([ - [6.59008, 46.500283], [6.59008, 46.551542], - [6.694794, 46.551542], [6.694794, 46.500283], - [6.59008, 46.500283] - ])], srid=4326)) + [2534861.402730483, 1150239.0030801909], + [2534923.142683635, 1155936.9007802252], + [2542953.5027929996, 1155855.250616442], + [2542899.372803648, 1150157.2750742848], + [2534861.402730483, 1150239.0030801909] + ])])) self.switzerlandDataOwner = ProductOwnership.objects.create( user_group=Group.objects.create(name="switzerland_data_owner"), product=self.products['free'], geom=MultiPolygon([Polygon([ - [5.50415, 45.713851], [5.50415, 47.857403], - [10.667725, 47.857403], [10.667725, 45.713851], - [5.50415, 45.713851] - ])], srid=4326)) + [2449355.7225977806, 1064320.9468696574], + [2455237.7729970617, 1302556.4191441573], + [2841601.3432626957, 1305741.504752999], + [2851408.7686695675, 1067635.8085796747], + [2449355.7225977806, 1064320.9468696574] + ])])) zurichOwners = self.zurichDataOwner.user_group zurichOwners.user_set.add(self.user_private) diff --git a/api/tests/test_order.py b/api/tests/test_order.py index d40ce489..00ad1e92 100644 --- a/api/tests/test_order.py +++ b/api/tests/test_order.py @@ -489,7 +489,7 @@ def test_order_geom_is_fine(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.content) @override_settings(MAX_ORDER_AREA = 100) - def test_order_owned_contains(self): + def test_order_owned_noExcluded(self): url = reverse('order-list') self.client.credentials(HTTP_AUTHORIZATION='Bearer ' + self.config.client_token) self.order_data['items'] = [{ @@ -507,11 +507,11 @@ def test_order_owned_contains(self): response = self.client.post(url, self.order_data, format='json') order = json.loads(response.content) - self.assertTrue(areasEqual(order["geom"], order["actualGeom"])) + self.assertEqual(len(order["excludedGeom"]["coordinates"]), 0) self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.content) - @override_settings(MAX_ORDER_AREA = 1000) - def test_order_owned_intersects(self): + @override_settings(MAX_ORDER_AREA = 1001) + def test_order_owned_intersects_toobig(self): url = reverse('order-list') self.client.credentials(HTTP_AUTHORIZATION='Bearer ' + self.config.client_token) self.order_data['items'] = [{ @@ -528,11 +528,12 @@ def test_order_owned_intersects(self): ]} response = self.client.post(url, self.order_data, format='json') - order = json.loads(response.content) - self.assertFalse(areasEqual(order["geom"], order["actualGeom"])) - self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.content) + errorDetails = json.loads(response.content) + self.assertEqual(errorDetails['message'], ['Order area is too large']) + self.assertEqual(errorDetails['expected'], ['1001']) + self.assertTrue(errorDetails['actual'][0].startswith('203800502.0')) - @override_settings(MAX_ORDER_AREA = 100) + @override_settings(MAX_ORDER_AREA = 10000) def test_order_unowned_limited(self): url = reverse('order-list') self.client.credentials(HTTP_AUTHORIZATION='Bearer ' + self.config.client_token) @@ -542,9 +543,13 @@ def test_order_unowned_limited(self): self.order_data['geom'] = { 'type': 'Polygon', 'coordinates': [ - [[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]] + [[2682058.9416315095, 1246529.024343783], + [2682052.9914918, 1246958.8119991398], + [2682208.5772218467, 1246960.9680303528], + [2682214.538653401, 1246531.1805305541], + [2682058.9416315095, 1246529.024343783]] ]} response = self.client.post(url, self.order_data, format='json') order = json.loads(response.content) - self.assertEqual(len(order["actualGeom"]["coordinates"]), 0) + self.assertEqual(len(order["excludedGeom"]["coordinates"]), 1) self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.content)