diff --git a/django_project/_version.txt b/django_project/_version.txt index 8a9ecc2e..7bcd0e36 100644 --- a/django_project/_version.txt +++ b/django_project/_version.txt @@ -1 +1 @@ -0.0.1 \ No newline at end of file +0.0.2 \ No newline at end of file diff --git a/django_project/gap/admin/crop_insight.py b/django_project/gap/admin/crop_insight.py index 83941199..fcb30d67 100644 --- a/django_project/gap/admin/crop_insight.py +++ b/django_project/gap/admin/crop_insight.py @@ -117,17 +117,12 @@ class CropInsightRequestAdmin(admin.ModelAdmin): """Admin for CropInsightRequest.""" list_display = ( - 'requested_at', 'farm_count', 'file_url', 'last_task_status', + 'requested_at', 'farm_group', 'file_url', 'last_task_status', 'background_tasks' ) - filter_horizontal = ('farms',) actions = (generate_insight_report_action,) readonly_fields = ('file',) - def farm_count(self, obj: CropInsightRequest): - """Return farm list.""" - return obj.farms.count() - def file_url(self, obj): """Return file url.""" if obj.file: diff --git a/django_project/gap/admin/farm.py b/django_project/gap/admin/farm.py index fe9acaeb..85c60f4c 100644 --- a/django_project/gap/admin/farm.py +++ b/django_project/gap/admin/farm.py @@ -6,12 +6,68 @@ """ from django.contrib import admin, messages +from django.utils.html import format_html from core.admin import AbstractDefinitionAdmin from gap.models import ( FarmCategory, FarmRSVPStatus, Farm ) -from gap.tasks.crop_insight import generate_spw +from gap.models.farm_group import FarmGroup, FarmGroupCropInsightField +from gap.tasks.crop_insight import generate_spw, generate_crop_plan + + +class FarmGroupCropInsightFieldInline(admin.TabularInline): + """Inline list for model output in FarmGroupCropInsightField.""" + + model = FarmGroupCropInsightField + extra = 0 + + +@admin.action(description='Recreate fields') +def recreate_farm_group_fields(modeladmin, request, queryset): + """Recreate farm group fields.""" + for group in queryset.all(): + group.prepare_fields() + + +@admin.action(description='Run crop insight') +def run_crop_insight(modeladmin, request, queryset): + """Run crop insight.""" + generate_crop_plan.delay() + + +@admin.register(FarmGroup) +class FarmGroupAdmin(AbstractDefinitionAdmin): + """FarmGroup admin.""" + + list_display = ( + 'name', 'description', 'farm_count' + ) + + filter_horizontal = ('farms', 'users') + inlines = (FarmGroupCropInsightFieldInline,) + actions = (recreate_farm_group_fields, run_crop_insight) + readonly_fields = ('displayed_headers',) + + def farm_count(self, obj: FarmGroup): + """Return farm list.""" + return obj.farms.count() + + def displayed_headers(self, obj: FarmGroup): + """Display headers as a table.""" + columns = "".join( + f'{header}' + for header in obj.headers + ) + return format_html( + '
' + ' ' + f" {columns}" + '
' + '
' + ) + + displayed_headers.allow_tags = True @admin.action(description='Generate farms spw') diff --git a/django_project/gap/factories/farm.py b/django_project/gap/factories/farm.py index 0285b914..6358b2fe 100644 --- a/django_project/gap/factories/farm.py +++ b/django_project/gap/factories/farm.py @@ -11,6 +11,7 @@ from gap.models import ( FarmCategory, FarmRSVPStatus, Farm ) +from gap.models.farm_group import FarmGroup class FarmCategoryFactory(DjangoModelFactory): @@ -51,3 +52,12 @@ class Meta: # noqa category = factory.SubFactory(FarmCategoryFactory) crop = factory.SubFactory('gap.factories.crop_insight.CropFactory') phone_number = '123-456-7936' + + +class FarmGroupFactory(DjangoModelFactory): + """Factory class for FarmGroup model.""" + + class Meta: # noqa + model = FarmGroup + + name = factory.Sequence(lambda n: f'name-{n}') diff --git a/django_project/gap/migrations/0020_farmgroup_remove_cropinsightrequest_farms_and_more.py b/django_project/gap/migrations/0020_farmgroup_remove_cropinsightrequest_farms_and_more.py new file mode 100644 index 00000000..c0e43325 --- /dev/null +++ b/django_project/gap/migrations/0020_farmgroup_remove_cropinsightrequest_farms_and_more.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.7 on 2024-09-20 05:51 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('gap', '0019_grid'), + ] + + operations = [ + migrations.CreateModel( + name='FarmGroup', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=512)), + ('description', models.TextField(blank=True, null=True)), + ('farms', models.ManyToManyField(to='gap.farm')), + ('users', models.ManyToManyField(to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['name'], + 'abstract': False, + }, + ), + migrations.RemoveField( + model_name='cropinsightrequest', + name='farms', + ), + migrations.CreateModel( + name='FarmGroupCropInsightField', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('field', models.CharField(max_length=256)), + ('column_number', models.PositiveSmallIntegerField(default=1, help_text='Column number on the csv. It is start from 1')), + ('label', models.CharField(blank=True, help_text='Change the field name to a other label. Keep it empty if does not want to override', max_length=256, null=True)), + ('active', models.BooleanField(default=True, help_text='Check if the field is active and included on the csv')), + ('farm_group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gap.farmgroup')), + ], + options={ + 'ordering': ['column_number'], + }, + ), + migrations.AddField( + model_name='cropinsightrequest', + name='farm_group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='gap.farmgroup'), + ), + ] diff --git a/django_project/gap/models/crop_insight.py b/django_project/gap/models/crop_insight.py index 57a6f95d..79d33edc 100644 --- a/django_project/gap/models/crop_insight.py +++ b/django_project/gap/models/crop_insight.py @@ -5,7 +5,6 @@ .. note:: Models """ -import json import uuid from datetime import date, timedelta @@ -16,10 +15,10 @@ from django.core.mail import EmailMessage from django.utils import timezone -from core.group_email_receiver import crop_plan_receiver from core.models.background_task import BackgroundTask, TaskStatus from core.models.common import Definition -from gap.models import Farm +from gap.models.farm import Farm +from gap.models.farm_group import FarmGroup from gap.models.lookup import RainfallClassification from gap.models.measurement import DatasetAttribute from gap.models.preferences import Preferences @@ -263,7 +262,7 @@ class CropPlanData: """The report model for the Insight Request Report.""" @staticmethod - def default_fields(): + def forecast_default_fields(): """Return shortterm default fields.""" from gap.providers.tio import tomorrowio_shortterm_forecast_dataset @@ -279,6 +278,18 @@ def default_fields(): forecast_fields.append('rainAccumulationType') return forecast_fields + @staticmethod + def default_fields(): + """Return default fields for Farm Plan Data.""" + return [ + 'farmID', + 'phoneNumber', + 'latitude', + 'longitude', + 'SPWTopMessage', + 'SPWDescription', + ] + def __init__( self, farm: Farm, generated_date: date, forecast_days: int = 13, forecast_fields: list = None @@ -322,7 +333,7 @@ def __init__( # Make default forecast_fields if not forecast_fields: - forecast_fields = CropPlanData.default_fields() + forecast_fields = CropPlanData.forecast_default_fields() self.forecast = self.forecast.filter( dataset_attribute__source__in=forecast_fields @@ -331,6 +342,19 @@ def __init__( self.forecast_fields = forecast_fields self.forecast_days = forecast_days + @staticmethod + def forecast_key(day_n, field): + """Return key for forecast field.""" + return f'day{day_n}_{field}' + + @staticmethod + def forecast_fields_used(): + """Return list of forecast fields that being used by crop insight.""" + return [ + 'rainAccumulationSum', 'precipitationProbability', + 'rainAccumulationType' + ] + @property def data(self) -> dict: """Return the data.""" @@ -365,7 +389,7 @@ def data(self) -> dict: # Short term forecast data for idx in range(self.forecast_days): for field in self.forecast_fields: - output[f'day{idx + 1}_{field}'] = '' + output[CropPlanData.forecast_key(idx + 1, field)] = '' first_date = None if self.forecast.first(): @@ -376,7 +400,7 @@ def data(self) -> dict: for data in self.forecast: var = data.dataset_attribute.source day_n = (data.value_date - first_date).days + 1 - output[f'day{day_n}_{var}'] = data.value + output[CropPlanData.forecast_key(day_n, var)] = data.value if (var == 'rainAccumulationSum' and 'rainAccumulationType' in self.forecast_fields): @@ -384,7 +408,9 @@ def data(self) -> dict: _class = RainfallClassification.classify(data.value) if _class: output[ - f'day{day_n}_rainAccumulationType' + CropPlanData.forecast_key( + day_n, 'rainAccumulationType' + ) ] = _class.name return output @@ -402,7 +428,9 @@ class CropInsightRequest(models.Model): default=timezone.now, help_text='The time when the request is made' ) - farms = models.ManyToManyField(Farm) + farm_group = models.ForeignKey( + FarmGroup, null=True, blank=True, on_delete=models.CASCADE + ) file = models.FileField( upload_to=ingestor_file_path, null=True, blank=True @@ -500,8 +528,11 @@ def title(self) -> str: """Return the title of the request.""" east_africa_timezone = Preferences.east_africa_timezone() east_africa_time = self.requested_at.astimezone(east_africa_timezone) + group = '' + if self.farm_group: + group = f' {self.farm_group} -' return ( - "GAP - Crop Plan Generator Results - " + f"GAP - Crop Plan Generator Results -{group} " f"{east_africa_time.strftime('%A-%d-%m-%Y')} " f"({east_africa_timezone})" ) @@ -509,12 +540,16 @@ def title(self) -> str: def _generate_report(self): """Generate reports.""" from spw.generator.crop_insight import CropInsightFarmGenerator - output = [] # If farm is empty, put empty farm - farms = self.farms.all() - if not farms.count(): - farms = [Farm()] + farms = [] + if self.farm_group: + farms = self.farm_group.farms.all() + + output = [ + self.farm_group.headers + ] + fields = self.farm_group.fields # Get farms for farm in farms: @@ -532,21 +567,19 @@ def _generate_report(self): ).data # Create header - if len(output) == 0: - output.append(list(data.keys())) - output.append([val for key, val in data.items()]) + row_data = [] + for field in fields: + try: + row_data.append(data[field.field]) + except KeyError: + row_data.append('') + output.append(row_data) # Render csv self.update_note('Generate CSV') csv_content = '' - # Replace header - output[0] = json.loads( - json.dumps(output[0]). - replace('rainAccumulationSum', 'mm'). - replace('rainAccumulationType', 'Type'). - replace('precipitationProbability', 'Chance') - ) + # Save to csv for row in output: csv_content += ','.join(map(str, row)) + '\n' content_file = ContentFile(csv_content) @@ -566,11 +599,7 @@ def _generate_report(self): Best regards ''', from_email=settings.DEFAULT_FROM_EMAIL, - to=[ - email for email in - crop_plan_receiver().values_list('email', flat=True) - if email - ] + to=self.farm_group.email_recipients() ) email.attach( f'{self.unique_id}.csv', diff --git a/django_project/gap/models/farm_group.py b/django_project/gap/models/farm_group.py new file mode 100644 index 00000000..a215ae73 --- /dev/null +++ b/django_project/gap/models/farm_group.py @@ -0,0 +1,134 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Farm models +""" + +from django.contrib.auth import get_user_model +from django.contrib.gis.db import models + +from core.group_email_receiver import crop_plan_receiver +from core.models.common import Definition +from gap.models.farm import Farm + +User = get_user_model() + + +class FarmGroup(Definition): + """Model representing group of farms.""" + + farms = models.ManyToManyField(Farm) + users = models.ManyToManyField(User) + + def email_recipients(self) -> list: + """Return list of email addresses of farm recipients.""" + return [ + user.email for user in + crop_plan_receiver().filter( + id__in=self.users.all().values_list('id', flat=True) + ) + if user.email + ] + + def save(self, *args, **kwargs): + """Save the group.""" + created = not self.pk + super(FarmGroup, self).save(*args, **kwargs) + if created: + self.prepare_fields() + + def prepare_fields(self): + """Prepare fields.""" + from gap.models.crop_insight import CropPlanData + from gap.providers.tio import TomorrowIODatasetReader + TomorrowIODatasetReader.init_provider() + + self.farmgroupcropinsightfield_set.all().delete() + column_num = 1 + for default_field in CropPlanData.default_fields(): + FarmGroupCropInsightField.objects.update_or_create( + farm_group=self, + field=default_field, + defaults={ + 'column_number': column_num + } + ) + column_num += 1 + + # Create for forecast + forecast_day_n = 13 + for idx in range(forecast_day_n): + for default_field in ( + CropPlanData.forecast_fields_used() + + CropPlanData.forecast_default_fields() + ): + field = CropPlanData.forecast_key(idx + 1, default_field) + label = field.replace( + 'rainAccumulationSum', 'mm' + ).replace( + 'rainAccumulationType', 'Type' + ).replace( + 'precipitationProbability', 'Chance' + ) + active = default_field in CropPlanData.forecast_fields_used() + FarmGroupCropInsightField.objects.update_or_create( + farm_group=self, + field=field, + defaults={ + 'column_number': column_num, + 'label': label if label != field else None, + 'active': active + } + ) + column_num += 1 + + @property + def headers(self): + """Return headers.""" + return [ + field.name for field in self.fields + ] + + @property + def fields(self): + """Return headers.""" + return self.farmgroupcropinsightfield_set.filter(active=True) + + +class FarmGroupCropInsightField(models.Model): + """Model representing the fields on the crop insight file.""" + + farm_group = models.ForeignKey( + FarmGroup, on_delete=models.CASCADE + ) + field = models.CharField( + max_length=256 + ) + column_number = models.PositiveSmallIntegerField( + help_text='Column number on the csv. It is start from 1', + default=1 + ) + label = models.CharField( + max_length=256, + help_text=( + 'Change the field name to a other label. ' + 'Keep it empty if does not want to override' + ), + blank=True, + null=True + ) + active = models.BooleanField( + default=True, + help_text='Check if the field is active and included on the csv' + ) + + class Meta: # noqa: D106 + ordering = ['column_number'] + + @property + def name(self): + """Return field key.""" + if self.label: + return self.label + return self.field diff --git a/django_project/gap/tasks/crop_insight.py b/django_project/gap/tasks/crop_insight.py index 96fbbca0..46a3788e 100644 --- a/django_project/gap/tasks/crop_insight.py +++ b/django_project/gap/tasks/crop_insight.py @@ -11,6 +11,7 @@ from core.celery import app from gap.models.crop_insight import CropInsightRequest from gap.models.farm import Farm +from gap.models.farm_group import FarmGroup from spw.generator.crop_insight import CropInsightFarmGenerator logger = get_task_logger(__name__) @@ -36,10 +37,13 @@ def generate_crop_plan(): """Generate crop plan for registered farms.""" # create report request user = User.objects.filter(is_superuser=True).first() - request = CropInsightRequest.objects.create(requested_by=user) - request.farms.set(Farm.objects.all().order_by('id')) - # generate report - request.run() + for group in FarmGroup.objects.all(): + request = CropInsightRequest.objects.create( + requested_by=user, + farm_group=group, + ) + # generate report + request.run() @app.task(name="retry_crop_plan_generators") diff --git a/django_project/gap/tests/crop_insight/test_crop_insight_model.py b/django_project/gap/tests/crop_insight/test_crop_insight_model.py index c128f03e..23f7340e 100644 --- a/django_project/gap/tests/crop_insight/test_crop_insight_model.py +++ b/django_project/gap/tests/crop_insight/test_crop_insight_model.py @@ -10,7 +10,7 @@ from django.test import TestCase -from gap.factories import CropInsightRequestFactory +from gap.factories import CropInsightRequestFactory, FarmGroupFactory from gap.models import CropInsightRequest @@ -50,7 +50,10 @@ def test_delete_object(self): def test_title(self): """Test update object.""" - obj = self.Factory() + group = FarmGroupFactory() + obj = self.Factory( + farm_group=group + ) obj.requested_at = datetime( 2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc @@ -58,7 +61,7 @@ def test_title(self): obj.save() self.assertEqual( obj.title, ( - "GAP - Crop Plan Generator Results - " + f"GAP - Crop Plan Generator Results - {group.name} - " "Monday-01-01-2024 (UTC+03:00)" ) ) @@ -71,7 +74,7 @@ def test_title(self): obj.save() self.assertEqual( obj.title, ( - "GAP - Crop Plan Generator Results - " + f"GAP - Crop Plan Generator Results - {group.name} - " "Tuesday-02-01-2024 (UTC+03:00)" ) ) @@ -84,7 +87,7 @@ def test_title(self): obj.save() self.assertEqual( obj.title, ( - "GAP - Crop Plan Generator Results - " + f"GAP - Crop Plan Generator Results - {group.name} - " "Tuesday-02-01-2024 (UTC+03:00)" ) ) @@ -97,7 +100,7 @@ def test_title(self): obj.save() self.assertEqual( obj.title, ( - "GAP - Crop Plan Generator Results - " + f"GAP - Crop Plan Generator Results - {group.name} - " "Monday-01-01-2024 (UTC+03:00)" ) ) diff --git a/django_project/gap/tests/farm_group/__init__.py b/django_project/gap/tests/farm_group/__init__.py new file mode 100644 index 00000000..4a9c2b85 --- /dev/null +++ b/django_project/gap/tests/farm_group/__init__.py @@ -0,0 +1,6 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Farm tests +""" diff --git a/django_project/gap/tests/farm_group/test_models.py b/django_project/gap/tests/farm_group/test_models.py new file mode 100644 index 00000000..5bdd4914 --- /dev/null +++ b/django_project/gap/tests/farm_group/test_models.py @@ -0,0 +1,74 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Unit tests for GAP Models. +""" + +from django.test import TestCase + +from gap.factories import ( + FarmGroupFactory, FarmFactory, UserF +) +from gap.models import FarmGroup + + +class FarmGroupCRUDTest(TestCase): + """Farm Group test case.""" + + Factory = FarmGroupFactory + Model = FarmGroup + + def test_create_object(self): + """Test create object.""" + obj = self.Factory() + self.assertIsInstance(obj, self.Model) + self.assertTrue(self.Model.objects.filter(id=obj.id).exists()) + + def test_read_object(self): + """Test read object.""" + obj = self.Factory() + fetched_obj = self.Model.objects.get(id=obj.id) + self.assertEqual(obj, fetched_obj) + + def test_update_object(self): + """Test update object.""" + obj = self.Factory() + new_name = "New ID" + obj.name = new_name + obj.save() + updated_obj = self.Model.objects.get(id=obj.id) + self.assertEqual(updated_obj.name, new_name) + + def test_delete_object(self): + """Test delete object.""" + obj = self.Factory() + _id = obj.id + obj.delete() + self.assertFalse(self.Model.objects.filter(id=_id).exists()) + + +class FarmGroupFUnctionalityCRUDTest(TestCase): + """Farm Group test case.""" + + Factory = FarmGroupFactory + Model = FarmGroup + + def setUp(self) -> None: + """Set test class.""" + group_1 = self.Factory() + group_1.farms.add(FarmFactory(), FarmFactory(), FarmFactory()) + group_1.users.add(UserF(), UserF(), UserF()) + group_2 = self.Factory() + group_2.farms.add(FarmFactory()) + group_2.users.add(UserF()) + + self.group_1 = group_1 + self.group_2 = group_2 + + def test_check_many_to_many(self): + """Test read object.""" + self.assertEqual(self.group_1.farms.count(), 3) + self.assertEqual(self.group_2.farms.count(), 1) + self.assertEqual(self.group_1.users.count(), 3) + self.assertEqual(self.group_2.users.count(), 1) diff --git a/django_project/gap_api/api_views/crop_insight.py b/django_project/gap_api/api_views/crop_insight.py index 2b97c6ea..93a076f6 100644 --- a/django_project/gap_api/api_views/crop_insight.py +++ b/django_project/gap_api/api_views/crop_insight.py @@ -6,9 +6,9 @@ """ from datetime import datetime +from django.conf import settings from django.db.utils import ProgrammingError from django.http import HttpResponseBadRequest -from django.conf import settings from django.utils import timezone from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -33,7 +33,7 @@ def default_fields(): if settings.DEBUG: return try: - return CropPlanData.default_fields() + return CropPlanData.forecast_default_fields() except ProgrammingError: return [] diff --git a/django_project/spw/tests/test_crop_insight_generator.py b/django_project/spw/tests/test_crop_insight_generator.py index e5755a75..659a2c61 100644 --- a/django_project/spw/tests/test_crop_insight_generator.py +++ b/django_project/spw/tests/test_crop_insight_generator.py @@ -14,13 +14,13 @@ from core.factories import UserF from core.group_email_receiver import _group_crop_plan_receiver -from gap.factories import PreferencesFactory from gap.factories.crop_insight import CropInsightRequestFactory -from gap.factories.farm import FarmFactory +from gap.factories.farm import FarmFactory, FarmGroupFactory from gap.factories.grid import GridFactory from gap.models.crop_insight import ( FarmSuitablePlantingWindowSignal, CropInsightRequest ) +from gap.models.preferences import Preferences from gap.tasks.crop_insight import ( generate_insight_report, generate_crop_plan @@ -119,6 +119,10 @@ def setUp(self): geometry=Point(50.22222222, 50), grid=grid_3, ) + self.farm_group = FarmGroupFactory() + self.farm_group.farms.add( + self.farm, self.farm_2, self.farm_3, self.farm_4, self.farm_5 + ) self.r_model = RModelFactory.create(name='test') self.today = date.today() self.superuser = UserF.create( @@ -131,8 +135,10 @@ def setUp(self): self.user_1 = UserF(email='user_1@email.com') self.user_2 = UserF(email='user_2@email.com') self.user_3 = UserF(email='user_3@email.com') + self.user_4 = UserF(email='') + self.farm_group.users.add(self.user_1, self.user_2, self.user_3) group.user_set.add(self.user_1, self.user_2) - self.preferences = PreferencesFactory() + self.preferences = Preferences().load() self.preferences.crop_plan_config = { 'lat_lon_decimal_digits': 4 } @@ -263,12 +269,9 @@ def create_timeline_data( mock_fetch_timelines_data.return_value = {} # Crop insight report - self.request = CropInsightRequestFactory.create() - self.request.farms.add(self.farm) - self.request.farms.add(self.farm_2) - self.request.farms.add(self.farm_3) - self.request.farms.add(self.farm_4) - self.request.farms.add(self.farm_5) + self.request = CropInsightRequestFactory.create( + farm_group=self.farm_group + ) generate_insight_report(self.request.id) self.request.refresh_from_db() with self.request.file.open(mode='r') as csv_file: @@ -382,11 +385,16 @@ def mock_send_fn(self, fail_silently=False): with patch( "django.core.mail.EmailMessage.send", mock_send_fn ): - request = CropInsightRequestFactory.create() + FarmGroupFactory() + used_group = FarmGroupFactory() + used_group.users.add(parent.user_1, parent.user_2, parent.user_4) + request = CropInsightRequestFactory.create( + farm_group=used_group + ) request.run() parent.assertEqual(len(self.recipients), 2) parent.assertEqual( self.recipients, - [parent.user_1.email, parent.user_2.email] + [parent.user_2.email, parent.user_1.email] )