Merge pull request #4282 from unicef/create-tp-from-admin

[216917] Program Cycle required for TP, adjust create TP from list in admin, remove eDopomoga

codecov.yml

@@ -9,3 +9,19 @@ coverage:
     patch:
       default:
         target: 95%
+  ignore:
+    - "**/forms.py"
+    - "*/selenium_tests/**"
+    - "*/tests/**"
+    - "*/migrations/*"
+    - "*/apps.py"
+    - "*/admin/*.py"
+    - "*/admin.py"
+    - "**/fixtures.py"
+    - "hct_mis_api/one_time_scripts/*"
+    - "hct_mis_api/libs/*"
+    - "hct_mis_api/settings/*"
+    - "hct_mis_api/settings.py"
+    - "hct_mis_api/conftest.py"
+    - "hct_mis_api/config/settings.py"
+    - "hct_mis_api/apps/core/management/commands/*"

src/hct_mis_api/apps/core/admin.py

@@ -45,10 +45,9 @@
 from hct_mis_api.apps.account.models import Role, User
 from hct_mis_api.apps.administration.widgets import JsonWidget
 from hct_mis_api.apps.core.celery_tasks import (
-    create_target_population_task,
     upload_new_kobo_template_and_update_flex_fields_task,
 )
-from hct_mis_api.apps.core.forms import DataCollectingTypeForm, ProgramForm
+from hct_mis_api.apps.core.forms import DataCollectingTypeForm
 from hct_mis_api.apps.core.models import (
     BusinessArea,
     CountryCodeMap,
@@ -66,7 +65,6 @@
 from hct_mis_api.apps.household.models import DocumentType
 from hct_mis_api.apps.payment.forms import AcceptanceProcessThresholdForm
 from hct_mis_api.apps.payment.models import AcceptanceProcessThreshold
-from hct_mis_api.apps.targeting.models import TargetPopulation
 from hct_mis_api.apps.utils.admin import (
     HOPEModelAdminBase,
     LastSyncDateResetMixin,
@@ -692,30 +690,6 @@ def has_view_permission(self, request: HttpRequest, obj: Optional[Any] = None) -
     def has_add_permission(self, request: HttpRequest) -> bool:
         return request.user.can_download_storage_files()
 
-    @button(label="Create eDopomoga TP")
-    def create_tp(self, request: HttpRequest, pk: "UUID") -> Union[TemplateResponse, HttpResponsePermanentRedirect]:
-        storage_obj = StorageFile.objects.get(pk=pk)
-        context = self.get_common_context(
-            request,
-            pk,
-        )
-        if request.method == "GET":
-            if TargetPopulation.objects.filter(storage_file=storage_obj).exists():
-                self.message_user(request, "TargetPopulation for this storageFile have been created", messages.ERROR)
-                return redirect("..")
-
-            form = ProgramForm(business_area_id=storage_obj.business_area_id)
-            context["form"] = form
-            return TemplateResponse(request, "core/admin/create_tp.html", context)
-        else:
-            program_id = request.POST.get("program")
-            tp_name = request.POST.get("name")
-
-            create_target_population_task.delay(storage_obj.pk, program_id, tp_name)
-
-            self.message_user(request, "Creation of TargetPopulation started")
-            return redirect("..")
-
 
 @admin.register(MigrationStatus)
 class MigrationStatusAdmin(admin.ModelAdmin):

src/hct_mis_api/apps/core/celery_tasks.py

@@ -1,42 +1,16 @@
-import csv
 import logging
-import os
-import tempfile
-from datetime import datetime
 from functools import wraps
 from typing import Any, Callable
 
 from django.db import transaction
-from django.utils import timezone
 
 from hct_mis_api.apps.core.celery import app
-from hct_mis_api.apps.core.models import StorageFile, XLSXKoboTemplate
+from hct_mis_api.apps.core.models import XLSXKoboTemplate
 from hct_mis_api.apps.core.tasks.upload_new_template_and_update_flex_fields import (
     KoboRetriableError,
 )
-from hct_mis_api.apps.core.utils import IDENTIFICATION_TYPE_TO_KEY_MAPPING
-from hct_mis_api.apps.household.models import (
-    COLLECT_TYPE_SIZE_ONLY,
-    HEAD,
-    IDENTIFICATION_TYPE_NATIONAL_PASSPORT,
-    IDENTIFICATION_TYPE_TAX_ID,
-    MALE,
-    ROLE_PRIMARY,
-    BankAccountInfo,
-    Document,
-    DocumentType,
-    Household,
-    Individual,
-    IndividualRoleInHousehold,
-)
-from hct_mis_api.apps.periodic_data_update.utils import populate_pdu_with_null_values
-from hct_mis_api.apps.program.models import Program
-from hct_mis_api.apps.registration_data.models import RegistrationDataImport
-from hct_mis_api.apps.targeting.models import TargetPopulation
-from hct_mis_api.apps.targeting.services.targeting_stats_refresher import refresh_stats
 from hct_mis_api.apps.utils.logs import log_start_and_end
-from hct_mis_api.apps.utils.models import MergeStatusModel
-from hct_mis_api.apps.utils.sentry import sentry_tags, set_sentry_business_area_tag
+from hct_mis_api.apps.utils.sentry import sentry_tags
 
 logger = logging.getLogger(__name__)
 
@@ -100,173 +74,3 @@ def upload_new_kobo_template_and_update_flex_fields_task(self: Any, xlsx_kobo_te
     except Exception as e:
         logger.exception(e)
         raise self.retry(exc=e)
-
-
-@app.task(bind=True, default_retry_delay=60, max_retries=3)
-@log_start_and_end
-@sentry_tags
-def create_target_population_task(self: Any, storage_id: str, program_id: str, tp_name: str) -> None:
-    storage_obj = StorageFile.objects.get(id=storage_id)
-    file_path = None
-    program = Program.objects.get(id=program_id)
-    set_sentry_business_area_tag(program.business_area.name)
-
-    try:
-        with transaction.atomic():
-            registration_data_import = RegistrationDataImport.objects.create(
-                name=f"{storage_obj.file.name}_{program.name}",
-                number_of_individuals=0,
-                number_of_households=0,
-                business_area=program.business_area,
-                data_source=RegistrationDataImport.EDOPOMOGA,
-                program=program,
-            )
-            if program.biometric_deduplication_enabled:
-                registration_data_import.deduplication_engine_status = RegistrationDataImport.DEDUP_ENGINE_PENDING
-
-            business_area = storage_obj.business_area
-            country = business_area.countries.first()
-
-            passport_type = DocumentType.objects.get(
-                key=IDENTIFICATION_TYPE_TO_KEY_MAPPING[IDENTIFICATION_TYPE_NATIONAL_PASSPORT]
-            )
-            tax_type = DocumentType.objects.get(key=IDENTIFICATION_TYPE_TO_KEY_MAPPING[IDENTIFICATION_TYPE_TAX_ID])
-
-            first_registration_date = timezone.now()
-            last_registration_date = first_registration_date
-
-            families = {}
-            individuals, documents, bank_infos = [], [], []
-
-            storage_obj.status = StorageFile.STATUS_PROCESSING
-            storage_obj.save(update_fields=["status"])
-            rows_count = 0
-
-            # TODO fix to use Azure storage override AzureStorageFile open method
-            with storage_obj.file.open("rb") as original_file, tempfile.NamedTemporaryFile(delete=False) as tmp:
-                tmp.write(original_file.read())
-                file_path = tmp.name
-
-            with open(file_path, encoding="cp1251") as file:
-                reader = csv.DictReader(file, delimiter=";")
-                for row in reader:
-                    rows_count += 1
-                    family_id = row["ID_FAM"]
-                    iban = row["IBAN"]
-                    tax_id = row["N_ID"]
-                    passport_id = row["PASSPORT"]
-                    size = row["FAM_NUM"]
-
-                    individual_data = {
-                        "given_name": row.get("NAME", ""),
-                        "middle_name": row.get("PATRONYMIC", ""),
-                        "family_name": row.get("SURNAME", ""),
-                        "full_name": f'{row.get("NAME", "")} {row.get("PATRONYMIC", "")} {row.get("SURNAME", "")}',
-                        "birth_date": datetime.strptime(row["BDATE"], "%d.%m.%Y").date(),
-                        "phone_no": row.get("PHONЕ", ""),
-                        "business_area": business_area,
-                        "first_registration_date": first_registration_date,
-                        "last_registration_date": last_registration_date,
-                        "sex": MALE,
-                        "relationship": HEAD,
-                        "rdi_merge_status": MergeStatusModel.MERGED,
-                        "flex_fields": populate_pdu_with_null_values(program),
-                        "registration_data_import": registration_data_import,
-                    }
-                    if family_id in families:
-                        individual = Individual(**individual_data, household_id=families.get(family_id))
-                        individuals.append(individual)
-                    else:
-                        individual = Individual.objects.create(**individual_data)
-                        individual.refresh_from_db()
-
-                        household = Household.objects.create(
-                            head_of_household=individual,
-                            business_area=business_area,
-                            first_registration_date=first_registration_date,
-                            last_registration_date=last_registration_date,
-                            registration_data_import=registration_data_import,
-                            size=size,
-                            family_id=family_id,
-                            storage_obj=storage_obj,
-                            collect_individual_data=COLLECT_TYPE_SIZE_ONLY,
-                            country=country,
-                            rdi_merge_status=MergeStatusModel.MERGED,
-                        )
-
-                        individual.household = household
-                        individual.save(update_fields=("household",))
-
-                        IndividualRoleInHousehold.objects.create(
-                            role=ROLE_PRIMARY,
-                            individual=individual,
-                            household=household,
-                            rdi_merge_status=MergeStatusModel.MERGED,
-                        )
-
-                        families[family_id] = household.id
-
-                    passport = Document(
-                        document_number=passport_id,
-                        type=passport_type,
-                        individual=individual,
-                        status=Document.STATUS_INVALID,
-                        country=country,
-                        rdi_merge_status=MergeStatusModel.MERGED,
-                    )
-
-                    tax = Document(
-                        document_number=tax_id,
-                        type=tax_type,
-                        individual=individual,
-                        status=Document.STATUS_INVALID,
-                        country=country,
-                        rdi_merge_status=MergeStatusModel.MERGED,
-                    )
-
-                    bank_account_info = BankAccountInfo(
-                        bank_account_number=iban, individual=individual, rdi_merge_status=MergeStatusModel.MERGED
-                    )
-
-                    documents.append(passport)
-                    documents.append(tax)
-                    bank_infos.append(bank_account_info)
-
-                    if rows_count % 1000 == 0:
-                        Individual.objects.bulk_create(individuals)
-                        Document.objects.bulk_create(documents)
-                        BankAccountInfo.objects.bulk_create(bank_infos)
-                        individuals = []
-                        documents = []
-                        bank_infos = []
-
-            Individual.objects.bulk_create(individuals)
-            Document.objects.bulk_create(documents)
-            BankAccountInfo.objects.bulk_create(bank_infos)
-
-            households = Household.objects.filter(family_id__in=list(families.keys()))
-            households.update(withdrawn=True, withdrawn_date=timezone.now())
-            Individual.objects.filter(household__in=households).update(withdrawn=True, withdrawn_date=timezone.now())
-
-            target_population = TargetPopulation.objects.create(
-                name=tp_name,
-                created_by=storage_obj.created_by,
-                program=program,
-                status=TargetPopulation.STATUS_LOCKED,
-                build_status=TargetPopulation.BUILD_STATUS_OK,
-                business_area=business_area,
-                storage_file=storage_obj,
-            )
-            target_population.households.set(households)
-            refresh_stats(target_population)
-            target_population.save()
-
-            storage_obj.status = StorageFile.STATUS_FINISHED
-            storage_obj.save(update_fields=["status"])
-    except Exception as e:
-        storage_obj.status = StorageFile.STATUS_FAILED
-        storage_obj.save(update_fields=["status"])
-        raise self.retry(exc=e)
-    finally:
-        if file_path:
-            os.remove(file_path)

src/hct_mis_api/apps/household/forms.py

@@ -15,7 +15,7 @@
     PendingIndividual,
     XlsxUpdateFile,
 )
-from hct_mis_api.apps.program.models import Program
+from hct_mis_api.apps.program.models import Program, ProgramCycle
 from hct_mis_api.apps.registration_data.models import RegistrationDataImport
 from hct_mis_api.apps.targeting.models import TargetingCriteria, TargetPopulation
 
@@ -189,13 +189,18 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         assert self.program is not None
         read_only = kwargs.pop("read_only", False)
         super().__init__(*args, **kwargs)
+        self.fields["program_cycle"] = forms.ModelChoiceField(
+            queryset=ProgramCycle.objects.filter(program=self.program),
+            label="Programme Cycle",
+        )
 
         if read_only:
             self.fields["name"].widget = HiddenInput()
             self.fields["criteria"].widget = HiddenInput()
             self.fields["target_field"].widget = HiddenInput()
             self.fields["separator"].widget = HiddenInput()
             self.fields["targeting_criteria"].widget = HiddenInput()
+            self.fields["program_cycle"].widget = HiddenInput()
 
     def clean_criteria(self) -> Optional[List]:
         try:

src/hct_mis_api/apps/payment/fixtures.py

@@ -959,6 +959,7 @@ def generate_payment_plan() -> None:
         business_area=afghanistan,
         program=program,
         created_by=root,
+        program_cycle=program_cycle,
     )[0]
     full_rebuild(target_population)
     target_population.save()

src/hct_mis_api/apps/targeting/celery_tasks.py

@@ -139,6 +139,7 @@ def create_tp_from_list(form_data: Dict[str, str], user_id: str, program_pk: str
                     name=form.cleaned_data["name"],
                     business_area=program.business_area,
                     program=program,
+                    program_cycle=form.cleaned_data["program_cycle"],
                 )
                 tp.households.set(population)
                 refresh_stats(tp)

src/hct_mis_api/apps/targeting/fixtures.py

@@ -13,7 +13,10 @@
 from hct_mis_api.apps.core.models import BusinessArea
 from hct_mis_api.apps.household.fixtures import HouseholdFactory
 from hct_mis_api.apps.household.models import RESIDENCE_STATUS_CHOICE
-from hct_mis_api.apps.program.fixtures import get_program_with_dct_type_and_name
+from hct_mis_api.apps.program.fixtures import (
+    ProgramCycleFactory,
+    get_program_with_dct_type_and_name,
+)
 from hct_mis_api.apps.program.models import Program
 from hct_mis_api.apps.targeting.models import (
     HouseholdSelection,
@@ -94,6 +97,7 @@ class Meta:
         lambda t: Program.objects.filter(status=Program.ACTIVE).first() or get_program_with_dct_type_and_name()
     )
     business_area = factory.LazyAttribute(lambda t: BusinessArea.objects.first())
+    program_cycle = factory.LazyAttribute(lambda t: t.program.cycles.first() or ProgramCycleFactory(program=t.program))
 
     @factory.post_generation
     def households(self, create: bool, extracted: Iterable, **kwargs: Any) -> None:

src/hct_mis_api/apps/targeting/migrations/0048_migration.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.25 on 2024-10-03 01:07
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('program', '0053_migration'),
+        ('targeting', '0047_migration'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='targetpopulation',
+            name='program_cycle',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='target_populations', to='program.programcycle'),
+        ),
+    ]

src/hct_mis_api/apps/targeting/models.py

@@ -180,7 +180,7 @@ class TargetPopulation(SoftDeletableModel, TimeStampedUUIDModel, ConcurrencyMode
         on_delete=models.SET_NULL,
     )
     program_cycle = models.ForeignKey(
-        "program.ProgramCycle", on_delete=models.CASCADE, related_name="target_populations", null=True, blank=True
+        "program.ProgramCycle", on_delete=models.CASCADE, related_name="target_populations"
     )
     targeting_criteria = models.OneToOneField(
         "TargetingCriteria",