From 522579b8b2aeff90d367b83bf725060d38485767 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 3 Oct 2024 04:40:48 +0200 Subject: [PATCH 01/14] Program cycle requred for TP, adjust create TP from list in admin --- src/hct_mis_api/apps/household/forms.py | 7 ++++++- src/hct_mis_api/apps/payment/fixtures.py | 1 + .../apps/targeting/celery_tasks.py | 1 + src/hct_mis_api/apps/targeting/fixtures.py | 3 ++- .../targeting/migrations/0048_migration.py | 20 +++++++++++++++++++ src/hct_mis_api/apps/targeting/models.py | 2 +- 6 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 src/hct_mis_api/apps/targeting/migrations/0048_migration.py diff --git a/src/hct_mis_api/apps/household/forms.py b/src/hct_mis_api/apps/household/forms.py index 4b582b5bc9..30a3b19e78 100644 --- a/src/hct_mis_api/apps/household/forms.py +++ b/src/hct_mis_api/apps/household/forms.py @@ -15,7 +15,7 @@ PendingIndividual, XlsxUpdateFile, ) -from hct_mis_api.apps.program.models import Program +from hct_mis_api.apps.program.models import Program, ProgramCycle from hct_mis_api.apps.registration_data.models import RegistrationDataImport from hct_mis_api.apps.targeting.models import TargetingCriteria, TargetPopulation @@ -189,6 +189,10 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: assert self.program is not None read_only = kwargs.pop("read_only", False) super().__init__(*args, **kwargs) + self.fields["program_cycle"] = forms.ModelChoiceField( + queryset=ProgramCycle.objects.filter(program=self.program), + label="Programme Cycle", + ) if read_only: self.fields["name"].widget = HiddenInput() @@ -196,6 +200,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.fields["target_field"].widget = HiddenInput() self.fields["separator"].widget = HiddenInput() self.fields["targeting_criteria"].widget = HiddenInput() + self.fields["program_cycle"].widget = HiddenInput() def clean_criteria(self) -> Optional[List]: try: diff --git a/src/hct_mis_api/apps/payment/fixtures.py b/src/hct_mis_api/apps/payment/fixtures.py index 5d8a3095b7..bb5878a813 100644 --- a/src/hct_mis_api/apps/payment/fixtures.py +++ b/src/hct_mis_api/apps/payment/fixtures.py @@ -959,6 +959,7 @@ def generate_payment_plan() -> None: business_area=afghanistan, program=program, created_by=root, + program_cycle=program_cycle, )[0] full_rebuild(target_population) target_population.save() diff --git a/src/hct_mis_api/apps/targeting/celery_tasks.py b/src/hct_mis_api/apps/targeting/celery_tasks.py index 12131a0fb6..65d8abd1d5 100644 --- a/src/hct_mis_api/apps/targeting/celery_tasks.py +++ b/src/hct_mis_api/apps/targeting/celery_tasks.py @@ -139,6 +139,7 @@ def create_tp_from_list(form_data: Dict[str, str], user_id: str, program_pk: str name=form.cleaned_data["name"], business_area=program.business_area, program=program, + program_cycle=form.cleaned_data["program_cycle"], ) tp.households.set(population) refresh_stats(tp) diff --git a/src/hct_mis_api/apps/targeting/fixtures.py b/src/hct_mis_api/apps/targeting/fixtures.py index a3f063a7e8..2953abb0e4 100644 --- a/src/hct_mis_api/apps/targeting/fixtures.py +++ b/src/hct_mis_api/apps/targeting/fixtures.py @@ -13,7 +13,7 @@ from hct_mis_api.apps.core.models import BusinessArea from hct_mis_api.apps.household.fixtures import HouseholdFactory from hct_mis_api.apps.household.models import RESIDENCE_STATUS_CHOICE -from hct_mis_api.apps.program.fixtures import get_program_with_dct_type_and_name +from hct_mis_api.apps.program.fixtures import get_program_with_dct_type_and_name, ProgramCycleFactory from hct_mis_api.apps.program.models import Program from hct_mis_api.apps.targeting.models import ( HouseholdSelection, @@ -94,6 +94,7 @@ class Meta: lambda t: Program.objects.filter(status=Program.ACTIVE).first() or get_program_with_dct_type_and_name() ) business_area = factory.LazyAttribute(lambda t: BusinessArea.objects.first()) + program_cycle = factory.LazyAttribute(lambda t: t.program.cycles.first() or ProgramCycleFactory(program=t.program)) @factory.post_generation def households(self, create: bool, extracted: Iterable, **kwargs: Any) -> None: diff --git a/src/hct_mis_api/apps/targeting/migrations/0048_migration.py b/src/hct_mis_api/apps/targeting/migrations/0048_migration.py new file mode 100644 index 0000000000..fca6eed297 --- /dev/null +++ b/src/hct_mis_api/apps/targeting/migrations/0048_migration.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.25 on 2024-10-03 01:07 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('program', '0053_migration'), + ('targeting', '0047_migration'), + ] + + operations = [ + migrations.AlterField( + model_name='targetpopulation', + name='program_cycle', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='target_populations', to='program.programcycle'), + ), + ] diff --git a/src/hct_mis_api/apps/targeting/models.py b/src/hct_mis_api/apps/targeting/models.py index fb41545f1b..18accfa09d 100644 --- a/src/hct_mis_api/apps/targeting/models.py +++ b/src/hct_mis_api/apps/targeting/models.py @@ -180,7 +180,7 @@ class TargetPopulation(SoftDeletableModel, TimeStampedUUIDModel, ConcurrencyMode on_delete=models.SET_NULL, ) program_cycle = models.ForeignKey( - "program.ProgramCycle", on_delete=models.CASCADE, related_name="target_populations", null=True, blank=True + "program.ProgramCycle", on_delete=models.CASCADE, related_name="target_populations" ) targeting_criteria = models.OneToOneField( "TargetingCriteria", From e434edd397fcf560958af858fb5056dc32efd339 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 3 Oct 2024 04:55:57 +0200 Subject: [PATCH 02/14] linter --- src/hct_mis_api/apps/targeting/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hct_mis_api/apps/targeting/fixtures.py b/src/hct_mis_api/apps/targeting/fixtures.py index 2953abb0e4..57d4d62962 100644 --- a/src/hct_mis_api/apps/targeting/fixtures.py +++ b/src/hct_mis_api/apps/targeting/fixtures.py @@ -13,7 +13,7 @@ from hct_mis_api.apps.core.models import BusinessArea from hct_mis_api.apps.household.fixtures import HouseholdFactory from hct_mis_api.apps.household.models import RESIDENCE_STATUS_CHOICE -from hct_mis_api.apps.program.fixtures import get_program_with_dct_type_and_name, ProgramCycleFactory +from hct_mis_api.apps.program.fixtures import ProgramCycleFactory, get_program_with_dct_type_and_name from hct_mis_api.apps.program.models import Program from hct_mis_api.apps.targeting.models import ( HouseholdSelection, From b9ffb29e9ab71db539d3d14a69a127f0b7ec997b Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 3 Oct 2024 06:05:42 +0200 Subject: [PATCH 03/14] fix tests creating TPs without cycles --- src/hct_mis_api/apps/targeting/fixtures.py | 5 +- .../test_pull_from_datahub.py | 1 + .../test_data_send_tp_to_datahub.py | 2 + ...t_external_collector_send_tp_to_datahub.py | 2 + .../mis_datahub/test_send_tp_to_datahub.py | 5 + .../payment/test_payment_plan_services.py | 18 -- .../test_copy_target_population_mutation.py | 14 +- ...tatus_change_target_population_mutation.py | 18 +- ...test_target_population_households_query.py | 4 + .../unit/apps/targeting/test_target_query.py | 6 + .../test_update_target_population_mutation.py | 5 +- .../test_program_cycle_data_migration.py | 275 ------------------ 12 files changed, 57 insertions(+), 298 deletions(-) delete mode 100644 tests/unit/one_time_scripts/test_program_cycle_data_migration.py diff --git a/src/hct_mis_api/apps/targeting/fixtures.py b/src/hct_mis_api/apps/targeting/fixtures.py index 57d4d62962..91a13cb578 100644 --- a/src/hct_mis_api/apps/targeting/fixtures.py +++ b/src/hct_mis_api/apps/targeting/fixtures.py @@ -13,7 +13,10 @@ from hct_mis_api.apps.core.models import BusinessArea from hct_mis_api.apps.household.fixtures import HouseholdFactory from hct_mis_api.apps.household.models import RESIDENCE_STATUS_CHOICE -from hct_mis_api.apps.program.fixtures import ProgramCycleFactory, get_program_with_dct_type_and_name +from hct_mis_api.apps.program.fixtures import ( + ProgramCycleFactory, + get_program_with_dct_type_and_name, +) from hct_mis_api.apps.program.models import Program from hct_mis_api.apps.targeting.models import ( HouseholdSelection, diff --git a/tests/unit/apps/cash_assist_datahub/test_pull_from_datahub.py b/tests/unit/apps/cash_assist_datahub/test_pull_from_datahub.py index f820e0301d..e954b4c0cf 100644 --- a/tests/unit/apps/cash_assist_datahub/test_pull_from_datahub.py +++ b/tests/unit/apps/cash_assist_datahub/test_pull_from_datahub.py @@ -75,6 +75,7 @@ def _setup_in_app_data(cls) -> None: status=TargetPopulation.STATUS_PROCESSING, program=cls.program, business_area=cls.business_area, + program_cycle=cls.program.cycles.first() ) program = ProgramFactory( diff --git a/tests/unit/apps/mis_datahub/test_data_send_tp_to_datahub.py b/tests/unit/apps/mis_datahub/test_data_send_tp_to_datahub.py index 88e310adc0..4d9e2e18e5 100644 --- a/tests/unit/apps/mis_datahub/test_data_send_tp_to_datahub.py +++ b/tests/unit/apps/mis_datahub/test_data_send_tp_to_datahub.py @@ -88,6 +88,7 @@ def setUpTestData(cls) -> None: ca_hash_id=uuid.uuid4(), ca_id="TEST", ) + cls.program_cycle = cls.program.cycles.first() rdi = RegistrationDataImportFactory(program=cls.program) cls.create_first_household(admin_area2, rdi) @@ -98,6 +99,7 @@ def setUpTestData(cls) -> None: program=cls.program, business_area=business_area_with_data_sharing, status=TargetPopulation.STATUS_PROCESSING, + program_cycle=cls.program_cycle, ) cls.target_population.households.set([cls.household]) cls.target_population: TargetPopulation = refresh_stats(cls.target_population) diff --git a/tests/unit/apps/mis_datahub/test_external_collector_send_tp_to_datahub.py b/tests/unit/apps/mis_datahub/test_external_collector_send_tp_to_datahub.py index 48d5705826..584f3d57d3 100644 --- a/tests/unit/apps/mis_datahub/test_external_collector_send_tp_to_datahub.py +++ b/tests/unit/apps/mis_datahub/test_external_collector_send_tp_to_datahub.py @@ -90,6 +90,7 @@ def setUpTestData(cls) -> None: program=cls.program_individual_data_needed_true, business_area=business_area_with_data_sharing, status=TargetPopulation.STATUS_PROCESSING, + program_cycle=cls.program_individual_data_needed_true.cycles.first(), ) cls.target_population_with_individuals.households.set([cls.household, cls.household_second]) cls.target_population_with_individuals = refresh_stats(cls.target_population_with_individuals) @@ -101,6 +102,7 @@ def setUpTestData(cls) -> None: program=cls.program_individual_data_needed_false, business_area=business_area_with_data_sharing, status=TargetPopulation.STATUS_PROCESSING, + program_cycle=cls.program_individual_data_needed_false.cycles.first(), ) cls.target_population_without_individuals.households.set([cls.household, cls.household_second]) cls.target_population_without_individuals = refresh_stats(cls.target_population_without_individuals) diff --git a/tests/unit/apps/mis_datahub/test_send_tp_to_datahub.py b/tests/unit/apps/mis_datahub/test_send_tp_to_datahub.py index b37c356fdb..66c2c96e9a 100644 --- a/tests/unit/apps/mis_datahub/test_send_tp_to_datahub.py +++ b/tests/unit/apps/mis_datahub/test_send_tp_to_datahub.py @@ -204,6 +204,7 @@ def setUpTestData(cls) -> None: program=cls.program_individual_data_needed_true, business_area=business_area_with_data_sharing, status=TargetPopulation.STATUS_PROCESSING, + program_cycle=cls.program_individual_data_needed_true.cycles.first(), ) cls.target_population_first.households.set([cls.household]) cls.target_population_first = refresh_stats(cls.target_population_first) @@ -215,6 +216,7 @@ def setUpTestData(cls) -> None: program=cls.program_individual_data_needed_false, business_area=business_area_with_data_sharing, status=TargetPopulation.STATUS_PROCESSING, + program_cycle=cls.program_individual_data_needed_false.cycles.first(), ) cls.target_population_second.households.set([cls.household]) cls.target_population_second = refresh_stats(cls.target_population_second) @@ -226,6 +228,7 @@ def setUpTestData(cls) -> None: program=cls.program_third, business_area=business_area_with_data_sharing, status=TargetPopulation.STATUS_PROCESSING, + program_cycle=cls.program_third.cycles.first(), ) cls.target_population_third.households.set([cls.household_second]) cls.target_population_third = refresh_stats(cls.target_population_third) @@ -292,6 +295,7 @@ def test_send_two_times_household_with_different(self) -> None: program=program_individual_data_needed_false, business_area=business_area_with_data_sharing, status=TargetPopulation.STATUS_PROCESSING, + program_cycle=program_individual_data_needed_false.cycles.first(), ) target_population_first.households.set([household]) target_population_first = refresh_stats(target_population_first) @@ -301,6 +305,7 @@ def test_send_two_times_household_with_different(self) -> None: program=program_individual_data_needed_true, business_area=business_area_with_data_sharing, status=TargetPopulation.STATUS_PROCESSING, + program_cycle=program_individual_data_needed_true.cycles.first(), ) target_population_second.households.set([household]) target_population_second = refresh_stats(target_population_second) diff --git a/tests/unit/apps/payment/test_payment_plan_services.py b/tests/unit/apps/payment/test_payment_plan_services.py index 4887a78472..e3dbbc32b6 100644 --- a/tests/unit/apps/payment/test_payment_plan_services.py +++ b/tests/unit/apps/payment/test_payment_plan_services.py @@ -600,21 +600,3 @@ def test_create_with_program_cycle_validation_error(self) -> None: PaymentPlanService.create(input_data=input_data, user=self.user) cycle.refresh_from_db() assert cycle.status == ProgramCycle.ACTIVE - - def test_create_pp_validation_errors_if_tp_without_cycle(self) -> None: - self.business_area.is_payment_plan_applicable = True - self.business_area.save() - targeting_without_cycle = TargetPopulationFactory( - program=ProgramFactory(), status=TargetPopulation.STATUS_READY_FOR_PAYMENT_MODULE - ) - input_data = dict( - business_area_slug=self.business_area.slug, - targeting_id=self.id_to_base64(targeting_without_cycle.id, "TargetingNode"), - dispersion_start_date=parse_date("2020-09-10"), - dispersion_end_date=parse_date("2020-09-11"), - currency="USD", - ) - - self.assertIsNone(targeting_without_cycle.program_cycle) - with self.assertRaisesMessage(GraphQLError, "Target Population should have assigned Programme Cycle"): - PaymentPlanService.create(input_data=input_data, user=self.user) diff --git a/tests/unit/apps/targeting/test_copy_target_population_mutation.py b/tests/unit/apps/targeting/test_copy_target_population_mutation.py index b2894ead74..f8c9638aac 100644 --- a/tests/unit/apps/targeting/test_copy_target_population_mutation.py +++ b/tests/unit/apps/targeting/test_copy_target_population_mutation.py @@ -76,7 +76,11 @@ def setUpTestData(cls) -> None: cls.household = household cls.update_partner_access_to_program(partner, cls.program) tp = TargetPopulation( - name="Original Target Population", status="LOCKED", business_area=cls.business_area, program=cls.program + name="Original Target Population", + status="LOCKED", + business_area=cls.business_area, + program=cls.program, + program_cycle=cls.cycle, ) tp.targeting_criteria = cls.get_targeting_criteria_for_rule( @@ -86,7 +90,11 @@ def setUpTestData(cls) -> None: tp.households.add(cls.household) cls.target_population = tp cls.empty_target_population_1 = TargetPopulation( - name="emptyTargetPopulation1", status="LOCKED", business_area=cls.business_area, program=cls.program + name="emptyTargetPopulation1", + status="LOCKED", + business_area=cls.business_area, + program=cls.program, + program_cycle=cls.cycle, ) cls.empty_target_population_1.save() @@ -95,6 +103,7 @@ def setUpTestData(cls) -> None: status="LOCKED", business_area=cls.business_area, program=cls.program, + program_cycle=cls.cycle, ) targeting_criteria_hh_ids = TargetingCriteria(household_ids=[cls.household.unicef_id]) targeting_criteria_hh_ids.save() @@ -106,6 +115,7 @@ def setUpTestData(cls) -> None: status="LOCKED", business_area=cls.business_area, program=cls.program, + program_cycle=cls.cycle, ) targeting_criteria_hh_ids = TargetingCriteria(individual_ids=[individual.unicef_id]) targeting_criteria_hh_ids.save() diff --git a/tests/unit/apps/targeting/test_status_change_target_population_mutation.py b/tests/unit/apps/targeting/test_status_change_target_population_mutation.py index c956353b29..fe3503b017 100644 --- a/tests/unit/apps/targeting/test_status_change_target_population_mutation.py +++ b/tests/unit/apps/targeting/test_status_change_target_population_mutation.py @@ -65,6 +65,7 @@ def setUpTestData(cls) -> None: cls.households.append(cls.household_size_1) cls.households.append(cls.household_size_2) cls.program = ProgramFactory(status=Program.ACTIVE, business_area=cls.business_area) + cls.program_cycle = cls.program.cycles.first() cls.update_partner_access_to_program(partner, cls.program) tp = TargetPopulation( @@ -72,6 +73,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_OPEN, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) tp.targeting_criteria = cls.get_targeting_criteria_for_rule( @@ -86,6 +88,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) tp.targeting_criteria = cls.get_targeting_criteria_for_rule( @@ -100,6 +103,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) tp.targeting_criteria = cls.get_targeting_criteria_for_rule( @@ -183,6 +187,7 @@ def setUpTestData(cls) -> None: cls.households.append(cls.household_size_1) cls.households.append(cls.household_size_2) cls.program = ProgramFactory(status=Program.ACTIVE, business_area=cls.business_area) + cls.program_cycle = cls.program.cycles.first() cls.update_partner_access_to_program(partner, cls.program) tp = TargetPopulation( @@ -190,6 +195,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_OPEN, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) tp.targeting_criteria = cls.get_targeting_criteria_for_rule( @@ -203,6 +209,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) tp.targeting_criteria = cls.get_targeting_criteria_for_rule( @@ -217,6 +224,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) tp.targeting_criteria = cls.get_targeting_criteria_for_rule( @@ -313,6 +321,7 @@ def setUpTestData(cls) -> None: cls.households.append(cls.household_size_1) cls.households.append(cls.household_size_2) cls.program = ProgramFactory(status=Program.ACTIVE, business_area=cls.business_area) + cls.program_cycle = cls.program.cycles.first() cls.update_partner_access_to_program(partner, cls.program) tp = TargetPopulation( @@ -320,6 +329,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_OPEN, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) tp.targeting_criteria = cls.get_targeting_criteria_for_rule( @@ -332,6 +342,8 @@ def setUpTestData(cls) -> None: name="Approved Target Population with final filters", status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, + program=cls.program, + program_cycle=cls.program_cycle, ) tp.targeting_criteria = cls.get_targeting_criteria_for_rule( @@ -344,7 +356,11 @@ def setUpTestData(cls) -> None: cls.target_population_approved_with_final_rule = tp tp = TargetPopulation( - name="Approved Target Population", status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area + name="Approved Target Population", + status=TargetPopulation.STATUS_LOCKED, + business_area=cls.business_area, + program=cls.program, + program_cycle=cls.program_cycle, ) tp.targeting_criteria = cls.get_targeting_criteria_for_rule( diff --git a/tests/unit/apps/targeting/test_target_population_households_query.py b/tests/unit/apps/targeting/test_target_population_households_query.py index 16d9fd7add..c2115e4b1c 100644 --- a/tests/unit/apps/targeting/test_target_population_households_query.py +++ b/tests/unit/apps/targeting/test_target_population_households_query.py @@ -66,6 +66,7 @@ def setUpTestData(cls) -> None: cls.partner = PartnerFactory(name="TestPartner") cls.user = UserFactory(partner=cls.partner) cls.program = get_program_with_dct_type_and_name() + cls.program_cycle = cls.program.cycles.first() targeting_criteria = cls.get_targeting_criteria_for_rule( {"field_name": "size", "arguments": [2], "comparison_method": "EQUALS"} ) @@ -75,6 +76,7 @@ def setUpTestData(cls) -> None: targeting_criteria=targeting_criteria, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) cls.target_population_size_2.save() targeting_criteria = cls.get_targeting_criteria_for_rule( @@ -86,6 +88,7 @@ def setUpTestData(cls) -> None: targeting_criteria=targeting_criteria, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) cls.target_population_residence_status.save() @@ -99,6 +102,7 @@ def setUpTestData(cls) -> None: status="LOCKED", business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) cls.target_population_size_1_approved.save() HouseholdSelection.objects.create( diff --git a/tests/unit/apps/targeting/test_target_query.py b/tests/unit/apps/targeting/test_target_query.py index 32d6fde118..2204eefbc2 100644 --- a/tests/unit/apps/targeting/test_target_query.py +++ b/tests/unit/apps/targeting/test_target_query.py @@ -145,6 +145,7 @@ def setUpTestData(cls) -> None: targeting_criteria=targeting_criteria, business_area=cls.business_area, program=cls.program, + program_cycle=cls.cycle_2, ) cls.target_population_size_2.save() cls.target_population_size_2 = full_rebuild(cls.target_population_size_2) @@ -174,6 +175,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program, + program_cycle=cls.cycle_2, ) cls.target_population_size_1_approved.save() cls.target_population_size_1_approved = full_rebuild(cls.target_population_size_1_approved) @@ -220,6 +222,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program, + program_cycle=cls.cycle_2, ) cls.target_population_with_pdu_filter.save() cls.target_population_with_pdu_filter = full_rebuild(cls.target_population_with_pdu_filter) @@ -254,6 +257,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program, + program_cycle=cls.cycle_2, ) cls.target_population_with_individual_filter.save() cls.target_population_with_individual_filter = full_rebuild(cls.target_population_with_individual_filter) @@ -266,6 +270,7 @@ def setUpTestData(cls) -> None: business_area=cls.business_area, data_collecting_type__type=DataCollectingType.Type.SOCIAL, ) + cls.cycle_sw = cls.program_sw.cycles.first() pdu_data_string_sw = PeriodicFieldDataFactory( subtype=PeriodicFieldData.STRING, number_of_rounds=2, @@ -292,6 +297,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program_sw, + program_cycle=cls.cycle_sw, ) cls.target_population_with_pdu_filter_for_sw.save() cls.target_population_with_pdu_filter_for_sw = full_rebuild(cls.target_population_with_pdu_filter_for_sw) diff --git a/tests/unit/apps/targeting/test_update_target_population_mutation.py b/tests/unit/apps/targeting/test_update_target_population_mutation.py index 6b6571ee34..6620da653d 100644 --- a/tests/unit/apps/targeting/test_update_target_population_mutation.py +++ b/tests/unit/apps/targeting/test_update_target_population_mutation.py @@ -168,6 +168,7 @@ def setUpTestData(cls) -> None: create_household({"size": 3, "residence_status": "HOST", "business_area": cls.business_area}) create_household({"size": 3, "residence_status": "HOST", "business_area": cls.business_area}) cls.program = ProgramFactory(status=Program.ACTIVE, business_area=cls.business_area) + cls.program_cycle = cls.program.cycles.first() cls.update_partner_access_to_program(partner, cls.program) cls.draft_target_population = TargetPopulation( name="draft_target_population", @@ -177,6 +178,7 @@ def setUpTestData(cls) -> None: created_by=cls.user, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) cls.draft_target_population.save() cls.approved_target_population = TargetPopulation( @@ -188,11 +190,11 @@ def setUpTestData(cls) -> None: created_by=cls.user, business_area=cls.business_area, program=cls.program, + program_cycle=cls.program_cycle, ) cls.approved_target_population.save() cls.approved_target_population.households.set(Household.objects.all()) cls.target_populations = [cls.draft_target_population, cls.approved_target_population] - cls.program_cycle = cls.program.cycles.first() @staticmethod def get_targeting_criteria_for_rule(rule_filter: Dict) -> TargetingCriteria: @@ -292,6 +294,7 @@ def test_fail_update_for_incorrect_status(self) -> None: business_area=self.business_area, program=self.program, status=TargetPopulation.STATUS_PROCESSING, + program_cycle=self.program_cycle, ) target_population_with_incorrect_status.save() diff --git a/tests/unit/one_time_scripts/test_program_cycle_data_migration.py b/tests/unit/one_time_scripts/test_program_cycle_data_migration.py deleted file mode 100644 index 0949b58fc6..0000000000 --- a/tests/unit/one_time_scripts/test_program_cycle_data_migration.py +++ /dev/null @@ -1,275 +0,0 @@ -from django.test.testcases import TestCase - -from hct_mis_api.apps.core.fixtures import create_afghanistan -from hct_mis_api.apps.household.fixtures import create_household_and_individuals -from hct_mis_api.apps.payment.fixtures import PaymentFactory, PaymentPlanFactory -from hct_mis_api.apps.payment.models import PaymentPlan -from hct_mis_api.apps.program.fixtures import ProgramCycleFactory, ProgramFactory -from hct_mis_api.apps.program.models import Program, ProgramCycle -from hct_mis_api.apps.targeting.fixtures import TargetPopulationFactory -from hct_mis_api.apps.targeting.models import TargetPopulation -from hct_mis_api.one_time_scripts.program_cycle_data_migration import ( - program_cycle_data_migration, -) - - -class TestProgramCycleDataMigration(TestCase): - @classmethod - def setUpTestData(cls) -> None: - super().setUpTestData() - ba = create_afghanistan() - start_date = "2022-10-10" - end_date = "2023-10-10" - program_finished = ProgramFactory( - name="Finished 001", - business_area=ba, - start_date="2022-10-10", - end_date="2022-10-29", - status=Program.FINISHED, - cycle__title="Already Created Cycle for program_finished", - cycle__status=ProgramCycle.DRAFT, - cycle__start_date="2022-10-11", - cycle__end_date="2022-10-12", - ) - program_finished2 = ProgramFactory( - name="Finished 002", - business_area=ba, - start_date="2022-11-10", - end_date="2022-11-30", - status=Program.FINISHED, - cycle__title="Cycle_program_finished2", - cycle__status=ProgramCycle.DRAFT, - cycle__start_date="2022-10-11", - cycle__end_date="2022-10-12", - ) - # remove default program cycle for program_finished2 - ProgramCycle.objects.filter(title="Cycle_program_finished2").delete() - tp_1 = TargetPopulationFactory(program=program_finished, program_cycle=None) - tp_2 = TargetPopulationFactory(program=program_finished2, program_cycle=None) - PaymentPlanFactory( - program=program_finished, - target_population=tp_1, - program_cycle=None, - start_date=start_date, - end_date=end_date, - ) - PaymentPlanFactory( - program=program_finished2, - target_population=tp_2, - program_cycle=None, - start_date=start_date, - end_date=end_date, - ) - - # active programs - program_active_001 = ProgramFactory( - name="Active 001", - business_area=ba, - start_date="2023-01-01", - end_date="2022-01-30", - status=Program.ACTIVE, - cycle__title="Cycle for program_active_001", - cycle__status=ProgramCycle.DRAFT, - cycle__start_date="2023-01-01", - cycle__end_date="2023-01-30", - ) - program_active_002 = ProgramFactory( - name="Active 002", - business_area=ba, - start_date="2023-02-01", - end_date="2023-02-25", - status=Program.ACTIVE, - cycle__title="Cycle for program_active_002", - cycle__status=ProgramCycle.DRAFT, - cycle__start_date="2023-02-01", - cycle__end_date="2023-02-25", - ) - ProgramCycle.objects.filter(title="Cycle for program_active_002").delete() - cls.tp_3 = TargetPopulationFactory(program=program_active_001, program_cycle=None) - cls.tp_4 = TargetPopulationFactory(program=program_active_002, program_cycle=None) - ProgramCycleFactory( - program=program_active_002, - title="Cycle 01", - start_date="2023-02-10", - end_date=None, - ) - household_1, inds = create_household_and_individuals( - household_data={ - "business_area": ba, - "program": program_finished, - }, - individuals_data=[ - { - "business_area": ba, - "program": program_finished, - }, - ], - ) - household_2, inds = create_household_and_individuals( - household_data={ - "business_area": ba, - "program": program_finished, - }, - individuals_data=[ - { - "business_area": ba, - "program": program_finished, - }, - ], - ) - household_3, inds = create_household_and_individuals( - household_data={ - "business_area": ba, - "program": program_finished, - }, - individuals_data=[ - { - "business_area": ba, - "program": program_finished, - }, - ], - ) - household_4, inds = create_household_and_individuals( - household_data={ - "business_area": ba, - "program": program_finished, - }, - individuals_data=[ - { - "business_area": ba, - "program": program_finished, - }, - ], - ) - household_5, inds = create_household_and_individuals( - household_data={ - "business_area": ba, - "program": program_finished, - }, - individuals_data=[ - { - "business_area": ba, - "program": program_finished, - }, - ], - ) - household_6, inds = create_household_and_individuals( - household_data={ - "business_area": ba, - "program": program_finished, - }, - individuals_data=[ - { - "business_area": ba, - "program": program_finished, - }, - ], - ) - - cls.pp_1 = PaymentPlanFactory( - name="Payment Plan pp1", - program=program_active_001, - target_population=cls.tp_3, - program_cycle=None, - start_date=start_date, - end_date=end_date, - ) - cls.pp_2 = PaymentPlanFactory( - name="Payment Plan pp2", - program=program_active_001, - target_population=cls.tp_3, - program_cycle=None, - start_date=start_date, - end_date=end_date, - ) - PaymentFactory(household=household_1, parent=cls.pp_1, status="Distribution Successful") - PaymentFactory(household=household_2, parent=cls.pp_2, status="Distribution Successful") - - cls.pp_3 = PaymentPlanFactory( - name="Payment Plan pp3", - program=program_active_002, - target_population=cls.tp_4, - program_cycle=None, - start_date=start_date, - end_date=end_date, - ) - cls.pp_4 = PaymentPlanFactory( - name="Payment Plan pp4", - program=program_active_002, - target_population=cls.tp_4, - program_cycle=None, - start_date=start_date, - end_date=end_date, - ) - cls.pp_5 = PaymentPlanFactory( - name="Payment Plan pp5", - program=program_active_002, - target_population=cls.tp_4, - program_cycle=None, - start_date=start_date, - end_date=end_date, - ) - - # cycle 1 = Cycle 01 - PaymentFactory(household=household_3, parent=cls.pp_3, status="Distribution Successful") - PaymentFactory(household=household_4, parent=cls.pp_3, status="Distribution Successful") - - # cycle 2 = new created - PaymentFactory(household=household_4, parent=cls.pp_4, status="Distribution Successful") - PaymentFactory(household=household_5, parent=cls.pp_4, status="Distribution Successful") - - # cycle 3 = new created - PaymentFactory(household=household_5, parent=cls.pp_5, status="Distribution Successful") - PaymentFactory(household=household_6, parent=cls.pp_5, status="Distribution Successful") - PaymentFactory(household=household_3, parent=cls.pp_5, status="Distribution Successful") - - def test_program_cycle_data_migration(self) -> None: - # check cycle for program_active_002 - self.assertEqual(ProgramCycle.objects.filter(program=self.pp_3.program).count(), 1) - self.assertEqual(ProgramCycle.objects.filter(program=self.pp_3.program).first().title, "Cycle 01") - - # run script - program_cycle_data_migration() - - program_finished = Program.objects.get(name="Finished 001") - program_finished2 = Program.objects.get(name="Finished 002") - cycle_for_program_finished = program_finished.cycles.first() - self.assertEqual(program_finished.start_date, cycle_for_program_finished.start_date) - self.assertEqual(program_finished.end_date, cycle_for_program_finished.end_date) - self.assertEqual(cycle_for_program_finished.status, ProgramCycle.FINISHED) - self.assertEqual(TargetPopulation.objects.filter(program_cycle=cycle_for_program_finished).count(), 1) - self.assertEqual(PaymentPlan.objects.filter(program_cycle=cycle_for_program_finished).count(), 1) - - cycle_for_program_finished2 = program_finished2.cycles.first() - self.assertEqual(program_finished2.start_date, cycle_for_program_finished2.start_date) - self.assertEqual(program_finished2.end_date, cycle_for_program_finished2.end_date) - self.assertEqual(cycle_for_program_finished2.status, ProgramCycle.FINISHED) - self.assertEqual(TargetPopulation.objects.filter(program_cycle=cycle_for_program_finished2).count(), 1) - self.assertEqual(PaymentPlan.objects.filter(program_cycle=cycle_for_program_finished2).count(), 1) - - # check with active program - self.pp_1.refresh_from_db() - self.pp_2.refresh_from_db() - self.tp_3.refresh_from_db() - - self.assertEqual(self.pp_1.program_cycle.status, ProgramCycle.ACTIVE) - # new default name starts with "Cycle {PaymentPlan.start_date} ({random 4 digits})" - self.assertTrue(self.pp_1.program_cycle.title.startswith("Cycle 2022-10-10 (")) - self.assertTrue(self.tp_3.program_cycle.title.startswith("Cycle 2022-10-10 (")) - - self.pp_3.refresh_from_db() - self.pp_4.refresh_from_db() - self.pp_5.refresh_from_db() - self.tp_4.refresh_from_db() - - self.assertEqual(self.pp_3.program_cycle.status, ProgramCycle.ACTIVE) - self.assertTrue(self.pp_3.program_cycle.title.startswith("Cycle 2022-10-10 (")) - self.assertEqual(self.pp_4.program_cycle.status, ProgramCycle.ACTIVE) - self.assertEqual(self.pp_5.program_cycle.status, ProgramCycle.ACTIVE) - self.assertEqual(self.tp_4.program_cycle.status, ProgramCycle.ACTIVE) - - program_active_002 = self.pp_3.program - values_cycles = ProgramCycle.objects.filter(program=program_active_002).values( - "title", "start_date", "end_date" - ) - self.assertEqual(values_cycles.count(), 3) From 372b465b95c3cf96d4bee3c3dddf1340f7596977 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 3 Oct 2024 06:14:24 +0200 Subject: [PATCH 04/14] skip forms --- tests/.coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/.coveragerc b/tests/.coveragerc index 8e5e354932..96ca8cc7c1 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -11,6 +11,7 @@ omit = */admin/*.py */admin.py **/fixtures.py + **/forms.py hct_mis_api/one_time_scripts/* hct_mis_api/libs/* hct_mis_api/settings/* From 01679655753a5ebd021aaade01d94073cbe704b9 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 3 Oct 2024 10:19:10 +0200 Subject: [PATCH 05/14] bring back nighlt runs for tests marked before --- .../selenium/programme_management/test_programme_management.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/selenium/programme_management/test_programme_management.py b/tests/selenium/programme_management/test_programme_management.py index dc39300971..fb79de1f6d 100644 --- a/tests/selenium/programme_management/test_programme_management.py +++ b/tests/selenium/programme_management/test_programme_management.py @@ -395,6 +395,7 @@ def test_create_programme_cancel_scenario( # ToDo: Check Unicef partner! and delete classes +@pytest.mark.night @pytest.mark.usefixtures("login") class TestBusinessAreas: @pytest.mark.parametrize( @@ -522,6 +523,7 @@ def test_copy_programme( assert "New Programme" in pageProgrammeDetails.getHeaderTitle().text +@pytest.mark.night @pytest.mark.usefixtures("login") class TestAdminAreas: @pytest.mark.parametrize( @@ -659,6 +661,7 @@ def test_create_programme_back_scenarios( assert "UNHCR" in pageProgrammeDetails.getLabelPartnerName().text +@pytest.mark.night @pytest.mark.usefixtures("login") class TestManualCalendar: @pytest.mark.parametrize( From 2355700d42b3de5f642f620de85840e48b8ae276 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 3 Oct 2024 10:40:56 +0200 Subject: [PATCH 06/14] text --- tests/unit/apps/core/test_edopomoga_tp_creation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/apps/core/test_edopomoga_tp_creation.py b/tests/unit/apps/core/test_edopomoga_tp_creation.py index 5a55c73dd2..5cea47b5c4 100644 --- a/tests/unit/apps/core/test_edopomoga_tp_creation.py +++ b/tests/unit/apps/core/test_edopomoga_tp_creation.py @@ -5,6 +5,8 @@ from django.core.files import File from django.core.management import call_command +import pytest + from hct_mis_api.apps.account.fixtures import UserFactory from hct_mis_api.apps.core.base_test_case import APITestCase from hct_mis_api.apps.core.celery_tasks import create_target_population_task @@ -18,6 +20,7 @@ from hct_mis_api.apps.targeting.models import TargetPopulation +@pytest.mark.skip(reason="Functionality probably can be removed.") class TestEdopomogaCreation(APITestCase): databases = ("default", "cash_assist_datahub_mis") From a714960f9495c8fbb652df840861ce9f8ea0265c Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 3 Oct 2024 18:49:14 +0200 Subject: [PATCH 07/14] adjust test to testcase --- tests/unit/apps/targeting/test_target_query.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/apps/targeting/test_target_query.py b/tests/unit/apps/targeting/test_target_query.py index 2204eefbc2..3efdc3eeb7 100644 --- a/tests/unit/apps/targeting/test_target_query.py +++ b/tests/unit/apps/targeting/test_target_query.py @@ -115,6 +115,7 @@ def setUpTestData(cls) -> None: cls.partner = PartnerFactory(name="TestPartner") cls.business_area = BusinessArea.objects.get(slug="afghanistan") cls.program = ProgramFactory(name="test_program", status=Program.ACTIVE) + cls.cycle = cls.program.cycles.first() cls.cycle_2 = ProgramCycleFactory(program=cls.program) _ = create_household( @@ -145,7 +146,7 @@ def setUpTestData(cls) -> None: targeting_criteria=targeting_criteria, business_area=cls.business_area, program=cls.program, - program_cycle=cls.cycle_2, + program_cycle=cls.cycle, ) cls.target_population_size_2.save() cls.target_population_size_2 = full_rebuild(cls.target_population_size_2) @@ -175,7 +176,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program, - program_cycle=cls.cycle_2, + program_cycle=cls.cycle, ) cls.target_population_size_1_approved.save() cls.target_population_size_1_approved = full_rebuild(cls.target_population_size_1_approved) @@ -222,7 +223,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program, - program_cycle=cls.cycle_2, + program_cycle=cls.cycle, ) cls.target_population_with_pdu_filter.save() cls.target_population_with_pdu_filter = full_rebuild(cls.target_population_with_pdu_filter) @@ -257,7 +258,7 @@ def setUpTestData(cls) -> None: status=TargetPopulation.STATUS_LOCKED, business_area=cls.business_area, program=cls.program, - program_cycle=cls.cycle_2, + program_cycle=cls.cycle, ) cls.target_population_with_individual_filter.save() cls.target_population_with_individual_filter = full_rebuild(cls.target_population_with_individual_filter) From 154e9d8aa6e08f0d0a173828413dca8e3d654505 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 3 Oct 2024 18:56:32 +0200 Subject: [PATCH 08/14] remove edopomoga --- src/hct_mis_api/apps/core/admin.py | 31 +--- src/hct_mis_api/apps/core/celery_tasks.py | 170 ------------------ .../core/templates/core/admin/create_tp.html | 10 -- .../apps/core/test_edopomoga_tp_creation.py | 100 ----------- 4 files changed, 2 insertions(+), 309 deletions(-) delete mode 100644 src/hct_mis_api/apps/core/templates/core/admin/create_tp.html delete mode 100644 tests/unit/apps/core/test_edopomoga_tp_creation.py diff --git a/src/hct_mis_api/apps/core/admin.py b/src/hct_mis_api/apps/core/admin.py index 12272eeab1..9b9343e777 100644 --- a/src/hct_mis_api/apps/core/admin.py +++ b/src/hct_mis_api/apps/core/admin.py @@ -44,11 +44,8 @@ from hct_mis_api.apps.account.models import Role, User from hct_mis_api.apps.administration.widgets import JsonWidget -from hct_mis_api.apps.core.celery_tasks import ( - create_target_population_task, - upload_new_kobo_template_and_update_flex_fields_task, -) -from hct_mis_api.apps.core.forms import DataCollectingTypeForm, ProgramForm +from hct_mis_api.apps.core.celery_tasks import upload_new_kobo_template_and_update_flex_fields_task +from hct_mis_api.apps.core.forms import DataCollectingTypeForm from hct_mis_api.apps.core.models import ( BusinessArea, CountryCodeMap, @@ -692,30 +689,6 @@ def has_view_permission(self, request: HttpRequest, obj: Optional[Any] = None) - def has_add_permission(self, request: HttpRequest) -> bool: return request.user.can_download_storage_files() - @button(label="Create eDopomoga TP") - def create_tp(self, request: HttpRequest, pk: "UUID") -> Union[TemplateResponse, HttpResponsePermanentRedirect]: - storage_obj = StorageFile.objects.get(pk=pk) - context = self.get_common_context( - request, - pk, - ) - if request.method == "GET": - if TargetPopulation.objects.filter(storage_file=storage_obj).exists(): - self.message_user(request, "TargetPopulation for this storageFile have been created", messages.ERROR) - return redirect("..") - - form = ProgramForm(business_area_id=storage_obj.business_area_id) - context["form"] = form - return TemplateResponse(request, "core/admin/create_tp.html", context) - else: - program_id = request.POST.get("program") - tp_name = request.POST.get("name") - - create_target_population_task.delay(storage_obj.pk, program_id, tp_name) - - self.message_user(request, "Creation of TargetPopulation started") - return redirect("..") - @admin.register(MigrationStatus) class MigrationStatusAdmin(admin.ModelAdmin): diff --git a/src/hct_mis_api/apps/core/celery_tasks.py b/src/hct_mis_api/apps/core/celery_tasks.py index 05813521f4..27ea7bea47 100644 --- a/src/hct_mis_api/apps/core/celery_tasks.py +++ b/src/hct_mis_api/apps/core/celery_tasks.py @@ -100,173 +100,3 @@ def upload_new_kobo_template_and_update_flex_fields_task(self: Any, xlsx_kobo_te except Exception as e: logger.exception(e) raise self.retry(exc=e) - - -@app.task(bind=True, default_retry_delay=60, max_retries=3) -@log_start_and_end -@sentry_tags -def create_target_population_task(self: Any, storage_id: str, program_id: str, tp_name: str) -> None: - storage_obj = StorageFile.objects.get(id=storage_id) - file_path = None - program = Program.objects.get(id=program_id) - set_sentry_business_area_tag(program.business_area.name) - - try: - with transaction.atomic(): - registration_data_import = RegistrationDataImport.objects.create( - name=f"{storage_obj.file.name}_{program.name}", - number_of_individuals=0, - number_of_households=0, - business_area=program.business_area, - data_source=RegistrationDataImport.EDOPOMOGA, - program=program, - ) - if program.biometric_deduplication_enabled: - registration_data_import.deduplication_engine_status = RegistrationDataImport.DEDUP_ENGINE_PENDING - - business_area = storage_obj.business_area - country = business_area.countries.first() - - passport_type = DocumentType.objects.get( - key=IDENTIFICATION_TYPE_TO_KEY_MAPPING[IDENTIFICATION_TYPE_NATIONAL_PASSPORT] - ) - tax_type = DocumentType.objects.get(key=IDENTIFICATION_TYPE_TO_KEY_MAPPING[IDENTIFICATION_TYPE_TAX_ID]) - - first_registration_date = timezone.now() - last_registration_date = first_registration_date - - families = {} - individuals, documents, bank_infos = [], [], [] - - storage_obj.status = StorageFile.STATUS_PROCESSING - storage_obj.save(update_fields=["status"]) - rows_count = 0 - - # TODO fix to use Azure storage override AzureStorageFile open method - with storage_obj.file.open("rb") as original_file, tempfile.NamedTemporaryFile(delete=False) as tmp: - tmp.write(original_file.read()) - file_path = tmp.name - - with open(file_path, encoding="cp1251") as file: - reader = csv.DictReader(file, delimiter=";") - for row in reader: - rows_count += 1 - family_id = row["ID_FAM"] - iban = row["IBAN"] - tax_id = row["N_ID"] - passport_id = row["PASSPORT"] - size = row["FAM_NUM"] - - individual_data = { - "given_name": row.get("NAME", ""), - "middle_name": row.get("PATRONYMIC", ""), - "family_name": row.get("SURNAME", ""), - "full_name": f'{row.get("NAME", "")} {row.get("PATRONYMIC", "")} {row.get("SURNAME", "")}', - "birth_date": datetime.strptime(row["BDATE"], "%d.%m.%Y").date(), - "phone_no": row.get("PHONЕ", ""), - "business_area": business_area, - "first_registration_date": first_registration_date, - "last_registration_date": last_registration_date, - "sex": MALE, - "relationship": HEAD, - "rdi_merge_status": MergeStatusModel.MERGED, - "flex_fields": populate_pdu_with_null_values(program), - "registration_data_import": registration_data_import, - } - if family_id in families: - individual = Individual(**individual_data, household_id=families.get(family_id)) - individuals.append(individual) - else: - individual = Individual.objects.create(**individual_data) - individual.refresh_from_db() - - household = Household.objects.create( - head_of_household=individual, - business_area=business_area, - first_registration_date=first_registration_date, - last_registration_date=last_registration_date, - registration_data_import=registration_data_import, - size=size, - family_id=family_id, - storage_obj=storage_obj, - collect_individual_data=COLLECT_TYPE_SIZE_ONLY, - country=country, - rdi_merge_status=MergeStatusModel.MERGED, - ) - - individual.household = household - individual.save(update_fields=("household",)) - - IndividualRoleInHousehold.objects.create( - role=ROLE_PRIMARY, - individual=individual, - household=household, - rdi_merge_status=MergeStatusModel.MERGED, - ) - - families[family_id] = household.id - - passport = Document( - document_number=passport_id, - type=passport_type, - individual=individual, - status=Document.STATUS_INVALID, - country=country, - rdi_merge_status=MergeStatusModel.MERGED, - ) - - tax = Document( - document_number=tax_id, - type=tax_type, - individual=individual, - status=Document.STATUS_INVALID, - country=country, - rdi_merge_status=MergeStatusModel.MERGED, - ) - - bank_account_info = BankAccountInfo( - bank_account_number=iban, individual=individual, rdi_merge_status=MergeStatusModel.MERGED - ) - - documents.append(passport) - documents.append(tax) - bank_infos.append(bank_account_info) - - if rows_count % 1000 == 0: - Individual.objects.bulk_create(individuals) - Document.objects.bulk_create(documents) - BankAccountInfo.objects.bulk_create(bank_infos) - individuals = [] - documents = [] - bank_infos = [] - - Individual.objects.bulk_create(individuals) - Document.objects.bulk_create(documents) - BankAccountInfo.objects.bulk_create(bank_infos) - - households = Household.objects.filter(family_id__in=list(families.keys())) - households.update(withdrawn=True, withdrawn_date=timezone.now()) - Individual.objects.filter(household__in=households).update(withdrawn=True, withdrawn_date=timezone.now()) - - target_population = TargetPopulation.objects.create( - name=tp_name, - created_by=storage_obj.created_by, - program=program, - status=TargetPopulation.STATUS_LOCKED, - build_status=TargetPopulation.BUILD_STATUS_OK, - business_area=business_area, - storage_file=storage_obj, - ) - target_population.households.set(households) - refresh_stats(target_population) - target_population.save() - - storage_obj.status = StorageFile.STATUS_FINISHED - storage_obj.save(update_fields=["status"]) - except Exception as e: - storage_obj.status = StorageFile.STATUS_FAILED - storage_obj.save(update_fields=["status"]) - raise self.retry(exc=e) - finally: - if file_path: - os.remove(file_path) diff --git a/src/hct_mis_api/apps/core/templates/core/admin/create_tp.html b/src/hct_mis_api/apps/core/templates/core/admin/create_tp.html deleted file mode 100644 index ee43184ab4..0000000000 --- a/src/hct_mis_api/apps/core/templates/core/admin/create_tp.html +++ /dev/null @@ -1,10 +0,0 @@ -{% extends "admin_extra_buttons/action_page.html" %} -{% block content %} -

Target Population

-
- {% csrf_token %} - {{ form.as_p }} -
- -
-{% endblock %} \ No newline at end of file diff --git a/tests/unit/apps/core/test_edopomoga_tp_creation.py b/tests/unit/apps/core/test_edopomoga_tp_creation.py deleted file mode 100644 index 5cea47b5c4..0000000000 --- a/tests/unit/apps/core/test_edopomoga_tp_creation.py +++ /dev/null @@ -1,100 +0,0 @@ -from io import BytesIO -from pathlib import Path - -from django.conf import settings -from django.core.files import File -from django.core.management import call_command - -import pytest - -from hct_mis_api.apps.account.fixtures import UserFactory -from hct_mis_api.apps.core.base_test_case import APITestCase -from hct_mis_api.apps.core.celery_tasks import create_target_population_task -from hct_mis_api.apps.core.fixtures import create_afghanistan -from hct_mis_api.apps.core.models import BusinessArea, StorageFile -from hct_mis_api.apps.geo import models as geo_models -from hct_mis_api.apps.household.models import STATUS_INACTIVE, Household, Individual -from hct_mis_api.apps.mis_datahub import models as dh_mis_models -from hct_mis_api.apps.mis_datahub.tasks.send_tp_to_datahub import SendTPToDatahubTask -from hct_mis_api.apps.program.fixtures import ProgramFactory -from hct_mis_api.apps.targeting.models import TargetPopulation - - -@pytest.mark.skip(reason="Functionality probably can be removed.") -class TestEdopomogaCreation(APITestCase): - databases = ("default", "cash_assist_datahub_mis") - - @classmethod - def setUpTestData(cls) -> None: - super().setUpTestData() - cls.business_area = create_afghanistan() - call_command("loadcountries") - cls.generate_document_types_for_all_countries() - cls.user = UserFactory.create() - cls.business_area.countries.add(geo_models.Country.objects.get(name="Afghanistan")) - cls.program = ProgramFactory( - name="Test program ONE", - business_area=BusinessArea.objects.first(), - ) - content = Path(f"{settings.TESTS_ROOT}/apps/core/test_files/edopomoga_sample.csv") - cls.storage_file = StorageFile.objects.create( - created_by=cls.user, - business_area=cls.business_area, - status=StorageFile.STATUS_NOT_PROCESSED, - file=File(BytesIO(content.read_bytes()), name="edopomoga_sample.csv"), - ) - - def test_edopomoga_tp_creation(self) -> None: - create_target_population_inner = create_target_population_task.__wrapped__ - create_target_population_inner(self.storage_file.id, self.program.id, "test_edopomoga") - - self.assertEqual(Household.objects.count(), 3) - self.assertEqual(Individual.objects.count(), 5) - self.assertEqual(TargetPopulation.objects.count(), 1) - self.assertEqual(Household.objects.filter(withdrawn=True).count(), 3) - self.assertEqual(Individual.objects.filter(withdrawn=True).count(), 5) - - self.storage_file.refresh_from_db() - self.assertEqual(self.storage_file.status, StorageFile.STATUS_FINISHED) - - def test_calculate_household_size(self) -> None: - create_target_population_inner = create_target_population_task.__wrapped__ - create_target_population_inner(self.storage_file.id, self.program.id, "test_edopomoga") - - household1 = Household.objects.get(family_id="1281191") - household2 = Household.objects.get(family_id="1281375") - household3 = Household.objects.get(family_id="1281383") - - self.assertEqual(household1.size, 4) - self.assertEqual(household2.size, 4) - self.assertEqual(household3.size, 4) - - def test_create_collector(self) -> None: - create_target_population_inner = create_target_population_task.__wrapped__ - create_target_population_inner(self.storage_file.id, self.program.id, "test_edopomoga") - - household1 = Household.objects.get(family_id="1281191") - household2 = Household.objects.get(family_id="1281375") - household3 = Household.objects.get(family_id="1281383") - - self.assertEqual(household1.representatives.count(), 1) - self.assertEqual(household2.representatives.count(), 1) - self.assertEqual(household3.representatives.count(), 1) - - def test_edopomoga_tp_send_to_ca_clear_withdrawn(self) -> None: - # set clear_withdrawn flag - self.business_area.custom_fields = {"clear_withdrawn": True} - self.business_area.save() - tp_name = "New eDopomoga test clear_withdrawn data" - create_target_population_inner = create_target_population_task.__wrapped__ - create_target_population_inner(self.storage_file.id, self.program.id, tp_name) - - target_population = TargetPopulation.objects.get(name=tp_name) - - SendTPToDatahubTask().execute(target_population) - - self.assertEqual(Household.objects.filter(withdrawn=True).count(), 3) - self.assertEqual(Individual.objects.filter(withdrawn=True).count(), 5) - - self.assertEqual(dh_mis_models.Household.objects.filter(status=STATUS_INACTIVE).count(), 0) - self.assertEqual(dh_mis_models.Individual.objects.filter(status=STATUS_INACTIVE).count(), 0) From aa6ee62e740cd276c62333e11f8716558e0f5293 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 3 Oct 2024 19:08:33 +0200 Subject: [PATCH 09/14] include src in path --- tests/.coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/.coveragerc b/tests/.coveragerc index 96ca8cc7c1..370816c587 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -11,7 +11,7 @@ omit = */admin/*.py */admin.py **/fixtures.py - **/forms.py + src/hct_mis_api/**/forms.py hct_mis_api/one_time_scripts/* hct_mis_api/libs/* hct_mis_api/settings/* From a3041d118b6468caf1f5fe0e4286c9c7f5959ed4 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 3 Oct 2024 20:37:30 +0200 Subject: [PATCH 10/14] linter --- src/hct_mis_api/apps/core/admin.py | 5 ++-- src/hct_mis_api/apps/core/celery_tasks.py | 30 ++----------------- .../test_pull_from_datahub.py | 2 +- 3 files changed, 6 insertions(+), 31 deletions(-) diff --git a/src/hct_mis_api/apps/core/admin.py b/src/hct_mis_api/apps/core/admin.py index 9b9343e777..91fd4c590e 100644 --- a/src/hct_mis_api/apps/core/admin.py +++ b/src/hct_mis_api/apps/core/admin.py @@ -44,7 +44,9 @@ from hct_mis_api.apps.account.models import Role, User from hct_mis_api.apps.administration.widgets import JsonWidget -from hct_mis_api.apps.core.celery_tasks import upload_new_kobo_template_and_update_flex_fields_task +from hct_mis_api.apps.core.celery_tasks import ( + upload_new_kobo_template_and_update_flex_fields_task, +) from hct_mis_api.apps.core.forms import DataCollectingTypeForm from hct_mis_api.apps.core.models import ( BusinessArea, @@ -63,7 +65,6 @@ from hct_mis_api.apps.household.models import DocumentType from hct_mis_api.apps.payment.forms import AcceptanceProcessThresholdForm from hct_mis_api.apps.payment.models import AcceptanceProcessThreshold -from hct_mis_api.apps.targeting.models import TargetPopulation from hct_mis_api.apps.utils.admin import ( HOPEModelAdminBase, LastSyncDateResetMixin, diff --git a/src/hct_mis_api/apps/core/celery_tasks.py b/src/hct_mis_api/apps/core/celery_tasks.py index 27ea7bea47..59e4112f54 100644 --- a/src/hct_mis_api/apps/core/celery_tasks.py +++ b/src/hct_mis_api/apps/core/celery_tasks.py @@ -1,42 +1,16 @@ -import csv import logging -import os -import tempfile -from datetime import datetime from functools import wraps from typing import Any, Callable from django.db import transaction -from django.utils import timezone from hct_mis_api.apps.core.celery import app -from hct_mis_api.apps.core.models import StorageFile, XLSXKoboTemplate +from hct_mis_api.apps.core.models import XLSXKoboTemplate from hct_mis_api.apps.core.tasks.upload_new_template_and_update_flex_fields import ( KoboRetriableError, ) -from hct_mis_api.apps.core.utils import IDENTIFICATION_TYPE_TO_KEY_MAPPING -from hct_mis_api.apps.household.models import ( - COLLECT_TYPE_SIZE_ONLY, - HEAD, - IDENTIFICATION_TYPE_NATIONAL_PASSPORT, - IDENTIFICATION_TYPE_TAX_ID, - MALE, - ROLE_PRIMARY, - BankAccountInfo, - Document, - DocumentType, - Household, - Individual, - IndividualRoleInHousehold, -) -from hct_mis_api.apps.periodic_data_update.utils import populate_pdu_with_null_values -from hct_mis_api.apps.program.models import Program -from hct_mis_api.apps.registration_data.models import RegistrationDataImport -from hct_mis_api.apps.targeting.models import TargetPopulation -from hct_mis_api.apps.targeting.services.targeting_stats_refresher import refresh_stats from hct_mis_api.apps.utils.logs import log_start_and_end -from hct_mis_api.apps.utils.models import MergeStatusModel -from hct_mis_api.apps.utils.sentry import sentry_tags, set_sentry_business_area_tag +from hct_mis_api.apps.utils.sentry import sentry_tags logger = logging.getLogger(__name__) diff --git a/tests/unit/apps/cash_assist_datahub/test_pull_from_datahub.py b/tests/unit/apps/cash_assist_datahub/test_pull_from_datahub.py index e954b4c0cf..d3994159d5 100644 --- a/tests/unit/apps/cash_assist_datahub/test_pull_from_datahub.py +++ b/tests/unit/apps/cash_assist_datahub/test_pull_from_datahub.py @@ -75,7 +75,7 @@ def _setup_in_app_data(cls) -> None: status=TargetPopulation.STATUS_PROCESSING, program=cls.program, business_area=cls.business_area, - program_cycle=cls.program.cycles.first() + program_cycle=cls.program.cycles.first(), ) program = ProgramFactory( From 4f664e967906053b7a316a38befa3eeb00e4d652 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Fri, 4 Oct 2024 00:52:00 +0200 Subject: [PATCH 11/14] fix test admin --- tests/unit/apps/account/test_admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/apps/account/test_admin.py b/tests/unit/apps/account/test_admin.py index 8088a9917e..de60521556 100644 --- a/tests/unit/apps/account/test_admin.py +++ b/tests/unit/apps/account/test_admin.py @@ -5,7 +5,7 @@ from django_webtest import WebTest -from hct_mis_api.apps.account.fixtures import UserFactory +from hct_mis_api.apps.account.fixtures import RoleFactory, UserFactory from hct_mis_api.apps.account.models import User @@ -13,6 +13,7 @@ class RoleTest(WebTest): @classmethod def setUpTestData(cls) -> None: super().setUpTestData() + RoleFactory() cls.superuser: User = UserFactory(is_superuser=True, is_staff=True) def test_role_perm_matrix(self) -> None: @@ -24,7 +25,6 @@ def test_role_sync(self) -> None: url = reverse("admin:account_role_dumpdata_qs") res = self.app.get(url, user=self.superuser) assert res.status_code == 200 - print(res.json) jres = json.loads(unquote(res.json["data"])) models = set([item["model"] for item in jres]) assert len(models) == 1 From 77ec188e7e8656a03998929d9694baf08551cae0 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Fri, 4 Oct 2024 02:49:35 +0200 Subject: [PATCH 12/14] fix coveragerc --- tests/.coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/.coveragerc b/tests/.coveragerc index 370816c587..7942061b4e 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -11,7 +11,6 @@ omit = */admin/*.py */admin.py **/fixtures.py - src/hct_mis_api/**/forms.py hct_mis_api/one_time_scripts/* hct_mis_api/libs/* hct_mis_api/settings/* @@ -19,6 +18,7 @@ omit = hct_mis_api/conftest.py hct_mis_api/config/settings.py hct_mis_api/apps/core/management/commands/* + hct_mis_api/apps/household/forms.py [report] # Regexes for lines to exclude from consideration From 4ba14c0f4bb8703b38432ee4526fe08840ea7a65 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Fri, 4 Oct 2024 03:20:39 +0200 Subject: [PATCH 13/14] add ignore to codecov files --- codecov.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codecov.yml b/codecov.yml index 02b73140fc..8a09aba375 100644 --- a/codecov.yml +++ b/codecov.yml @@ -9,3 +9,5 @@ coverage: patch: default: target: 95% + ignore: + - "**/forms.py" From fc309cc2a0f3f44f8fa7d43eb910a69d0e609c10 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Fri, 4 Oct 2024 03:51:43 +0200 Subject: [PATCH 14/14] move ignored paths to codecov.yml --- codecov.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/codecov.yml b/codecov.yml index 8a09aba375..72f96c556e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -11,3 +11,17 @@ coverage: target: 95% ignore: - "**/forms.py" + - "*/selenium_tests/**" + - "*/tests/**" + - "*/migrations/*" + - "*/apps.py" + - "*/admin/*.py" + - "*/admin.py" + - "**/fixtures.py" + - "hct_mis_api/one_time_scripts/*" + - "hct_mis_api/libs/*" + - "hct_mis_api/settings/*" + - "hct_mis_api/settings.py" + - "hct_mis_api/conftest.py" + - "hct_mis_api/config/settings.py" + - "hct_mis_api/apps/core/management/commands/*"