Merge pull request #3852 from unicef/staging

[Merge] Staging->Develop

backend/hct_mis_api/api/endpoints/rdi/push_people.py

@@ -31,6 +31,7 @@
     ImportedIndividual,
     RegistrationDataImportDatahub,
 )
+from hct_mis_api.apps.utils.phone import calculate_phone_numbers_validity
 
 PEOPLE_TYPE_CHOICES = (
     (BLANK, "None"),
@@ -54,6 +55,9 @@ class PushPeopleSerializer(serializers.ModelSerializer):
     residence_status = serializers.ChoiceField(choices=RESIDENCE_STATUS_CHOICE)
     village = serializers.CharField(allow_blank=True, required=False)
 
+    phone_no = serializers.CharField(allow_null=True, allow_blank=True, required=False)
+    phone_no_alternative = serializers.CharField(allow_null=True, allow_blank=True, required=False)
+
     class Meta:
         model = ImportedIndividual
         exclude = [
@@ -118,6 +122,9 @@ def _create_individual(
             relationship=relationship,
             **individual_data,
         )
+        if ind.phone_no or ind.phone_no_alternative:
+            ind = self._validate_phone_number(ind)
+            ind.save(update_fields=("phone_no_valid", "phone_no_alternative_valid"))
 
         if person_type is not NON_BENEFICIARY:
             hh.head_of_household = ind
@@ -128,6 +135,9 @@ def _create_individual(
             self._create_document(ind, doc)
         return ind
 
+    def _validate_phone_number(self, individual: ImportedIndividual) -> ImportedIndividual:
+        return calculate_phone_numbers_validity(individual)
+
     def _create_document(self, member: ImportedIndividual, doc: Dict) -> None:
         ImportedDocument.objects.create(
             document_number=doc["document_number"],

backend/hct_mis_api/api/tests/test_push_people.py

@@ -1,3 +1,6 @@
+from typing import Any
+
+from parameterized import parameterized
 from rest_framework import status
 from rest_framework.reverse import reverse
 
@@ -241,3 +244,38 @@ def test_upload_with_errors(self) -> None:
                 {"birth_date": ["This field is required."], "type": ["This field is required."]},
             ],
         )
+
+    @parameterized.expand(
+        [
+            ("invalid_phone_no", "phone_no", "invalid", False),
+            ("invalid_phone_no_alternative", "phone_no", "invalid", False),
+            ("valid_phone_no", "phone_no_alternative", "+48 632 215 789", True),
+            ("valid_phone_no_alternative", "phone_no_alternative", "+48 632 215 789", True),
+        ]
+    )
+    def test_upload_single_person_with_phone_number(
+        self, _: Any, field_name: str, phone_number: str, expected_value: bool
+    ) -> None:
+        data = [
+            {
+                "residence_status": "IDP",
+                "village": "village1",
+                "country": "AF",
+                "collect_individual_data": COLLECT_TYPE_FULL,
+                "full_name": "John Doe",
+                "birth_date": "2000-01-01",
+                "sex": "MALE",
+                "type": "",
+                field_name: phone_number,
+            }
+        ]
+        response = self.client.post(self.url, data, format="json")
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED, str(response.json()))
+        response_json = response.json()
+
+        rdi_datahub = RegistrationDataImportDatahub.objects.filter(id=response_json["id"]).first()
+        self.assertIsNotNone(rdi_datahub)
+        ind = ImportedIndividual.objects.filter(registration_data_import=rdi_datahub).first()
+        self.assertIsNotNone(ind)
+        self.assertEqual(ind.full_name, "John Doe")
+        self.assertEqual(getattr(ind, f"{field_name}_valid"), expected_value)

backend/hct_mis_api/apps/registration_data/admin.py

@@ -180,10 +180,14 @@ def delete_rdi(self, request: HttpRequest, pk: UUID) -> Any:  # TODO: typing
             self.message_user(request, "An error occurred while processing RDI delete", messages.ERROR)
 
     @staticmethod
-    def delete_merged_rdi_visible(o: Any, r: Any) -> bool:
-        is_correct_status = o.status == RegistrationDataImport.MERGED
-        is_not_used_in_targeting = HouseholdSelection.objects.filter(household__registration_data_import=o).count() == 0
-        is_not_used_by_payment_record = PaymentRecord.objects.filter(household__registration_data_import=o).count() == 0
+    def delete_merged_rdi_visible(rdi: RegistrationDataImport) -> bool:
+        is_correct_status = rdi.status == RegistrationDataImport.MERGED
+        is_not_used_in_targeting = (
+            HouseholdSelection.objects.filter(household__registration_data_import=rdi).count() == 0
+        )
+        is_not_used_by_payment_record = (
+            PaymentRecord.objects.filter(household__registration_data_import=rdi).count() == 0
+        )
         return is_correct_status and is_not_used_in_targeting and is_not_used_by_payment_record
 
     @staticmethod
@@ -212,7 +216,7 @@ def generate_query_for_all_grievances_tickets(rdi: RegistrationDataImport) -> Q:
 
     @button(
         permission=is_root,
-        visible=lambda o, r: RegistrationDataImportAdmin.delete_merged_rdi_visible(o, r),
+        visible=lambda btn: RegistrationDataImportAdmin.delete_merged_rdi_visible(btn.original),
     )
     def delete_merged_rdi(self, request: HttpRequest, pk: UUID) -> Optional[HttpResponse]:
         try:

backend/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py

@@ -43,7 +43,10 @@
     logger,
 )
 from hct_mis_api.apps.registration_datahub.tasks.utils import get_submission_metadata
-from hct_mis_api.apps.registration_datahub.utils import find_attachment_in_kobo
+from hct_mis_api.apps.registration_datahub.utils import (
+    calculate_hash_for_kobo_submission,
+    find_attachment_in_kobo,
+)
 from hct_mis_api.apps.utils.age_at_registration import calculate_age_at_registration
 
 
@@ -203,9 +206,17 @@ def execute(
         individuals_ids_hash_dict = {}
         bank_accounts_to_create = []
         collectors_to_create = defaultdict(list)
+        household_hash_list = []
         household_batch_size = 50
         for reduced_submission_chunk in chunks(self.reduced_submissions, household_batch_size):
             for household in reduced_submission_chunk:
+                # AB#199540
+                household_hash = calculate_hash_for_kobo_submission(household)
+                submission_duplicate = household_hash in household_hash_list
+                if submission_duplicate:
+                    continue
+                household_hash_list.append(household_hash)
+
                 submission_meta_data = get_submission_metadata(household)
                 if self.business_area.get_sys_option("ignore_amended_kobo_submissions"):
                     submission_meta_data["amended"] = False

backend/hct_mis_api/apps/registration_datahub/tests/test_file/kobo_submissions_collectors.json

@@ -408,5 +408,147 @@
     "enumerator/org_enumerator": "unicef",
     "_id": 102612403,
     "living_conditions_questions/living_situation_h_f": "own"
+  },
+  {
+    "_notes": [],
+    "wash_questions/score_num_items": "8",
+    "household_questions/household_location/address_h_c": "Some Street 123",
+    "wash_questions/bed_hhsize": "NaN",
+    "monthly_income_questions/total_inc_h_f": "0",
+    "household_questions/m_0_5_age_group_h_c": "0",
+    "_xform_id_string": "aPkhoRMrkkDwgsvWuwi39s",
+    "_bamboo_dataset_id": "",
+    "_tags": [],
+    "health_questions/pregnant_member_h_c": "0",
+    "household_questions/f_0_5_disability_h_c": "0",
+    "end": "2020-05-28T14:35:43.469+02:00",
+    "household_questions/size_h_c": "3",
+    "household_questions/household_location/admin2_h_c": "SO2502",
+    "monthly_expenditures_questions/total_expense_h_f": "0",
+    "individual_questions": [
+      {
+        "individual_questions/role_i_c": "primary",
+        "individual_questions/age": "60",
+        "individual_questions/given_name_i_c": "Test",
+        "individual_questions/gender_i_c": "male",
+        "individual_questions/more_information/marital_status_i_c": "married",
+        "individual_questions/more_information/id_type_i_c": "birth_certificate",
+        "individual_questions/more_information/phone_no_i_c": "333111412",
+        "individual_questions/individual_index": "1",
+        "individual_questions/full_name_i_c": "XLast XFull XName",
+        "individual_questions/relationship_i_c": "head",
+        "individual_questions/individual_vulnerabilities/work_status_i_c": "1",
+        "individual_questions/family_name_i_c": "XLast",
+        "individual_questions/individual_vulnerabilities/disability_i_c": "not disabled",
+        "individual_questions/more_information/birth_certificate_no_i_c": "123123133",
+        "individual_questions/more_information/birth_certificate_issuer_i_c": "AFG",
+        "individual_questions/birth_date_i_c": "1969-09-10"
+      },
+      {
+        "individual_questions/role_i_c": "alternate",
+        "individual_questions/age": "32",
+        "individual_questions/given_name_i_c": "Tesa",
+        "individual_questions/gender_i_c": "female",
+        "individual_questions/more_information/marital_status_i_c": "married",
+        "individual_questions/more_information/pregnant_i_f": "0",
+        "individual_questions/family_name_i_c": "XLast",
+        "individual_questions/more_information/phone_no_i_c": "333444555",
+        "individual_questions/individual_index": "2",
+        "individual_questions/full_name_i_c": "Tesa XLast",
+        "individual_questions/relationship_i_c": "wife_husband",
+        "individual_questions/individual_vulnerabilities/work_status_i_c": "1",
+        "individual_questions/estimated_birth_date_i_c": "0",
+        "individual_questions/more_information/id_type_i_c": "birth_certificate",
+        "individual_questions/individual_vulnerabilities/disability_i_c": "not disabled",
+        "individual_questions/more_information/birth_certificate_no_i_c": "444111123",
+        "individual_questions/more_information/birth_certificate_issuer_i_c": "AFG",
+        "individual_questions/birth_date_i_c": "1988-03-23"
+      }
+    ],
+    "wash_questions/score_bed": "5",
+    "meta/instanceID": "uuid:c09130af-6c9c-4dba-8c7f-1b2ff1970d19",
+    "wash_questions/blanket_hhsize": "NaN",
+    "household_questions/household_location/country_h_c": "AFG",
+    "household_questions/household_location/residence_status_h_c": "refugee",
+    "household_questions/f_adults_disability_h_c": "0",
+    "wash_questions/score_childclothes": "5",
+    "humanitarian_assistance_questions/assistance_h_f": "0",
+    "household_questions/m_12_17_age_group_h_c": "0",
+    "household_questions/f_adults_h_c": "1",
+    "household_questions/f_12_17_disability_h_c": "0",
+    "household_questions/f_0_5_age_group_h_c": "0",
+    "household_questions/m_6_11_age_group_h_c": "0",
+    "wash_questions/score_womencloth": "5",
+    "household_questions/f_12_17_age_group_h_c": "0",
+    "wash_questions/score_jerrycan": "5",
+    "start": "2020-06-02T14:59:08.962+02:00",
+    "wash_questions/sufficient_water_h_f": "sufficientwater",
+    "_attachments": [
+      {
+        "mimetype": "image/png",
+        "download_small_url": "https://kc.humanitarianresponse.info/media/small?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png",
+        "download_large_url": "https://kc.humanitarianresponse.info/media/large?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png",
+        "download_url": "https://kc.humanitarianresponse.info/media/original?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png",
+        "filename": "wnosal/attachments/b83407aca1d647a5bf65a3383ee761d4/c09130af-6c9c-4dba-8c7f-1b2ff1970d19/signature-14_59_24_Wrong_name.png",
+        "instance": 102612403,
+        "download_medium_url": "https://kc.humanitarianresponse.info/media/medium?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png",
+        "id": 35027752,
+        "xform": 549831
+      },
+      {
+        "mimetype": "image/png",
+        "download_small_url": "https://kc.humanitarianresponse.info/media/small?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png",
+        "download_large_url": "https://kc.humanitarianresponse.info/media/large?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png",
+        "download_url": "https://kc.humanitarianresponse.info/media/original?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png",
+        "filename": "wnosal/attachments/b83407aca1d647a5bf65a3383ee761d4/c09130af-6c9c-4dba-8c7f-1b2ff1970d19/signature-14_59_24_Wrong_name.png",
+        "instance": 102612403,
+        "download_medium_url": "https://kc.humanitarianresponse.info/media/medium?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png",
+        "id": 35027752,
+        "xform": 549831
+      }
+    ],
+    "_status": "submitted_via_web",
+    "__version__": "vrBoKHPPCWpiRNvCbmnXCK",
+    "household_questions/m_12_17_disability_h_c": "0",
+    "wash_questions/score_tool": "5",
+    "wash_questions/total_liter_yesterday_h_f": "0",
+    "wash_questions/score_NFI_h_f": "5",
+    "food_security_questions/FCS_h_f": "NaN",
+    "wash_questions/jerrycan_hhsize": "NaN",
+    "household_questions/household_location/admin1_h_c": "SO25",
+    "wash_questions/score_bassin": "5",
+    "_validation_status": {},
+    "_uuid": "c09130af-6c9c-4dba-8c7f-1b2ff1970d19",
+    "household_questions/m_adults_h_c": "1",
+    "enumerator/name_enumerator": "Tester123",
+    "consent/consent_sign_h_c": "signature-14_59_24.png",
+    "wash_questions/score_cookingpot": "5",
+    "_submitted_by": null,
+    "individual_questions_count": "2",
+    "wash_questions/water_source_h_f": "piped_water",
+    "household_questions/pregnant_h_c": "0",
+    "household_questions/m_6_11_disability_h_c": "0",
+    "household_questions/m_0_5_disability_h_c": "0",
+    "formhub/uuid": "b83407aca1d647a5bf65a3383ee761d4",
+    "end": "2020-06-03T15:05:08.220+02:00",
+    "household_questions/household_location/hh_geopoint_h_c": "4.747176 43.942756 100 100",
+    "monthly_income_questions/round_total_income_h_f": "0",
+    "wash_questions/score_total": "40",
+    "household_questions/m_adults_disability_h_c": "0",
+    "_submission_time": "2020-06-04T13:05:10+00:00",
+    "household_questions/household_location/country_origin_h_c": "AFG",
+    "_geolocation": [
+      4.747176,
+      43.942756
+    ],
+    "monthly_expenditures_questions/round_total_expense_h_f": "0",
+    "deviceid": "ee.humanitarianresponse.info:AqAb03KLuEfWXes0",
+    "food_security_questions/cereals_tuber_score_h_f": "NaN",
+    "household_questions/f_6_11_disability_h_c": "0",
+    "wash_questions/score_blanket": "5",
+    "household_questions/f_6_11_age_group_h_c": "0",
+    "enumerator/org_enumerator": "unicef",
+    "_id": 132642406,
+    "living_conditions_questions/living_situation_h_f": "own"
   }
 ]