From a81b00c5da405cd0b40717374db576263a19d950 Mon Sep 17 00:00:00 2001 From: Pavlo Mokiichuk Date: Mon, 6 May 2024 10:34:16 +0200 Subject: [PATCH 1/6] fix Payment plan filtering --- .../apps/reporting/services/generate_report_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/hct_mis_api/apps/reporting/services/generate_report_service.py b/backend/hct_mis_api/apps/reporting/services/generate_report_service.py index ef351365e0..2af33d86c2 100644 --- a/backend/hct_mis_api/apps/reporting/services/generate_report_service.py +++ b/backend/hct_mis_api/apps/reporting/services/generate_report_service.py @@ -289,8 +289,8 @@ def format_payment_verification_row(cls, payment_verification: PaymentVerificati def get_payment_plans(report: Report) -> QuerySet[PaymentPlan]: filter_vars = { "business_area": report.business_area, - "dispersion_start_date__gte": report.date_from, - "dispersion_end_date__lte": report.date_to, + "start_date__gte": report.date_from, + "end_date__lte": report.date_to, } if report.program: filter_vars["program_cycle__program"] = report.program From 96a50e8c536517a6cc27919128c31e707bbf0aa7 Mon Sep 17 00:00:00 2001 From: Patryk Dabrowski Date: Mon, 6 May 2024 14:27:48 +0200 Subject: [PATCH 2/6] Fix validate phone number --- .../api/endpoints/rdi/push_people.py | 10 +++++ .../hct_mis_api/api/tests/test_push_people.py | 38 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/backend/hct_mis_api/api/endpoints/rdi/push_people.py b/backend/hct_mis_api/api/endpoints/rdi/push_people.py index 5759fcd57e..b98380bbd0 100644 --- a/backend/hct_mis_api/api/endpoints/rdi/push_people.py +++ b/backend/hct_mis_api/api/endpoints/rdi/push_people.py @@ -31,6 +31,7 @@ ImportedIndividual, RegistrationDataImportDatahub, ) +from hct_mis_api.apps.utils.phone import calculate_phone_numbers_validity PEOPLE_TYPE_CHOICES = ( (BLANK, "None"), @@ -54,6 +55,9 @@ class PushPeopleSerializer(serializers.ModelSerializer): residence_status = serializers.ChoiceField(choices=RESIDENCE_STATUS_CHOICE) village = serializers.CharField(allow_blank=True, required=False) + phone_no = serializers.CharField(allow_null=True, allow_blank=True, required=False) + phone_no_alternative = serializers.CharField(allow_null=True, allow_blank=True, required=False) + class Meta: model = ImportedIndividual exclude = [ @@ -118,6 +122,9 @@ def _create_individual( relationship=relationship, **individual_data, ) + if ind.phone_no or ind.phone_no_alternative: + ind = self._validate_phone_number(ind) + ind.save(update_fields=("phone_no_valid", "phone_no_alternative_valid")) if person_type is not NON_BENEFICIARY: hh.head_of_household = ind @@ -128,6 +135,9 @@ def _create_individual( self._create_document(ind, doc) return ind + def _validate_phone_number(self, individual: ImportedIndividual) -> ImportedIndividual: + return calculate_phone_numbers_validity(individual) + def _create_document(self, member: ImportedIndividual, doc: Dict) -> None: ImportedDocument.objects.create( document_number=doc["document_number"], diff --git a/backend/hct_mis_api/api/tests/test_push_people.py b/backend/hct_mis_api/api/tests/test_push_people.py index 01e04c027f..3a83601730 100644 --- a/backend/hct_mis_api/api/tests/test_push_people.py +++ b/backend/hct_mis_api/api/tests/test_push_people.py @@ -1,3 +1,6 @@ +from typing import Any + +from parameterized import parameterized from rest_framework import status from rest_framework.reverse import reverse @@ -241,3 +244,38 @@ def test_upload_with_errors(self) -> None: {"birth_date": ["This field is required."], "type": ["This field is required."]}, ], ) + + @parameterized.expand( + [ + ("invalid_phone_no", "phone_no", "invalid", False), + ("invalid_phone_no_alternative", "phone_no", "invalid", False), + ("valid_phone_no", "phone_no_alternative", "+48 632 215 789", True), + ("valid_phone_no_alternative", "phone_no_alternative", "+48 632 215 789", True), + ] + ) + def test_upload_single_person_with_phone_number( + self, _: Any, field_name: str, phone_number: str, expected_value: bool + ) -> None: + data = [ + { + "residence_status": "IDP", + "village": "village1", + "country": "AF", + "collect_individual_data": COLLECT_TYPE_FULL, + "full_name": "John Doe", + "birth_date": "2000-01-01", + "sex": "MALE", + "type": "", + field_name: phone_number, + } + ] + response = self.client.post(self.url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_201_CREATED, str(response.json())) + response_json = response.json() + + rdi_datahub = RegistrationDataImportDatahub.objects.filter(id=response_json["id"]).first() + self.assertIsNotNone(rdi_datahub) + ind = ImportedIndividual.objects.filter(registration_data_import=rdi_datahub).first() + self.assertIsNotNone(ind) + self.assertEqual(ind.full_name, "John Doe") + self.assertEqual(getattr(ind, f"{field_name}_valid"), expected_value) From 1e1adc215d03216ea08dad2a03418ad1867cce64 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Tue, 7 May 2024 14:16:28 +0200 Subject: [PATCH 3/6] AB#199540 Ignore duplicates in kobo import --- .../tasks/rdi_kobo_create.py | 10 +- .../kobo_submissions_collectors.json | 142 ++++++++++++++++ ...t_calculate_hash_for_kobo_submission1.json | 159 ++++++++++++++++++ ...t_calculate_hash_for_kobo_submission2.json | 126 ++++++++++++++ ...t_calculate_hash_for_kobo_submission3.json | 126 ++++++++++++++ .../tests/test_rdi_create.py | 2 +- .../registration_datahub/tests/test_utils.py | 29 ++++ .../apps/registration_datahub/utils.py | 22 +++ .../apps/registration_datahub/validators.py | 10 +- 9 files changed, 620 insertions(+), 6 deletions(-) create mode 100644 backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission1.json create mode 100644 backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission2.json create mode 100644 backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission3.json create mode 100644 backend/hct_mis_api/apps/registration_datahub/tests/test_utils.py diff --git a/backend/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py b/backend/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py index 11b4e7167f..bebbced52e 100644 --- a/backend/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py +++ b/backend/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py @@ -43,7 +43,7 @@ logger, ) from hct_mis_api.apps.registration_datahub.tasks.utils import get_submission_metadata -from hct_mis_api.apps.registration_datahub.utils import find_attachment_in_kobo +from hct_mis_api.apps.registration_datahub.utils import find_attachment_in_kobo, calculate_hash_for_kobo_submission from hct_mis_api.apps.utils.age_at_registration import calculate_age_at_registration @@ -203,9 +203,17 @@ def execute( individuals_ids_hash_dict = {} bank_accounts_to_create = [] collectors_to_create = defaultdict(list) + household_hash_list = [] household_batch_size = 50 for reduced_submission_chunk in chunks(self.reduced_submissions, household_batch_size): for household in reduced_submission_chunk: + # AB#199540 + household_hash = calculate_hash_for_kobo_submission(household) + submission_duplicate = household_hash in household_hash_list + if submission_duplicate: + continue + household_hash_list.append(household_hash) + submission_meta_data = get_submission_metadata(household) if self.business_area.get_sys_option("ignore_amended_kobo_submissions"): submission_meta_data["amended"] = False diff --git a/backend/hct_mis_api/apps/registration_datahub/tests/test_file/kobo_submissions_collectors.json b/backend/hct_mis_api/apps/registration_datahub/tests/test_file/kobo_submissions_collectors.json index f48dc6e166..e8385eee8c 100644 --- a/backend/hct_mis_api/apps/registration_datahub/tests/test_file/kobo_submissions_collectors.json +++ b/backend/hct_mis_api/apps/registration_datahub/tests/test_file/kobo_submissions_collectors.json @@ -408,5 +408,147 @@ "enumerator/org_enumerator": "unicef", "_id": 102612403, "living_conditions_questions/living_situation_h_f": "own" + }, + { + "_notes": [], + "wash_questions/score_num_items": "8", + "household_questions/household_location/address_h_c": "Some Street 123", + "wash_questions/bed_hhsize": "NaN", + "monthly_income_questions/total_inc_h_f": "0", + "household_questions/m_0_5_age_group_h_c": "0", + "_xform_id_string": "aPkhoRMrkkDwgsvWuwi39s", + "_bamboo_dataset_id": "", + "_tags": [], + "health_questions/pregnant_member_h_c": "0", + "household_questions/f_0_5_disability_h_c": "0", + "end": "2020-05-28T14:35:43.469+02:00", + "household_questions/size_h_c": "3", + "household_questions/household_location/admin2_h_c": "SO2502", + "monthly_expenditures_questions/total_expense_h_f": "0", + "individual_questions": [ + { + "individual_questions/role_i_c": "primary", + "individual_questions/age": "60", + "individual_questions/given_name_i_c": "Test", + "individual_questions/gender_i_c": "male", + "individual_questions/more_information/marital_status_i_c": "married", + "individual_questions/more_information/id_type_i_c": "birth_certificate", + "individual_questions/more_information/phone_no_i_c": "333111412", + "individual_questions/individual_index": "1", + "individual_questions/full_name_i_c": "XLast XFull XName", + "individual_questions/relationship_i_c": "head", + "individual_questions/individual_vulnerabilities/work_status_i_c": "1", + "individual_questions/family_name_i_c": "XLast", + "individual_questions/individual_vulnerabilities/disability_i_c": "not disabled", + "individual_questions/more_information/birth_certificate_no_i_c": "123123133", + "individual_questions/more_information/birth_certificate_issuer_i_c": "AFG", + "individual_questions/birth_date_i_c": "1969-09-10" + }, + { + "individual_questions/role_i_c": "alternate", + "individual_questions/age": "32", + "individual_questions/given_name_i_c": "Tesa", + "individual_questions/gender_i_c": "female", + "individual_questions/more_information/marital_status_i_c": "married", + "individual_questions/more_information/pregnant_i_f": "0", + "individual_questions/family_name_i_c": "XLast", + "individual_questions/more_information/phone_no_i_c": "333444555", + "individual_questions/individual_index": "2", + "individual_questions/full_name_i_c": "Tesa XLast", + "individual_questions/relationship_i_c": "wife_husband", + "individual_questions/individual_vulnerabilities/work_status_i_c": "1", + "individual_questions/estimated_birth_date_i_c": "0", + "individual_questions/more_information/id_type_i_c": "birth_certificate", + "individual_questions/individual_vulnerabilities/disability_i_c": "not disabled", + "individual_questions/more_information/birth_certificate_no_i_c": "444111123", + "individual_questions/more_information/birth_certificate_issuer_i_c": "AFG", + "individual_questions/birth_date_i_c": "1988-03-23" + } + ], + "wash_questions/score_bed": "5", + "meta/instanceID": "uuid:c09130af-6c9c-4dba-8c7f-1b2ff1970d19", + "wash_questions/blanket_hhsize": "NaN", + "household_questions/household_location/country_h_c": "AFG", + "household_questions/household_location/residence_status_h_c": "refugee", + "household_questions/f_adults_disability_h_c": "0", + "wash_questions/score_childclothes": "5", + "humanitarian_assistance_questions/assistance_h_f": "0", + "household_questions/m_12_17_age_group_h_c": "0", + "household_questions/f_adults_h_c": "1", + "household_questions/f_12_17_disability_h_c": "0", + "household_questions/f_0_5_age_group_h_c": "0", + "household_questions/m_6_11_age_group_h_c": "0", + "wash_questions/score_womencloth": "5", + "household_questions/f_12_17_age_group_h_c": "0", + "wash_questions/score_jerrycan": "5", + "start": "2020-06-02T14:59:08.962+02:00", + "wash_questions/sufficient_water_h_f": "sufficientwater", + "_attachments": [ + { + "mimetype": "image/png", + "download_small_url": "https://kc.humanitarianresponse.info/media/small?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png", + "download_large_url": "https://kc.humanitarianresponse.info/media/large?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png", + "download_url": "https://kc.humanitarianresponse.info/media/original?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png", + "filename": "wnosal/attachments/b83407aca1d647a5bf65a3383ee761d4/c09130af-6c9c-4dba-8c7f-1b2ff1970d19/signature-14_59_24_Wrong_name.png", + "instance": 102612403, + "download_medium_url": "https://kc.humanitarianresponse.info/media/medium?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png", + "id": 35027752, + "xform": 549831 + }, + { + "mimetype": "image/png", + "download_small_url": "https://kc.humanitarianresponse.info/media/small?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png", + "download_large_url": "https://kc.humanitarianresponse.info/media/large?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png", + "download_url": "https://kc.humanitarianresponse.info/media/original?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png", + "filename": "wnosal/attachments/b83407aca1d647a5bf65a3383ee761d4/c09130af-6c9c-4dba-8c7f-1b2ff1970d19/signature-14_59_24_Wrong_name.png", + "instance": 102612403, + "download_medium_url": "https://kc.humanitarianresponse.info/media/medium?media_file=wnosal%2Fattachments%2Fb83407aca1d647a5bf65a3383ee761d4%2Fc09130af-6c9c-4dba-8c7f-1b2ff1970d19%2Fsignature-14_59_24.png", + "id": 35027752, + "xform": 549831 + } + ], + "_status": "submitted_via_web", + "__version__": "vrBoKHPPCWpiRNvCbmnXCK", + "household_questions/m_12_17_disability_h_c": "0", + "wash_questions/score_tool": "5", + "wash_questions/total_liter_yesterday_h_f": "0", + "wash_questions/score_NFI_h_f": "5", + "food_security_questions/FCS_h_f": "NaN", + "wash_questions/jerrycan_hhsize": "NaN", + "household_questions/household_location/admin1_h_c": "SO25", + "wash_questions/score_bassin": "5", + "_validation_status": {}, + "_uuid": "c09130af-6c9c-4dba-8c7f-1b2ff1970d19", + "household_questions/m_adults_h_c": "1", + "enumerator/name_enumerator": "Tester123", + "consent/consent_sign_h_c": "signature-14_59_24.png", + "wash_questions/score_cookingpot": "5", + "_submitted_by": null, + "individual_questions_count": "2", + "wash_questions/water_source_h_f": "piped_water", + "household_questions/pregnant_h_c": "0", + "household_questions/m_6_11_disability_h_c": "0", + "household_questions/m_0_5_disability_h_c": "0", + "formhub/uuid": "b83407aca1d647a5bf65a3383ee761d4", + "end": "2020-06-03T15:05:08.220+02:00", + "household_questions/household_location/hh_geopoint_h_c": "4.747176 43.942756 100 100", + "monthly_income_questions/round_total_income_h_f": "0", + "wash_questions/score_total": "40", + "household_questions/m_adults_disability_h_c": "0", + "_submission_time": "2020-06-04T13:05:10+00:00", + "household_questions/household_location/country_origin_h_c": "AFG", + "_geolocation": [ + 4.747176, + 43.942756 + ], + "monthly_expenditures_questions/round_total_expense_h_f": "0", + "deviceid": "ee.humanitarianresponse.info:AqAb03KLuEfWXes0", + "food_security_questions/cereals_tuber_score_h_f": "NaN", + "household_questions/f_6_11_disability_h_c": "0", + "wash_questions/score_blanket": "5", + "household_questions/f_6_11_age_group_h_c": "0", + "enumerator/org_enumerator": "unicef", + "_id": 132642406, + "living_conditions_questions/living_situation_h_f": "own" } ] diff --git a/backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission1.json b/backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission1.json new file mode 100644 index 0000000000..7427e03422 --- /dev/null +++ b/backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission1.json @@ -0,0 +1,159 @@ +{ + "_id": 742217, + "formhub/uuid": "2a12f0fff2744cf8ab8117948f6add22", + "start": "2024-04-01T15:26:44.285+06:00", + "end": "2024-04-01T15:39:45.890+06:00", + "deviceid": "collect:jKI1p9BIx2CvS7TH", + "today": "2024-04-01", + "username": "data_collection_bgd", + "enumerator/name_enumerator_h_c": "Mr. Jan Romaniak", + "enumerator/org_enumerator_h_c": "partner", + "enumerator/org_name_enumerator_h_c": "ttt", + "enumerator/position_enumerator_h_c": "Volunteer", + "enumerator/cell_enumerator_h_c": "+48605888015", + "enumerator/first_registration_date_h_c": "2024-04-01", + "consent/consent_h_c": "1", + "household_location/country_h_c": "BGD", + "household_location/admin1_h_c": "BD55", + "household_location/admin2_h_c": "BD5549", + "household_location/admin3_h_c": "BD554979", + "household_location/admin4_h_c": "BD55497971", + "household_location/village_h_c": "Uttor para", + "household_location/hh_geopoint_h_c": "26.5804162 79.8380654 0.0 2099.999", + "individual_questions": [ + { + "individual_questions/individual_index": "1", + "individual_questions/individual_details/role_i_c": "primary", + "individual_questions/individual_details/role_primary": "1", + "individual_questions/individual_details/role_alternate": "0", + "individual_questions/individual_details/relationship_i_c": "wife_husband", + "individual_questions/individual_details/full_name_i_c": "Mst. Test Okoe", + "individual_questions/individual_details/birth_date_i_c": "2007-08-02", + "individual_questions/individual_details/age": "16", + "individual_questions/individual_details/estimated_birth_date_i_c": "1", + "individual_questions/individual_details/gender_i_c": "female", + "individual_questions/individual_details/marital_status_i_c": "married", + "individual_questions/individual_details/residence_i_c": "1", + "individual_questions/individual_details/highest_education_i_c": "secondary", + "individual_questions/individual_details/pregnant_i_c": "1", + "individual_questions/individual_details/pregnant_period_i_c": "7", + "individual_questions/individual_details/identification/id_type_i_c": "birth_certificate", + "individual_questions/individual_details/identification/birth_certificate_no_i_c": "123", + "individual_questions/individual_details/identification/birth_certificate_issuer_i_c": "BGD", + "individual_questions/individual_details/identification/birth_certificate_permission_i_c": "1", + "individual_questions/individual_details/identification/birth_certificate_photo_i_c": "1711963979920.jpg", + "individual_questions/individual_details/Disability/observed_disability_i_c": "none", + "individual_questions/individual_details/Disability/chronical_disease_i_c": "none", + "individual_questions/contact_information/photo_i_c": "1711964003171.jpg", + "individual_questions/contact_information/pc_mobile": "1", + "individual_questions/contact_information/phone_no_i_c": "+48605888015", + "individual_questions/contact_information/pc_mobile_banking_i_c": "1", + "individual_questions/contact_information/pc_mobile_banking_type_i_c": "bkash", + "individual_questions/contact_information/alternate_collector": "1" + }, + { + "individual_questions/individual_index": "2", + "individual_questions/individual_details/role_i_c": "alternate", + "individual_questions/individual_details/role_primary": "0", + "individual_questions/individual_details/role_alternate": "1", + "individual_questions/individual_details/relationship_i_c": "head", + "individual_questions/individual_details/full_name_i_c": "Md. Test Elo", + "individual_questions/individual_details/birth_date_i_c": "1995-01-10", + "individual_questions/individual_details/age": "29", + "individual_questions/individual_details/estimated_birth_date_i_c": "1", + "individual_questions/individual_details/gender_i_c": "male", + "individual_questions/individual_details/marital_status_i_c": "married", + "individual_questions/individual_details/residence_i_c": "1", + "individual_questions/individual_details/highest_education_i_c": "primary", + "individual_questions/individual_details/identification/id_type_i_c": "birth_certificate", + "individual_questions/individual_details/identification/birth_certificate_no_i_c": "123454", + "individual_questions/individual_details/identification/birth_certificate_issuer_i_c": "BGD", + "individual_questions/individual_details/identification/birth_certificate_permission_i_c": "1", + "individual_questions/individual_details/identification/birth_certificate_photo_i_c": "1711964188191.jpg", + "individual_questions/individual_details/Disability/observed_disability_i_c": "none", + "individual_questions/contact_information/pc_mobile": "1", + "individual_questions/contact_information/phone_no_i_c": "+48605888015", + "individual_questions/contact_information/pc_mobile_banking_i_c": "1", + "individual_questions/contact_information/pc_mobile_banking_type_i_c": "bkash" + } + ], + "number_primary": "1", + "number_alternate": "1", + "household_living_condition/size_h_c": "3", + "household_living_condition/hh_tubewell_h_c": "1", + "household_living_condition/hh_latrine_h_c": "1", + "household_living_condition/hh_hygiene_h_c": "1", + "household_living_condition/hh_electricity_h_c": "1", + "household_living_condition/hh_structure_h_c": "kucha_cgi_sheet", + "livelihood_information/currency_h_c": "BDT", + "livelihood_information/hh_land_decimal_h_c": "4", + "livelihood_information/hh_income_bdt_h_c": "23444", + "livelihood_information/hh_income_source_h_c": "small_business", + "livelihood_information/primary_expenditures_h_f": "food health_costs shelter utilities transport debt", + "livelihood_information/hh_nearest_health_facility_h_c": "Upazila_health_complex", + "financial_assistance/social_protection_support_h_c": "0", + "financial_assistance/humanitarian_assistance_h_c": "0", + "preferred_payment_method/payment_method_h_c": "mobile_money", + "preferred_payment_method/cash_collection_time_h_c": "1hour", + "preferred_payment_method/cash_collection_safe_h_c": "completely_safe", + "feedback_communication/knowledge_complain_h_c": "0", + "feedback_communication/communication_method_h_c": "community_meetings", + "__version__": "v35KF93esisEFCUCdXQt3h", + "meta/audit": "audit.csv", + "meta/instanceID": "uuid:d8ae6b50-6fae-429d-a0f5-66807d5a198b", + "_xform_id_string": "a64aUtFpY8PbooJQQsFAmN", + "_uuid": "d8ae6b50-6fae-429d-a0f5-66807d5a198b", + "_attachments": [ + { + "download_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619355/?format=json", + "download_large_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619355/?format=json", + "download_medium_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619355/?format=json", + "download_small_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619355/?format=json", + "mimetype": "text/comma-separated-values", + "filename": "bgd_admin/attachments/2a12f0fff2744cf8ab8117948f6add22/d8ae6b50-6fae-429d-a0f5-66807d5a198b/audit.csv", + "instance": 742217, + "xform": 824, + "id": 1619355 + }, + { + "download_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619356/?format=json", + "download_large_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619356/?format=json", + "download_medium_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619356/?format=json", + "download_small_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619356/?format=json", + "mimetype": "image/jpeg", + "filename": "bgd_admin/attachments/2a12f0fff2744cf8ab8117948f6add22/d8ae6b50-6fae-429d-a0f5-66807d5a198b/1711963979920.jpg", + "instance": 742217, + "xform": 824, + "id": 1619356 + }, + { + "download_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619357/?format=json", + "download_large_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619357/?format=json", + "download_medium_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619357/?format=json", + "download_small_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619357/?format=json", + "mimetype": "image/jpeg", + "filename": "bgd_admin/attachments/2a12f0fff2744cf8ab8117948f6add22/d8ae6b50-6fae-429d-a0f5-66807d5a198b/1711964003171.jpg", + "instance": 742217, + "xform": 824, + "id": 1619357 + }, + { + "download_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619358/?format=json", + "download_large_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619358/?format=json", + "download_medium_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619358/?format=json", + "download_small_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742217/attachments/1619358/?format=json", + "mimetype": "image/jpeg", + "filename": "bgd_admin/attachments/2a12f0fff2744cf8ab8117948f6add22/d8ae6b50-6fae-429d-a0f5-66807d5a198b/1711964188191.jpg", + "instance": 742217, + "xform": 824, + "id": 1619358 + } + ], + "_status": "submitted_via_web", + "_geolocation": [28.5804162, 79.8380654], + "_submission_time": "2024-04-01T11:04:10", + "_tags": [], + "_notes": [], + "_validation_status": {}, + "_submitted_by": "data_collection_bgd" +} diff --git a/backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission2.json b/backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission2.json new file mode 100644 index 0000000000..487496c872 --- /dev/null +++ b/backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission2.json @@ -0,0 +1,126 @@ +{ + "_id": 742220, + "formhub/uuid": "2a12f0fff2744cf8ab8117948f6add22", + "start": "2024-04-01T15:26:44.285+06:00", + "end": "2024-04-01T17:04:33.180+06:00", + "deviceid": "collect:jKI1p9BIx2CvS7TH", + "today": "2024-04-01", + "username": "data_collection_bgd", + "enumerator/name_enumerator_h_c": "Mr. Jan Romaniak", + "enumerator/org_enumerator_h_c": "partner", + "enumerator/org_name_enumerator_h_c": "ttt", + "enumerator/position_enumerator_h_c": "Volunteer", + "enumerator/cell_enumerator_h_c": "+48605888015", + "enumerator/first_registration_date_h_c": "2024-04-01", + "consent/consent_h_c": "1", + "household_location/country_h_c": "BGD", + "household_location/admin1_h_c": "BD55", + "household_location/admin2_h_c": "BD5549", + "household_location/admin3_h_c": "BD554979", + "household_location/admin4_h_c": "BD55497971", + "household_location/village_h_c": "Uttor para", + "household_location/hh_geopoint_h_c": "26.5804162 79.8380654 0.0 2099.999", + "individual_questions": [ + { + "individual_questions/individual_index": "1", + "individual_questions/individual_details/role_i_c": "primary", + "individual_questions/individual_details/role_primary": "1", + "individual_questions/individual_details/role_alternate": "0", + "individual_questions/individual_details/relationship_i_c": "wife_husband", + "individual_questions/individual_details/full_name_i_c": "Mst. Test Okoe", + "individual_questions/individual_details/birth_date_i_c": "2007-08-02", + "individual_questions/individual_details/age": "16", + "individual_questions/individual_details/estimated_birth_date_i_c": "1", + "individual_questions/individual_details/gender_i_c": "female", + "individual_questions/individual_details/marital_status_i_c": "married", + "individual_questions/individual_details/residence_i_c": "1", + "individual_questions/individual_details/highest_education_i_c": "secondary", + "individual_questions/individual_details/pregnant_i_c": "1", + "individual_questions/individual_details/pregnant_period_i_c": "7", + "individual_questions/individual_details/identification/id_type_i_c": "birth_certificate", + "individual_questions/individual_details/identification/birth_certificate_no_i_c": "123", + "individual_questions/individual_details/identification/birth_certificate_issuer_i_c": "BGD", + "individual_questions/individual_details/identification/birth_certificate_permission_i_c": "1", + "individual_questions/individual_details/identification/birth_certificate_photo_i_c": "1711963979920.jpg", + "individual_questions/individual_details/Disability/observed_disability_i_c": "none", + "individual_questions/individual_details/Disability/chronical_disease_i_c": "none", + "individual_questions/contact_information/photo_i_c": "1711964003171.jpg", + "individual_questions/contact_information/pc_mobile": "1", + "individual_questions/contact_information/phone_no_i_c": "+48605888015", + "individual_questions/contact_information/pc_mobile_banking_i_c": "1", + "individual_questions/contact_information/pc_mobile_banking_type_i_c": "bkash", + "individual_questions/contact_information/alternate_collector": "1" + }, + { + "individual_questions/individual_index": "2", + "individual_questions/individual_details/role_i_c": "alternate", + "individual_questions/individual_details/role_primary": "0", + "individual_questions/individual_details/role_alternate": "1", + "individual_questions/individual_details/relationship_i_c": "head", + "individual_questions/individual_details/full_name_i_c": "Md. Test Elo", + "individual_questions/individual_details/birth_date_i_c": "1995-01-10", + "individual_questions/individual_details/age": "29", + "individual_questions/individual_details/estimated_birth_date_i_c": "1", + "individual_questions/individual_details/gender_i_c": "male", + "individual_questions/individual_details/marital_status_i_c": "married", + "individual_questions/individual_details/residence_i_c": "1", + "individual_questions/individual_details/highest_education_i_c": "primary", + "individual_questions/individual_details/identification/id_type_i_c": "birth_certificate", + "individual_questions/individual_details/identification/birth_certificate_no_i_c": "123454", + "individual_questions/individual_details/identification/birth_certificate_issuer_i_c": "BGD", + "individual_questions/individual_details/identification/birth_certificate_permission_i_c": "1", + "individual_questions/individual_details/identification/birth_certificate_photo_i_c": "1711964188191.jpg", + "individual_questions/individual_details/Disability/observed_disability_i_c": "none", + "individual_questions/contact_information/pc_mobile": "1", + "individual_questions/contact_information/phone_no_i_c": "+48605888015", + "individual_questions/contact_information/pc_mobile_banking_i_c": "1", + "individual_questions/contact_information/pc_mobile_banking_type_i_c": "bkash" + } + ], + "number_primary": "1", + "number_alternate": "1", + "household_living_condition/size_h_c": "3", + "household_living_condition/hh_tubewell_h_c": "1", + "household_living_condition/hh_latrine_h_c": "1", + "household_living_condition/hh_hygiene_h_c": "1", + "household_living_condition/hh_electricity_h_c": "1", + "household_living_condition/hh_structure_h_c": "kucha_cgi_sheet", + "livelihood_information/currency_h_c": "BDT", + "livelihood_information/hh_land_decimal_h_c": "4", + "livelihood_information/hh_income_bdt_h_c": "23444", + "livelihood_information/hh_income_source_h_c": "small_business", + "livelihood_information/primary_expenditures_h_f": "food health_costs shelter utilities transport debt", + "livelihood_information/hh_nearest_health_facility_h_c": "Upazila_health_complex", + "financial_assistance/social_protection_support_h_c": "0", + "financial_assistance/humanitarian_assistance_h_c": "0", + "preferred_payment_method/payment_method_h_c": "mobile_money", + "preferred_payment_method/cash_collection_time_h_c": "1hour", + "preferred_payment_method/cash_collection_safe_h_c": "completely_safe", + "feedback_communication/knowledge_complain_h_c": "0", + "feedback_communication/communication_method_h_c": "community_meetings", + "__version__": "v35KF93esisEFCUCdXQt3h", + "meta/audit": "audit.csv", + "meta/instanceID": "uuid:d8ae6b50-6fae-429d-a0f5-66807d5a198b", + "_xform_id_string": "a64aUtFpY8PbooJQQsFAmN", + "_uuid": "d8ae6b50-6fae-429d-a0f5-66807d5a198b", + "_attachments": [ + { + "download_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742220/attachments/1619367/?format=json", + "download_large_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742220/attachments/1619367/?format=json", + "download_medium_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742220/attachments/1619367/?format=json", + "download_small_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742220/attachments/1619367/?format=json", + "mimetype": "text/comma-separated-values", + "filename": "bgd_admin/attachments/2a12f0fff2744cf8ab8117948f6add22/d8ae6b50-6fae-429d-a0f5-66807d5a198b/audit_IaSSXpM.csv", + "instance": 742220, + "xform": 824, + "id": 1619367 + } + ], + "_status": "submitted_via_web", + "_geolocation": [28.5804162, 79.8380654], + "_submission_time": "2024-04-01T11:04:36", + "_tags": [], + "_notes": [], + "_validation_status": {}, + "_submitted_by": "data_collection_bgd" +} diff --git a/backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission3.json b/backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission3.json new file mode 100644 index 0000000000..6b704206c9 --- /dev/null +++ b/backend/hct_mis_api/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission3.json @@ -0,0 +1,126 @@ +{ + "_id": 742220, + "formhub/uuid": "2a12f0fff2744cf8ab8117948f6add22", + "start": "2024-04-01T15:26:44.285+06:00", + "end": "2024-04-01T17:04:33.180+06:00", + "deviceid": "collect:jKI1p9BIx2CvS7TH", + "today": "2024-04-01", + "username": "data_collection_bgd", + "enumerator/name_enumerator_h_c": "Mr. Jan Romaniak", + "enumerator/org_enumerator_h_c": "partner", + "enumerator/org_name_enumerator_h_c": "ttt", + "enumerator/position_enumerator_h_c": "Volunteer", + "enumerator/cell_enumerator_h_c": "+48605888015", + "enumerator/first_registration_date_h_c": "2024-04-01", + "consent/consent_h_c": "1", + "household_location/country_h_c": "BGD", + "household_location/admin1_h_c": "BD55", + "household_location/admin2_h_c": "BD5549", + "household_location/admin3_h_c": "BD554979", + "household_location/admin4_h_c": "BD55497971", + "household_location/village_h_c": "Uttor para", + "household_location/hh_geopoint_h_c": "26.5804162 79.8380654 0.0 2099.999", + "individual_questions": [ + { + "individual_questions/individual_index": "1", + "individual_questions/individual_details/role_i_c": "primary", + "individual_questions/individual_details/role_primary": "1", + "individual_questions/individual_details/role_alternate": "0", + "individual_questions/individual_details/relationship_i_c": "wife_husband", + "individual_questions/individual_details/full_name_i_c": "Mst. Test Okoe", + "individual_questions/individual_details/birth_date_i_c": "2007-08-02", + "individual_questions/individual_details/age": "16", + "individual_questions/individual_details/estimated_birth_date_i_c": "1", + "individual_questions/individual_details/gender_i_c": "female", + "individual_questions/individual_details/marital_status_i_c": "married", + "individual_questions/individual_details/residence_i_c": "1", + "individual_questions/individual_details/highest_education_i_c": "secondary", + "individual_questions/individual_details/pregnant_i_c": "1", + "individual_questions/individual_details/pregnant_period_i_c": "7", + "individual_questions/individual_details/identification/id_type_i_c": "birth_certificate", + "individual_questions/individual_details/identification/birth_certificate_no_i_c": "1237", + "individual_questions/individual_details/identification/birth_certificate_issuer_i_c": "BGD", + "individual_questions/individual_details/identification/birth_certificate_permission_i_c": "1", + "individual_questions/individual_details/identification/birth_certificate_photo_i_c": "1711963979920.jpg", + "individual_questions/individual_details/Disability/observed_disability_i_c": "none", + "individual_questions/individual_details/Disability/chronical_disease_i_c": "none", + "individual_questions/contact_information/photo_i_c": "1711964003171.jpg", + "individual_questions/contact_information/pc_mobile": "1", + "individual_questions/contact_information/phone_no_i_c": "+48605888015", + "individual_questions/contact_information/pc_mobile_banking_i_c": "1", + "individual_questions/contact_information/pc_mobile_banking_type_i_c": "bkash", + "individual_questions/contact_information/alternate_collector": "1" + }, + { + "individual_questions/individual_index": "2", + "individual_questions/individual_details/role_i_c": "alternate", + "individual_questions/individual_details/role_primary": "0", + "individual_questions/individual_details/role_alternate": "1", + "individual_questions/individual_details/relationship_i_c": "head", + "individual_questions/individual_details/full_name_i_c": "Md. Test Elo", + "individual_questions/individual_details/birth_date_i_c": "1995-01-10", + "individual_questions/individual_details/age": "29", + "individual_questions/individual_details/estimated_birth_date_i_c": "1", + "individual_questions/individual_details/gender_i_c": "male", + "individual_questions/individual_details/marital_status_i_c": "married", + "individual_questions/individual_details/residence_i_c": "1", + "individual_questions/individual_details/highest_education_i_c": "primary", + "individual_questions/individual_details/identification/id_type_i_c": "birth_certificate", + "individual_questions/individual_details/identification/birth_certificate_no_i_c": "123454", + "individual_questions/individual_details/identification/birth_certificate_issuer_i_c": "BGD", + "individual_questions/individual_details/identification/birth_certificate_permission_i_c": "1", + "individual_questions/individual_details/identification/birth_certificate_photo_i_c": "1711964188191.jpg", + "individual_questions/individual_details/Disability/observed_disability_i_c": "none", + "individual_questions/contact_information/pc_mobile": "1", + "individual_questions/contact_information/phone_no_i_c": "+48605888015", + "individual_questions/contact_information/pc_mobile_banking_i_c": "1", + "individual_questions/contact_information/pc_mobile_banking_type_i_c": "bkash" + } + ], + "number_primary": "1", + "number_alternate": "1", + "household_living_condition/size_h_c": "3", + "household_living_condition/hh_tubewell_h_c": "1", + "household_living_condition/hh_latrine_h_c": "1", + "household_living_condition/hh_hygiene_h_c": "1", + "household_living_condition/hh_electricity_h_c": "1", + "household_living_condition/hh_structure_h_c": "kucha_cgi_sheet", + "livelihood_information/currency_h_c": "BDT", + "livelihood_information/hh_land_decimal_h_c": "4", + "livelihood_information/hh_income_bdt_h_c": "23444", + "livelihood_information/hh_income_source_h_c": "small_business", + "livelihood_information/primary_expenditures_h_f": "food health_costs shelter utilities transport debt", + "livelihood_information/hh_nearest_health_facility_h_c": "Upazila_health_complex", + "financial_assistance/social_protection_support_h_c": "0", + "financial_assistance/humanitarian_assistance_h_c": "0", + "preferred_payment_method/payment_method_h_c": "mobile_money", + "preferred_payment_method/cash_collection_time_h_c": "1hour", + "preferred_payment_method/cash_collection_safe_h_c": "completely_safe", + "feedback_communication/knowledge_complain_h_c": "0", + "feedback_communication/communication_method_h_c": "community_meetings", + "__version__": "v35KF93esisEFCUCdXQt3h", + "meta/audit": "audit.csv", + "meta/instanceID": "uuid:d8ae6b50-6fae-429d-a0f5-66807d5a198b", + "_xform_id_string": "a64aUtFpY8PbooJQQsFAmN", + "_uuid": "d8ae6b50-6fae-429d-a0f5-66807d5a198b", + "_attachments": [ + { + "download_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742220/attachments/1619367/?format=json", + "download_large_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742220/attachments/1619367/?format=json", + "download_medium_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742220/attachments/1619367/?format=json", + "download_small_url": "https://kf.hope.unicef.org/api/v2/assets/a64aUtFpY8PbooJQQsFAmN/data/742220/attachments/1619367/?format=json", + "mimetype": "text/comma-separated-values", + "filename": "bgd_admin/attachments/2a12f0fff2744cf8ab8117948f6add22/d8ae6b50-6fae-429d-a0f5-66807d5a198b/audit_IaSSXpM.csv", + "instance": 742220, + "xform": 824, + "id": 1619367 + } + ], + "_status": "submitted_via_web", + "_geolocation": [28.5804162, 79.8380654], + "_submission_time": "2024-04-01T11:04:36", + "_tags": [], + "_notes": [], + "_validation_status": {}, + "_submitted_by": "data_collection_bgd" +} diff --git a/backend/hct_mis_api/apps/registration_datahub/tests/test_rdi_create.py b/backend/hct_mis_api/apps/registration_datahub/tests/test_rdi_create.py index 91c287beef..99af95f11d 100644 --- a/backend/hct_mis_api/apps/registration_datahub/tests/test_rdi_create.py +++ b/backend/hct_mis_api/apps/registration_datahub/tests/test_rdi_create.py @@ -428,7 +428,7 @@ def test_create_receiver_poi_document(self) -> None: self.assertEqual(list(documents), expected) -@disabled_locally_test + class TestRdiKoboCreateTask(BaseElasticSearchTestCase): databases = { "default", diff --git a/backend/hct_mis_api/apps/registration_datahub/tests/test_utils.py b/backend/hct_mis_api/apps/registration_datahub/tests/test_utils.py new file mode 100644 index 0000000000..8396df3858 --- /dev/null +++ b/backend/hct_mis_api/apps/registration_datahub/tests/test_utils.py @@ -0,0 +1,29 @@ +import json + +from django.conf import settings +from django.test import TestCase + +from hct_mis_api.apps.core.fixtures import create_afghanistan +from hct_mis_api.apps.core.models import BusinessArea +from hct_mis_api.apps.geo.fixtures import AreaFactory, AreaTypeFactory +from hct_mis_api.apps.registration_datahub.fixtures import ImportedHouseholdFactory +from hct_mis_api.apps.registration_datahub.utils import calculate_hash_for_kobo_submission + + +class TestRdiUtils(TestCase): + + def test_calculate_hash_for_kobo_submission(self): + test_data1 = json.load( + open( + f"{settings.PROJECT_ROOT}/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission1.json")) + test_data2 = json.load( + open( + f"{settings.PROJECT_ROOT}/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission2.json")) + test_data3 = json.load( + open( + f"{settings.PROJECT_ROOT}/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission3.json")) + hash1 = calculate_hash_for_kobo_submission(test_data1) + hash2 = calculate_hash_for_kobo_submission(test_data2) + hash3 = calculate_hash_for_kobo_submission(test_data3) + self.assertEqual(hash1, hash2) + self.assertNotEqual(hash1, hash3) diff --git a/backend/hct_mis_api/apps/registration_datahub/utils.py b/backend/hct_mis_api/apps/registration_datahub/utils.py index acf2072b7b..601bc77ee9 100644 --- a/backend/hct_mis_api/apps/registration_datahub/utils.py +++ b/backend/hct_mis_api/apps/registration_datahub/utils.py @@ -1,3 +1,5 @@ +import hashlib +import json import re import sys from typing import Any, Dict, List, Optional @@ -51,3 +53,23 @@ def find_attachment_in_kobo(attachments: List[Dict], value: str) -> Optional[Dic if regex_name.match(get_field_name(attachment["filename"])): return attachment return None + + +def calculate_hash_for_kobo_submission(submission: Dict) -> str: + keys_to_remove = [ + "_id", + "start", + "end", + "_submission_time", + "_attachments", + ] + submission_copy = submission.copy() + for key in keys_to_remove: + if key in submission_copy: + del submission_copy[key] + + d_string = json.dumps(submission_copy, sort_keys=True) + d_bytes = d_string.encode('utf-8') + hash_object = hashlib.sha256(d_bytes) + hex_dig = hash_object.hexdigest() + return hex_dig diff --git a/backend/hct_mis_api/apps/registration_datahub/validators.py b/backend/hct_mis_api/apps/registration_datahub/validators.py index 8da326b041..7354d1eb76 100644 --- a/backend/hct_mis_api/apps/registration_datahub/validators.py +++ b/backend/hct_mis_api/apps/registration_datahub/validators.py @@ -45,7 +45,7 @@ ) from hct_mis_api.apps.registration_datahub.models import KoboImportedSubmission from hct_mis_api.apps.registration_datahub.tasks.utils import collectors_str_ids_to_list -from hct_mis_api.apps.registration_datahub.utils import find_attachment_in_kobo +from hct_mis_api.apps.registration_datahub.utils import find_attachment_in_kobo, calculate_hash_for_kobo_submission logger = logging.getLogger(__name__) @@ -1374,15 +1374,17 @@ def validate_everything(self, submissions: List, business_area: BusinessArea) -> item.append(submission["kobo_submission_time"].isoformat()) all_saved_submissions_dict[str(submission["kobo_submission_uuid"])] = item household: Dict[str, Any] + household_hash_list: List[str] = [] for household in reduced_submissions: household_uuid = str(household.get("_uuid")) - + household_hash = calculate_hash_for_kobo_submission(household) submission_exists = household.get("_submission_time") in all_saved_submissions_dict.get( household_uuid, [] ) - if submission_exists: + submission_duplicate = household_hash in household_hash_list + if submission_exists or submission_duplicate: continue - + household_hash_list.append(household_hash) head_of_hh_counter = 0 primary_collector_counter = 0 alternate_collector_counter = 0 From 8a7c3baffcb81cd5206ddd03124e144bc89c829d Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Tue, 7 May 2024 14:38:23 +0200 Subject: [PATCH 4/6] fixed code style --- .../tasks/rdi_kobo_create.py | 5 +++- .../tests/test_rdi_create.py | 1 - .../registration_datahub/tests/test_utils.py | 23 +++++++++++-------- .../apps/registration_datahub/utils.py | 2 +- .../apps/registration_datahub/validators.py | 5 +++- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/backend/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py b/backend/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py index bebbced52e..c6679e66ff 100644 --- a/backend/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py +++ b/backend/hct_mis_api/apps/registration_datahub/tasks/rdi_kobo_create.py @@ -43,7 +43,10 @@ logger, ) from hct_mis_api.apps.registration_datahub.tasks.utils import get_submission_metadata -from hct_mis_api.apps.registration_datahub.utils import find_attachment_in_kobo, calculate_hash_for_kobo_submission +from hct_mis_api.apps.registration_datahub.utils import ( + calculate_hash_for_kobo_submission, + find_attachment_in_kobo, +) from hct_mis_api.apps.utils.age_at_registration import calculate_age_at_registration diff --git a/backend/hct_mis_api/apps/registration_datahub/tests/test_rdi_create.py b/backend/hct_mis_api/apps/registration_datahub/tests/test_rdi_create.py index 99af95f11d..dfcdfefce8 100644 --- a/backend/hct_mis_api/apps/registration_datahub/tests/test_rdi_create.py +++ b/backend/hct_mis_api/apps/registration_datahub/tests/test_rdi_create.py @@ -428,7 +428,6 @@ def test_create_receiver_poi_document(self) -> None: self.assertEqual(list(documents), expected) - class TestRdiKoboCreateTask(BaseElasticSearchTestCase): databases = { "default", diff --git a/backend/hct_mis_api/apps/registration_datahub/tests/test_utils.py b/backend/hct_mis_api/apps/registration_datahub/tests/test_utils.py index 8396df3858..fa458a9f0e 100644 --- a/backend/hct_mis_api/apps/registration_datahub/tests/test_utils.py +++ b/backend/hct_mis_api/apps/registration_datahub/tests/test_utils.py @@ -3,25 +3,28 @@ from django.conf import settings from django.test import TestCase -from hct_mis_api.apps.core.fixtures import create_afghanistan -from hct_mis_api.apps.core.models import BusinessArea -from hct_mis_api.apps.geo.fixtures import AreaFactory, AreaTypeFactory -from hct_mis_api.apps.registration_datahub.fixtures import ImportedHouseholdFactory -from hct_mis_api.apps.registration_datahub.utils import calculate_hash_for_kobo_submission +from hct_mis_api.apps.registration_datahub.utils import ( + calculate_hash_for_kobo_submission, +) class TestRdiUtils(TestCase): - - def test_calculate_hash_for_kobo_submission(self): + def test_calculate_hash_for_kobo_submission(self) -> None: test_data1 = json.load( open( - f"{settings.PROJECT_ROOT}/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission1.json")) + f"{settings.PROJECT_ROOT}/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission1.json" + ) + ) test_data2 = json.load( open( - f"{settings.PROJECT_ROOT}/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission2.json")) + f"{settings.PROJECT_ROOT}/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission2.json" + ) + ) test_data3 = json.load( open( - f"{settings.PROJECT_ROOT}/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission3.json")) + f"{settings.PROJECT_ROOT}/apps/registration_datahub/tests/test_file/test_calculate_hash_for_kobo_submission3.json" + ) + ) hash1 = calculate_hash_for_kobo_submission(test_data1) hash2 = calculate_hash_for_kobo_submission(test_data2) hash3 = calculate_hash_for_kobo_submission(test_data3) diff --git a/backend/hct_mis_api/apps/registration_datahub/utils.py b/backend/hct_mis_api/apps/registration_datahub/utils.py index 601bc77ee9..8969fee623 100644 --- a/backend/hct_mis_api/apps/registration_datahub/utils.py +++ b/backend/hct_mis_api/apps/registration_datahub/utils.py @@ -69,7 +69,7 @@ def calculate_hash_for_kobo_submission(submission: Dict) -> str: del submission_copy[key] d_string = json.dumps(submission_copy, sort_keys=True) - d_bytes = d_string.encode('utf-8') + d_bytes = d_string.encode("utf-8") hash_object = hashlib.sha256(d_bytes) hex_dig = hash_object.hexdigest() return hex_dig diff --git a/backend/hct_mis_api/apps/registration_datahub/validators.py b/backend/hct_mis_api/apps/registration_datahub/validators.py index 7354d1eb76..ce8e9d9b28 100644 --- a/backend/hct_mis_api/apps/registration_datahub/validators.py +++ b/backend/hct_mis_api/apps/registration_datahub/validators.py @@ -45,7 +45,10 @@ ) from hct_mis_api.apps.registration_datahub.models import KoboImportedSubmission from hct_mis_api.apps.registration_datahub.tasks.utils import collectors_str_ids_to_list -from hct_mis_api.apps.registration_datahub.utils import find_attachment_in_kobo, calculate_hash_for_kobo_submission +from hct_mis_api.apps.registration_datahub.utils import ( + calculate_hash_for_kobo_submission, + find_attachment_in_kobo, +) logger = logging.getLogger(__name__) From f9a10ea6c7814c62a97ee6a947efa3689bad49c5 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Tue, 7 May 2024 15:18:33 +0200 Subject: [PATCH 5/6] added validator test --- .../tests/test_kobo_validators_methods.py | 101 +++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/backend/hct_mis_api/apps/registration_datahub/tests/test_kobo_validators_methods.py b/backend/hct_mis_api/apps/registration_datahub/tests/test_kobo_validators_methods.py index 03c2e87c63..8bf4ad6dad 100644 --- a/backend/hct_mis_api/apps/registration_datahub/tests/test_kobo_validators_methods.py +++ b/backend/hct_mis_api/apps/registration_datahub/tests/test_kobo_validators_methods.py @@ -136,7 +136,106 @@ class TestKoboSaveValidatorsMethods(TestCase): "wash_questions/score_blanket": "5", "household_questions/f_6_11_age_group_h_c": "0", "_id": 101804069, - } + }, + { + "_notes": [], + "wash_questions/score_num_items": "8", + "wash_questions/bed_hhsize": "NaN", + "monthly_income_questions/total_inc_h_f": "0", + "household_questions/m_0_5_age_group_h_c": "0", + "_xform_id_string": "aPkhoRMrkkDwgsvWuwi39s", + "_bamboo_dataset_id": "", + "_tags": [], + "health_questions/pregnant_member_h_c": "0", + "health_questions/start": "2020-04-28T17:34:22.979+02:00", + "health_questions/end": "2020-05-28T18:56:33.979+02:00", + "household_questions/first_registration_date_h_c": "2020-07-18", + "household_questions/f_0_5_disability_h_c": "0", + "household_questions/size_h_c": "1", + "household_questions/country_h_c": "AFG", + "monthly_expenditures_questions/total_expense_h_f": "0", + "individual_questions": [ + { + "individual_questions/preferred_language_i_c": "pl-pl", + "individual_questions/role_i_c": "primary", + "individual_questions/age": "40", + "individual_questions/first_registration_date_i_c": "2020-07-18", + "individual_questions/more_information/marital_status_i_c": "married", + "individual_questions/individual_index": "1", + "individual_questions/birth_date_i_c": "1980-07-18", + "individual_questions/estimated_birth_date_i_c": "0", + "individual_questions/relationship_i_c": "head", + "individual_questions/gender_i_c": "male", + "individual_questions/individual_vulnerabilities/disability_i_c": "not disabled", + "individual_questions/full_name_i_c": "Test Testowy", + "individual_questions/is_only_collector": "NO", + "individual_questions/mas_treatment_i_f": "1", + "individual_questions/arm_picture_i_f": "signature-17_32_52.png", + "individual_questions/identification/tax_id_no_i_c": "45638193", + "individual_questions/identification/tax_id_issuer_i_c": "UKR", + "individual_questions/identification/bank_account_number_i_c": "UA3481939838393949", + "individual_questions/identification/bank_name_i_c": "Privat", + "individual_questions/identification/account_holder_name_i_c": "Name 123", + } + ], + "wash_questions/score_bed": "5", + "meta/instanceID": "uuid:512ca816-5cab-45a6-a676-1f47cfe7658e", + "wash_questions/blanket_hhsize": "NaN", + "household_questions/f_adults_disability_h_c": "0", + "wash_questions/score_childclothes": "5", + "household_questions/org_enumerator_h_c": "UNICEF", + "household_questions/specific_residence_h_f": "returnee", + "household_questions/org_name_enumerator_h_c": "Test", + "household_questions/name_enumerator_h_c": "Test", + "household_questions/consent_h_c": "0", + "household_questions/consent_sharing_h_c": "UNICEF", + "household_questions/m_12_17_age_group_h_c": "0", + "household_questions/f_adults_h_c": "0", + "household_questions/f_12_17_disability_h_c": "0", + "household_questions/f_0_5_age_group_h_c": "0", + "household_questions/m_6_11_age_group_h_c": "0", + "wash_questions/score_womencloth": "5", + "household_questions/f_12_17_age_group_h_c": "0", + "wash_questions/score_jerrycan": "5", + "start": "2020-05-28T17:32:43.054+02:00", + "_attachments": [], + "_status": "submitted_via_web", + "__version__": "vrBoKHPPCWpiRNvCbmnXCK", + "household_questions/m_12_17_disability_h_c": "0", + "wash_questions/score_tool": "5", + "wash_questions/total_liter_yesterday_h_f": "0", + "wash_questions/score_NFI_h_f": "5", + "food_security_questions/FCS_h_f": "NaN", + "wash_questions/jerrycan_hhsize": "NaN", + "enumerator/name_enumerator": "Test", + "wash_questions/score_bassin": "5", + "_validation_status": {}, + "_uuid": "512ca816-5cab-45a6-a676-1f47cfe7658e", + "household_questions/m_adults_h_c": "1", + "consent/consent_sign_h_c": "signature-17_32_52.png", + "wash_questions/score_total": "40", + "_submitted_by": None, + "individual_questions_count": "1", + "end": "2020-05-28T17:34:22.979+02:00", + "household_questions/pregnant_h_c": "0", + "household_questions/m_6_11_disability_h_c": "0", + "household_questions/m_0_5_disability_h_c": "0", + "formhub/uuid": "b83407aca1d647a5bf65a3383ee761d4", + "enumerator/org_enumerator": "unicef", + "monthly_income_questions/round_total_income_h_f": "0", + "wash_questions/score_cookingpot": "5", + "household_questions/m_adults_disability_h_c": "0", + "_submission_time": "2020-05-28T15:34:37", + "household_questions/household_location/residence_status_h_c": "refugee", + "_geolocation": [None, None], + "monthly_expenditures_questions/round_total_expense_h_f": "0", + "deviceid": "ee.humanitarianresponse.info:AqAb03KLuEfWXes0", + "food_security_questions/cereals_tuber_score_h_f": "NaN", + "household_questions/f_6_11_disability_h_c": "0", + "wash_questions/score_blanket": "5", + "household_questions/f_6_11_age_group_h_c": "0", + "_id": 23456, + }, ] INVALID_JSON = [ From f05b3d5cd5dba10b4c1e2410cebacaa7d8f14319 Mon Sep 17 00:00:00 2001 From: Patryk Dabrowski Date: Tue, 7 May 2024 15:23:15 +0200 Subject: [PATCH 6/6] Fix visible delete rdi button --- .../hct_mis_api/apps/registration_data/admin.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/backend/hct_mis_api/apps/registration_data/admin.py b/backend/hct_mis_api/apps/registration_data/admin.py index 504054394d..addfb7305c 100644 --- a/backend/hct_mis_api/apps/registration_data/admin.py +++ b/backend/hct_mis_api/apps/registration_data/admin.py @@ -180,10 +180,14 @@ def delete_rdi(self, request: HttpRequest, pk: UUID) -> Any: # TODO: typing self.message_user(request, "An error occurred while processing RDI delete", messages.ERROR) @staticmethod - def delete_merged_rdi_visible(o: Any, r: Any) -> bool: - is_correct_status = o.status == RegistrationDataImport.MERGED - is_not_used_in_targeting = HouseholdSelection.objects.filter(household__registration_data_import=o).count() == 0 - is_not_used_by_payment_record = PaymentRecord.objects.filter(household__registration_data_import=o).count() == 0 + def delete_merged_rdi_visible(rdi: RegistrationDataImport) -> bool: + is_correct_status = rdi.status == RegistrationDataImport.MERGED + is_not_used_in_targeting = ( + HouseholdSelection.objects.filter(household__registration_data_import=rdi).count() == 0 + ) + is_not_used_by_payment_record = ( + PaymentRecord.objects.filter(household__registration_data_import=rdi).count() == 0 + ) return is_correct_status and is_not_used_in_targeting and is_not_used_by_payment_record @staticmethod @@ -212,7 +216,7 @@ def generate_query_for_all_grievances_tickets(rdi: RegistrationDataImport) -> Q: @button( permission=is_root, - visible=lambda o, r: RegistrationDataImportAdmin.delete_merged_rdi_visible(o, r), + visible=lambda btn: RegistrationDataImportAdmin.delete_merged_rdi_visible(btn.original), ) def delete_merged_rdi(self, request: HttpRequest, pk: UUID) -> Optional[HttpResponse]: try: