Merge branch 'develop' into rdi-import-validation-messages-universal-…

…formatting

backend/hct_mis_api/apps/account/export_users_xlsx.py

@@ -55,7 +55,7 @@ def _add_headers(self) -> None:
         for i in range(1, len(self.FIELDS_TO_COLUMNS_MAPPING) + 1):
             self.ws.column_dimensions[get_column_letter(i)].width = 20
 
-    @transaction.atomic(using="default")
+    @transaction.atomic()
     def get_exported_users_file(self) -> Optional[Workbook]:
         fields = self.FIELDS_TO_COLUMNS_MAPPING.values()
         users = (

backend/hct_mis_api/apps/administration/tests/test_admin.py

@@ -54,7 +54,6 @@ class TestAdminSite(WebTest):
         "cash_assist_datahub_ca",
         "cash_assist_datahub_erp",
         "cash_assist_datahub_mis",
-        "registration_datahub",
     ]
 
     @classmethod

backend/hct_mis_api/apps/cash_assist_datahub/admin.py

@@ -101,7 +101,7 @@ def simulate_import(self, request: "HttpRequest", pk: "UUID") -> Optional[Templa
 
             runner = PullFromDatahubTask()
             try:
-                with transaction.atomic(using="default"), transaction.atomic(using="cash_assist_datahub_ca"):
+                with transaction.atomic(), transaction.atomic(using="cash_assist_datahub_ca"):
                     runner.copy_session(session)
                     raise RollbackException()
             except RollbackException:

backend/hct_mis_api/apps/cash_assist_datahub/tasks/pull_from_datahub.py

@@ -139,7 +139,7 @@ def copy_session(self, session: "AbstractSession") -> None:
             session.save(update_fields=("status",))
             self.clear_cache(session)
             try:
-                with transaction.atomic(using="default"), transaction.atomic(using="cash_assist_datahub_ca"):
+                with transaction.atomic(), transaction.atomic(using="cash_assist_datahub_ca"):
                     self.copy_service_providers(session)
                     self.copy_programs(session)
                     self.copy_target_population(session)

backend/hct_mis_api/apps/core/celery_tasks.py

@@ -112,7 +112,7 @@ def create_target_population_task(self: Any, storage_id: str, program_id: str, t
     set_sentry_business_area_tag(program.business_area.name)
 
     try:
-        with transaction.atomic(), transaction.atomic("registration_datahub"):
+        with transaction.atomic():
             registration_data_import = RegistrationDataImport.objects.create(
                 name=f"{storage_obj.file.name}_{program.name}",
                 number_of_individuals=0,

backend/hct_mis_api/apps/core/field_attributes/core_fields_attributes.py

@@ -92,8 +92,10 @@
     SEX_CHOICE,
     WORK_STATUS_CHOICE,
 )
-from hct_mis_api.apps.registration_data.models import RegistrationDataImport
-from hct_mis_api.apps.registration_datahub.models import COLLECT_TYPES
+from hct_mis_api.apps.registration_data.models import (
+    COLLECT_TYPES,
+    RegistrationDataImport,
+)
 
 logger = logging.getLogger(__name__)
 

backend/hct_mis_api/apps/core/kobo/common.py

@@ -71,9 +71,6 @@ def count_population(results: list, business_area: BusinessArea) -> tuple[int, i
 
     from hct_mis_api.apps.core.utils import rename_dict_keys
     from hct_mis_api.apps.registration_data.models import KoboImportedSubmission
-    from hct_mis_api.apps.registration_datahub.tasks.utils import (
-        get_submission_metadata,
-    )
 
     total_households_count = 0
     total_individuals_count = 0
@@ -119,3 +116,16 @@ def filter_by_owner(data: List, business_area: BusinessArea) -> List:
     if data:
         return [element for element in data if element["owner__username"] == kobo_username]
     return []
+
+
+def get_submission_metadata(household_data_dict: Dict) -> Dict:
+    meta_fields_mapping = {
+        "_uuid": "kobo_submission_uuid",
+        "_xform_id_string": "kobo_asset_id",
+        "_submission_time": "kobo_submission_time",
+    }
+    submission_meta_data = {}
+    for meta_field, model_field_name in meta_fields_mapping.items():
+        submission_meta_data[model_field_name] = household_data_dict.get(meta_field)
+
+    return submission_meta_data

backend/hct_mis_api/apps/core/management/commands/createadminpermissiongroups.py

@@ -76,18 +76,6 @@ def handle(self, *args: Any, **options: Any) -> Any:
                 "targetpopulation",
             ],
             "erp_datahub": ["downpayment", "fundscommitment"],
-            "registration_datahub": [
-                "importdata",
-                "importedindividualidentity",
-                "importeddocumenttype",
-                "importeddocument",
-                "importedhousehold",
-                "importedindividualroleinhousehold",
-                "importedindividual",
-                "koboimportedsubmission",
-                "record",
-                "registrationdataimportdatahub",
-            ],
             "mis_datahub": [
                 "document",
                 "downpayment",

backend/hct_mis_api/apps/core/management/commands/generatedocumenttypes.py

@@ -5,9 +5,6 @@
 
 from hct_mis_api.apps.core.utils import IDENTIFICATION_TYPE_TO_KEY_MAPPING
 from hct_mis_api.apps.household.models import IDENTIFICATION_TYPE_CHOICE, DocumentType
-from hct_mis_api.apps.registration_datahub.models import (
-    ImportedDocumentType as RDHDocumentType,
-)
 
 
 class Command(BaseCommand):
@@ -17,9 +14,6 @@ class Command(BaseCommand):
     def handle(self, *args: Any, **options: Any) -> None:
         identification_type_choice = tuple((doc_type, label) for doc_type, label in IDENTIFICATION_TYPE_CHOICE)
         document_types = []
-        rdh_document_types = []
         for doc_type, label in identification_type_choice:
             document_types.append(DocumentType(label=label, key=IDENTIFICATION_TYPE_TO_KEY_MAPPING[doc_type]))
-            rdh_document_types.append(RDHDocumentType(label=label, key=IDENTIFICATION_TYPE_TO_KEY_MAPPING[doc_type]))
         DocumentType.objects.bulk_create(document_types, ignore_conflicts=True)
-        RDHDocumentType.objects.bulk_create(rdh_document_types, ignore_conflicts=True)

backend/hct_mis_api/apps/core/management/commands/generatefixtures.py

@@ -37,12 +37,7 @@
 )
 from hct_mis_api.apps.program.fixtures import ProgramFactory
 from hct_mis_api.apps.program.models import Program
-from hct_mis_api.apps.registration_data.fixtures import (
-    RegistrationDataImportDatahubFactory,
-    RegistrationDataImportFactory,
-)
-from hct_mis_api.apps.registration_data.models import RegistrationDataImport
-from hct_mis_api.apps.registration_datahub.fixtures import create_imported_household
+from hct_mis_api.apps.registration_data.fixtures import RegistrationDataImportFactory
 from hct_mis_api.apps.targeting.fixtures import (
     TargetingCriteriaFactory,
     TargetingCriteriaRuleFactory,
@@ -209,7 +204,6 @@ def handle(self, *args: Any, **options: Any) -> None:
             call_command("flush", "--noinput", database="cash_assist_datahub_mis")
             call_command("flush", "--noinput", database="cash_assist_datahub_ca")
             call_command("flush", "--noinput", database="cash_assist_datahub_erp")
-            call_command("flush", "--noinput", database="registration_datahub")
             call_command(
                 "loaddata",
                 "hct_mis_api/apps/account/fixtures/superuser.json",
@@ -251,19 +245,6 @@ def handle(self, *args: Any, **options: Any) -> None:
             for _ in range(programs_amount):
                 self._generate_program_with_dependencies(options, index)
 
-        # Data imports generation
-
-        for rdi in RegistrationDataImport.objects.all():
-            rdi_datahub = RegistrationDataImportDatahubFactory(
-                name=rdi.name, hct_id=rdi.id, business_area_slug=rdi.business_area.slug
-            )
-            rdi.datahub_id = rdi_datahub.id
-            rdi.save()
-            for _ in range(10):
-                create_imported_household(
-                    {"registration_data_import": rdi_datahub},
-                    {"registration_data_import": rdi_datahub},
-                )
         session = Session(source=Session.SOURCE_CA, status=Session.STATUS_READY)
         session.save()
         cash_assist_datahub_fixtures.ServiceProviderFactory.create_batch(10, session=session)

backend/hct_mis_api/apps/core/management/commands/initcypress.py

@@ -30,18 +30,12 @@ def handle(self, *args: Any, **options: Any) -> None:
         call_command("flush", "--noinput", database="cash_assist_datahub_mis")
         call_command("flush", "--noinput", database="cash_assist_datahub_ca")
         call_command("flush", "--noinput", database="cash_assist_datahub_erp")
-        call_command("flush", "--noinput", database="registration_datahub")
 
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/geo/fixtures/data.json")
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/core/fixtures/data.json")
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/account/fixtures/data.json")
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/program/fixtures/data-cypress.json")
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/registration_data/fixtures/data-cypress.json")
-        call_command(
-            "loaddata",
-            f"{settings.PROJECT_ROOT}/apps/registration_datahub/fixtures/data-cypress.json",
-            database="registration_datahub",
-        )
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/household/fixtures/documenttype.json")
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/household/fixtures/data-cypress.json")
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/targeting/fixtures/data-cypress.json")

backend/hct_mis_api/apps/core/management/commands/initdemo.py

@@ -54,7 +54,6 @@ def handle(self, *args: Any, **options: Any) -> None:
         call_command("flush", "--noinput", database="cash_assist_datahub_mis")
         call_command("flush", "--noinput", database="cash_assist_datahub_ca")
         call_command("flush", "--noinput", database="cash_assist_datahub_erp")
-        call_command("flush", "--noinput", database="registration_datahub")
 
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/geo/fixtures/data.json")
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/core/fixtures/data.json")
@@ -65,12 +64,6 @@ def handle(self, *args: Any, **options: Any) -> None:
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/household/fixtures/documenttype.json")
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/household/fixtures/data.json")
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/accountability/fixtures/data.json")
-
-        call_command(
-            "loaddata",
-            f"{settings.PROJECT_ROOT}/apps/registration_datahub/fixtures/data.json",
-            database="registration_datahub",
-        )
         call_command("loaddata", f"{settings.PROJECT_ROOT}/apps/steficon/fixtures/data.json")
         call_command("loaddata", f"{settings.PROJECT_ROOT}/aurora/fixtures/data.json")
 

backend/hct_mis_api/apps/core/tasks_schedules.py

@@ -41,10 +41,6 @@
         "task": "hct_mis_api.apps.registration_datahub.celery_tasks.check_rdi_import_periodic_task",
         "schedule": crontab(minute="*/15"),
     },
-    # "remove_old_rdi_links_task": {
-    #     "task": "hct_mis_api.apps.registration_datahub.celery_tasks.remove_old_rdi_links_task",
-    #     "schedule": crontab(0, 0, day_of_month="1,15"),
-    # },
     "clean_old_record_files_task": {
         "task": "hct_mis_api.aurora.celery_tasks.clean_old_record_files_task",
         "schedule": crontab(month_of_year="2-12/2"),

backend/hct_mis_api/apps/core/tests/test_edopomoga_tp_creation.py

@@ -19,7 +19,7 @@
 
 
 class TestEdopomogaCreation(APITestCase):
-    databases = ("default", "registration_datahub", "cash_assist_datahub_mis")
+    databases = ("default", "cash_assist_datahub_mis")
 
     @classmethod
     def setUpTestData(cls) -> None:

backend/hct_mis_api/apps/erp_datahub/admin.py

@@ -128,7 +128,7 @@ class FundsCommitmentAdmin(HOPEModelAdminBase):
     search_fields = ("rec_serial_number", "vendor_id", "wbs_element", "funds_commitment_number")
 
     @atomic(using="cash_assist_datahub_erp")
-    @atomic(using="default")
+    @atomic()
     @button(permission=should_show_assign_business_office)
     def assign_business_office(
         self, request: "HttpRequest", pk: "UUID"
@@ -224,7 +224,7 @@ class DownPaymentAdmin(HOPEModelAdminBase):
     date_hierarchy = "create_date"
 
     @atomic(using="cash_assist_datahub_erp")
-    @atomic(using="default")
+    @atomic()
     @button(permission=should_show_assign_business_office)
     def assign_business_office(
         self, request: "HttpRequest", pk: "UUID"

backend/hct_mis_api/apps/erp_datahub/tasks/sync_to_mis_datahub.py

@@ -21,7 +21,7 @@ def get_model_dict(model: Model) -> Dict:
         return model_dict
 
     @atomic(using="cash_assist_datahub_erp")
-    @atomic(using="default")
+    @atomic()
     def execute(self) -> None:
         # have to be list because it used in another database
 

backend/hct_mis_api/apps/grievance/schema.py

@@ -75,7 +75,7 @@
 from hct_mis_api.apps.payment.schema import PaymentRecordAndPaymentNode
 from hct_mis_api.apps.program.models import Program
 from hct_mis_api.apps.program.schema import ProgramNode
-from hct_mis_api.apps.registration_datahub.schema import DeduplicationResultNode
+from hct_mis_api.apps.registration_data.nodes import DeduplicationResultNode
 from hct_mis_api.apps.utils.exceptions import log_and_raise
 from hct_mis_api.apps.utils.schema import Arg, ChartDatasetNode
 

backend/hct_mis_api/apps/grievance/tasks/deduplicate_and_check_sanctions.py

@@ -26,7 +26,7 @@
 
 
 class DeduplicateAndCheckAgainstSanctionsListTask:
-    @transaction.atomic(using="default")
+    @transaction.atomic()
     def execute(self, should_populate_index: bool, individuals_ids: List[str]) -> None:
         individuals = Individual.objects.filter(id__in=individuals_ids)
         business_area = individuals.first().business_area

backend/hct_mis_api/apps/grievance/tests/services/data_change/test_add_individual_service.py

@@ -22,8 +22,6 @@
 
 
 class TestAddIndividualService(BaseElasticSearchTestCase, TestCase):
-    databases = {"default", "registration_datahub"}
-
     @classmethod
     def setUpTestData(cls) -> None:
         create_afghanistan()

backend/hct_mis_api/apps/grievance/tests/services/data_change/test_change_individual_role.py

@@ -23,8 +23,6 @@
 
 
 class TestChangeIndividualRole(BaseElasticSearchTestCase, TestCase):
-    databases = {"default", "registration_datahub"}
-
     @classmethod
     def setUpTestData(cls) -> None:
         business_area = BusinessAreaFactory()

backend/hct_mis_api/apps/grievance/tests/services/data_change/test_update_individual_data_service.py

@@ -20,8 +20,6 @@
 
 
 class TestUpdateIndividualDataService(BaseElasticSearchTestCase, TestCase):
-    databases = {"default", "registration_datahub"}
-
     @classmethod
     def setUpTestData(cls) -> None:
         cls.business_area = BusinessAreaFactory()

backend/hct_mis_api/apps/grievance/tests/test_change_head_of_household.py

@@ -22,8 +22,6 @@
 
 
 class TestChangeHeadOfHousehold(BaseElasticSearchTestCase, APITestCase):
-    databases = {"default", "registration_datahub"}
-
     STATUS_CHANGE_MUTATION = """
     mutation GrievanceStatusChange($grievanceTicketId: ID!, $status: Int) {
       grievanceStatusChange(grievanceTicketId: $grievanceTicketId, status: $status) {

backend/hct_mis_api/apps/grievance/tests/test_close_data_change_tickets.py

@@ -54,8 +54,6 @@
 
 @flaky
 class TestCloseDataChangeTickets(BaseElasticSearchTestCase, APITestCase):
-    databases = {"default", "registration_datahub"}
-
     STATUS_CHANGE_MUTATION = """
     mutation GrievanceStatusChange($grievanceTicketId: ID!, $status: Int) {
       grievanceStatusChange(grievanceTicketId: $grievanceTicketId, status: $status) {

backend/hct_mis_api/apps/grievance/tests/test_close_grievance_ticket_and_disable_decuplication.py

@@ -34,7 +34,6 @@
 
 
 class TestCloseGrievanceTicketAndDisableDeduplication(BaseElasticSearchTestCase, APITestCase):
-    databases = {"default", "registration_datahub"}
     fixtures = (f"{settings.PROJECT_ROOT}/apps/geo/fixtures/data.json",)
 
     UPDATE_GRIEVANCE_TICKET_STATUS_CHANGE_MUTATION = """

backend/hct_mis_api/apps/grievance/tests/test_grievance_data_change_mutations.py

@@ -40,8 +40,6 @@
 
 
 class TestGrievanceCreateDataChangeMutation(BaseElasticSearchTestCase, APITestCase):
-    databases = {"default", "registration_datahub"}
-
     CREATE_DATA_CHANGE_GRIEVANCE_MUTATION = """
     mutation createGrievanceTicket($input:CreateGrievanceTicketInput!){
       createGrievanceTicket(input:$input){

backend/hct_mis_api/apps/household/models.py

@@ -53,7 +53,7 @@
 )
 
 if TYPE_CHECKING:
-    from hct_mis_api.apps.registration_datahub.models import Record
+    from hct_mis_api.aurora.models import Record
 
 BLANK = ""
 IDP = "IDP"
@@ -658,7 +658,7 @@ def alternate_collector(self) -> Optional["Individual"]:
 
     @property
     def flex_registrations_record(self) -> Optional["Record"]:
-        from hct_mis_api.apps.registration_datahub.models import Record
+        from hct_mis_api.aurora.models import Record
 
         return Record.objects.filter(id=self.flex_registrations_record_id).first()
 

backend/hct_mis_api/apps/household/schema.py

@@ -84,7 +84,7 @@
 )
 from hct_mis_api.apps.payment.utils import get_payment_items_for_dashboard
 from hct_mis_api.apps.program.models import Program
-from hct_mis_api.apps.registration_datahub.schema import DeduplicationResultNode
+from hct_mis_api.apps.registration_data.nodes import DeduplicationResultNode
 from hct_mis_api.apps.targeting.models import HouseholdSelection
 from hct_mis_api.apps.utils.graphql import does_path_exist_in_query
 from hct_mis_api.apps.utils.schema import (

backend/hct_mis_api/apps/household/serializers.py

@@ -10,22 +10,21 @@
 
 
 def get_household_status(household: Household) -> Tuple[str, datetime]:
-    if isinstance(household, Household):
-        payment_records = PaymentRecord.objects.filter(household=household)
-        if payment_records.exists():
-            return "paid", payment_records.first().updated_at
-
-        selections = HouseholdSelection.objects.filter(household=household)
-        if selections.exists():
-            selection = selections.order_by("updated_at").first()
-            if selection.target_population.status == TargetPopulation.STATUS_PROCESSING:
-                return "sent to cash assist", selection.updated_at
-            return "targeted", selection.updated_at
-
-        return "merged to population", household.created_at
-
-    # if is not Household, then must be ImportedHousehold, so it was imported
-    return "imported", household.updated_at
+    if household.rdi_merge_status == Household.PENDING:
+        return "imported", household.updated_at
+
+    payment_records = PaymentRecord.objects.filter(household=household)
+    if payment_records.exists():
+        return "paid", payment_records.first().updated_at
+
+    selections = HouseholdSelection.objects.filter(household=household)
+    if selections.exists():
+        selection = selections.order_by("updated_at").first()
+        if selection.target_population.status == TargetPopulation.STATUS_PROCESSING:
+            return "sent to cash assist", selection.updated_at
+        return "targeted", selection.updated_at
+
+    return "merged to population", household.created_at
 
 
 def get_individual_info(individual: "Individual", tax_id: Optional[str]) -> Dict: