Master into stg (#4534)

* AB#226656 Error Cannot read properties of undefined Grievance ticket

* Fixes to managers for migretable models

* added unicef id

* fix frontend lint

* unique unicef_id constraint

* fix test_double_entries

* fix household test_models

* change exclusion for importing hh and inds to another program

* fix test_create_pending_objects_from_objects

* fix linters

* fix test_create_targeting_for_people

* add models for new constraints

* fix test

* remove one time scripts which are not valid with new contraint

* fixed format

* change unique name

* fixed tests

* fix more tests

* fix laast e2e tests

---------

Co-authored-by: Jan Romaniak <[email protected]>
Co-authored-by: Jan Romaniak <[email protected]>
Co-authored-by: Paulina Kujawa <[email protected]>
Co-authored-by: Paulina Kujawa <[email protected]>

pyproject.toml

@@ -137,7 +137,7 @@ distribution = true
 
 [project]
 name = "hope"
-version = "2.15.0"
+version = "2.16.0"
 description = "HCT MIS is UNICEF's humanitarian cash transfer platform."
 authors = [
     { name = "Tivix" },

src/frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "frontend",
-  "version": "2.15.0",
+  "version": "2.16.0",
   "private": true,
   "type": "module",
   "scripts": {

src/frontend/src/components/grievances/LookUps/LookUpReassignRole/LookUpReassignRole.tsx

@@ -119,7 +119,7 @@ export function LookUpReassignRole({
   return (
     <Formik
       initialValues={{
-        selectedIndividual: individualData.individual,
+        selectedIndividual: individualData?.individual,
         selectedHousehold,
         role: individualRole.role,
       }}

src/frontend/src/components/grievances/ReassignMultipleRoleBox.tsx

@@ -52,7 +52,6 @@ export function ReassignMultipleRoleBox({
       (el) =>
         el.role === IndividualRoleInHouseholdRole.Primary || el.role === 'HEAD',
     );
-
   const mappedReassignLookups = (): ReactElement => (
     <>
       {selectedIndividualsToReassign.map((selectedIndividualToReassign) => {
@@ -97,7 +96,7 @@ export function ReassignMultipleRoleBox({
                 ticket={ticket}
                 household={householdAndRole.household}
                 individualToReassign={selectedIndividualToReassign}
-                initialSelectedIndividualId={reassignDataDictByIndividualId[selectedIndividualToReassign.id].new_individual}
+                initialSelectedIndividualId={reassignDataDictByIndividualId[selectedIndividualToReassign.id]?.new_individual}
               />
             </Box>
           ));
@@ -137,7 +136,7 @@ export function ReassignMultipleRoleBox({
                   ticket={ticket}
                   household={household}
                   individualToReassign={selectedIndividualToReassign}
-                  initialSelectedIndividualId={reassignDataDictByIndividualId[selectedIndividualToReassign.id].new_individual}
+                  initialSelectedIndividualId={reassignDataDictByIndividualId[selectedIndividualToReassign.id]?.new_individual}
                 />
               </Box>
             )}

src/frontend/src/containers/tables/rdi/ImportedHouseholdsTable/ImportedHouseholdTableRow.tsx

@@ -57,7 +57,7 @@ export function ImportedHouseholdTableRow({
       </TableCell>
       <TableCell align="left">
         <StyledLink onClick={() => handleClick()}>
-          {isMerged ? household.unicefId : household.importId}
+          {household.unicefId}
         </StyledLink>
       </TableCell>
       <AnonTableCell>{household?.headOfHousehold?.fullName}</AnonTableCell>

src/frontend/src/containers/tables/rdi/ImportedIndividualsTable/ImportedIndividualsTableRow.tsx

@@ -20,7 +20,6 @@ interface ImportedIndividualsTableRowProps {
 export function ImportedIndividualsTableRow({
   individual,
   choices,
-  isMerged,
   rdi,
 }: ImportedIndividualsTableRowProps): ReactElement {
   const navigate = useNavigate();
@@ -56,7 +55,7 @@ export function ImportedIndividualsTableRow({
     >
       <TableCell align="left">
         <BlackLink to={individualDetailsPath}>
-          {isMerged ? individual.unicefId : individual.importId}
+          {individual.unicefId}
         </BlackLink>
       </TableCell>
       <AnonTableCell>{individual.fullName}</AnonTableCell>

src/hct_mis_api/api/endpoints/rdi/upload.py

@@ -140,6 +140,7 @@ class Meta:
             "updated_at",
             "version",
             "vector_column",
+            "unicef_id",
         ]
 
     def validate_role(self, value: str) -> Optional[str]:
@@ -175,6 +176,7 @@ class Meta:
             "geopoint",
             "detail_id",
             "version",
+            "unicef_id",
         ]
         validators = [HouseholdValidator()]
 

src/hct_mis_api/apps/account/permissions.py

@@ -378,9 +378,12 @@ def check_node_permission(cls, info: Any, object_instance: Any) -> None:
             raise PermissionDenied("Permission Denied")
 
     @classmethod
-    def get_node(cls, info: Any, object_id: str) -> Optional[Model]:
+    def get_node(cls, info: Any, object_id: str, **kwargs: Any) -> Optional[Model]:
         try:
-            object_instance = cls._meta.model.objects.get(pk=object_id)
+            if "get_object_queryset" in kwargs:
+                object_instance = kwargs.get("get_object_queryset").get(pk=object_id)
+            else:
+                object_instance = cls.get_queryset(cls._meta.model.objects, info).get(pk=object_id)
             cls.check_node_permission(info, object_instance)
         except cls._meta.model.DoesNotExist:
             object_instance = None

src/hct_mis_api/apps/household/migrations/0005_migration.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.25 on 2024-12-19 11:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('household', '0004_migration'),
+    ]
+
+    operations = [
+        migrations.AddConstraint(
+            model_name='household',
+            constraint=models.UniqueConstraint(condition=models.Q(('is_removed', False)), fields=('unicef_id', 'program'), name='unique_hh_unicef_id_in_program'),
+        ),
+        migrations.AddConstraint(
+            model_name='individual',
+            constraint=models.UniqueConstraint(condition=models.Q(('is_removed', False), ('duplicate', False)), fields=('unicef_id', 'program'), name='unique_ind_unicef_id_in_program'),
+        ),
+    ]

src/hct_mis_api/apps/household/models.py

@@ -576,6 +576,13 @@ class CollectType(models.TextChoices):
     class Meta:
         verbose_name = "Household"
         permissions = (("can_withdrawn", "Can withdrawn Household"),)
+        constraints = [
+            UniqueConstraint(
+                fields=["unicef_id", "program"],
+                condition=Q(is_removed=False),
+                name="unique_hh_unicef_id_in_program",
+            )
+        ]
 
     def save(self, *args: Any, **kwargs: Any) -> None:
         from hct_mis_api.apps.targeting.models import (
@@ -1187,6 +1194,13 @@ def __str__(self) -> str:
     class Meta:
         verbose_name = "Individual"
         indexes = (GinIndex(fields=["vector_column"]),)
+        constraints = [
+            UniqueConstraint(
+                fields=["unicef_id", "program"],
+                condition=Q(is_removed=False) & Q(duplicate=False),
+                name="unique_ind_unicef_id_in_program",
+            )
+        ]
 
     def recalculate_data(self, save: bool = True) -> Tuple[Any, List[str]]:
         update_fields = ["disability"]

src/hct_mis_api/apps/household/schema.py

@@ -5,6 +5,7 @@
     Case,
     F,
     Func,
+    Model,
     OuterRef,
     Prefetch,
     Q,
@@ -264,13 +265,17 @@ class IndividualNode(BaseNodePermissionMixin, AdminUrlNodeMixin, DjangoObjectTyp
         IndividualIdentityNode,
     )
 
+    @classmethod
+    def get_node(cls, info: Any, object_id: str, **kwargs: Any) -> Optional[Model]:
+        return super().get_node(info, object_id, get_object_queryset=Individual.all_merge_status_objects)
+
     @staticmethod
     def resolve_documents(parent: Individual, info: Any) -> QuerySet[Document]:
-        return Document.objects.filter(pk__in=parent.documents.values("id"))
+        return parent.documents(manager="all_merge_status_objects")
 
     @staticmethod
     def resolve_identities(parent: Individual, info: Any) -> QuerySet[IndividualIdentity]:
-        return IndividualIdentity.objects.filter(pk__in=parent.identities.values("id"))
+        return parent.identities(manager="all_merge_status_objects")
 
     @staticmethod
     def resolve_import_id(parent: Individual, info: Any) -> str:
@@ -282,16 +287,16 @@ def resolve_preferred_language(parent: Individual, info: Any) -> Optional[str]:
 
     @staticmethod
     def resolve_payment_channels(parent: Individual, info: Any) -> QuerySet[BankAccountInfo]:
-        return BankAccountInfo.objects.filter(individual=parent).annotate(type=Value("BANK_TRANSFER"))
+        return BankAccountInfo.all_merge_status_objects.filter(individual=parent).annotate(type=Value("BANK_TRANSFER"))
 
     def resolve_bank_account_info(parent, info: Any) -> Optional[BankAccountInfo]:
-        bank_account_info = parent.bank_account_info.first()
+        bank_account_info = parent.bank_account_info(manager="all_merge_status_objects").first()  # type: ignore
         if bank_account_info:
             return bank_account_info
         return None
 
     def resolve_role(parent, info: Any) -> str:
-        role = parent.households_and_roles.first()
+        role = parent.households_and_roles(manager="all_merge_status_objects").first()
         if role is not None:
             return role.role
         return ROLE_NO_ROLE
@@ -481,13 +486,14 @@ def resolve_selection(parent: Household, info: Any) -> HouseholdSelection:
 
     @staticmethod
     def resolve_individuals(parent: Household, info: Any, *arg: Any, **kwargs: Any) -> QuerySet:
-        individuals_ids = list(parent.individuals.values_list("id", flat=True))
-        collectors_ids = list(parent.representatives.values_list("id", flat=True))
+        individuals_ids = list(parent.individuals(manager="all_merge_status_objects").values_list("id", flat=True))
+
+        collectors_ids = list(parent.representatives(manager="all_merge_status_objects").values_list("id", flat=True))
         ids = list(set(individuals_ids + collectors_ids))
-        return Individual.objects.filter(id__in=ids).prefetch_related(
+        return Individual.all_merge_status_objects.filter(id__in=ids).prefetch_related(
             Prefetch(
                 "households_and_roles",
-                queryset=IndividualRoleInHousehold.objects.filter(household=parent.id),
+                queryset=IndividualRoleInHousehold.all_merge_status_objects.filter(household=parent.id),
             )
         )
 
@@ -578,6 +584,10 @@ def get_queryset(cls, queryset: QuerySet[Household], info: Any) -> QuerySet[Hous
         qs = super().get_queryset(queryset, info)
         return qs
 
+    @classmethod
+    def get_node(cls, info: Any, object_id: str, **kwargs: Any) -> Optional[Model]:
+        return super().get_node(info, object_id, get_object_queryset=Household.all_merge_status_objects)
+
     class Meta:
         model = Household
         filter_fields = []
@@ -706,7 +716,7 @@ def resolve_all_individuals(self, info: Any, **kwargs: Any) -> QuerySet[Individu
             if program and program.status == Program.DRAFT:
                 return Individual.objects.none()
 
-        queryset = Individual.objects.all()
+        queryset = Individual.all_merge_status_objects.all()
         if does_path_exist_in_query("edges.node.household", info):
             queryset = queryset.select_related("household")
         if does_path_exist_in_query("edges.node.household.admin2", info):
@@ -762,7 +772,7 @@ def resolve_all_households(self, info: Any, **kwargs: Any) -> QuerySet:
             if program and program.status == Program.DRAFT:
                 return Household.objects.none()
 
-        queryset = Household.objects.all()
+        queryset = Household.all_merge_status_objects.all()
 
         if not user.partner.is_unicef:  # Unicef partner has full access to all AdminAreas
             business_area_id = BusinessArea.objects.get(slug=business_area_slug).id

src/hct_mis_api/apps/registration_datahub/mutations.py

@@ -115,15 +115,21 @@ def create_registration_data_import_for_import_program_population(
     pull_pictures = registration_data_import_data.pop("pull_pictures", True)
     screen_beneficiary = registration_data_import_data.pop("screen_beneficiary", False)
     import_from_program_id = registration_data_import_data.pop("import_from_program_id", None)
+    households_to_exclude = Household.all_merge_status_objects.filter(
+        program=import_to_program_id,
+    ).values_list("unicef_id", flat=True)
     households = Household.objects.filter(
         program_id=import_from_program_id,
         withdrawn=False,
-    ).exclude(household_collection__households__program=import_to_program_id)
+    ).exclude(unicef_id__in=households_to_exclude)
+    individuals_to_exclude = Individual.all_merge_status_objects.filter(
+        program=import_to_program_id,
+    ).values_list("unicef_id", flat=True)
     individuals = Individual.objects.filter(
         program_id=import_from_program_id,
         withdrawn=False,
         duplicate=False,
-    ).exclude(individual_collection__individuals__program=import_to_program_id)
+    ).exclude(unicef_id__in=individuals_to_exclude)
     created_obj_hct = RegistrationDataImport(
         status=RegistrationDataImport.IMPORTING,
         imported_by=user,

src/hct_mis_api/apps/registration_datahub/tasks/import_program_population.py

@@ -8,17 +8,23 @@
 def import_program_population(
     import_from_program_id: str, import_to_program_id: str, rdi: RegistrationDataImport
 ) -> None:
+    households_to_exclude = Household.all_merge_status_objects.filter(
+        program=import_to_program_id,
+    ).values_list("unicef_id", flat=True)
     copy_from_households = Household.objects.filter(
         program=import_from_program_id,
         withdrawn=False,
-    ).exclude(household_collection__households__program_id=import_to_program_id)
+    ).exclude(unicef_id__in=households_to_exclude)
+    individuals_to_exclude = Individual.all_merge_status_objects.filter(
+        program=import_to_program_id,
+    ).values_list("unicef_id", flat=True)
     copy_from_individuals = (
         Individual.objects.filter(
             program_id=import_from_program_id,
             withdrawn=False,
             duplicate=False,
         )
-        .exclude(individual_collection__individuals__program_id=import_to_program_id)
+        .exclude(unicef_id__in=individuals_to_exclude)
         .order_by("first_registration_date")
     )
     import_to_program = Program.objects.get(id=import_to_program_id)

src/hct_mis_api/apps/utils/models.py

@@ -152,9 +152,7 @@ class SoftDeletableRepresentationMergeStatusModel(MergeStatusModel):
     class Meta:
         abstract = True
 
-    # objects = SoftDeletableRepresentationMergedManager(_emit_deprecation_warnings=True)
-    # now we use 'rdi_merge_status' field for filtering
-    objects = SoftDeletableRepresentationManager()
+    objects = SoftDeletableRepresentationMergedManager(_emit_deprecation_warnings=True)
     all_merge_status_objects = SoftDeletableRepresentationManager()
     available_objects = SoftDeletableRepresentationMergedManager()
     all_objects = models.Manager()