move full_rebuild & refactoring

src/hct_mis_api/apps/payment/celery_tasks.py

@@ -5,9 +5,11 @@
 from django.contrib.admin.options import get_content_type_for_model
 from django.contrib.auth import get_user_model
 from django.contrib.contenttypes.models import ContentType
+from django.core.cache import cache
 from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
 from django.db import transaction
+from django.shortcuts import get_object_or_404
 from django.utils import timezone
 
 from concurrency.api import disable_concurrency
@@ -321,10 +323,10 @@ def payment_plan_apply_engine_rule(self: Any, payment_plan_id: str, engine_rule_
     from hct_mis_api.apps.payment.models import Payment, PaymentPlan
     from hct_mis_api.apps.steficon.models import Rule, RuleCommit
 
-    payment_plan = PaymentPlan.objects.get(id=payment_plan_id)
+    payment_plan = get_object_or_404(PaymentPlan, id=payment_plan_id)
     set_sentry_business_area_tag(payment_plan.business_area.name)
-    engine_rule = Rule.objects.get(id=engine_rule_id)
-    rule: RuleCommit = engine_rule.latest
+    engine_rule = get_object_or_404(Rule, id=engine_rule_id)
+    rule: Optional["RuleCommit"] = engine_rule.latest
     if rule.id != payment_plan.steficon_rule_id:
         payment_plan.steficon_rule = rule
         payment_plan.save()
@@ -671,3 +673,111 @@ def periodic_sync_payment_gateway_delivery_mechanisms(self: Any) -> None:
     except Exception as e:
         logger.exception(e)
         raise self.retry(exc=e)
+
+
+# TODO: copied from TP 'target_population_apply_steficon'
+@app.task(bind=True, queue="priority", default_retry_delay=60, max_retries=3)
+@log_start_and_end
+@sentry_tags
+def payment_plan_apply_steficon_hh_selection(self: Any, payment_plan_id: str, engine_rule_id: str) -> None:
+    from hct_mis_api.apps.payment.models import Payment, PaymentPlan
+    from hct_mis_api.apps.steficon.models import Rule, RuleCommit
+
+    payment_plan = get_object_or_404(PaymentPlan, id=payment_plan_id)
+    set_sentry_business_area_tag(payment_plan.business_area.name)
+    engine_rule = get_object_or_404(Rule, id=engine_rule_id)
+    rule: Optional["RuleCommit"] = engine_rule.latest
+    if rule.id != payment_plan.steficon_rule_id:
+        payment_plan.steficon_rule = rule
+        payment_plan.save(update_fields=["steficon_rule"])
+
+    try:
+        payment_plan.status = PaymentPlan.Status.TP_STEFICON_RUN
+        payment_plan.steficon_applied_date = timezone.now()
+        payment_plan.save(update_fields=["status", "steficon_applied_date"])
+        updates = []
+        with transaction.atomic():
+            payment: Payment
+            for payment in payment_plan.eligible_payments:
+                result = rule.execute(
+                    {
+                        "household": payment.household,
+                        "payment_plan": payment_plan,
+                    }
+                )
+                payment.vulnerability_score = result.value
+                updates.append(payment)
+            Payment.objects.bulk_update(updates, ["vulnerability_score"])
+        payment_plan.status = PaymentPlan.Status.TP_STEFICON_COMPLETED
+        payment_plan.steficon_applied_date = timezone.now()
+        with disable_concurrency(payment_plan):
+            payment_plan.save(update_fields=["status", "steficon_applied_date"])
+    except Exception as e:
+        logger.exception(e)
+        payment_plan.steficon_applied_date = timezone.now()
+        payment_plan.status = PaymentPlan.Status.TP_STEFICON_ERROR
+        payment_plan.save(update_fields=["status", "steficon_applied_date"])
+        raise self.retry(exc=e)
+
+
+# TODO: copied from TP 'target_population_rebuild_stats'
+@app.task(bind=True, queue="priority", default_retry_delay=60, max_retries=3)
+@log_start_and_end
+@sentry_tags
+def payment_plan_rebuild_stats(self: Any, payment_plan_id: str) -> None:
+    with cache.lock(
+        f"payment_plan_rebuild_stats_{payment_plan_id}",
+        blocking_timeout=60 * 10,
+        timeout=60 * 60 * 2,
+    ):
+        payment_plan = get_object_or_404(PaymentPlan, id=payment_plan_id)
+        set_sentry_business_area_tag(payment_plan.business_area.name)
+        payment_plan.build_status = PaymentPlan.BuildStatus.BUILD_STATUS_BUILDING
+        payment_plan.save(update_fields=["build_status"])
+        try:
+            with transaction.atomic():
+                payment_plan.update_population_count_fields()
+                payment_plan.build_status = PaymentPlan.BuildStatus.BUILD_STATUS_OK
+                payment_plan.built_at = timezone.now()
+                payment_plan.save(update_fields=["build_status", "built_at"])
+        except Exception as e:
+            logger.exception(e)
+            payment_plan.refresh_from_db()
+            payment_plan.build_status = PaymentPlan.BuildStatus.BUILD_STATUS_FAILED
+            payment_plan.save()
+            raise self.retry(exc=e)
+
+
+# TODO: copied from TP 'target_population_full_rebuild'
+@app.task(bind=True, queue="priority", default_retry_delay=60, max_retries=3)
+@log_start_and_end
+@sentry_tags
+def payment_plan_full_rebuild(self: Any, payment_plan_id: str) -> None:
+    from hct_mis_api.apps.payment.services.payment_plan_services import (
+        PaymentPlanService,
+    )
+
+    with cache.lock(
+        f"payment_plan_full_rebuild_{payment_plan_id}",
+        blocking_timeout=60 * 10,
+        timeout=60 * 60 * 2,
+    ):
+        payment_plan = get_object_or_404(PaymentPlan, id=payment_plan_id)
+        set_sentry_business_area_tag(payment_plan.business_area.name)
+        payment_plan.build_status = PaymentPlan.BuildStatus.BUILD_STATUS_BUILDING
+        payment_plan.save(update_fields=["build_status"])
+        try:
+            with transaction.atomic():
+                if payment_plan.status not in [
+                    PaymentPlan.Status.DRAFT,
+                    PaymentPlan.Status.PREPARING,
+                    PaymentPlan.Status.OPEN,
+                ]:
+                    raise Exception("Payment Plan is not in correct status")
+                PaymentPlanService(payment_plan).full_rebuild()
+        except Exception as e:
+            logger.exception(e)
+            payment_plan.refresh_from_db()
+            payment_plan.build_status = PaymentPlan.BuildStatus.BUILD_STATUS_FAILED
+            payment_plan.save()
+            raise self.retry(exc=e)

src/hct_mis_api/apps/payment/migrations/0147_migration.py

@@ -1,24 +1,83 @@
-# Generated by Django 3.2.25 on 2024-10-25 11:09
+# Generated by Django 3.2.25 on 2024-10-31 13:36
 
+import django.core.validators
 from django.db import migrations, models
+import django.db.models.deletion
 import django_fsm
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
+        ('targeting', '0049_migration'),
+        ('core', '0088_migration'),
         ('payment', '0146_migration'),
     ]
 
     operations = [
+        migrations.AddField(
+            model_name='payment',
+            name='vulnerability_score',
+            field=models.DecimalField(blank=True, db_index=True, decimal_places=3, help_text='Written by Steficon', max_digits=6, null=True),
+        ),
+        migrations.AddField(
+            model_name='paymentplan',
+            name='build_status',
+            field=models.CharField(choices=[('PENDING', 'Pending'), ('BUILDING', 'Building'), ('FAILED', 'Failed'), ('OK', 'Ok')], db_index=True, default='PENDING', max_length=256),
+        ),
+        migrations.AddField(
+            model_name='paymentplan',
+            name='built_at',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='paymentplan',
+            name='excluded_ids',
+            field=models.TextField(blank=True),
+        ),
         migrations.AddField(
             model_name='paymentplan',
             name='internal_data',
             field=models.JSONField(default=dict),
         ),
+        migrations.AddField(
+            model_name='paymentplan',
+            name='storage_file',
+            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.storagefile'),
+        ),
+        migrations.AddField(
+            model_name='paymentplan',
+            name='targeting_criteria',
+            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='payment_plans', to='targeting.targetingcriteria'),
+        ),
+        migrations.AddField(
+            model_name='paymentplan',
+            name='vulnerability_score_max',
+            field=models.DecimalField(blank=True, decimal_places=3, help_text='Written by a tool such as Corticon.', max_digits=6, null=True),
+        ),
+        migrations.AddField(
+            model_name='paymentplan',
+            name='vulnerability_score_min',
+            field=models.DecimalField(blank=True, decimal_places=3, help_text='Written by a tool such as Corticon.', max_digits=6, null=True),
+        ),
+        migrations.AlterField(
+            model_name='payment',
+            name='status_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='paymentplan',
+            name='name',
+            field=models.CharField(blank=True, max_length=255, null=True, validators=[django.core.validators.MinLengthValidator(3), django.core.validators.MaxLengthValidator(255), django.core.validators.RegexValidator('\\s{2,}', 'Double spaces characters are not allowed.', code='double_spaces_characters_not_allowed', inverse_match=True), django.core.validators.RegexValidator('(^\\s+)|(\\s+$)', 'Leading or trailing spaces characters are not allowed.', code='leading_trailing_spaces_characters_not_allowed', inverse_match=True), django.core.validators.ProhibitNullCharactersValidator()]),
+        ),
         migrations.AlterField(
             model_name='paymentplan',
             name='status',
             field=django_fsm.FSMField(choices=[('TP_OPEN', 'Open'), ('TP_LOCKED', 'Locked'), ('PROCESSING', 'Processing'), ('STEFICON_WAIT', 'Steficon Wait'), ('STEFICON_RUN', 'Steficon Run'), ('STEFICON_COMPLETED', 'Steficon Completed'), ('STEFICON_ERROR', 'Steficon Error'), ('DRAFT', 'Draft'), ('PREPARING', 'Preparing'), ('OPEN', 'Open'), ('LOCKED', 'Locked'), ('LOCKED_FSP', 'Locked FSP'), ('IN_APPROVAL', 'In Approval'), ('IN_AUTHORIZATION', 'In Authorization'), ('IN_REVIEW', 'In Review'), ('ACCEPTED', 'Accepted'), ('FINISHED', 'Finished')], db_index=True, default='OPEN', max_length=50),
         ),
+        migrations.AlterField(
+            model_name='paymentrecord',
+            name='status_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
     ]

src/hct_mis_api/apps/payment/models.py

@@ -520,18 +520,6 @@ class PaymentPlan(
         },
     )
 
-    BUILD_STATUS_PENDING = "PENDING"
-    BUILD_STATUS_BUILDING = "BUILDING"
-    BUILD_STATUS_FAILED = "FAILED"
-    BUILD_STATUS_OK = "OK"
-
-    BUILD_STATUS_CHOICES = (
-        (BUILD_STATUS_PENDING, _("Pending")),
-        (BUILD_STATUS_BUILDING, _("Building")),
-        (BUILD_STATUS_FAILED, _("Failed")),
-        (BUILD_STATUS_OK, _("Ok")),
-    )
-
     class Status(models.TextChoices):
         # new from TP
         TP_OPEN = "TP_OPEN", "Open"
@@ -553,6 +541,12 @@ class Status(models.TextChoices):
         ACCEPTED = "ACCEPTED", "Accepted"
         FINISHED = "FINISHED", "Finished"
 
+    class BuildStatus(models.TextChoices):
+        BUILD_STATUS_PENDING = "PENDING", "Pending"
+        BUILD_STATUS_BUILDING = "BUILDING", "Building"
+        BUILD_STATUS_FAILED = "FAILED", "Failed"
+        BUILD_STATUS_OK = "OK", "Ok"
+
     class BackgroundActionStatus(models.TextChoices):
         RULE_ENGINE_RUN = "RULE_ENGINE_RUN", "Rule Engine Running"
         RULE_ENGINE_ERROR = "RULE_ENGINE_ERROR", "Rule Engine Errored"
@@ -606,9 +600,10 @@ class Action(models.TextChoices):
         choices=BackgroundActionStatus.choices,
     )
     build_status = models.CharField(
-        max_length=256, choices=BUILD_STATUS_CHOICES, default=BUILD_STATUS_PENDING, db_index=True
+        max_length=256, choices=BuildStatus.choices, default=BuildStatus.BUILD_STATUS_PENDING, db_index=True
     )
     built_at = models.DateTimeField(null=True, blank=True)
+    # TODO: remove this field after migrations
     target_population = models.ForeignKey(
         "targeting.TargetPopulation",
         on_delete=models.CASCADE,
@@ -674,11 +669,11 @@ class Action(models.TextChoices):
         blank=True,
     )
     targeting_criteria = models.OneToOneField(
-        "TargetingCriteria",
+        "targeting.TargetingCriteria",
         blank=True,
         null=True,
         on_delete=models.SET_NULL,
-        related_name="target_population",
+        related_name="payment_plans",
     )
     vulnerability_score_min = models.DecimalField(
         null=True,
@@ -1940,6 +1935,9 @@ class Payment(SoftDeletableModel, GenericPayment, UnicefIdentifiedModel, AdminUr
         max_length=128, blank=True, null=True, help_text="Use this field for reconciliation data"
     )
     fsp_auth_code = models.CharField(max_length=128, blank=True, null=True, help_text="FSP Auth Code")
+    vulnerability_score = models.DecimalField(
+        blank=True, null=True, decimal_places=3, max_digits=6, help_text="Written by Steficon", db_index=True
+    )
 
     @property
     def full_name(self) -> str:

src/hct_mis_api/apps/payment/services/payment_plan_services.py

@@ -312,6 +312,13 @@ def check_payment_plan_and_update_status(self, approval_process: ApprovalProcess
 
     @staticmethod
     def create_payments(payment_plan: PaymentPlan) -> None:
+        # TODO: copied from TP?
+        # households = Household.objects.filter(
+        #     business_area=payment_plan.business_area, program=payment_plan.program
+        # )
+        # households = households.filter(payment_plan.targeting_criteria.get_query())
+        # households = households.only("id")
+
         payments_to_create = []
         households = (
             payment_plan.target_population.households.annotate(
@@ -745,3 +752,13 @@ def split(self, split_type: str, chunks_no: Optional[int] = None) -> PaymentPlan
                 payment_plan_splits_to_create[i].payments.add(*chunk)
 
         return self.payment_plan
+
+    def full_rebuild(self) -> None:
+        payment_plan: PaymentPlan = self.payment_plan
+        # remove all payment and recreate
+        payment_plan.payment_items.all().delete()
+
+        # TODO: get all new HH list?
+        self.create_payments(payment_plan)
+
+        payment_plan.update_population_count_fields()

src/hct_mis_api/apps/steficon/models.py

@@ -1,4 +1,4 @@
-from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type, Union
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type
 
 from django.conf import settings
 from django.contrib.postgres.fields import ArrayField, CICharField
@@ -146,14 +146,14 @@ def release(self) -> Optional["RuleCommit"]:
         return commit
 
     @property
-    def latest(self) -> Union[QuerySet, None]:
+    def latest(self) -> Optional["RuleCommit"]:
         try:
             return self.history.filter(is_release=True).order_by("-version").first()
         except RuleCommit.DoesNotExist:
             return None
 
     @property
-    def latest_commit(self) -> Optional[QuerySet]:
+    def latest_commit(self) -> Optional["RuleCommit"]:
         try:
             return self.history.order_by("version").last()
         except RuleCommit.DoesNotExist: