feature: New endpoint to return total counters for subject (M2-7889) (#…

…1669)

* feature: New endpoint to return total counters for subject (M2- 7889)

* Refactoring to better understading code

* Improvments on tests

* Fix change in old endpoint response

* Fix duplicate acitivity ids and activities counts

* Update src/apps/activities/router.py

Co-authored-by: Farmer Paul <[email protected]>

* Change to union to be more clear

* fix: refactory code to count correctly

* fix lint

* Refactoring code

* Fix in related endpoint

* Filtering out soft deleted subjects in submissions

* fix: use internal classes

* fix: counting submissions when have soft deleted subjects

* fix lint

* Add counting for deleted activities

* Filtering out deleted subject for submissions queries to avoid wroing activity listing

* Update src/apps/activities/api/activities.py

Co-authored-by: Farmer Paul <[email protected]>

* Update src/apps/activities/api/activities.py

Co-authored-by: Farmer Paul <[email protected]>

* Removing limited account from auto-assigned activites

* Rename endpoint to Metadata

* fix: use `submit_id`s for flow submission counts

* fix: exclude flows that have not been completed

For all the PDP queries, a "submission" for a flow should only be
**completed** ones. A flow that has only been partially completed (say,
only the 1st activity of 5 activities have been submitted) should not
count towards having any submissions, and thus should NOT be:

- listed on the PDP pages for the associated subjects
- included in the tallying of activities/flows for that tab
- included in the submissions count on the flow level within each tab
- included in that flow's expanded view's submissions counts

* Add more tests

---------

Co-authored-by: Farmer Paul <[email protected]>

Pipfile

@@ -4,24 +4,20 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-aio-pika = "==9.4.3"
+aio-pika = "==9.5.3"
 aiofiles = "==24.1.0"
-aiohttp = "==3.10.10"
+aiohttp = "==3.11.9"
 alembic = "==1.13.3"
 asyncpg = "==0.30.0"
 azure-storage-blob = "==12.23.1"
 bcrypt = "==4.2.0"
-boto3 = "==1.35.47"
-fastapi = "==0.115.3"
-# The latest version of the fastapi is not taken because of the issue
-# starlette version for those deps ==0.21.0
-# with fastapi-mail that requires 0.21 < starlette < 0.22
-
+boto3 = "==1.35.54"
+fastapi = "==0.115.4"
 fastapi-mail = "==1.2.9"
 firebase-admin = "==6.5.0"
-httpx = "==0.27.2"
+httpx = "==0.28.0"
 jinja2 = "==3.1.4"
-nh3 = "==0.2.18"
+nh3 = "==0.2.19"
 opentelemetry-api = "==1.27.0"
 opentelemetry-distro = "==0.48b0"
 opentelemetry-exporter-otlp = "==1.27.0"
@@ -44,8 +40,8 @@ opentelemetry-util-http = "==0.48b0"
 pyOpenSSL = "==24.2.1"
 pydantic = { extras = ["email"], version = "==1.10.18" }
 pymongo = "*"
-python-multipart = "==0.0.12"
-redis = "==5.1.1"
+python-multipart = "==0.0.19"
+redis = "==5.2.0"
 sentry-sdk = "~=2.13"
 sqlalchemy = { extras = ["asyncio"], version = "==1.4.53" }
 sqlalchemy-utils = "==0.41.2"
@@ -55,23 +51,24 @@ taskiq-fastapi = "==0.3.2"
 taskiq-redis = "==1.0.2"
 typer = "==0.12.5"
 uvicorn = { extras = ["standard"], version = "==0.32.0" }
-pyjwt = "==2.9.0"
+pyjwt = "==2.10.1"
+more-itertools = "==10.5.0"
 
 [dev-packages]
 ipdb = "==0.13.13"
 pudb = "==2024.1.3"
 pre-commit = "==4.0.1"
-ruff = "==0.7.0"
+ruff = "==0.7.2"
 allure-pytest = "==2.13.5"
 pydantic-factories = "==1.17.3"
-pytest = "==8.3.3"
+pytest = "==8.3.4"
 pytest-asyncio = "~=0.19"
-pytest-cov = "==5.0.0"
+pytest-cov = "==6.0.0"
 pytest-env = "==1.1.5"
 pytest-lazy-fixtures = "==1.1.1"
 pytest-mock = "==3.14.0"
 nest-asyncio = "==1.6.0"
-gevent = "==24.2.1"
+gevent = "==24.11.1"
 mypy = "==1.13.0"
 types-python-dateutil = "==2.9.0.20241003"
 typing-extensions = "==4.12.2"
@@ -80,7 +77,7 @@ types-pytz = "==2024.2.0.20241003"
 types-aiofiles = "==24.1.0.20240626"
 types-cachetools = "==5.5.0.20240820"
 greenlet = "==3.1.0"
-reproschema = "*"
+reproschema = "0.9.0"
 cachetools = "==5.3.0"
 pyld = "==2.0.4"
 

src/apps/activities/api/activities.py

@@ -5,14 +5,17 @@
 
 from apps.activities.crud import ActivitiesCRUD
 from apps.activities.domain.activity import (
+    ActivitiesMetadata,
     ActivityLanguageWithItemsMobileDetailPublic,
     ActivityOrFlowWithAssignmentsPublic,
     ActivitySingleLanguageWithItemsDetailPublic,
+    ActivitySubjectMetadata,
     ActivityWithAssignmentDetailsPublic,
 )
 from apps.activities.filters import AppletActivityFilter
 from apps.activities.services.activity import ActivityItemService, ActivityService
 from apps.activity_assignments.service import ActivityAssignmentService
+from apps.activity_flows.crud import FlowsCRUD
 from apps.activity_flows.domain.flow import FlowWithAssignmentDetailsPublic
 from apps.activity_flows.service.flow import FlowService
 from apps.answers.deps.preprocess_arbitrary import get_answer_session
@@ -169,7 +172,8 @@ async def applet_activities_for_target_subject(
     applet_service = AppletService(session, user.id)
     await applet_service.exist_by_id(applet_id)
 
-    await SubjectsService(session, user.id).exist_by_id(subject_id)
+    subject = await SubjectsService(session, user.id).exist_by_id(subject_id)
+    is_limited_respondent = subject.user_id is None
 
     # Restrict the endpoint access to owners, managers, coordinators, and assigned reviewers
     await CheckAccessService(session, user.id).check_subject_subject_access(applet_id, subject_id)
@@ -191,7 +195,7 @@ async def applet_activities_for_target_subject(
     activities_and_flows = await ActivityService(session, user.id).get_activity_and_flow_basic_info_by_ids_or_auto(
         applet_id=applet_id,
         ids=activity_and_flow_ids_from_submissions + activity_and_flow_ids_from_assignments,
-        include_auto=True,
+        include_auto=not is_limited_respondent,
         language=language,
     )
 
@@ -358,3 +362,110 @@ async def __filter_activities(
                     activities.remove(activity)
 
     return activities
+
+
+async def applet_activities_metadata_for_subject(
+    applet_id: uuid.UUID,
+    subject_id: uuid.UUID,
+    user: User = Depends(get_current_user),
+    language: str = Depends(get_language),
+    session=Depends(get_session),
+    answer_session=Depends(get_answer_session),
+) -> Response[ActivitiesMetadata]:
+    applet_service = AppletService(session, user.id)
+    await applet_service.exist_by_id(applet_id)
+    await CheckAccessService(session, user.id).check_applet_detail_access(applet_id)
+
+    subject = await SubjectsService(session, user.id).exist_by_id(subject_id)
+    is_limited_respondent = subject.user_id is None
+
+    # Fetch assigned activity or flow IDs for the subject
+    assigned_activities = await ActivityAssignmentService(session).get_assigned_activity_or_flow_ids_for_subject(
+        subject_id
+    )
+
+    # Fetch activities submissions by the subject
+    submitted_activities = await AnswerService(session, user.id, answer_session).get_submissions_by_subject(subject_id)
+
+    # Fetch auto-assigned activity and flow IDs by applet ID
+    auto_activity_ids = await ActivityService(session, user.id).get_activity_and_flow_ids_by_applet_id_auto(applet_id)
+
+    # Combine all assigned IDs and submitted activity IDs
+    all_activity_ids = (
+        set(assigned_activities.activities.keys())
+        | set(submitted_activities.activities.keys())
+        | set(auto_activity_ids)
+    )
+
+    activities = await ActivitiesCRUD(session).get_by_applet_id_and_activities_ids(applet_id, list(all_activity_ids))
+    flows = await FlowsCRUD(session).get_by_applet_id_and_flows_ids(applet_id, list(all_activity_ids))
+
+    activities_state = {activity.id: activity.soft_exists() for activity in activities}
+    flows_state = {flow.id: flow.soft_exists() for flow in flows}
+
+    # Initialize ActivitiesCounters with zero counts
+    activities_metadata = ActivitiesMetadata(subject_id=subject_id)
+
+    # Iterate over all activity or flow IDs
+    for activity_or_flow_id in all_activity_ids:
+        is_auto = activity_or_flow_id in auto_activity_ids
+
+        # Get submission and assignment data if available
+        submission_data = submitted_activities.activities.get(activity_or_flow_id)
+        assignments_data = assigned_activities.activities.get(activity_or_flow_id)
+
+        # Initialize sets for respondents and subjects
+        respondents = set()
+        subjects = set()
+
+        # Initialize submission counts
+        respondent_submissions_count = 0
+        subject_submissions_count = 0
+
+        # Update from submission data
+        if submission_data:
+            respondents.update(submission_data.respondents)
+            subjects.update(submission_data.subjects)
+            respondent_submissions_count = submission_data.respondent_submissions_count
+            subject_submissions_count = submission_data.subject_submissions_count
+
+        # Update from assignment data
+        if assignments_data:
+            respondents.update(assignments_data.respondents)
+            subjects.update(assignments_data.subjects)
+
+        # Include the subject for auto-assigned activities, excluding limited accounts
+        if is_auto and not is_limited_respondent:
+            respondents.add(subject_id)
+            subjects.add(subject_id)
+
+        # Calculate counts
+        respondents_count = len(respondents)
+        subjects_count = len(subjects)
+
+        activity_or_flow_exists = activities_state.get(activity_or_flow_id) or flows_state.get(activity_or_flow_id)
+
+        # Update activities counters counts
+        if subjects_count > 0:
+            if activity_or_flow_exists:
+                activities_metadata.respondent_activities_count_existing += 1
+            else:
+                activities_metadata.respondent_activities_count_deleted += 1
+        if respondents_count > 0:
+            if activity_or_flow_exists:
+                activities_metadata.target_activities_count_existing += 1
+            else:
+                activities_metadata.target_activities_count_deleted += 1
+
+        # Append the activity subject counters
+        activities_metadata.activities_or_flows.append(
+            ActivitySubjectMetadata(
+                activity_or_flow_id=activity_or_flow_id,
+                respondents_count=respondents_count,
+                subjects_count=subjects_count,
+                respondent_submissions_count=respondent_submissions_count,
+                subject_submissions_count=subject_submissions_count,
+            )
+        )
+
+    return Response(result=activities_metadata)

src/apps/activities/crud/activity.py

@@ -107,6 +107,12 @@ async def get_ids_by_applet_id(self, applet_id: uuid.UUID) -> list[uuid.UUID]:
         result = await self._execute(query)
         return result.scalars().all()
 
+    async def get_ids_by_applet_id_auto(self, applet_id: uuid.UUID) -> list[uuid.UUID]:
+        query: Query = select(ActivitySchema.id)
+        query = query.where(ActivitySchema.applet_id == applet_id, ActivitySchema.auto_assign.is_(True))
+        result = await self._execute(query)
+        return result.scalars().all()
+
     async def get_activity_and_flow_basic_info_by_ids_or_auto(
         self, applet_id: uuid.UUID, ids: list[uuid.UUID], include_auto: bool, language: str
     ) -> list[ActivityOrFlowBasicInfoInternal]:

src/apps/activities/domain/activity.py

@@ -156,3 +156,20 @@ class ActivityBaseInfo(ActivityMinimumInfo, InternalModel):
     contains_response_types: list[ResponseType]
     item_count: int
     auto_assign: bool
+
+
+class ActivitySubjectMetadata(PublicModel):
+    activity_or_flow_id: uuid.UUID
+    respondents_count: int
+    respondent_submissions_count: int
+    subjects_count: int
+    subject_submissions_count: int
+
+
+class ActivitiesMetadata(PublicModel):
+    subject_id: uuid.UUID
+    respondent_activities_count_existing: int = 0
+    respondent_activities_count_deleted: int = 0
+    target_activities_count_existing: int = 0
+    target_activities_count_deleted: int = 0
+    activities_or_flows: list[ActivitySubjectMetadata] = Field(default_factory=list)

src/apps/activities/router.py

@@ -8,10 +8,12 @@
     applet_activities_for_respondent_subject,
     applet_activities_for_subject,
     applet_activities_for_target_subject,
+    applet_activities_metadata_for_subject,
     public_activity_retrieve,
 )
 from apps.activities.api.reusable_item_choices import item_choice_create, item_choice_delete, item_choice_retrieve
 from apps.activities.domain.activity import (
+    ActivitiesMetadata,
     ActivityOrFlowWithAssignmentsPublic,
     ActivitySingleLanguageWithItemsDetailPublic,
 )
@@ -133,3 +135,16 @@
         **DEFAULT_OPENAPI_RESPONSE,
     },
 )(applet_activities_for_respondent_subject)
+
+router.get(
+    "/applet/{applet_id}/subject/{subject_id}/metadata",
+    description="""Get metadata, like number of submissions and participants for activities and activity 
+                  flows associated with the given participant, either as respondent or target subject.
+                """,
+    status_code=status.HTTP_200_OK,
+    responses={
+        status.HTTP_200_OK: {"model": Response[ActivitiesMetadata]},
+        **AUTHENTICATION_ERROR_RESPONSES,
+        **DEFAULT_OPENAPI_RESPONSE,
+    },
+)(applet_activities_metadata_for_subject)

src/apps/activities/services/activity.py

@@ -1,3 +1,4 @@
+import asyncio
 import uuid
 
 from apps.activities.crud import ActivitiesCRUD, ActivityHistoriesCRUD
@@ -20,6 +21,7 @@
 from apps.activities.errors import ActivityAccessDeniedError, ActivityDoeNotExist
 from apps.activities.services.activity_item import ActivityItemService
 from apps.activity_assignments.service import ActivityAssignmentService
+from apps.activity_flows.crud import FlowsCRUD
 from apps.applets.crud import AppletsCRUD, UserAppletAccessCRUD
 from apps.schedule.crud.events import ActivityEventsCRUD, EventCRUD
 from apps.schedule.service.schedule import ScheduleService
@@ -462,3 +464,11 @@ async def get_activity_and_flow_basic_info_by_ids_or_auto(
         return await ActivitiesCRUD(self.session).get_activity_and_flow_basic_info_by_ids_or_auto(
             applet_id, ids, include_auto, language
         )
+
+    async def get_activity_and_flow_ids_by_applet_id_auto(self, applet_id: uuid.UUID) -> list[uuid.UUID]:
+        activity_ids_coro = ActivitiesCRUD(self.session).get_ids_by_applet_id_auto(applet_id)
+        flow_ids_coro = FlowsCRUD(self.session).get_ids_by_applet_id_auto(applet_id)
+
+        activity_ids, flow_ids = await asyncio.gather(activity_ids_coro, flow_ids_coro)
+
+        return activity_ids + flow_ids