Merge remote-tracking branch 'origin/datadog' into datadog

.github/workflows/update-jira-tickets.yaml

@@ -103,6 +103,8 @@ jobs:
           echo "tickets=${jiraTickets}" >> $GITHUB_OUTPUT
 
       - name: Periodically ping Jenkins for current tag build status
+        env:
+          REPO_URL: "${{ github.server_url }}/${{ github.repository }}"
         run: |
           repoName=${GITHUB_REPOSITORY##*/}
           currentTag="${{ steps.get-tag.outputs.tag }}"
@@ -124,7 +126,7 @@ jobs:
             if [[ "$result" == "SUCCESS" ]]; then
               echo "Build successful! Submitting ticket numbers to Jira"
               tickets="${{ steps.jira-tickets.outputs.tickets }}"
-              json="{ \"issues\": $(echo "${tickets}" | jq -R -s -c 'split(" ")[:-1]') }"
+              json="{ \"issues\": $(echo "${tickets}" | jq -R -s -c 'split(" ")[:-1]'), \"data\": { \"tag\": \"${currentTag}\", \"repository\": \"${REPO_URL}\" } }"
               curl -X POST -H 'Content-Type: application/json' --url "${JIRA_WEBHOOK_URL}" --data "$json"
               break
             elif [[ "$result" != "null" ]]; then

Pipfile

@@ -4,20 +4,21 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-aio-pika = "==9.4.3"
+aio-pika = "==9.5.3"
 aiofiles = "==24.1.0"
-aiohttp = "==3.10.10"
-alembic = "==1.13.3"
+aiohttp = "==3.11.9"
+alembic = "==1.14.0"
 asyncpg = "==0.30.0"
-azure-storage-blob = "==12.23.1"
-bcrypt = "==4.2.0"
-boto3 = "==1.35.47"
-fastapi = "==0.115.3"
+azure-storage-blob = "==12.24.0"
+bcrypt = "==4.2.1"
+boto3 = "==1.35.77"
+fastapi = "==0.115.6"
 fastapi-mail = "==1.2.9"
 firebase-admin = "==6.5.0"
-httpx = "==0.27.2"
+httpx = "==0.28.0"
 jinja2 = "==3.1.4"
-nh3 = "==0.2.18"
+more-itertools = "==10.5.0"
+nh3 = "==0.2.19"
 opentelemetry-api = "==1.27.0"
 opentelemetry-distro = "==0.48b0"
 opentelemetry-exporter-otlp = "==1.27.0"
@@ -37,11 +38,12 @@ opentelemetry-sdk-extension-aws = "==2.0.2"
 opentelemetry-semantic-conventions = "==0.48b0"
 opentelemetry-test-utils = "==0.48b0"
 opentelemetry-util-http = "==0.48b0"
-pyOpenSSL = "==24.2.1"
 pydantic = { extras = ["email"], version = "==1.10.18" }
-pymongo = "*"
-python-multipart = "==0.0.12"
-redis = "==5.1.1"
+pyjwt = "==2.10.1"
+pymongo = "==4.10.1"
+pyOpenSSL = "==24.2.1"
+python-multipart = "==0.0.19"
+redis = "==5.2.0"
 sentry-sdk = "~=2.13"
 sqlalchemy = { extras = ["asyncio"], version = "==1.4.53" }
 sqlalchemy-utils = "==0.41.2"
@@ -57,32 +59,33 @@ bytecode = "==0.16.0"
 structlog = "==24.4.0"
 asgi-correlation-id = "==4.3.4"
 
+
 [dev-packages]
+allure-pytest = "==2.13.5"
+cachetools = "==5.3.0"
+gevent = "==24.2.1"
+greenlet = "==3.1.0"
 ipdb = "==0.13.13"
-pudb = "==2024.1.3"
+mypy = "==1.13.0"
+nest-asyncio = "==1.6.0"
 pre-commit = "==4.0.1"
-ruff = "==0.7.0"
-allure-pytest = "==2.13.5"
+pudb = "==2024.1.3"
 pydantic-factories = "==1.17.3"
-pytest = "==8.3.3"
+pyld = "==2.0.4"
+pytest = "==8.3.4"
 pytest-asyncio = "~=0.19"
-pytest-cov = "==5.0.0"
+pytest-cov = "==6.0.0"
 pytest-env = "==1.1.5"
 pytest-lazy-fixtures = "==1.1.1"
 pytest-mock = "==3.14.0"
-nest-asyncio = "==1.6.0"
-gevent = "==24.2.1"
-mypy = "==1.13.0"
-types-python-dateutil = "==2.9.0.20241003"
-typing-extensions = "==4.12.2"
-types-requests = "==2.32.0.20241016"
-types-pytz = "==2024.2.0.20241003"
+reproschema = "==0.6.2"
+ruff = "==0.8.2"
 types-aiofiles = "==24.1.0.20240626"
 types-cachetools = "==5.5.0.20240820"
-greenlet = "==3.1.0"
-reproschema = "*"
-cachetools = "==5.3.0"
-pyld = "==2.0.4"
+types-python-dateutil = "==2.9.0.20241206   "
+types-pytz = "==2024.2.0.20241003"
+types-requests = "==2.32.0.20241016"
+typing-extensions = "==4.12.2"
 
 [requires]
 python_version = "3.11"

src/apps/activities/api/activities.py

@@ -5,14 +5,17 @@
 
 from apps.activities.crud import ActivitiesCRUD
 from apps.activities.domain.activity import (
+    ActivitiesMetadata,
     ActivityLanguageWithItemsMobileDetailPublic,
     ActivityOrFlowWithAssignmentsPublic,
     ActivitySingleLanguageWithItemsDetailPublic,
+    ActivitySubjectMetadata,
     ActivityWithAssignmentDetailsPublic,
 )
 from apps.activities.filters import AppletActivityFilter
 from apps.activities.services.activity import ActivityItemService, ActivityService
 from apps.activity_assignments.service import ActivityAssignmentService
+from apps.activity_flows.crud import FlowsCRUD
 from apps.activity_flows.domain.flow import FlowWithAssignmentDetailsPublic
 from apps.activity_flows.service.flow import FlowService
 from apps.answers.deps.preprocess_arbitrary import get_answer_session
@@ -169,7 +172,8 @@ async def applet_activities_for_target_subject(
     applet_service = AppletService(session, user.id)
     await applet_service.exist_by_id(applet_id)
 
-    await SubjectsService(session, user.id).exist_by_id(subject_id)
+    subject = await SubjectsService(session, user.id).exist_by_id(subject_id)
+    is_limited_respondent = subject.user_id is None
 
     # Restrict the endpoint access to owners, managers, coordinators, and assigned reviewers
     await CheckAccessService(session, user.id).check_subject_subject_access(applet_id, subject_id)
@@ -191,7 +195,7 @@ async def applet_activities_for_target_subject(
     activities_and_flows = await ActivityService(session, user.id).get_activity_and_flow_basic_info_by_ids_or_auto(
         applet_id=applet_id,
         ids=activity_and_flow_ids_from_submissions + activity_and_flow_ids_from_assignments,
-        include_auto=True,
+        include_auto=not is_limited_respondent,
         language=language,
     )
 
@@ -358,3 +362,118 @@ async def __filter_activities(
                     activities.remove(activity)
 
     return activities
+
+
+async def applet_activities_metadata_for_subject(
+    applet_id: uuid.UUID,
+    subject_id: uuid.UUID,
+    user: User = Depends(get_current_user),
+    language: str = Depends(get_language),
+    session=Depends(get_session),
+    answer_session=Depends(get_answer_session),
+) -> Response[ActivitiesMetadata]:
+    applet_service = AppletService(session, user.id)
+    await applet_service.exist_by_id(applet_id)
+    await CheckAccessService(session, user.id).check_applet_detail_access(applet_id)
+
+    subject = await SubjectsService(session, user.id).exist_by_id(subject_id)
+    is_limited_respondent = subject.user_id is None
+
+    # Fetch assigned activity or flow IDs for the subject
+    assigned_activities = await ActivityAssignmentService(session).get_assigned_activity_or_flow_ids_for_subject(
+        subject_id
+    )
+
+    # Fetch activities submissions by the subject
+    submissions_metadata = await AnswerService(session, user.id, answer_session).get_submissions_metadata_by_subject(
+        subject_id
+    )
+
+    # Fetch auto-assigned activity and flow IDs by applet ID
+    auto_activity_ids = await ActivityService(session, user.id).get_activity_and_flow_ids_by_applet_id_auto(applet_id)
+
+    # Combine all assigned IDs and submitted activity IDs
+    all_activity_ids = (
+        set(assigned_activities.activities.keys())
+        | set(submissions_metadata.activities.keys())
+        | set(auto_activity_ids)
+    )
+
+    activities = await ActivitiesCRUD(session).get_by_applet_id_and_activities_ids(applet_id, list(all_activity_ids))
+    flows = await FlowsCRUD(session).get_by_applet_id_and_flows_ids(applet_id, list(all_activity_ids))
+
+    activities_state = {activity.id: activity.soft_exists() for activity in activities}
+    flows_state = {flow.id: flow.soft_exists() for flow in flows}
+
+    # Initialize ActivitiesCounters with zero counts
+    activities_metadata = ActivitiesMetadata(subject_id=subject_id)
+
+    # Iterate over all activity or flow IDs
+    for activity_or_flow_id in all_activity_ids:
+        is_auto = activity_or_flow_id in auto_activity_ids
+
+        # Get submission and assignment data if available
+        submission_data = submissions_metadata.activities.get(activity_or_flow_id)
+        assignments_data = assigned_activities.activities.get(activity_or_flow_id)
+
+        # Initialize sets for respondents and subjects
+        respondents = set()
+        subjects = set()
+
+        # Initialize submission counts
+        respondent_submissions_count = 0
+        respondent_last_submission_date = None
+        subject_submissions_count = 0
+        subject_last_submission_date = None
+
+        # Update from submission data
+        if submission_data:
+            respondents.update(submission_data.respondents)
+            subjects.update(submission_data.subjects)
+            respondent_submissions_count = submission_data.respondent_submissions_count
+            respondent_last_submission_date = submission_data.respondent_last_submission_date
+            subject_submissions_count = submission_data.subject_submissions_count
+            subject_last_submission_date = submission_data.subject_last_submission_date
+
+        # Update from assignment data
+        if assignments_data:
+            respondents.update(assignments_data.respondents)
+            subjects.update(assignments_data.subjects)
+
+        # Include the subject for auto-assigned activities, excluding limited accounts
+        if is_auto and not is_limited_respondent:
+            respondents.add(subject_id)
+            subjects.add(subject_id)
+
+        # Calculate counts
+        respondents_count = len(respondents)
+        subjects_count = len(subjects)
+
+        activity_or_flow_exists = activities_state.get(activity_or_flow_id) or flows_state.get(activity_or_flow_id)
+
+        # Update activities counters counts
+        if subjects_count > 0:
+            if activity_or_flow_exists:
+                activities_metadata.respondent_activities_count_existing += 1
+            else:
+                activities_metadata.respondent_activities_count_deleted += 1
+        if respondents_count > 0:
+            if activity_or_flow_exists:
+                activities_metadata.target_activities_count_existing += 1
+            else:
+                activities_metadata.target_activities_count_deleted += 1
+
+        # Append the activity subject counters
+        activities_metadata.activities_or_flows.append(
+            ActivitySubjectMetadata(
+                activity_or_flow_id=activity_or_flow_id,
+                respondents_count=respondents_count,
+                subjects_count=subjects_count,
+                respondent_submissions_count=respondent_submissions_count,
+                respondent_last_submission_date=respondent_last_submission_date,
+                subject_submissions_count=subject_submissions_count,
+                subject_last_submission_date=subject_last_submission_date,
+            )
+        )
+
+    return Response(result=activities_metadata)

src/apps/activities/crud/activity.py

@@ -107,6 +107,12 @@ async def get_ids_by_applet_id(self, applet_id: uuid.UUID) -> list[uuid.UUID]:
         result = await self._execute(query)
         return result.scalars().all()
 
+    async def get_ids_by_applet_id_auto(self, applet_id: uuid.UUID) -> list[uuid.UUID]:
+        query: Query = select(ActivitySchema.id)
+        query = query.where(ActivitySchema.applet_id == applet_id, ActivitySchema.auto_assign.is_(True))
+        result = await self._execute(query)
+        return result.scalars().all()
+
     async def get_activity_and_flow_basic_info_by_ids_or_auto(
         self, applet_id: uuid.UUID, ids: list[uuid.UUID], include_auto: bool, language: str
     ) -> list[ActivityOrFlowBasicInfoInternal]:

src/apps/activities/domain/activity.py

@@ -156,3 +156,22 @@ class ActivityBaseInfo(ActivityMinimumInfo, InternalModel):
     contains_response_types: list[ResponseType]
     item_count: int
     auto_assign: bool
+
+
+class ActivitySubjectMetadata(PublicModel):
+    activity_or_flow_id: uuid.UUID
+    respondents_count: int
+    respondent_submissions_count: int
+    respondent_last_submission_date: datetime | None
+    subjects_count: int
+    subject_submissions_count: int
+    subject_last_submission_date: datetime | None
+
+
+class ActivitiesMetadata(PublicModel):
+    subject_id: uuid.UUID
+    respondent_activities_count_existing: int = 0
+    respondent_activities_count_deleted: int = 0
+    target_activities_count_existing: int = 0
+    target_activities_count_deleted: int = 0
+    activities_or_flows: list[ActivitySubjectMetadata] = Field(default_factory=list)

src/apps/activities/router.py

@@ -8,10 +8,12 @@
     applet_activities_for_respondent_subject,
     applet_activities_for_subject,
     applet_activities_for_target_subject,
+    applet_activities_metadata_for_subject,
     public_activity_retrieve,
 )
 from apps.activities.api.reusable_item_choices import item_choice_create, item_choice_delete, item_choice_retrieve
 from apps.activities.domain.activity import (
+    ActivitiesMetadata,
     ActivityOrFlowWithAssignmentsPublic,
     ActivitySingleLanguageWithItemsDetailPublic,
 )
@@ -133,3 +135,16 @@
         **DEFAULT_OPENAPI_RESPONSE,
     },
 )(applet_activities_for_respondent_subject)
+
+router.get(
+    "/applet/{applet_id}/subject/{subject_id}/metadata",
+    description="""Get metadata, like number of submissions and participants for activities and activity 
+                  flows associated with the given participant, either as respondent or target subject.
+                """,
+    status_code=status.HTTP_200_OK,
+    responses={
+        status.HTTP_200_OK: {"model": Response[ActivitiesMetadata]},
+        **AUTHENTICATION_ERROR_RESPONSES,
+        **DEFAULT_OPENAPI_RESPONSE,
+    },
+)(applet_activities_metadata_for_subject)

src/apps/activities/services/activity.py

@@ -1,3 +1,4 @@
+import asyncio
 import uuid
 
 from apps.activities.crud import ActivitiesCRUD, ActivityHistoriesCRUD
@@ -20,6 +21,7 @@
 from apps.activities.errors import ActivityAccessDeniedError, ActivityDoeNotExist
 from apps.activities.services.activity_item import ActivityItemService
 from apps.activity_assignments.service import ActivityAssignmentService
+from apps.activity_flows.crud import FlowsCRUD
 from apps.applets.crud import AppletsCRUD, UserAppletAccessCRUD
 from apps.schedule.crud.events import ActivityEventsCRUD, EventCRUD
 from apps.schedule.service.schedule import ScheduleService
@@ -462,3 +464,11 @@ async def get_activity_and_flow_basic_info_by_ids_or_auto(
         return await ActivitiesCRUD(self.session).get_activity_and_flow_basic_info_by_ids_or_auto(
             applet_id, ids, include_auto, language
         )
+
+    async def get_activity_and_flow_ids_by_applet_id_auto(self, applet_id: uuid.UUID) -> list[uuid.UUID]:
+        activity_ids_coro = ActivitiesCRUD(self.session).get_ids_by_applet_id_auto(applet_id)
+        flow_ids_coro = FlowsCRUD(self.session).get_ids_by_applet_id_auto(applet_id)
+
+        activity_ids, flow_ids = await asyncio.gather(activity_ids_coro, flow_ids_coro)
+
+        return activity_ids + flow_ids