feature: Add Last Submission Timestamp to metadata endpoint (M2-7859) (#1676)

* feature: Add Last Submission Timestamp to counters endpoint (M2-7859)

* Update due metadata endpoint refactoring

* Returning lastSubmissionDate separated between subject and respondent

* Update deps

Update deps

Author: rcmerlo

Pipfile

@@ -7,16 +7,17 @@ name = "pypi"
 aio-pika = "==9.5.3"
 aiofiles = "==24.1.0"
 aiohttp = "==3.11.9"
-alembic = "==1.13.3"
+alembic = "==1.14.0"
 asyncpg = "==0.30.0"
-azure-storage-blob = "==12.23.1"
-bcrypt = "==4.2.0"
-boto3 = "==1.35.54"
-fastapi = "==0.115.4"
+azure-storage-blob = "==12.24.0"
+bcrypt = "==4.2.1"
+boto3 = "==1.35.77"
+fastapi = "==0.115.6"
 fastapi-mail = "==1.2.9"
 firebase-admin = "==6.5.0"
 httpx = "==0.28.0"
 jinja2 = "==3.1.4"
+more-itertools = "==10.5.0"
 nh3 = "==0.2.19"
 opentelemetry-api = "==1.27.0"
 opentelemetry-distro = "==0.48b0"
@@ -37,9 +38,10 @@ opentelemetry-sdk-extension-aws = "==2.0.2"
 opentelemetry-semantic-conventions = "==0.48b0"
 opentelemetry-test-utils = "==0.48b0"
 opentelemetry-util-http = "==0.48b0"
-pyOpenSSL = "==24.2.1"
 pydantic = { extras = ["email"], version = "==1.10.18" }
-pymongo = "*"
+pyjwt = "==2.10.1"
+pymongo = "==4.10.1"
+pyOpenSSL = "==24.2.1"
 python-multipart = "==0.0.19"
 redis = "==5.2.0"
 sentry-sdk = "~=2.13"
@@ -49,37 +51,35 @@ taskiq = { extras = ["reload"], version = "==0.11.7" }
 taskiq-aio-pika = "==0.4.1"
 taskiq-fastapi = "==0.3.2"
 taskiq-redis = "==1.0.2"
-typer = "==0.12.5"
-uvicorn = { extras = ["standard"], version = "==0.32.0" }
-pyjwt = "==2.10.1"
-more-itertools = "==10.5.0"
+typer = "==0.15.1"
+uvicorn = { extras = ["standard"], version = "==0.32.1" }
 
 [dev-packages]
+allure-pytest = "==2.13.5"
+cachetools = "==5.3.0"
+gevent = "==24.2.1"
+greenlet = "==3.1.0"
 ipdb = "==0.13.13"
-pudb = "==2024.1.3"
+mypy = "==1.13.0"
+nest-asyncio = "==1.6.0"
 pre-commit = "==4.0.1"
-ruff = "==0.7.2"
-allure-pytest = "==2.13.5"
+pudb = "==2024.1.3"
 pydantic-factories = "==1.17.3"
+pyld = "==2.0.4"
 pytest = "==8.3.4"
 pytest-asyncio = "~=0.19"
 pytest-cov = "==6.0.0"
 pytest-env = "==1.1.5"
 pytest-lazy-fixtures = "==1.1.1"
 pytest-mock = "==3.14.0"
-nest-asyncio = "==1.6.0"
-gevent = "==24.11.1"
-mypy = "==1.13.0"
-types-python-dateutil = "==2.9.0.20241003"
-typing-extensions = "==4.12.2"
-types-requests = "==2.32.0.20241016"
-types-pytz = "==2024.2.0.20241003"
+reproschema = "==0.6.2"
+ruff = "==0.8.2"
 types-aiofiles = "==24.1.0.20240626"
 types-cachetools = "==5.5.0.20240820"
-greenlet = "==3.1.0"
-reproschema = "0.9.0"
-cachetools = "==5.3.0"
-pyld = "==2.0.4"
+types-python-dateutil = "==2.9.0.20241206   "
+types-pytz = "==2024.2.0.20241003"
+types-requests = "==2.32.0.20241016"
+typing-extensions = "==4.12.2"
 
 [requires]
 python_version = "3.11"

Pipfile.lock

@@ -385,15 +385,17 @@ async def applet_activities_metadata_for_subject(
     )
 
     # Fetch activities submissions by the subject
-    submitted_activities = await AnswerService(session, user.id, answer_session).get_submissions_by_subject(subject_id)
+    submissions_metadata = await AnswerService(session, user.id, answer_session).get_submissions_metadata_by_subject(
+        subject_id
+    )
 
     # Fetch auto-assigned activity and flow IDs by applet ID
     auto_activity_ids = await ActivityService(session, user.id).get_activity_and_flow_ids_by_applet_id_auto(applet_id)
 
     # Combine all assigned IDs and submitted activity IDs
     all_activity_ids = (
         set(assigned_activities.activities.keys())
-        | set(submitted_activities.activities.keys())
+        | set(submissions_metadata.activities.keys())
         | set(auto_activity_ids)
     )
 
@@ -411,7 +413,7 @@ async def applet_activities_metadata_for_subject(
         is_auto = activity_or_flow_id in auto_activity_ids
 
         # Get submission and assignment data if available
-        submission_data = submitted_activities.activities.get(activity_or_flow_id)
+        submission_data = submissions_metadata.activities.get(activity_or_flow_id)
         assignments_data = assigned_activities.activities.get(activity_or_flow_id)
 
         # Initialize sets for respondents and subjects
@@ -420,14 +422,18 @@ async def applet_activities_metadata_for_subject(
 
         # Initialize submission counts
         respondent_submissions_count = 0
+        respondent_last_submission_date = None
         subject_submissions_count = 0
+        subject_last_submission_date = None
 
         # Update from submission data
         if submission_data:
             respondents.update(submission_data.respondents)
             subjects.update(submission_data.subjects)
             respondent_submissions_count = submission_data.respondent_submissions_count
+            respondent_last_submission_date = submission_data.respondent_last_submission_date
             subject_submissions_count = submission_data.subject_submissions_count
+            subject_last_submission_date = submission_data.subject_last_submission_date
 
         # Update from assignment data
         if assignments_data:
@@ -464,7 +470,9 @@ async def applet_activities_metadata_for_subject(
                 respondents_count=respondents_count,
                 subjects_count=subjects_count,
                 respondent_submissions_count=respondent_submissions_count,
+                respondent_last_submission_date=respondent_last_submission_date,
                 subject_submissions_count=subject_submissions_count,
+                subject_last_submission_date=subject_last_submission_date,
             )
         )
 

src/apps/activities/api/activities.py

@@ -162,8 +162,10 @@ class ActivitySubjectMetadata(PublicModel):
     activity_or_flow_id: uuid.UUID
     respondents_count: int
     respondent_submissions_count: int
+    respondent_last_submission_date: datetime | None
     subjects_count: int
     subject_submissions_count: int
+    subject_last_submission_date: datetime | None
 
 
 class ActivitiesMetadata(PublicModel):

src/apps/activities/domain/activity.py

@@ -1353,18 +1353,22 @@ async def test_assigned_activities_auto_assigned(
         assert result["targetActivitiesCountExisting"] == 2
         assert result["targetActivitiesCountDeleted"] == 0
         assert len(result["activitiesOrFlows"]) == 2
-        flow_counters = next(item for item in result["activitiesOrFlows"] if item["activityOrFlowId"] == str(flow.id))
-        activity_counters = next(
+        flow_metadata = next(item for item in result["activitiesOrFlows"] if item["activityOrFlowId"] == str(flow.id))
+        activity_metadata = next(
             item for item in result["activitiesOrFlows"] if item["activityOrFlowId"] == str(activity.id)
         )
-        assert flow_counters["respondentsCount"] == 1
-        assert flow_counters["respondentSubmissionsCount"] == 0
-        assert flow_counters["subjectsCount"] == 1
-        assert flow_counters["subjectSubmissionsCount"] == 0
-        assert activity_counters["respondentsCount"] == 1
-        assert activity_counters["respondentSubmissionsCount"] == 0
-        assert activity_counters["subjectsCount"] == 1
-        assert activity_counters["subjectSubmissionsCount"] == 0
+        assert flow_metadata["respondentsCount"] == 1
+        assert flow_metadata["respondentSubmissionsCount"] == 0
+        assert flow_metadata["respondentLastSubmissionDate"] is None
+        assert flow_metadata["subjectsCount"] == 1
+        assert flow_metadata["subjectSubmissionsCount"] == 0
+        assert flow_metadata["subjectLastSubmissionDate"] is None
+        assert activity_metadata["respondentsCount"] == 1
+        assert activity_metadata["respondentSubmissionsCount"] == 0
+        assert activity_metadata["respondentLastSubmissionDate"] is None
+        assert activity_metadata["subjectsCount"] == 1
+        assert activity_metadata["subjectSubmissionsCount"] == 0
+        assert activity_metadata["subjectLastSubmissionDate"] is None
 
     @pytest.mark.parametrize(
         "subject_type,result_order",
@@ -1754,8 +1758,18 @@ async def test_assigned_activities_from_submission(
         activityOrFlow = result["activitiesOrFlows"][0]
         assert activityOrFlow["respondentsCount"] == (1 if subject_type == "target" else 0)
         assert activityOrFlow["respondentSubmissionsCount"] == (0 if subject_type == "target" else 1)
+        assert (
+            activityOrFlow["respondentLastSubmissionDate"] is None
+            if subject_type == "target"
+            else activityOrFlow["respondentLastSubmissionDate"] is not None
+        )
         assert activityOrFlow["subjectsCount"] == (0 if subject_type == "target" else 1)
         assert activityOrFlow["subjectSubmissionsCount"] == (1 if subject_type == "target" else 0)
+        assert (
+            activityOrFlow["subjectLastSubmissionDate"] is not None
+            if subject_type == "target"
+            else activityOrFlow["subjectLastSubmissionDate"] is None
+        )
 
     @pytest.mark.parametrize("subject_type", ["target", "respondent"])
     async def test_assigned_hidden_activities(

src/apps/activities/tests/test_activities.py

@@ -1011,7 +1011,7 @@ async def get_activity_and_flow_ids_by_source_subject(self, source_subject_id: u
         return res.mappings().all()
 
     @staticmethod
-    def _query_submissions_by_subject(subject_column: InstrumentedAttribute, subject_id: uuid.UUID) -> Query:
+    def _query_submissions_metadata_by_subject(subject_column: InstrumentedAttribute, subject_id: uuid.UUID) -> Query:
         query: Query = (
             select(
                 func.count(func.distinct(AnswerSchema.submit_id)).label("submission_count"),
@@ -1027,6 +1027,7 @@ def _query_submissions_by_subject(subject_column: InstrumentedAttribute, subject
                     if subject_column == AnswerSchema.target_subject_id
                     else AnswerSchema.target_subject_id
                 ).label("subject_id"),
+                func.max(AnswerSchema.created_at).label("last_submission_date"),
             )
             .where(
                 subject_column == subject_id,
@@ -1035,17 +1036,20 @@ def _query_submissions_by_subject(subject_column: InstrumentedAttribute, subject
             )
             .group_by("activity_id", "subject_id")
         )
+
         return query
 
-    async def get_submissions_by_target_subject(self, target_subject_id: uuid.UUID) -> list[dict]:
-        query: Query = self._query_submissions_by_subject(AnswerSchema.target_subject_id, target_subject_id)
+    async def get_submissions_metadata_by_target_subject(self, target_subject_id: uuid.UUID) -> list[dict]:
+        query: Query = self._query_submissions_metadata_by_subject(AnswerSchema.target_subject_id, target_subject_id)
 
         res = await self._execute(query)
 
         return res.mappings().all()
 
-    async def get_submissions_by_respondent_subject(self, respondent_subject_id: uuid.UUID) -> list[dict]:
-        query: Query = self._query_submissions_by_subject(AnswerSchema.source_subject_id, respondent_subject_id)
+    async def get_submissions_metadata_by_respondent_subject(self, respondent_subject_id: uuid.UUID) -> list[dict]:
+        query: Query = self._query_submissions_metadata_by_subject(
+            AnswerSchema.source_subject_id, respondent_subject_id
+        )
 
         res = await self._execute(query)
 

src/apps/answers/crud/answers.py

@@ -719,9 +719,11 @@ class SubmissionsSubjectCounters(InternalModel):
     respondents: set[uuid.UUID] = Field(default_factory=set)
     subjects: set[uuid.UUID] = Field(default_factory=set)
     subject_submissions_count: int = 0
+    subject_last_submission_date: datetime.datetime | None = None
     respondent_submissions_count: int = 0
+    respondent_last_submission_date: datetime.datetime | None = None
 
 
-class SubmissionsActivityCountBySubject(InternalModel):
+class SubmissionsActivityMetadataBySubject(InternalModel):
     subject_id: uuid.UUID
     activities: dict[uuid.UUID, SubmissionsSubjectCounters] = Field(default_factory=dict)

src/apps/answers/domain/answers.py

@@ -67,7 +67,7 @@
     AppletSubmission,
     FilesCopyCheckResult,
     RespondentAnswerData,
-    SubmissionsActivityCountBySubject,
+    SubmissionsActivityMetadataBySubject,
     SubmissionsSubjectCounters,
 )
 from apps.answers.errors import (
@@ -1979,37 +1979,55 @@ async def _filter_out_soft_deleted_subjects(self, submissions: list[dict]) -> se
 
         return existing_subject_ids
 
-    async def get_submissions_by_subject(self, subject_id: uuid.UUID) -> SubmissionsActivityCountBySubject:
-        submissions_target_coro = AnswersCRUD(self.answer_session).get_submissions_by_target_subject(subject_id)
-        submissions_respondent_coro = AnswersCRUD(self.answer_session).get_submissions_by_respondent_subject(subject_id)
+    async def get_submissions_metadata_by_subject(self, subject_id: uuid.UUID) -> SubmissionsActivityMetadataBySubject:
+        submissions_target_coro = AnswersCRUD(self.answer_session).get_submissions_metadata_by_target_subject(
+            subject_id
+        )
+        submissions_respondent_coro = AnswersCRUD(self.answer_session).get_submissions_metadata_by_respondent_subject(
+            subject_id
+        )
 
         submissions_target, submissions_respondent = await asyncio.gather(
             submissions_target_coro, submissions_respondent_coro
         )
 
         existing_subject_ids = await self._filter_out_soft_deleted_subjects(submissions_target + submissions_respondent)
 
-        submissions_activity_count = SubmissionsActivityCountBySubject(subject_id=subject_id)
+        submissions_activity_metadata = SubmissionsActivityMetadataBySubject(subject_id=subject_id)
 
         for activity_submissions in submissions_target:
-            activity_counters = submissions_activity_count.activities.setdefault(
+            activity_metadata = submissions_activity_metadata.activities.setdefault(
                 uuid.UUID(activity_submissions["activity_id"]), SubmissionsSubjectCounters()
             )
             respondent_subject_id = activity_submissions["subject_id"]
             if respondent_subject_id in existing_subject_ids:
-                activity_counters.respondents.add(respondent_subject_id)
-                activity_counters.subject_submissions_count += activity_submissions["submission_count"]
+                activity_metadata.respondents.add(respondent_subject_id)
+                activity_metadata.subject_submissions_count += activity_submissions["submission_count"]
+                activity_metadata.subject_last_submission_date = (
+                    activity_submissions["last_submission_date"]
+                    if not activity_metadata.subject_last_submission_date
+                    else max(
+                        activity_metadata.subject_last_submission_date, activity_submissions["last_submission_date"]
+                    )
+                )
 
         for activity_submissions in submissions_respondent:
-            activity_counters = submissions_activity_count.activities.setdefault(
+            activity_metadata = submissions_activity_metadata.activities.setdefault(
                 uuid.UUID(activity_submissions["activity_id"]), SubmissionsSubjectCounters()
             )
             target_subject_id = activity_submissions["subject_id"]
             if target_subject_id in existing_subject_ids:
-                activity_counters.subjects.add(target_subject_id)
-                activity_counters.respondent_submissions_count += activity_submissions["submission_count"]
+                activity_metadata.subjects.add(target_subject_id)
+                activity_metadata.respondent_submissions_count += activity_submissions["submission_count"]
+                activity_metadata.respondent_last_submission_date = (
+                    activity_submissions["last_submission_date"]
+                    if not activity_metadata.respondent_last_submission_date
+                    else max(
+                        activity_metadata.respondent_last_submission_date, activity_submissions["last_submission_date"]
+                    )
+                )
 
-        return submissions_activity_count
+        return submissions_activity_metadata
 
 
 class ReportServerService:

src/apps/answers/service.py

@@ -1,5 +1,5 @@
 import uuid
-from datetime import datetime
+from datetime import datetime, timezone
 
 import jwt
 from fastapi import Body, Depends
@@ -86,10 +86,9 @@ async def refresh_access_token(
                 # check transition key
                 transition_key = settings.authentication.refresh_token.transition_key
                 transition_expire_date = settings.authentication.refresh_token.transition_expire_date
+                today = datetime.now(timezone.utc).date()
 
-                if not (
-                    transition_key and transition_expire_date and transition_expire_date > datetime.utcnow().date()
-                ):
+                if not (transition_key and transition_expire_date and transition_expire_date > today):
                     raise
                 payload = jwt.decode(
                     schema.refresh_token,

src/apps/authentication/api/auth.py

@@ -109,7 +109,7 @@ async def test_refresh_token_key_transition(self, client, tom: User, tom_create:
         token_data = TokenPayload(**payload)
 
         new_token_key = "new token key"
-        transition_expire_date = datetime.date.today() + datetime.timedelta(days=1)
+        transition_expire_date = datetime.datetime.now(datetime.timezone.utc).date() + datetime.timedelta(days=1)
 
         # refresh access token, check refresh token not changed
         _status_code, _token = await self._request_refresh_token(client, refresh_token)
@@ -129,7 +129,7 @@ async def test_refresh_token_key_transition(self, client, tom: User, tom_create:
 
             # check transition expire date
             with mock.patch("apps.authentication.api.auth.datetime") as date_mock:
-                date_mock.utcnow().date.return_value = transition_expire_date + datetime.timedelta(days=1)
+                date_mock.now().date.return_value = transition_expire_date + datetime.timedelta(days=1)
                 _status_code, _ = await self._request_refresh_token(client, refresh_token)
                 assert _status_code == http.HTTPStatus.BAD_REQUEST