Merge remote-tracking branch 'origin/develop' into develop

src/apps/answers/service.py

@@ -1034,6 +1034,7 @@ async def get_summary_activities(
         self, applet_id: uuid.UUID, filters: SummaryActivityFilter
     ) -> list[SummaryActivity]:
         assert self.user_id
+        applet = await AppletsCRUD(self.session).get_by_id(applet_id)
         act_hst_crud = ActivityHistoriesCRUD(self.session)
         activities = await act_hst_crud.get_last_histories_by_applet(applet_id=applet_id)
         activity_ver_ids = [activity.id_version for activity in activities]
@@ -1047,11 +1048,21 @@ async def get_summary_activities(
             submitted_activities[activity_id] = max(submit_date, date) if date else submit_date
             submitted_activities[activity_history_id] = submit_date
 
+        current_activity_histories = await act_hst_crud.retrieve_by_applet_version(f"{applet.id}_{applet.version}")
+        current_activities_map = {str(ah.id): ah for ah in current_activity_histories}
         results = []
         for activity in activities:
             activity_history_answer_date = submitted_activities.get(
                 activity.id_version, submitted_activities.get(str(activity.id))
             )
+            has_answer = bool(activity_history_answer_date)
+            if not has_answer:
+                activity_curr = current_activities_map.get(str(activity.id))
+                if not activity_curr:
+                    continue
+                elif activity_curr.is_reviewable:
+                    continue
+
             results.append(
                 SummaryActivity(
                     id=activity.id,
@@ -1067,6 +1078,7 @@ async def get_summary_activity_flows(
         self, applet_id: uuid.UUID, target_subject_id: uuid.UUID | None
     ) -> list[SummaryActivityFlow]:
         assert self.user_id
+        applet = await AppletsCRUD(self.session).get_by_id(applet_id)
         flow_crud = FlowsHistoryCRUD(self.session)
         answer_crud = AnswersCRUD(self.answer_session)
         flow_history_ids_with_date = await answer_crud.get_submitted_flows_with_last_date(applet_id, target_subject_id)
@@ -1079,11 +1091,17 @@ async def get_summary_activity_flows(
             submitted_activity_flows[flow_id] = max(submit_date, date) if date else submit_date
             submitted_activity_flows[version_id] = submit_date
 
+        flow_histories = await flow_crud.retrieve_by_applet_version(f"{applet.id}_{applet.version}")
+        flow_histories_curr = [flow_h.id for flow_h in flow_histories]
         results = []
         for flow_history in activity_flow_histories:
             flow_history_answer_date = submitted_activity_flows.get(
                 flow_history.id_version, submitted_activity_flows.get(str(flow_history.id))
             )
+            has_answer = bool(flow_history_answer_date)
+            if not has_answer and flow_history.id not in flow_histories_curr:
+                continue
+
             results.append(
                 SummaryActivityFlow(
                     id=flow_history.id,

src/apps/answers/tests/conftest.py

@@ -474,3 +474,67 @@ async def editor_user_reviewer_applet_one(user: UserSchema, session: AsyncSessio
     applet_id = uuid.UUID("92917a56-d586-4613-b7aa-991f2c4b15b1")
     srv = UserAppletAccessService(session, user.id, applet_id)
     await srv.add_role(user.id, Role.EDITOR)
+
+
+@pytest.fixture
+async def applet__activity_turned_into_assessment(
+    session: AsyncSession, tom: User, applet_data: AppletCreate
+) -> AppletFull:
+    """
+    Applet with one Activity0 updated into applet with Activity0(is_reviewable=True) and Activity1
+    All activities has no answers
+    """
+    srv = AppletService(session, tom.id)
+    applet = await srv.create(applet_data)
+    data = AppletUpdate(**applet.dict())
+
+    activity_new = data.activities[0].copy(deep=True)
+    activity_new.id = uuid.uuid4()
+    activity_new.key = uuid.uuid4()
+    for i in range(len(activity_new.items)):
+        activity_new.items[i].id = uuid.uuid4()
+    activity_new.name = "New activity"
+    data.activities.append(activity_new)
+
+    data.activities[0].is_reviewable = True
+    data.activities[0].name = "Reviewer assessment"
+    updated_applet = await srv.update(applet.id, data)
+    return updated_applet
+
+
+@pytest.fixture
+async def applet__deleted_activity_without_answers(
+    session: AsyncSession, tom: User, applet_data: AppletCreate
+) -> AppletFull:
+    """
+    Applet with one Activity0 updated into applet with Activity1 and deleted Activity1
+    All activities has no answers
+    """
+    srv = AppletService(session, tom.id)
+    applet = await srv.create(applet_data)
+    data = AppletUpdate(**applet.dict())
+
+    activity_new = data.activities[0].copy(deep=True)
+    activity_new.id = uuid.uuid4()
+    activity_new.key = uuid.uuid4()
+    for i in range(len(activity_new.items)):
+        activity_new.items[i].id = uuid.uuid4()
+    activity_new.name = "New activity"
+    data.activities = [activity_new]
+    updated_applet = await srv.update(applet.id, data)
+    return updated_applet
+
+
+@pytest.fixture
+async def applet__deleted_flow_without_answers(
+    session: AsyncSession, tom: User, applet_with_flow: AppletFull
+) -> AppletFull:
+    srv = AppletService(session, tom.id)
+    data = applet_with_flow.dict()
+    activity_flow = data["activity_flows"][0]
+    for i in range(len(activity_flow["items"])):
+        activity_flow["items"][i]["activity_key"] = data["activities"][0]["key"]
+    data["activity_flows"] = [activity_flow]
+    update_data = AppletUpdate(**data)
+    updated_applet = await srv.update(applet_with_flow.id, update_data)
+    return updated_applet

src/apps/answers/tests/test_answers.py

@@ -1948,3 +1948,61 @@ async def test_flow_submission_not_completed(
         data = data["result"]
         assert set(data.keys()) == {"flow", "submission", "summary"}
         assert data["submission"]["isCompleted"] is False
+
+    async def test_get_summary_activities_no_answer_no_empty_deleted_history(
+        self, client: TestClient, tom: User, applet: AppletFull
+    ):
+        client.login(tom)
+
+        response = await client.get(
+            self.summary_activities_url.format(
+                applet_id=str(applet.id),
+            )
+        )
+
+        assert response.status_code == http.HTTPStatus.OK
+        assert response.json()["count"] == 1
+        assert response.json()["result"][0]["name"] == applet.activities[0].name
+        assert response.json()["result"][0]["id"] == str(applet.activities[0].id)
+        assert not response.json()["result"][0]["isPerformanceTask"]
+        assert not response.json()["result"][0]["hasAnswer"]
+
+    async def test_activity_turned_into_assessment_not_included_in_list(
+        self, client: TestClient, tom: User, applet__activity_turned_into_assessment: AppletFull
+    ):
+        client.login(tom)
+        response = await client.get(
+            self.summary_activities_url.format(
+                applet_id=str(applet__activity_turned_into_assessment.id),
+            )
+        )
+
+        assert response.status_code == http.HTTPStatus.OK
+        assert response.json()["count"] == 1
+        assert response.json()["result"][0]["id"] == str(applet__activity_turned_into_assessment.activities[1].id)
+
+    async def test_deleted_activity_without_answers_not_included_in_list(
+        self, client: TestClient, tom: User, applet__deleted_activity_without_answers: AppletFull
+    ):
+        client.login(tom)
+        response = await client.get(
+            self.summary_activities_url.format(
+                applet_id=str(applet__deleted_activity_without_answers.id),
+            )
+        )
+
+        assert response.status_code == http.HTTPStatus.OK
+        assert response.json()["count"] == 1
+        assert response.json()["result"][0]["id"] == str(applet__deleted_activity_without_answers.activities[0].id)
+
+    async def test_deleted_flow_not_included_in_submission_list(
+        self, client: TestClient, tom: User, applet__deleted_flow_without_answers: AppletFull
+    ):
+        client.login(tom)
+        url = self.summary_activity_flows_url.format(applet_id=applet__deleted_flow_without_answers.id)
+        response = await client.get(url)
+        assert response.status_code == 200
+        payload = response.json()
+        assert applet__deleted_flow_without_answers.activity_flows[0].id
+        assert payload["count"] == 1
+        assert payload["result"][0]["id"] == str(applet__deleted_flow_without_answers.activity_flows[0].id)

src/apps/shared/commands/patch_commands.py

@@ -46,6 +46,14 @@
     description="Set proportion.enabled=True to Maki's applets",
     manage_session=False,
 )
+
+PatchRegister.register(
+    file_path="m2_6879_create_deleted_respondents.py",
+    task_id="M2-6879",
+    description="[Subject] Create deleted respondents roles",
+    manage_session=False,
+)
+
 PatchRegister.register(
     file_path="m2_5551_delete_invitations_of_existing_respondent.sql",
     task_id="M2-5551",

src/apps/shared/commands/patches/m2_6879_create_deleted_respondents.py

@@ -0,0 +1,146 @@
+import os
+import uuid
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import Query
+
+from apps.answers.db.schemas import AnswerSchema
+from apps.workspaces.crud.user_applet_access import UserAppletAccessCRUD
+from apps.workspaces.db.schemas import UserAppletAccessSchema
+from apps.workspaces.domain.constants import Role
+from apps.workspaces.service.workspace import WorkspaceService
+from infrastructure.database import session_manager
+
+LOCAL_DB_PATCH_SQL = """
+    insert into user_applet_accesses (
+        id,
+        is_deleted, 
+        "role", 
+        user_id, 
+        applet_id, 
+        owner_id, 
+        invitor_id, 
+        meta,
+        created_at,
+        updated_at,
+        migrated_date 
+    )
+    select
+        gen_random_uuid(),
+        true,
+        'respondent',
+        answers.respondent_id,
+        a.id,
+        owner.user_id,
+        owner.user_id,
+        '{"secretUserId": "#deleted#"}'::jsonb,
+        now() AT TIME ZONE 'UTC',
+        now() AT TIME ZONE 'UTC',
+        max(answers.migrated_date)
+    from applets a 
+    join user_applet_accesses owner on owner.applet_id = a.id and role = 'owner'
+    join answers on answers.applet_id = a.id
+    left join user_applet_accesses uaa 
+        on uaa.applet_id = answers.applet_id and uaa.user_id = answers.respondent_id and uaa."role" = 'respondent'
+    where 1=1
+        and uaa.id is null
+    group by a.id, answers.respondent_id, owner.user_id
+"""
+
+
+async def get_answer_count(session: AsyncSession):
+    query: Query = select(func.count(AnswerSchema.id))
+    db_result = await session.execute(query)
+    return db_result.first()[0]
+
+
+async def get_answers_applets_respondents(
+    session: AsyncSession, limit: int, offset: int
+) -> set[tuple[uuid.UUID, uuid.UUID]]:
+    query: Query = select(AnswerSchema.respondent_id, AnswerSchema.applet_id)
+    query = query.distinct(AnswerSchema.respondent_id, AnswerSchema.applet_id)
+    query = query.limit(limit)
+    query = query.offset(offset)
+    query = query.order_by(AnswerSchema.respondent_id.asc(), AnswerSchema.applet_id.asc())
+    db_result = await session.execute(query)
+    answer_applet_resp = db_result.all()
+    return {(a.respondent_id, a.applet_id) for a in answer_applet_resp}
+
+
+async def get_missing_applet_respondent(
+    session: AsyncSession, owner_id: uuid.UUID, arbitrary_applet_respondents: set[tuple[uuid.UUID, uuid.UUID]]
+) -> list[tuple[uuid.UUID, uuid.UUID]]:
+    query: Query = select(UserAppletAccessSchema.user_id, UserAppletAccessSchema.applet_id)
+    query = query.where(UserAppletAccessSchema.owner_id == owner_id, UserAppletAccessSchema.role == Role.RESPONDENT)
+    db_result = await session.execute(query)
+    roles_users_applets = db_result.all()
+    return list(arbitrary_applet_respondents - set(roles_users_applets))
+
+
+async def find_and_create_missing_roles_arbitrary(
+    session: AsyncSession, arbitrary_session: AsyncSession, owner_id: uuid.UUID
+):
+    count = await get_answer_count(arbitrary_session)
+    if not count:
+        print(f"Workspace: {owner_id}", f"answers count: {count}", "skip")
+        return
+
+    limit = int(os.environ.get("M2_6879_BATCH_SIZE", "1000"))
+    total_missing = 0
+    roles = []
+    for offset in range(0, count, limit):
+        arbitrary_applet_respondents = await get_answers_applets_respondents(arbitrary_session, limit, offset)
+        missing_users_applets = await get_missing_applet_respondent(session, owner_id, arbitrary_applet_respondents)
+        for user_id, applet_id in missing_users_applets:
+            schema = UserAppletAccessSchema(
+                user_id=user_id,
+                applet_id=applet_id,
+                role=Role.RESPONDENT,
+                owner_id=owner_id,
+                invitor_id=owner_id,
+                meta={"secretUserId": "#deleted#"},
+                is_deleted=True,
+            )
+            roles.append(schema)
+            total_missing += len(missing_users_applets)
+
+    await UserAppletAccessCRUD(session).create_many(roles)
+    print(
+        f"Workspace: {owner_id}",
+        f"answers count: {count}",
+        f"missing_roles: {total_missing}",
+        "done",
+    )
+
+
+async def main(session: AsyncSession, *args, **kwargs):
+    try:
+        await session.execute(LOCAL_DB_PATCH_SQL)
+        await session.commit()
+
+        workspaces = await WorkspaceService(session, uuid.uuid4()).get_arbitrary_list()
+        print(f"Found {len(workspaces)} workspaces with arbitrary servers")
+
+        processed = set()
+        for i, workspace in enumerate(workspaces):
+            if arb_uri := workspace.database_uri:
+                print(f"Processing workspace#{i + 1} {workspace.id}")
+                if arb_uri in processed:
+                    print(f"Workspace#{i + 1} DB already processed, skip...")
+                    continue
+                processed.add(arb_uri)
+                session_maker = session_manager.get_session(arb_uri)
+                async with session_maker() as arb_session:
+                    try:
+                        await find_and_create_missing_roles_arbitrary(session, arb_session, workspace.user_id)
+                        await arb_session.commit()
+                        print(f"Processing workspace#{i + 1} {workspace.id} " f"finished")
+                    except Exception:
+                        await arb_session.rollback()
+                        print(f"[bold red]Workspace#{i + 1} {workspace.id} " f"processing error[/bold red]")
+                        raise
+
+    except Exception as ex:
+        await session.rollback()
+        raise ex