From 209b789f6f62c91b2382ee332fccae8269e69585 Mon Sep 17 00:00:00 2001 From: Kenroy Gobourne Date: Tue, 6 Aug 2024 19:57:13 -0500 Subject: [PATCH 1/3] fix: Add temporary relation expiry date environment variable (M2-7523) (#1523) * Add temporary relation expiry date environment variable * Add variable to .env.default * Updated README.md * Updated default value of env var in README.md --------- Co-authored-by: Marty --- .env.default | 2 ++ README.md | 1 + src/apps/subjects/api.py | 3 ++- src/config/__init__.py | 3 +++ src/config/multiinformant.py | 5 +++++ 5 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 src/config/multiinformant.py diff --git a/.env.default b/.env.default index d7f05856e6b..5e7d7b5fb29 100644 --- a/.env.default +++ b/.env.default @@ -93,3 +93,5 @@ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT= # OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel:4317 # OTEL_TRACES_EXPORTER # OTEL_EXPORTER_OTLP_CERTIFICATE + +MULTI_INFORMANT__TEMP_RELATION_EXPIRY_SECS=86400 \ No newline at end of file diff --git a/README.md b/README.md index 76e1b87a798..26ff72b7246 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ pipenv --python /opt/homebrew/bin/python3.10 | MAILING\_\_MAIL\_\_USERNAME | mailhog | Mail service username | | MAILING\_\_MAIL\_\_PASSWORD | mailhog | Mail service password | | MAILING\_\_MAIL\_\_SERVER | mailhog | Mail service URL | +| MULTI\_INFORMANT\_\_TEMP\_RELATION\_EXPIRY\_SECS | 86400 | Expiry (sec) of temporary multi-informant participant take now relation | | SECRETS\_\_SECRET\_KEY | - | Secret key for data encryption. Use this key only for local development | ##### ✋ Mandatory: diff --git a/src/apps/subjects/api.py b/src/apps/subjects/api.py index 6fc54fc04b0..ec2c93bce23 100644 --- a/src/apps/subjects/api.py +++ b/src/apps/subjects/api.py @@ -33,6 +33,7 @@ from apps.workspaces.service.check_access import CheckAccessService from apps.workspaces.service.user_access import UserAccessService from apps.workspaces.service.user_applet_access import UserAppletAccessService +from config import settings from infrastructure.database import atomic from infrastructure.database.deps import get_session @@ -125,7 +126,7 @@ async def create_temporary_multiinformant_relation( await service.delete_relation(subject_id, source_subject_id) async with atomic(session): - expires_at = datetime.now() + timedelta(days=1) + expires_at = datetime.now() + timedelta(seconds=settings.multi_informant.temp_relation_expiry_secs) await service.create_relation(subject_id, source_subject_id, "take-now", {"expiresAt": expires_at.isoformat()}) return EmptyResponse() diff --git a/src/config/__init__.py b/src/config/__init__.py index d303f929f84..78522f864fd 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -11,6 +11,7 @@ from config.database import DatabaseSettings from config.logs import Logs from config.mailing import MailingSettings +from config.multiinformant import MultiInformantSettings from config.notification import FirebaseCloudMessagingSettings from config.opentelemetry import OpenTelemetrySettings from config.rabbitmq import RabbitMQSettings @@ -91,6 +92,8 @@ class Settings(BaseSettings): opentelemetry: OpenTelemetrySettings = OpenTelemetrySettings() + multi_informant: MultiInformantSettings = MultiInformantSettings() + @property def uploads_dir(self): return self.root_dir.parent / "uploads" diff --git a/src/config/multiinformant.py b/src/config/multiinformant.py new file mode 100644 index 00000000000..6c1dce969a4 --- /dev/null +++ b/src/config/multiinformant.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + + +class MultiInformantSettings(BaseModel): + temp_relation_expiry_secs: int = 86400 From 62f70cd5069d0f9316eeea93f0b506cb1a30ccef Mon Sep 17 00:00:00 2001 From: Marty Date: Tue, 6 Aug 2024 18:09:13 -0700 Subject: [PATCH 2/3] fix: incorrect relation value when non admin is source and respondent but not target (M2-7517) (#1530) * add an additional check for a temp relation to return 'other' in case it is a temp take_now relation * add another logical fix to the flow * add testing to validate the new changes * fix code quality checks duo to formating --------- Co-authored-by: Joud Awad --- .github/workflows/pr-open.yml | 2 +- src/apps/answers/service.py | 3 + src/apps/answers/tests/test_answers.py | 94 ++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml index c27989202b4..d22eea89343 100644 --- a/.github/workflows/pr-open.yml +++ b/.github/workflows/pr-open.yml @@ -42,7 +42,7 @@ jobs: create-preview-env: needs: [create-database] - uses: ./.github/workflows/create-preview-env.yaml + uses: ChildMindInstitute/mindlogger-backend-refactor/.github/workflows/create-preview-env.yaml@develop with: env-name: "pr-${{ github.event.number }}" env-snake-name: "pr_${{ github.event.number }}" diff --git a/src/apps/answers/service.py b/src/apps/answers/service.py index 5409859a44c..481725657ee 100644 --- a/src/apps/answers/service.py +++ b/src/apps/answers/service.py @@ -284,6 +284,9 @@ async def _get_answer_relation( return Relation.other + if is_take_now_relation(relation) and is_valid_take_now_relation(relation): + return Relation.other + return relation.relation async def _create_answer(self, applet_answer: AppletAnswerCreate) -> AnswerSchema: diff --git a/src/apps/answers/tests/test_answers.py b/src/apps/answers/tests/test_answers.py index 31dbf8e8a11..9942f40d84a 100644 --- a/src/apps/answers/tests/test_answers.py +++ b/src/apps/answers/tests/test_answers.py @@ -25,6 +25,7 @@ from apps.mailing.services import TestMail from apps.shared.test import BaseTest from apps.shared.test.client import TestClient +from apps.subjects.constants import Relation from apps.subjects.db.schemas import SubjectSchema from apps.subjects.domain import Subject, SubjectCreate from apps.subjects.services import SubjectsService @@ -1139,6 +1140,99 @@ async def test_answer_activity_items_create_temporary_relation_success( relation_exists = await subject_service.get_relation(applet_one_sam_subject.id, target_subject.id) assert not relation_exists + async def test_answer_activity_items_relation_equal_other_when_relation_is_temp( + self, + tom: User, + answer_create_applet_one: AppletAnswerCreate, + client: TestClient, + session: AsyncSession, + sam: User, + applet_one: AppletFull, + applet_one_sam_respondent, + applet_one_sam_subject, + ) -> None: + client.login(tom) + subject_service = SubjectsService(session, tom.id) + + data = answer_create_applet_one.copy(deep=True) + + client.login(sam) + subject_service = SubjectsService(session, sam.id) + target_subject = await subject_service.create( + SubjectCreate( + applet_id=applet_one.id, + creator_id=tom.id, + first_name="target", + last_name="subject", + secret_user_id=f"{uuid.uuid4()}", + ) + ) + await subject_service.create_relation( + relation="take-now", + source_subject_id=applet_one_sam_subject.id, + subject_id=target_subject.id, + meta={ + "expiresAt": (datetime.datetime.now() + datetime.timedelta(days=1)).isoformat(), + }, + ) + + data.source_subject_id = applet_one_sam_subject.id + data.target_subject_id = target_subject.id + data.input_subject_id = applet_one_sam_subject.id + + response = await client.post(self.answer_url, data=data) + + assert response.status_code == http.HTTPStatus.CREATED, response.json() + + answers, _ = await AnswersCRUD(session).get_applet_answers(applet_id=applet_one.id, page=1, limit=5) + + assert answers[0].relation == Relation.other + + async def test_answer_activity_items_relation_equal_other_when_relation_is_permanent( + self, + tom: User, + answer_create_applet_one: AppletAnswerCreate, + client: TestClient, + session: AsyncSession, + sam: User, + applet_one: AppletFull, + applet_one_sam_respondent, + applet_one_sam_subject, + ) -> None: + client.login(tom) + subject_service = SubjectsService(session, tom.id) + + data = answer_create_applet_one.copy(deep=True) + + client.login(sam) + subject_service = SubjectsService(session, sam.id) + target_subject = await subject_service.create( + SubjectCreate( + applet_id=applet_one.id, + creator_id=tom.id, + first_name="target", + last_name="subject", + secret_user_id=f"{uuid.uuid4()}", + ) + ) + await subject_service.create_relation( + relation="father", + source_subject_id=applet_one_sam_subject.id, + subject_id=target_subject.id, + ) + + data.source_subject_id = applet_one_sam_subject.id + data.target_subject_id = target_subject.id + data.input_subject_id = applet_one_sam_subject.id + + response = await client.post(self.answer_url, data=data) + + assert response.status_code == http.HTTPStatus.CREATED, response.json() + + answers, _ = await AnswersCRUD(session).get_applet_answers(applet_id=applet_one.id, page=1, limit=5) + + assert answers[0].relation == "father" + async def test_answer_activity_items_create_permanent_relation_success( self, tom: User, From a92d5bef7ebdeae73b488e33c8c64e9aa0546684 Mon Sep 17 00:00:00 2001 From: vshvechko Date: Wed, 7 Aug 2024 18:13:15 +0300 Subject: [PATCH 3/3] chore: Create patch to move applet answers to arbitrary server (M2-7543) --- src/apps/shared/commands/patch_commands.py | 5 + ...move_answers_from_internal_to_arbitrary.py | 118 ++++++++++++++++++ src/apps/workspaces/crud/workspaces.py | 18 +-- 3 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 src/apps/shared/commands/patches/m2_7543_move_answers_from_internal_to_arbitrary.py diff --git a/src/apps/shared/commands/patch_commands.py b/src/apps/shared/commands/patch_commands.py index 78eb53b9d13..eebed2d805b 100644 --- a/src/apps/shared/commands/patch_commands.py +++ b/src/apps/shared/commands/patch_commands.py @@ -108,6 +108,11 @@ task_id="M2-7203", description="[Migration] Migrate missed secret ids", ) +PatchRegister.register( + file_path="m2_7543_move_answers_from_internal_to_arbitrary.py", + task_id="M2-7543", + description="Move applet 'NIMH Rhythms and Mood Family Study EMA' answers to related arbitrary", +) app = typer.Typer() diff --git a/src/apps/shared/commands/patches/m2_7543_move_answers_from_internal_to_arbitrary.py b/src/apps/shared/commands/patches/m2_7543_move_answers_from_internal_to_arbitrary.py new file mode 100644 index 00000000000..cac683c7319 --- /dev/null +++ b/src/apps/shared/commands/patches/m2_7543_move_answers_from_internal_to_arbitrary.py @@ -0,0 +1,118 @@ +import asyncio +import uuid + +from rich import print +from sqlalchemy import func, select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from apps.answers.db.schemas import AnswerItemSchema, AnswerSchema +from apps.answers.deps.preprocess_arbitrary import preprocess_arbitrary_url +from apps.applets.service import AppletService +from infrastructure.database import atomic, session_manager + +APPLET_ID = uuid.UUID("62be21d7-cd01-4b9b-975a-39750d940f59") +INSERT_BATCH_SIZE = 1000 + + +def error_msg(msg: str): + print(f"[bold red]Error: {msg}[/bold red]") + + +async def copy_answer_items(session: AsyncSession, arb_session: AsyncSession, applet_id: uuid.UUID): + print("[green]Copy answer items...[/green]") + + query = ( + select(AnswerItemSchema.__table__) + .join(AnswerSchema, AnswerSchema.id == AnswerItemSchema.answer_id) + .where(AnswerSchema.applet_id == applet_id) + ) + + res = await session.execute(query) + data = res.all() + + print(f"Total records in internal DB: {len(data)}") + total_res = await arb_session.execute(query.with_only_columns(func.count(AnswerItemSchema.id))) + total_arb = total_res.scalar() + print(f"Total records in arbitrary DB: {total_arb}") + + for i in range(0, len(data), INSERT_BATCH_SIZE): + values = [dict(row) for row in data[i : i + INSERT_BATCH_SIZE]] + insert_query = ( + insert(AnswerItemSchema) + .values(values) + .on_conflict_do_nothing( + index_elements=[AnswerItemSchema.id], + ) + ) + await arb_session.execute(insert_query) + + print("[green]Copy answer items - DONE[/green]") + total_res = await arb_session.execute(query.with_only_columns(func.count(AnswerItemSchema.id))) + total_arb = total_res.scalar() + + print(f"Total records in arbitrary DB: {total_arb}\n") + + +async def copy_answers(session: AsyncSession, arb_session: AsyncSession, applet_id: uuid.UUID): + print("[green]Copy answers...[/green]") + + query = select(AnswerSchema.__table__).where(AnswerSchema.applet_id == applet_id) + res = await session.execute(query) + data = res.all() + + print(f"Total records in internal DB: {len(data)}") + total_res = await arb_session.execute(query.with_only_columns(func.count(AnswerSchema.id))) + total_arb = total_res.scalar() + print(f"Total records in arbitrary DB: {total_arb}") + + for i in range(0, len(data), INSERT_BATCH_SIZE): + values = [dict(row) for row in data[i : i + INSERT_BATCH_SIZE]] + insert_query = ( + insert(AnswerSchema) + .values(values) + .on_conflict_do_nothing( + index_elements=[AnswerSchema.id], + ) + ) + await arb_session.execute(insert_query) + + print("[green]Copy answers - Done[/green]") + + total_res = await arb_session.execute(query.with_only_columns(func.count(AnswerSchema.id))) + total_arb = total_res.scalar() + + print(f"Total records in arbitrary DB: {total_arb}") + + +async def main( + session: AsyncSession, + arbitrary_session: AsyncSession = None, + *args, + **kwargs, +): + applet = await AppletService(session, uuid.uuid4()).get(APPLET_ID) + arbitrary_uri = await preprocess_arbitrary_url(applet.id, session=session) + if not arbitrary_uri: + error_msg("Arbitrary db not set for the applet") + return + + print(f"[green]Move answers for applet '{applet.display_name}'({applet.id})[/green]") + + session_maker = session_manager.get_session(arbitrary_uri) + async with session_maker() as arb_session: + try: + print("Check DB availability...") + await arb_session.execute("select current_date") + print("[green]Database is available.[/green]") + except asyncio.TimeoutError: + error_msg("Timeout error") + return + except Exception as e: + error_msg(str(e)) + return + + async with atomic(arb_session): + await copy_answers(session, arb_session, applet.id) + async with atomic(arb_session): + await copy_answer_items(session, arb_session, applet.id) diff --git a/src/apps/workspaces/crud/workspaces.py b/src/apps/workspaces/crud/workspaces.py index 98f16faad7f..74ad3e74d3e 100644 --- a/src/apps/workspaces/crud/workspaces.py +++ b/src/apps/workspaces/crud/workspaces.py @@ -46,15 +46,19 @@ async def update_by_user_id(self, user_id: uuid.UUID, schema: UserWorkspaceSchem return instance async def get_by_applet_id(self, applet_id: uuid.UUID) -> UserWorkspaceSchema | None: - access_subquery: Query = select(UserAppletAccessSchema.owner_id) - access_subquery = access_subquery.where( - and_( - UserAppletAccessSchema.role == Role.OWNER, - UserAppletAccessSchema.applet_id == applet_id, + query: Query = ( + select(UserWorkspaceSchema) + .join( + UserAppletAccessSchema, + and_( + UserAppletAccessSchema.user_id == UserWorkspaceSchema.user_id, + UserAppletAccessSchema.role == Role.OWNER, + UserAppletAccessSchema.soft_exists(), + ), ) + .where(UserAppletAccessSchema.applet_id == applet_id) ) - query: Query = select(UserWorkspaceSchema) - query = query.where(UserWorkspaceSchema.user_id.in_(access_subquery)) + db_result = await self._execute(query) res = db_result.scalars().first() return res