Merge branch 'develop' into broker-structlog

Pipfile

@@ -4,21 +4,21 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-aio-pika = "==9.5.3"
+aio-pika = "==9.5.*"
 aiofiles = "==24.1.0"
-aiohttp = "==3.11.9"
+aiohttp = "==3.11.*"
 alembic = "==1.14.0"
 asyncpg = "==0.30.0"
 azure-storage-blob = "==12.24.0"
 bcrypt = "==4.2.1"
-boto3 = "==1.35.77"
+boto3 = "==1.35.*"
 fastapi = "==0.115.6"
 fastapi-mail = "==1.2.9"
-firebase-admin = "==6.5.0"
-httpx = "==0.28.0"
-jinja2 = "==3.1.4"
+firebase-admin = "==6.6.*"
+httpx = "==0.28.*"
+jinja2 = "==3.1.*"
 more-itertools = "==10.5.0"
-nh3 = "==0.2.19"
+nh3 = "==0.2.20"
 opentelemetry-api = "==1.27.0"
 opentelemetry-distro = "==0.48b0"
 opentelemetry-exporter-otlp = "==1.27.0"
@@ -41,19 +41,19 @@ opentelemetry-util-http = "==0.48b0"
 pydantic = { extras = ["email"], version = "==1.10.18" }
 pyjwt = "==2.10.1"
 pymongo = "==4.10.1"
-pyOpenSSL = "==24.2.1"
-python-multipart = "==0.0.19"
-redis = "==5.2.0"
+pyOpenSSL = "==24.3.*"
+python-multipart = "==0.0.20"
+redis = "==5.2.*"
 sentry-sdk = "~=2.13"
 sqlalchemy = { extras = ["asyncio"], version = "==1.4.53" }
 sqlalchemy-utils = "==0.41.2"
-taskiq = { extras = ["reload"], version = "==0.11.7" }
+taskiq = { extras = ["reload"], version = "==0.11.*" }
 taskiq-aio-pika = "==0.4.1"
-taskiq-fastapi = "==0.3.2"
+taskiq-fastapi = "==0.3.*"
 taskiq-redis = "==1.0.2"
-typer = "==0.12.5"
-uvicorn = { extras = ["standard"], version = "==0.32.0" }
-ddtrace = "==2.17.2"
+typer = "==0.15.*"
+uvicorn = { extras = ["standard"], version = "==0.34.*" }
+ddtrace = "==2.*"
 bytecode = "==0.16.0"
 structlog = "==24.4.0"
 asgi-correlation-id = "==4.3.4"
@@ -65,25 +65,25 @@ cachetools = "==5.3.0"
 gevent = "==24.2.1"
 greenlet = "==3.1.0"
 ipdb = "==0.13.13"
-mypy = "==1.13.0"
+mypy = "==1.14.*"
 nest-asyncio = "==1.6.0"
 pre-commit = "==4.0.1"
 pudb = "==2024.1.3"
 pydantic-factories = "==1.17.3"
 pyld = "==2.0.4"
 pytest = "==8.3.4"
-pytest-asyncio = "~=0.19"
+pytest-asyncio = "==0.25.*"
 pytest-cov = "==6.0.0"
 pytest-env = "==1.1.5"
-pytest-lazy-fixtures = "==1.1.1"
+pytest-lazy-fixtures = "==1.*"
 pytest-mock = "==3.14.0"
 reproschema = "==0.6.2"
-ruff = "==0.8.2"
-types-aiofiles = "==24.1.0.20240626"
-types-cachetools = "==5.5.0.20240820"
-types-python-dateutil = "==2.9.0.20241206   "
-types-pytz = "==2024.2.0.20241003"
-types-requests = "==2.32.0.20241016"
+ruff = "==0.8.*"
+types-aiofiles = "==24.1.0.*"
+types-cachetools = "==5.5.0.*"
+types-python-dateutil = "==2.9.0.*"
+types-pytz = "==2024.2.0.*"
+types-requests = "==2.32.0.*"
 typing-extensions = "==4.12.2"
 
 [requires]

copilot/mindlogger-backend/manifest.yml

@@ -180,7 +180,7 @@ logging:
     Name: "datadog"
     Host: "http-intake.logs.datadoghq.com"
     dd_service: "backend-api"
-    dd_source: "feature"
+    dd_source: "api-server"
     dd_message_key: "log"
     dd_tags: "version:${COPILOT_ENVIRONMENT_NAME},env:feature"
     TLS: "on"

src/apps/alerts/crud/alert.py

@@ -5,6 +5,7 @@
 
 from apps.alerts.db.schemas import AlertSchema
 from apps.applets.db.schemas import AppletHistorySchema, AppletSchema
+from apps.integrations.db.schemas import IntegrationsSchema
 from apps.shared.ordering import Ordering
 from apps.shared.paging import paging
 from apps.shared.searching import Searching
@@ -40,11 +41,23 @@ async def get_all_for_user(
         self, user_id: uuid.UUID, page: int, limit: int
     ) -> list[
         tuple[
-            AlertSchema, AppletHistorySchema, UserAppletAccessSchema, AppletSchema, UserWorkspaceSchema, SubjectSchema
+            AlertSchema,
+            AppletHistorySchema,
+            UserAppletAccessSchema,
+            AppletSchema,
+            UserWorkspaceSchema,
+            SubjectSchema,
+            IntegrationsSchema,
         ]
     ]:
         query: Query = select(
-            AlertSchema, AppletHistorySchema, UserAppletAccessSchema, AppletSchema, UserWorkspaceSchema, SubjectSchema
+            AlertSchema,
+            AppletHistorySchema,
+            UserAppletAccessSchema,
+            AppletSchema,
+            UserWorkspaceSchema,
+            SubjectSchema,
+            IntegrationsSchema.type,
         )
         query = query.join(
             UserAppletAccessSchema,
@@ -72,6 +85,7 @@ async def get_all_for_user(
             isouter=True,
         )
         query = query.outerjoin(SubjectSchema, SubjectSchema.id == AlertSchema.subject_id)
+        query = query.outerjoin(IntegrationsSchema, IntegrationsSchema.applet_id == UserAppletAccessSchema.applet_id)
         query = query.where(AlertSchema.user_id == user_id, AppletSchema.is_deleted.is_(False))
         query = query.order_by(AlertSchema.created_at.desc())
         query = paging(query, page, limit)

src/apps/alerts/db/schemas.py

@@ -33,3 +33,4 @@ class AlertSchema(Base):
     activity_item_id = Column(UUID(as_uuid=True))
     alert_message = Column(StringEncryptedType(Unicode, get_key), nullable=False)
     answer_id = Column(UUID(as_uuid=True))
+    type = Column(String())

src/apps/alerts/domain.py

@@ -1,7 +1,8 @@
 import datetime
 import uuid
+from enum import Enum
 
-from pydantic import validator
+from pydantic import Field, validator
 
 from apps.shared.domain import InternalModel, PublicModel, ResponseMulti, dict_keys_to_camel_case
 
@@ -12,15 +13,20 @@
 ]
 
 
+class AlertTypes(str, Enum):
+    ANSWER_ALERT = "answer"
+    INTEGRATION_ALERT = "integration"
+
+
 class Alert(InternalModel):
     id: uuid.UUID
     is_watched: bool
     applet_id: uuid.UUID
     applet_name: str
     version: str
     secret_id: str
-    activity_id: uuid.UUID
-    activity_item_id: uuid.UUID
+    activity_id: uuid.UUID | None
+    activity_item_id: uuid.UUID | None
     message: str
     created_at: datetime.datetime
     answer_id: uuid.UUID | None
@@ -29,6 +35,7 @@ class Alert(InternalModel):
     workspace: str
     respondent_id: uuid.UUID
     subject_id: uuid.UUID | None
+    type: AlertTypes = Field(default=AlertTypes.ANSWER_ALERT)
 
 
 class AlertPublic(PublicModel):
@@ -38,8 +45,8 @@ class AlertPublic(PublicModel):
     applet_name: str
     version: str
     secret_id: str
-    activity_id: uuid.UUID
-    activity_item_id: uuid.UUID
+    activity_id: uuid.UUID | None
+    activity_item_id: uuid.UUID | None
     message: str
     created_at: datetime.datetime
     answer_id: uuid.UUID | None
@@ -48,6 +55,7 @@ class AlertPublic(PublicModel):
     workspace: str
     respondent_id: uuid.UUID
     subject_id: uuid.UUID | None
+    type: AlertTypes = Field(default=AlertTypes.ANSWER_ALERT)
 
     @validator("encryption", pre=True)
     def convert_response_values_keys(cls, response_values):
@@ -64,9 +72,10 @@ class AlertMessage(InternalModel):
     version: str
     message: str
     created_at: datetime.datetime
-    activity_id: uuid.UUID
-    activity_item_id: uuid.UUID
-    answer_id: uuid.UUID
+    activity_id: uuid.UUID | None
+    activity_item_id: uuid.UUID | None
+    answer_id: uuid.UUID | None
+    type: AlertTypes = Field(default=AlertTypes.ANSWER_ALERT)
 
 
 class AlertHandlerResult(InternalModel):
@@ -85,6 +94,7 @@ class AlertHandlerResult(InternalModel):
     workspace: str
     respondent_id: str
     subject_id: str | None
+    type: AlertTypes = Field(default=AlertTypes.ANSWER_ALERT)
 
 
 class AlertResponseMulti(ResponseMulti[AlertPublic]):

src/apps/alerts/service.py

@@ -1,7 +1,7 @@
 import uuid
 
 from apps.alerts.crud.alert import AlertCRUD
-from apps.alerts.domain import Alert
+from apps.alerts.domain import Alert, AlertTypes
 from apps.shared.query_params import QueryParams
 
 
@@ -13,15 +13,19 @@ def __init__(self, session, user_id: uuid.UUID):
     async def get_all_alerts(self, filters: QueryParams) -> list[Alert]:
         alerts = []
         schemas = await AlertCRUD(self.session).get_all_for_user(self.user_id, filters.page, filters.limit)
-        for alert, applet_history, access, applet, workspace, subject in schemas:
+        for alert, applet_history, access, applet, workspace, subject, integrations in schemas:
+            if integrations and "LORIS" in integrations:
+                _secret_id = "Loris Integration"
+            else:
+                _secret_id = subject.secret_user_id if subject else "Anonymous"
             alerts.append(
                 Alert(
                     id=alert.id,
                     is_watched=alert.is_watched,
                     applet_id=alert.applet_id,
                     applet_name=applet_history.display_name,
                     version=alert.version,
-                    secret_id=subject.secret_user_id if subject else "Anonymous",
+                    secret_id=_secret_id,
                     activity_id=alert.activity_id,
                     activity_item_id=alert.activity_item_id,
                     message=alert.alert_message,
@@ -32,6 +36,7 @@ async def get_all_alerts(self, filters: QueryParams) -> list[Alert]:
                     workspace=workspace.workspace_name,
                     respondent_id=alert.respondent_id,
                     subject_id=alert.subject_id,
+                    type=alert.type if alert.type else AlertTypes.ANSWER_ALERT,
                 )
             )
         return alerts

src/apps/alerts/ws_api.py

@@ -64,12 +64,17 @@ async def _handle_websocket(websocket, user_id, session):
             traceback.print_tb(e.__traceback__)
             continue
         try:
+            if applet.integrations and "loris" in applet.integrations:
+                _secret_id = "Loris Integration"
+            else:
+                _secret_id = subject.secret_user_id if subject else "Anonymous"
+
             applet_alert = AlertHandlerResult(
                 id=str(alert_message.id),
                 applet_id=str(alert_message.applet_id),
                 applet_name=applet_history.display_name,
                 version=alert_message.version,
-                secret_id=subject.secret_user_id if subject else "Anonymous",
+                secret_id=_secret_id,
                 activity_id=str(alert_message.activity_id),
                 activity_item_id=str(alert_message.activity_item_id),
                 message=alert_message.message,
@@ -80,6 +85,7 @@ async def _handle_websocket(websocket, user_id, session):
                 workspace=workspace.workspace_name,
                 respondent_id=str(alert_message.respondent_id),
                 subject_id=str(alert_message.subject_id),
+                type=alert_message.type,
             )
             await websocket.send_json(applet_alert.dict())
         except ConnectionClosed:

src/apps/answers/crud/answers.py

@@ -1010,6 +1010,41 @@ async def get_activity_and_flow_ids_by_source_subject(self, source_subject_id: u
         )
         return res.mappings().all()
 
+    async def get_by_applet_id_and_readiness_to_share_data(
+        self, applet_id: uuid.UUID, respondent_id: uuid.UUID, answer_ids: list[uuid.UUID]
+    ) -> list[AnswerSchema] | None:
+        query: Query = select(AnswerSchema)
+        query = query.where(AnswerSchema.applet_id == applet_id)
+        query = query.where(AnswerSchema.respondent_id == respondent_id)
+        query = query.where(AnswerSchema.consent_to_share.is_(True))
+        query = query.where(AnswerSchema.id.in_(answer_ids))
+        query = query.order_by(AnswerSchema.created_at.asc())
+        db_result = await self._execute(query)
+        return db_result.scalars().all()
+
+    async def get_respondents_by_applet_id_and_readiness_to_share_data(
+        self, applet_id: uuid.UUID
+    ) -> list[AnswerSchema] | None:
+        query: Query = select(AnswerSchema.respondent_id)
+        query = query.where(AnswerSchema.applet_id == applet_id)
+        query = query.where(AnswerSchema.consent_to_share.is_(True))
+        query = query.order_by(AnswerSchema.created_at.asc())
+        db_result = await self._execute(query)
+        return db_result.scalars().all()
+
+    async def get_shareable_answers(self, applet_id: uuid.UUID):
+        query: Query = (
+            select(AnswerSchema)
+            .where(
+                AnswerSchema.applet_id == applet_id,
+                AnswerSchema.consent_to_share.is_(True),
+            )
+            .order_by(AnswerSchema.created_at)
+        )
+        result = await self._execute(query)
+
+        return result.scalars().all()
+
     @staticmethod
     def _query_submissions_metadata_by_subject(subject_column: InstrumentedAttribute, subject_id: uuid.UUID) -> Query:
         query: Query = (

src/apps/answers/db/schemas.py

@@ -41,6 +41,7 @@ class AnswerSchema(HistoryAware, Base):
     source_subject_id = Column(UUID(as_uuid=True), nullable=True, index=True)
     input_subject_id = Column(UUID(as_uuid=True), nullable=True, index=True)
     relation = Column(String(length=20), nullable=True)
+    consent_to_share = Column(Boolean(), default=False)
 
     answer_item = relationship(
         "AnswerItemSchema",

src/apps/answers/domain/answers.py

@@ -112,6 +112,7 @@ class AppletAnswerCreate(InternalModel):
     target_subject_id: uuid.UUID | None = None
     source_subject_id: uuid.UUID | None = None
     input_subject_id: uuid.UUID | None = None
+    consent_to_share: bool | None = False
 
     _dates_from_ms = validator("created_at", pre=True, allow_reuse=True)(datetime_from_ms)