Make send report message from discord a celery task. (LAION-AI#3089)

Fixes LAION-AI#2992

---------

Co-authored-by: Oliver Stanley <[email protected]>

backend/.env.example

@@ -1 +1,9 @@
 HUGGING_FACE_API_KEY=HF API KEY
+DATABASE_URI="postgresql://<username>:<password>@<host>/<database_name>"
+BACKEND_CORS_ORIGINS=["http://localhost", "http://localhost:4200", "http://localhost:3000", "http://localhost:8080", "https://localhost", "https://localhost:4200", "https://localhost:3000", "https://localhost:8080", "http://dev.oasst.laion.ai", "https://stag.oasst.laion.ai", "https://oasst.laion.ai"]
+REDIS_HOST=localhost
+REDIS_PORT=6379
+
+
+export DEBUG_SKIP_EMBEDDING_COMPUTATION=False
+export DEBUG_SKIP_TOXICITY_CALCULATION=False

backend/README.md

@@ -23,26 +23,21 @@ the `.python-version` in the project root directory.
 
 Next, to install all requirements, You can run
 
-1. `pip install -r requirements.txt` inside the `backend` folder; and
-2. `pip install -e .` inside the `oasst-shared` folder.
-3. `pip install -e .` inside the `oasst-data` folder.
-4. `../scripts/backend-development/run-local.sh` to run the backend. This will
+1. `pip install -r backend/requirements.txt`
+2. `pip install -e ./oasst-shared/.`
+3. `pip install -e ./oasst-data/.`
+4. `./scripts/backend-development/run-local.sh` to run the backend. This will
    start the backend server at `http://localhost:8080`.
 
 ## REST Server Configuration
 
-Please either use environment variables or create a `.env` file in the backend
-root directory (in which this readme file is located) to specify the
-`DATABASE_URI`.
+- Generate a new environment variable file `.env` by coping the content of the
+  [.env.example](.env.example) file.
 
-Example contents of a `.env` file for the backend:
+- Update the values of the environment variables in the `.env` file by setting
+  the DATABASE_URI to you local database URI.
 
-```
-DATABASE_URI="postgresql://<username>:<password>@<host>/<database_name>"
-BACKEND_CORS_ORIGINS=["http://localhost", "http://localhost:4200", "http://localhost:3000", "http://localhost:8080", "https://localhost", "https://localhost:4200", "https://localhost:3000", "https://localhost:8080", "http://dev.oasst.laion.ai", "https://stag.oasst.laion.ai", "https://oasst.laion.ai"]
-REDIS_HOST=localhost
-REDIS_PORT=6379
-```
+- Update the rest of the environment variables according to your needs.
 
 ## Running the REST Server locally for development
 

backend/oasst_backend/prompt_repository.py

@@ -30,8 +30,8 @@
 from oasst_backend.models.payload_column_type import PayloadContainer
 from oasst_backend.task_repository import TaskRepository, validate_frontend_message_id
 from oasst_backend.user_repository import UserRepository
-from oasst_backend.utils import discord
 from oasst_backend.utils.database_utils import CommitMode, db_lang_to_postgres_ts_lang, managed_tx_method
+from oasst_backend.utils.discord import send_new_report_message
 from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
 from oasst_shared.schemas.protocol import SystemStats
@@ -595,7 +595,7 @@ def store_text_labels(self, text_labels: protocol_schema.TextLabels) -> tuple[Te
                         message_id, protocol_schema.EmojiOp.add, protocol_schema.EmojiCode.red_flag
                     )
 
-                    discord.send_new_report_message(message=message, label_text=text_labels.text, user_id=self.user_id)
+                    send_new_report_message.delay(message=message, label_text=text_labels.text, user_id=self.user_id)
 
                 # update existing record for repeated updates (same user no task associated)
                 existing_text_label = self.fetch_non_task_text_labels(message_id, self.user_id)

backend/oasst_backend/utils/discord.py

@@ -2,13 +2,24 @@
 
 import requests
 from loguru import logger
+from oasst_backend.celery_worker import app as celery_app
 from oasst_backend.config import settings
 from oasst_backend.models.message import Message
 
 ROOT_ENDPOINT = "https://discord.com/api/v10"
 
 
-def send_new_report_message(message: Message, label_text: str, user_id: UUID) -> None:
+@celery_app.task(name="send_new_report_message")
+def send_new_report_message(message: Message, label_text: str, user_id: UUID):
+    """
+    Send a message to the Discord channel when a new message is flagged.
+    Note: this is a Celery task.
+
+    Args:
+        message (Message): the flagged message
+        label_text (str): the label text
+        user_id (UUID): the user ID
+    """
     if settings.DISCORD_API_KEY is None or settings.DISCORD_CHANNEL_ID is None:
         return
 

backend/requirements.txt

@@ -14,8 +14,8 @@ pydantic==1.10.4
 pydantic[email]==1.10.4
 python-dotenv==0.21.0
 python-jose[cryptography]==3.3.0
-redis
-requests
+redis==4.5.5
+requests==2.30.0
 scipy==1.8.1
 SQLAlchemy==1.4.41
 sqlmodel==0.0.8

backend/requirements_worker.txt

@@ -13,8 +13,8 @@ pydantic==1.9.1
 pydantic[email]==1.9.1
 python-dotenv==0.21.0
 python-jose[cryptography]==3.3.0
-redis
-requests
+redis==4.5.5
+requests==2.30.0
 scipy==1.8.1
 SQLAlchemy==1.4.41
 sqlmodel==0.0.8

backend/update_message_attributes.py

@@ -2,7 +2,7 @@
 
 from loguru import logger
 from oasst_backend.models import ApiClient, Message
-from oasst_backend.scheduled_tasks import hf_feature_extraction, toxicity
+from oasst_backend.scheduled_tasks import check_toxicity, hf_feature_extraction
 from oasst_backend.utils.database_utils import default_session_factory
 from sqlmodel import text
 
@@ -71,7 +71,7 @@ def find_and_update_toxicity(message_ids):
                     text = result.payload.payload.text
                     api_client = session.query(ApiClient).filter(ApiClient.id == api_client_id).first()
                     if api_client is not None and text is not None:
-                        toxicity(text=text, message_id=message_id, api_client=api_client.__dict__)
+                        check_toxicity(text=text, message_id=message_id, api_client=api_client.__dict__)
                         # to not get rate limited from HF
                         time.sleep(10)
     except Exception as e:

docker-compose.yaml

@@ -117,7 +117,7 @@ services:
     build:
       dockerfile: docker/Dockerfile.backend-worker
       context: .
-    command: celery -A oasst_backend.celery_worker worker -l info
+    command: celery -A oasst_backend.celery_worker worker -l info -E
     image: oasst-backend-worker
     environment:
       - CELERY_BROKER_URL=redis://redis:6379/0