diff --git a/backend/.env.example b/backend/.env.example index 4c0a1478ca..53149d3619 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1 +1,9 @@ HUGGING_FACE_API_KEY=HF API KEY +DATABASE_URI="postgresql://:@/" +BACKEND_CORS_ORIGINS=["http://localhost", "http://localhost:4200", "http://localhost:3000", "http://localhost:8080", "https://localhost", "https://localhost:4200", "https://localhost:3000", "https://localhost:8080", "http://dev.oasst.laion.ai", "https://stag.oasst.laion.ai", "https://oasst.laion.ai"] +REDIS_HOST=localhost +REDIS_PORT=6379 + + +export DEBUG_SKIP_EMBEDDING_COMPUTATION=False +export DEBUG_SKIP_TOXICITY_CALCULATION=False diff --git a/backend/README.md b/backend/README.md index f22f0ca827..053dd5760f 100644 --- a/backend/README.md +++ b/backend/README.md @@ -23,26 +23,21 @@ the `.python-version` in the project root directory. Next, to install all requirements, You can run -1. `pip install -r requirements.txt` inside the `backend` folder; and -2. `pip install -e .` inside the `oasst-shared` folder. -3. `pip install -e .` inside the `oasst-data` folder. -4. `../scripts/backend-development/run-local.sh` to run the backend. This will +1. `pip install -r backend/requirements.txt` +2. `pip install -e ./oasst-shared/.` +3. `pip install -e ./oasst-data/.` +4. `./scripts/backend-development/run-local.sh` to run the backend. This will start the backend server at `http://localhost:8080`. ## REST Server Configuration -Please either use environment variables or create a `.env` file in the backend -root directory (in which this readme file is located) to specify the -`DATABASE_URI`. +- Generate a new environment variable file `.env` by coping the content of the + [.env.example](.env.example) file. -Example contents of a `.env` file for the backend: +- Update the values of the environment variables in the `.env` file by setting + the DATABASE_URI to you local database URI. -``` -DATABASE_URI="postgresql://:@/" -BACKEND_CORS_ORIGINS=["http://localhost", "http://localhost:4200", "http://localhost:3000", "http://localhost:8080", "https://localhost", "https://localhost:4200", "https://localhost:3000", "https://localhost:8080", "http://dev.oasst.laion.ai", "https://stag.oasst.laion.ai", "https://oasst.laion.ai"] -REDIS_HOST=localhost -REDIS_PORT=6379 -``` +- Update the rest of the environment variables according to your needs. ## Running the REST Server locally for development diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py index d5f1a477aa..1ce0cab569 100644 --- a/backend/oasst_backend/prompt_repository.py +++ b/backend/oasst_backend/prompt_repository.py @@ -30,8 +30,8 @@ from oasst_backend.models.payload_column_type import PayloadContainer from oasst_backend.task_repository import TaskRepository, validate_frontend_message_id from oasst_backend.user_repository import UserRepository -from oasst_backend.utils import discord from oasst_backend.utils.database_utils import CommitMode, db_lang_to_postgres_ts_lang, managed_tx_method +from oasst_backend.utils.discord import send_new_report_message from oasst_shared.exceptions import OasstError, OasstErrorCode from oasst_shared.schemas import protocol as protocol_schema from oasst_shared.schemas.protocol import SystemStats @@ -595,7 +595,7 @@ def store_text_labels(self, text_labels: protocol_schema.TextLabels) -> tuple[Te message_id, protocol_schema.EmojiOp.add, protocol_schema.EmojiCode.red_flag ) - discord.send_new_report_message(message=message, label_text=text_labels.text, user_id=self.user_id) + send_new_report_message.delay(message=message, label_text=text_labels.text, user_id=self.user_id) # update existing record for repeated updates (same user no task associated) existing_text_label = self.fetch_non_task_text_labels(message_id, self.user_id) diff --git a/backend/oasst_backend/utils/discord.py b/backend/oasst_backend/utils/discord.py index 5bffe52ede..464ff6ae7a 100644 --- a/backend/oasst_backend/utils/discord.py +++ b/backend/oasst_backend/utils/discord.py @@ -2,13 +2,24 @@ import requests from loguru import logger +from oasst_backend.celery_worker import app as celery_app from oasst_backend.config import settings from oasst_backend.models.message import Message ROOT_ENDPOINT = "https://discord.com/api/v10" -def send_new_report_message(message: Message, label_text: str, user_id: UUID) -> None: +@celery_app.task(name="send_new_report_message") +def send_new_report_message(message: Message, label_text: str, user_id: UUID): + """ + Send a message to the Discord channel when a new message is flagged. + Note: this is a Celery task. + + Args: + message (Message): the flagged message + label_text (str): the label text + user_id (UUID): the user ID + """ if settings.DISCORD_API_KEY is None or settings.DISCORD_CHANNEL_ID is None: return diff --git a/backend/requirements.txt b/backend/requirements.txt index 3c841e8ff2..8d95299366 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -14,8 +14,8 @@ pydantic==1.10.4 pydantic[email]==1.10.4 python-dotenv==0.21.0 python-jose[cryptography]==3.3.0 -redis -requests +redis==4.5.5 +requests==2.30.0 scipy==1.8.1 SQLAlchemy==1.4.41 sqlmodel==0.0.8 diff --git a/backend/requirements_worker.txt b/backend/requirements_worker.txt index 3da8872149..a232125ee2 100644 --- a/backend/requirements_worker.txt +++ b/backend/requirements_worker.txt @@ -13,8 +13,8 @@ pydantic==1.9.1 pydantic[email]==1.9.1 python-dotenv==0.21.0 python-jose[cryptography]==3.3.0 -redis -requests +redis==4.5.5 +requests==2.30.0 scipy==1.8.1 SQLAlchemy==1.4.41 sqlmodel==0.0.8 diff --git a/backend/update_message_attributes.py b/backend/update_message_attributes.py index 8e09c92ab3..0290e029d1 100644 --- a/backend/update_message_attributes.py +++ b/backend/update_message_attributes.py @@ -2,7 +2,7 @@ from loguru import logger from oasst_backend.models import ApiClient, Message -from oasst_backend.scheduled_tasks import hf_feature_extraction, toxicity +from oasst_backend.scheduled_tasks import check_toxicity, hf_feature_extraction from oasst_backend.utils.database_utils import default_session_factory from sqlmodel import text @@ -71,7 +71,7 @@ def find_and_update_toxicity(message_ids): text = result.payload.payload.text api_client = session.query(ApiClient).filter(ApiClient.id == api_client_id).first() if api_client is not None and text is not None: - toxicity(text=text, message_id=message_id, api_client=api_client.__dict__) + check_toxicity(text=text, message_id=message_id, api_client=api_client.__dict__) # to not get rate limited from HF time.sleep(10) except Exception as e: diff --git a/docker-compose.yaml b/docker-compose.yaml index c3d1af4d58..6497af6a1b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -117,7 +117,7 @@ services: build: dockerfile: docker/Dockerfile.backend-worker context: . - command: celery -A oasst_backend.celery_worker worker -l info + command: celery -A oasst_backend.celery_worker worker -l info -E image: oasst-backend-worker environment: - CELERY_BROKER_URL=redis://redis:6379/0