Merge pull request #142 from mit-submit/main

Add sql database, feed ROOT code to A2rchi, run on HuggingFace embeddings

.github/workflows/dev-ci-cd.yaml

@@ -75,6 +75,9 @@ jobs:
           touch ${{ github.workspace }}/deploy/dev/secrets/hf_token.txt
           echo "${{ secrets.HF_TOKEN }}" >> ${{ github.workspace }}/deploy/dev/secrets/hf_token.txt
           chmod 400 ${{ github.workspace }}/deploy/dev/secrets/hf_token.txt
+          touch ${{ github.workspace }}/deploy/dev/secrets/pg_password.txt
+          echo "${{ secrets.DEV_PG_PASSWORD }}" >> ${{ github.workspace }}/deploy/dev/secrets/pg_password.txt
+          chmod 400 ${{ github.workspace }}/deploy/dev/secrets/pg_password.txt
 
       # create env file to set tag(s) for docker-compose
       - name: Create Env File
@@ -87,31 +90,25 @@ jobs:
       # stop any existing docker compose that's running
       - name: Stop Docker Compose
         run: |
-          ssh submit-t3desk 'bash -s' < ${{ github.workspace }}/deploy/dev/dev-stop.sh
+          ssh submit06 'bash -s' < ${{ github.workspace }}/deploy/dev/dev-stop.sh
 
       # copy repository to machine
       - name: Copy Repository
         run: |
-          rsync -e ssh -r ${{ github.workspace}}/* --exclude .git/ --delete submit-t3desk:~/A2rchi-dev/
+          rsync -e ssh -r ${{ github.workspace}}/* --exclude .git/ --delete submit06:~/A2rchi-dev/
 
       # run deploy script
       - name: Run Deploy Script
         run: |
           export tag="${GITHUB_REF#refs/heads/}"
           export tag="${tag//\//-}.${GITHUB_SHA}"
           sed -i "s/BASE_TAG/${tag}/" ${{ github.workspace }}/deploy/dev/dev-install.sh
-          ssh submit-t3desk 'bash -s' < ${{ github.workspace }}/deploy/dev/dev-install.sh
+          ssh submit06 'bash -s' < ${{ github.workspace }}/deploy/dev/dev-install.sh
 
       # clean up secret files
       - name: Remove Secrets from Runner
         run: |
-          rm ${{ github.workspace }}/deploy/dev/secrets/cleo_*.txt
-          rm ${{ github.workspace }}/deploy/dev/secrets/imap_*.txt
-          rm ${{ github.workspace }}/deploy/dev/secrets/sender_*.txt
-          rm ${{ github.workspace }}/deploy/dev/secrets/flask_uploader_app_secret_key.txt
-          rm ${{ github.workspace }}/deploy/dev/secrets/uploader_salt.txt
-          rm ${{ github.workspace }}/deploy/dev/secrets/openai_api_key.txt
-          rm ${{ github.workspace }}/deploy/dev/secrets/hf_token.txt
+          rm ${{ github.workspace }}/deploy/dev/secrets/*.txt
 
       # print job status
       - run: echo "🍏 This job's status is ${{ job.status }}."

.github/workflows/prod-801-ci-cd.yaml

@@ -42,6 +42,9 @@ jobs:
           touch ${{ github.workspace }}/deploy/prod-801/secrets/hf_token.txt
           echo "${{ secrets.HF_TOKEN }}" >> ${{ github.workspace }}/deploy/prod-801/secrets/hf_token.txt
           chmod 400 ${{ github.workspace }}/deploy/prod-801/secrets/hf_token.txt
+          touch ${{ github.workspace }}/deploy/prod-801/secrets/pg_password.txt
+          echo "${{ secrets.PROD_801_PG_PASSWORD }}" >> ${{ github.workspace }}/deploy/prod-801/secrets/pg_password.txt
+          chmod 400 ${{ github.workspace }}/deploy/prod-801/secrets/pg_password.txt
 
       # create env file to set tag(s) for docker-compose
       - name: Create Env File
@@ -72,10 +75,7 @@ jobs:
       # clean up secret files
       - name: Remove Secrets from Runner
         run: |
-          rm ${{ github.workspace }}/deploy/prod-801/secrets/flask_uploader_app_secret_key.txt
-          rm ${{ github.workspace }}/deploy/prod-801/secrets/uploader_salt.txt
-          rm ${{ github.workspace }}/deploy/prod-801/secrets/openai_api_key.txt
-          rm ${{ github.workspace }}/deploy/prod-801/secrets/hf_token.txt
+          rm ${{ github.workspace }}/deploy/prod-801/secrets/*.txt
 
       # print job status
       - run: echo "🍏 This job's status is ${{ job.status }}."

.github/workflows/prod-ci-cd.yaml

@@ -75,6 +75,9 @@ jobs:
           touch ${{ github.workspace }}/deploy/prod/secrets/hf_token.txt
           echo "${{ secrets.HF_TOKEN }}" >> ${{ github.workspace }}/deploy/prod/secrets/hf_token.txt
           chmod 400 ${{ github.workspace }}/deploy/prod/secrets/hf_token.txt
+          touch ${{ github.workspace }}/deploy/prod/secrets/pg_password.txt
+          echo "${{ secrets.PROD_PG_PASSWORD }}" >> ${{ github.workspace }}/deploy/prod/secrets/pg_password.txt
+          chmod 400 ${{ github.workspace }}/deploy/prod/secrets/pg_password.txt
 
       # create env file to set tag(s) for docker-compose
       - name: Create Env File
@@ -105,13 +108,7 @@ jobs:
       # clean up secret files
       - name: Remove Secrets from Runner
         run: |
-          rm ${{ github.workspace }}/deploy/prod/secrets/cleo_*.txt
-          rm ${{ github.workspace }}/deploy/prod/secrets/imap_*.txt
-          rm ${{ github.workspace }}/deploy/prod/secrets/sender_*.txt
-          rm ${{ github.workspace }}/deploy/prod/secrets/flask_uploader_app_secret_key.txt
-          rm ${{ github.workspace }}/deploy/prod/secrets/uploader_salt.txt
-          rm ${{ github.workspace }}/deploy/prod/secrets/openai_api_key.txt
-          rm ${{ github.workspace }}/deploy/prod/secrets/hf_token.txt
+          rm ${{ github.workspace }}/deploy/prod/secrets/*.txt
 
       # print job status
       - run: echo "🍏 This job's status is ${{ job.status }}."

.github/workflows/prod-root-ci-cd.yaml

@@ -42,6 +42,9 @@ jobs:
           touch ${{ github.workspace }}/deploy/prod-root/secrets/hf_token.txt
           echo "${{ secrets.HF_TOKEN }}" >> ${{ github.workspace }}/deploy/prod-root/secrets/hf_token.txt
           chmod 400 ${{ github.workspace }}/deploy/prod-root/secrets/hf_token.txt
+          touch ${{ github.workspace }}/deploy/prod-root/secrets/pg_password.txt
+          echo "${{ secrets.PROD_ROOT_PG_PASSWORD }}" >> ${{ github.workspace }}/deploy/prod-root/secrets/pg_password.txt
+          chmod 400 ${{ github.workspace }}/deploy/prod-root/secrets/pg_password.txt
 
       # create env file to set tag(s) for docker-compose
       - name: Create Env File
@@ -72,10 +75,7 @@ jobs:
       # clean up secret files
       - name: Remove Secrets from Runner
         run: |
-          rm ${{ github.workspace }}/deploy/prod-root/secrets/flask_uploader_app_secret_key.txt
-          rm ${{ github.workspace }}/deploy/prod-root/secrets/uploader_salt.txt
-          rm ${{ github.workspace }}/deploy/prod-root/secrets/openai_api_key.txt
-          rm ${{ github.workspace }}/deploy/prod-root/secrets/hf_token.txt
+          rm ${{ github.workspace }}/deploy/prod-root/secrets/*.txt
 
       # print job status
       - run: echo "🍏 This job's status is ${{ job.status }}."

.gitignore

@@ -11,3 +11,4 @@ venv
 *sqlite_db
 .vscode
 801-content/
+logos/

A2rchi/chains/chain.py

@@ -2,7 +2,6 @@
 
 from chromadb.config import Settings
 from langchain.vectorstores import Chroma
-from threading import Lock, Thread
 
 import chromadb
 import time
@@ -16,7 +15,6 @@ def __init__(self):
         into a format that the chain can use. Then, it creates the chain using 
         those documents.
         """
-        self.lock = Lock()
         self.kill = False
 
         from A2rchi.utils.config_loader import Config_Loader
@@ -55,8 +53,7 @@ def update_vectorstore_and_create_chain(self):
                 settings=Settings(allow_reset=True, anonymized_telemetry=False),  # NOTE: anonymized_telemetry doesn't actually do anything; need to build Chroma on our own without it
             )
 
-        # acquire lock and construct chain
-        self.lock.acquire()
+        # construct chain
         vectorstore = Chroma(
             client=client,
             collection_name=self.collection_name,
@@ -65,7 +62,6 @@ def update_vectorstore_and_create_chain(self):
         chain = BaseChain.from_llm(self.llm, vectorstore.as_retriever(), return_source_documents=True)
         print(f"N entries: {client.get_collection(self.collection_name).count()}")
         print("Updated chain with new vectorstore")
-        self.lock.release()
 
         return chain
 
@@ -135,10 +131,8 @@ def __call__(self, history):
         chat_history = history[:-1] if history is not None else None
 
         # make the request to the chain 
-        self.lock.acquire()
         answer = chain({"question": question, "chat_history": chat_history})
         print(f" INFO - answer: {answer}")
-        self.lock.release()
 
         # delete chain object to release chain, vectorstore, and client for garbage collection
         del chain