feat: Merge main and fix: toggle search bar notetaker (#3258)

# Description - Merge `main` into `release/notetaker-0.1` - Fix toggling between the editor and search bar on the notetaker on button click ## Checklist before requesting a review Please delete options that are not relevant. - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [ ] I have commented hard-to-understand areas - [ ] I have ideally added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged ## Screenshots (if appropriate): --------- Co-authored-by: Antoine Dewez <[email protected]> Co-authored-by: Stan Girard <[email protected]> Co-authored-by: chloedia <[email protected]> Co-authored-by: aminediro <[email protected]> Co-authored-by: Chloé Daems <[email protected]> Co-authored-by: Zewed <[email protected]> Co-authored-by: Stan Girard <[email protected]> Co-authored-by: aminediro <[email protected]> Co-authored-by: porter-deployment-app[bot] <87230664+porter-deployment-app[bot]@users.noreply.github.com> Co-authored-by: Jacopo Chevallard <[email protected]>
QuivrHQ · Sep 25, 2024 · cec7a0d · cec7a0d
1 parent 9045fe7
commit cec7a0d
Show file tree

Hide file tree

Showing 513 changed files with 15,279 additions and 36,789 deletions.
diff --git a/.env.example b/.env.example
@@ -18,7 +18,7 @@ NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzd
 NEXT_PUBLIC_CMS_URL=https://cms.quivr.app
 NEXT_PUBLIC_FRONTEND_URL=http://localhost:*
 NEXT_PUBLIC_AUTH_MODES=password
-NEXT_PUBLIC_SHOW_TOKENS=false 
+NEXT_PUBLIC_SHOW_TOKENS=false
 #NEXT_PUBLIC_PROJECT_NAME=<project-name>
 
 
@@ -28,6 +28,7 @@ NEXT_PUBLIC_SHOW_TOKENS=false
 
 LOG_LEVEL=INFO
 SUPABASE_URL=http://host.docker.internal:54321
+EXTERNAL_SUPABASE_URL=http://localhost:54321
 SUPABASE_SERVICE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU
 PG_DATABASE_URL=postgresql://postgres:[email protected]:54322/postgres
 PG_DATABASE_ASYNC_URL=postgresql+asyncpg://postgres:[email protected]:54322/postgres
@@ -39,6 +40,7 @@ CELERY_BROKER_URL=redis://redis:6379/0
 CELEBRY_BROKER_QUEUE_NAME=quivr-preview.fifo
 QUIVR_DOMAIN=http://localhost:3000/
 BACKEND_URL=http://localhost:5050
+EMBEDDING_DIM=1536
 #COHERE_API_KEY=CHANGE_ME
 DEACTIVATE_STRIPE=true
 

diff --git a/.github/workflows/backend-core-tests.yml b/.github/workflows/backend-core-tests.yml
@@ -20,20 +20,18 @@ jobs:
           - 9998:9998
 
     steps:
-      - name: Checkout code
+      - name: 👀 Checkout code
         uses: actions/checkout@v2
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
+      - name: 🔨 Install the latest version of rye
+        uses: eifinger/setup-rye@v4
         with:
-          python-version: "3.11"
-
-      - name: Install dependencies
+          enable-cache: true
+          working-directory: backend
+      - name: 🔄 Sync dependencies
         run: |
-          cd backend/core
-          python -m pip install --upgrade pip
-          pip install poetry
-          poetry install -E base --with dev,test
+          cd backend
+          UV_INDEX_STRATEGY=unsafe-first-match rye sync --no-lock
 
       - name: Run tests
         env:
@@ -42,6 +40,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y libmagic-dev poppler-utils libreoffice tesseract-ocr  pandoc
-          tesseract --version
-          cd backend/core
-          poetry run tox
+          cd backend
+          rye run python -c "from unstructured.nlp.tokenize import download_nltk_packages; download_nltk_packages()"
+          rye run python -c "import nltk;nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')"
+          rye test -p quivr-core
diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml
@@ -1,74 +1,70 @@
-# name: Backend Tests
+name: Run Tests API and Worker
 
-# on:
-#   push:
-#     branches: [main]
-#     paths:
-#       - 'backend/**'
-#   pull_request:
-#     branches: [main]
-#     paths:
-#       - 'backend/**'
-#   workflow_dispatch:
-#     inputs:
-#       logLevel:
-#         description: 'Log level'
-#         required: true
-#         default: 'warning'
-#         type: choice
-#         options:
-#         - info
-#         - warning
-#         - debug
-#       tags:
-#         description: 'Test scenario tags'
-#         required: false
-#         type: boolean
-#       environment:
-#         description: 'Environment to run tests against'
-#         type: environment
-#         required: false
+on:
+  pull_request:
+    paths:
+      - "backend/**"
+  workflow_dispatch:
 
-# jobs:
-#   build:
-#     runs-on: ubuntu-latest
-#     defaults:
-#       run:
-#         working-directory: ./backend
-#     environment: preview
-#     strategy:
-#       matrix:
-#         python-version: ["3.11"]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        project: [quivr-api, quivr-worker]
+    steps:
+      - name: 👀 Checkout code
+        uses: actions/checkout@v2
 
-#     steps:
-#       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
-#       - name: Set up Python ${{ matrix.python-version }}
-#         uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4
-#         with:
-#           python-version: ${{ matrix.python-version }}
-#           cache: "pip"
-#       - name: Install system dependencies
-#         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
-#       - name: Install dependencies
-#         run: |
-#           python -m pip install --upgrade pip
-#           pip install pytest pytest-emoji pytest-md pytest-mock
-#           pip install pyright
-#           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-#       - name: Run pytest
-#         uses: pavelzw/pytest-action@510c5e90c360a185039bea56ce8b3e7e51a16507 # v2
-#         with:
-#           verbose: true
-#           emoji: true
-#           job-summary: true
-#           click-to-expand: false
-#           report-title: 'Test Report'
-#         env:
-#           SUPABASE_URL: ${{secrets.SUPABASE_URL}}
-#           SUPABASE_SERVICE_KEY: ${{secrets.SUPABASE_SERVICE_KEY}}
-#           OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
-#           ANTHROPIC_API_KEY: ${{secrets.ANTHROPIC_API_KEY}}
-#           JWT_SECRET_KEY: ${{secrets.JWT_SECRET_KEY}}
-#           CI_TEST_API_KEY: ${{secrets.CI_TEST_API_KEY}}
-#           CELERY_BROKER_URL: ${{secrets.CELERY_BROKER_URL}}
+      - name: 🔨 Install the latest version of rye
+        uses: eifinger/setup-rye@v4
+        with:
+          enable-cache: true
+          working-directory: backend
 
+      - name: 🔄 Sync dependencies
+        run: |
+          cd backend
+          UV_INDEX_STRATEGY=unsafe-first-match rye sync --no-lock
+
+      - name: 🚤 Install Supabase CLI
+        run: |
+          ARCHITECTURE=$(uname -m)
+          if [ "$ARCHITECTURE" = "x86_64" ]; then
+            wget https://github.com/supabase/cli/releases/download/v1.163.6/supabase_1.163.6_linux_amd64.deb
+            sudo dpkg -i supabase_1.163.6_linux_amd64.deb
+          elif [ "$ARCHITECTURE" = "aarch64" ]; then
+            wget https://github.com/supabase/cli/releases/download/v1.163.6/supabase_1.163.6_linux_arm64.deb
+            sudo dpkg -i supabase_1.163.6_linux_arm64.deb
+          fi
+
+      - name: 😭 Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libmagic-dev poppler-utils libreoffice tesseract-ocr pandoc
+
+      - name: Install dependencies and run tests
+        env:
+          OPENAI_API_KEY: this-is-a-fake-openai-api-key
+          SUPABASE_URL: http://localhost:54321
+          SUPABASE_SERVICE_KEY: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU
+          PG_DATABASE_URL: postgresql://postgres:postgres@localhost:54322/postgres
+          PG_DATABASE_ASYNC_URL: postgresql+asyncpg://postgres:postgres@localhost:54322/postgres
+          ANTHROPIC_API_KEY: null
+          JWT_SECRET_KEY: super-secret-jwt-token-with-at-least-32-characters-long
+          AUTHENTICATE: true
+          TELEMETRY_ENABLED: true
+          CELERY_BROKER_URL: redis://redis:6379/0
+          CELEBRY_BROKER_QUEUE_NAME: quivr-preview.fifo
+          QUIVR_DOMAIN: http://localhost:3000/
+          BACKEND_URL: http://localhost:5050
+          EMBEDDING_DIM: 1536
+        run: |
+          cd backend
+          sed -i 's/enabled = true/enabled = false/g' supabase/config.toml
+          sed -i '/\[db\]/,/\[.*\]/s/enabled = false/enabled = true/' supabase/config.toml
+          sed -i '/\[storage\]/,/\[.*\]/s/enabled = false/enabled = true/' supabase/config.toml
+          supabase start
+          rye run python -c "from unstructured.nlp.tokenize import download_nltk_packages; download_nltk_packages()"
+          rye run python -c "import nltk;nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')"
+          rye test -p ${{ matrix.project }}
diff --git a/.github/workflows/porter_stack_raise-frontend.yml b/.github/workflows/porter_stack_raise-frontend.yml
@@ -1,7 +1,7 @@
 on:
   push:
     tags:
-      - "v*"
+      - 'v*'
 name: Deploy to raise-frontend
 jobs:
   porter-deploy:

diff --git a/.github/workflows/porter_stack_raise.yml b/.github/workflows/porter_stack_raise.yml
@@ -1,7 +1,7 @@
 on:
   push:
     tags:
-      - "v*"
+      - 'v*'
 name: Deploy to raise
 jobs:
   porter-deploy:

diff --git a/.github/workflows/release-please-core.yml b/.github/workflows/release-please-core.yml
@@ -40,26 +40,17 @@ jobs:
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: backend/core
+        working-directory: backend/
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v3
+      - name: Install Rye
+        uses: eifinger/setup-rye@v2
         with:
-          python-version: '3.11'
-      - name: Check working directory
-        run: pwd
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install poetry
-      - name: Install project dependencies
-        run: poetry install
-      - name: Build package
-        run: poetry build
-      - name: Publish package
-        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          packages_dir: backend/core/dist
+          enable-cache: true
+          working-directory: backend/
+      - name: Rye Sync
+        run: cd core/ && UV_INDEX_STRATEGY=unsafe-first-match rye sync --no-lock
+      - name: Rye Build
+        run: cd core/ && rye build
+      - name: Rye Publish
+        run: rye publish --token ${{ secrets.PYPI_API_TOKEN }} --yes --skip-existing
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 docker-compose.override.yml
 secondbrain/
 .env
+env.sh
 .streamlit/secrets.toml
 **/*.pyc
 toto.txt
@@ -18,6 +19,8 @@ backend/.python-version
 **/.pnp
 .pnp.js
 
+Pipfile
+
 # testing
 **/coverage
 
@@ -59,7 +62,7 @@ backend/core/local_models/*
 
 ## scripts
 package-lock.json
-backend/celerybeat-schedule
+celerybeat-schedule
 frontend/public/robots.txt
 frontend/public/sitemap*
 
@@ -98,3 +101,5 @@ backend/core/examples/chatbot/.chainlit/translations/en-US.json
 
 # Tox
 .tox
+Pipfile
+*.pkl
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -32,13 +32,6 @@ repos:
       - id: mypy
         name: mypy
         additional_dependencies: ["types-aiofiles"]
-  - repo: https://github.com/python-poetry/poetry
-    rev: "1.8.0"
-    hooks:
-      - id: poetry-check
-        args: ["-C", "./backend/core"]
-      - id: poetry-lock
-        args: ["-C", "./backend/core"]
 ci:
   autofix_commit_msg: |
     [pre-commit.ci] auto fixes from pre-commit.com hooks

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -0,0 +1,25 @@
+# Read the Docs configuration file for MkDocs projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+  commands:
+    - asdf plugin add uv
+    - asdf install uv latest
+    - asdf global uv latest
+    - uv venv
+    - cd backend && UV_INDEX_STRATEGY=unsafe-first-match uv pip install -r requirements.lock
+    - cd backend/ && ls -la && NO_COLOR=1 ../.venv/bin/mkdocs build --strict --site-dir $READTHEDOCS_OUTPUT/html --config-file docs/mkdocs.yml
+
+
+
+
+mkdocs:
+  configuration: backend/docs/mkdocs.yml
+
diff --git a/.release-please-manifest.json b/.release-please-manifest.json
@@ -1,4 +1,4 @@
 {
-    "backend/core": "0.0.13",
-    ".": "0.0.300"
+    "backend/core": "0.0.16",
+    ".": "0.0.315"
 }