Merge pull request #1464 from swirlai/main_prerelease_3_9_0

Main prerelease 3 9 0
swirlai · Nov 13, 2024 · 52086fc · 52086fc
2 parents 45c64f6 + 8cbd8e9
commit 52086fc
Show file tree

Hide file tree

Showing 27 changed files with 384 additions and 155 deletions.
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -14,8 +14,8 @@ jobs:
           - name: Login to Docker Hub
             uses: docker/login-action@v3
             with:
-              username: ${{ secrets.DOCKER_USERNAME }}
-              password: ${{ secrets.DOCKER_PASSWORD }}
+              username: ${{ secrets.SBS_DOCKER_USER }}
+              password: ${{ secrets.SBS_DOCKER_PAT }}
           - name: Build and Push Swirl Docker Image
             run: |
               BRANCH_NAME=${GITHUB_REF#refs/heads/}
@@ -25,8 +25,8 @@ jobs:
           - name: Update the Docker Repo Description
             uses: peter-evans/dockerhub-description@v4
             with:
-              username: ${{ secrets.DOCKER_USERNAME_X }}
-              password: ${{ secrets.DOCKER_PASSWORD_X }}
+              username: ${{ secrets.SBS_DOCKER_USER }}
+              password: ${{ secrets.SBS_DOCKER_PAT }}
               repository: swirlai/swirl-search
           - name: Upload Log Files
             if: always()

diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml
@@ -23,19 +23,19 @@ jobs:
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+          username: ${{ secrets.SBS_DOCKER_USER }}
+          password: ${{ secrets.SBS_DOCKER_PAT }}
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12.5'
+          python-version: '3.12.6'
           cache: 'pip'
       - name: Install Swirl
         run: ./install.sh
       - name: Setup Swirl
         run: python swirl.py setup
       - name: Install the Latest Galaxy UI
-        run: ./install-ui.sh
+        run: ./install-ui.sh -p
         env:
           MSAL_CB_PORT: 8000
           MSAL_HOST: localhost
@@ -46,17 +46,30 @@ jobs:
           python swirl.py start
         env:
           ALLOWED_HOSTS: localhost,host.docker.internal
+      - name: Decode BigQuery token and create JSON file
+        run: |
+          echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json"
+        env:
+          BIGQUERY_TOKEN_BASE64: ${{ secrets.QA_BIGQUERY_TOKEN_BASE64 }}
       - name: Run the QA Suite
         run: |
           echo "SWIRL_TEST_HOST=localhost" > .env.qa
           echo "MSAL_HOST=localhost" >> .env.qa
           echo "MSAL_CB_PORT=8000" >> .env.qa
           echo "QA_ADMIN_PW=${{ secrets.QA_ADMIN_PW }}" >> .env.qa
           echo "QA_OPENAI_KEY=${{ secrets.QA_OPENAI_KEY }}" >> .env.qa
+          echo "QA_NLR_USERNAME=${{ secrets.QA_NLR_USERNAME }}" >> .env.qa
+          echo "QA_NLR_PASSWORD=${{ secrets.QA_NLR_PASSWORD }}" >> .env.qa
+          echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa
+          echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa
+          echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa
+          echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa
+          echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> .env.qa
+          echo "QA_TRELLO_KEYS=${{ secrets.QA_TRELLO_KEYS }}" >> .env.qa
           echo "========"
           cat .env.qa
           echo "========"
-          docker run --net=host --env-file .env.qa -t swirlai/swirl-search-qa:automated-tests-master sh -c "behave --tags=qa_suite,community"
+          docker run --net=host --env-file .env.qa -t swirlai/swirl-search-qa:automated-tests-develop sh -c "behave --tags=qa_suite,community"
       - name: Upload Log Files
         if: always()
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/spell-checker.yml b/.github/workflows/spell-checker.yml
@@ -2,14 +2,8 @@ name: Check Spelling
 
 # Trigger to only run this workflow automatically on docs/ directory changes
 on:
-  push:
-    branches:
-      - "main"
-    paths:
-      - "docs/**"
-
   pull_request:
-    # Run for all PRs to develop - means PR cannot merge until unit tests pass
+    # Run for all PRs to develop or main - means PR cannot merge until unit tests pass
     branches:
       - develop
       - main

diff --git a/.github/workflows/test-build-pipeline.yml b/.github/workflows/test-build-pipeline.yml
@@ -20,7 +20,7 @@ jobs:
         - name: Set Up Python
           uses: actions/setup-python@v5
           with:
-            python-version: '3.12.5'
+            python-version: '3.12.6'
             cache: 'pip'
         - name: Install Swirl
           run: ./install.sh
@@ -48,12 +48,12 @@ jobs:
         - name: Login to Docker Hub
           uses: docker/login-action@v3
           with:
-            username: ${{ secrets.DOCKER_USERNAME }}
-            password: ${{ secrets.DOCKER_PASSWORD }}
+            username: ${{ secrets.SBS_DOCKER_USER }}
+            password: ${{ secrets.SBS_DOCKER_PAT }}
         - name: Set Up Python
           uses: actions/setup-python@v5
           with:
-            python-version: '3.12.5'
+            python-version: '3.12.6'
             cache: 'pip'
         - name: Install Swirl
           run: ./install.sh
@@ -71,13 +71,26 @@ jobs:
             python swirl.py start
           env:
             ALLOWED_HOSTS: localhost,host.docker.internal
+        - name: Decode BigQuery token and create JSON file
+          run: |
+            echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json"
+          env:
+            BIGQUERY_TOKEN_BASE64: ${{ secrets.QA_BIGQUERY_TOKEN_BASE64 }}
         - name: Run the QA Suite
           run: |
             echo "SWIRL_TEST_HOST=localhost" > .env.qa
             echo "MSAL_HOST=localhost" >> .env.qa
             echo "MSAL_CB_PORT=8000" >> .env.qa
             echo "QA_ADMIN_PW=${{ secrets.QA_ADMIN_PW }}" >> .env.qa
             echo "QA_OPENAI_KEY=${{ secrets.QA_OPENAI_KEY }}" >> .env.qa
+            echo "QA_NLR_USERNAME=${{ secrets.QA_NLR_USERNAME }}" >> .env.qa
+            echo "QA_NLR_PASSWORD=${{ secrets.QA_NLR_PASSWORD }}" >> .env.qa
+            echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa
+            echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa
+            echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa
+            echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa
+            echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> .env.qa
+            echo "QA_TRELLO_KEYS=${{ secrets.QA_TRELLO_KEYS }}" >> .env.qa
             echo "========"
             cat .env.qa
             echo "========"
@@ -92,8 +105,8 @@ jobs:
         - name: Login to Docker Hub
           uses: docker/login-action@v3
           with:
-            username: ${{ secrets.DOCKER_USERNAME }}
-            password: ${{ secrets.DOCKER_PASSWORD }}
+            username: ${{ secrets.SBS_DOCKER_USER }}
+            password: ${{ secrets.SBS_DOCKER_PAT }}
         - name: Build and Push Swirl Docker Image
           run: |
             BRANCH_NAME=${GITHUB_REF#refs/heads/}
@@ -103,8 +116,8 @@ jobs:
         - name: Update the Docker Repo Description
           uses: peter-evans/dockerhub-description@v4
           with:
-            username: ${{ secrets.DOCKER_USERNAME_X }}
-            password: ${{ secrets.DOCKER_PASSWORD_X }}
+            username: ${{ secrets.SBS_DOCKER_USER }}
+            password: ${{ secrets.SBS_DOCKER_PAT }}
             repository: swirlai/swirl-search
         - name: Upload Log Files
           if: always()

diff --git a/.github/workflows/testing-wip.yml b/.github/workflows/testing-wip.yml
@@ -36,12 +36,12 @@ jobs:
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+          username: ${{ secrets.SBS_DOCKER_USER }}
+          password: ${{ secrets.SBS_DOCKER_PAT }}
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12.5'
+          python-version: '3.12.6'
           cache: 'pip'
       - name: Install Swirl
         run: ./install.sh
@@ -59,13 +59,26 @@ jobs:
           python swirl.py start
         env:
           ALLOWED_HOSTS: localhost,host.docker.internal
+      - name: Decode BigQuery token and create JSON file
+        run: |
+          echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json"
+        env:
+          BIGQUERY_TOKEN_BASE64: ${{ secrets.QA_BIGQUERY_TOKEN_BASE64 }}
       - name: Run the QA Suite
         run: |
           echo "SWIRL_TEST_HOST=localhost" > .env.qa
           echo "MSAL_HOST=localhost" >> .env.qa
           echo "MSAL_CB_PORT=8000" >> .env.qa
           echo "QA_ADMIN_PW=${{ secrets.QA_ADMIN_PW }}" >> .env.qa
           echo "QA_OPENAI_KEY=${{ secrets.QA_OPENAI_KEY }}" >> .env.qa
+          echo "QA_NLR_USERNAME=${{ secrets.QA_NLR_USERNAME }}" >> .env.qa
+          echo "QA_NLR_PASSWORD=${{ secrets.QA_NLR_PASSWORD }}" >> .env.qa
+          echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa
+          echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa
+          echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa
+          echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa
+          echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> .env.qa
+          echo "QA_TRELLO_KEYS=${{ secrets.QA_TRELLO_KEYS }}" >> .env.qa
           echo "========"
           cat .env.qa
           echo "========"

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -31,7 +31,7 @@ jobs:
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12.5'
+          python-version: '3.12.6'
           cache: 'pip'
       - name: Install Swirl
         run: ./install.sh

diff --git a/.gitignore b/.gitignore
@@ -130,7 +130,6 @@ dmypy.json
 
 /static
 
-swirl/migrations/
 
 # emacs
 *~

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.12.5-slim-bookworm
+FROM python:3.12.6-slim-bookworm
 
 # Update, upgrade and install packages in a single RUN to reduce layers
 RUN apt-get update && apt-get install -y \

diff --git a/SearchProviders/atlassian.json b/SearchProviders/atlassian.json
@@ -22,7 +22,7 @@
         "tags": [
             "Jira",
             "Atlassian",
-            "Internal"
+            "Dev"
         ]
     },
     {
@@ -48,7 +48,7 @@
         "tags": [
             "Confluence",
             "Atlassian",
-            "Internal"
+            "Dev"
         ]
     },
     {

diff --git a/SearchProviders/company_data_bigquery.json b/SearchProviders/company_data_bigquery.json
@@ -0,0 +1,31 @@
+{
+    "name": "Company Data - BigQuery",
+    "active": false,
+    "default": false,
+    "connector": "BigQuery",
+    "url": "",
+    "query_template": "select {fields} from `{table}` where search({field1}, '{query_string}') or search({field2}, '{query_string}') or search({field3}, '{query_string}');",
+    "query_template_json": {},
+    "post_query_template": {},
+    "http_request_headers": {},
+    "page_fetch_config_json": {},
+    "query_processors": [
+        "AdaptiveQueryProcessor"
+    ],
+    "query_mappings": "fields=*,sort_by_date=year_founded,table=company_dataset.company,field1=name,field2=domain,field3=locality",
+    "result_grouping_field": "",
+    "result_processors": [
+        "MappingResultProcessor",
+        "CosineRelevancyResultProcessor"
+    ],
+    "response_mappings": "",
+    "result_mappings": "title=name,body='{name} was founded in {year_founded} and serves the {industry} industry. The company is located in {locality} and has approximately {current_employee_estimate} employees. The registered domain for this organization is: {domain}',url='https://www.{linkedin_url}',NO_PAYLOAD",
+    "results_per_query": 10,
+    "credentials": "/path/to/bigquery/token.json",
+    "eval_credentials": "",
+    "tags": [
+        "Company",
+        "BigQuery",
+        "Internal"
+    ]
+}
diff --git a/SearchProviders/elasticsearch.json b/SearchProviders/elasticsearch.json
@@ -13,7 +13,7 @@
         "MappingResultProcessor",
         "CosineRelevancyResultProcessor"
     ],
-    "result_mappings": "url=_source.url,date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD",
+    "result_mappings": "url='https://<your-elastic-instance>/email/_doc/{_id}',date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD",
     "credentials": "verify_certs=[True|False],ca_certs=/path/to/cert/file.crt,username:password",
     "tags": [
         "Enron",

diff --git a/SearchProviders/funding_db_bigquery.json b/SearchProviders/funding_db_bigquery.json
diff --git a/SearchProviders/github.json b/SearchProviders/github.json
@@ -28,7 +28,7 @@
             "GitHub",
             "Code",
             "Internal",
-            "Development"
+            "Dev"
         ]
     },
     {
@@ -60,7 +60,7 @@
             "GitHub",
             "Issues",
             "Internal",
-            "Development"
+            "Dev"
         ]
     },
     {
@@ -92,7 +92,7 @@
             "GitHub",
             "PullRequests",
             "Internal",
-            "Development"
+            "Dev"
         ]
     },
     {
@@ -124,7 +124,7 @@
             "GitHub",
             "Commits",
             "Internal",
-            "Development"
+            "Dev"
         ]
     }
 ]
diff --git a/SearchProviders/google.json b/SearchProviders/google.json
@@ -152,7 +152,8 @@
         "credentials": "key=AIzaSyDvVeE-L6nCC9u-TTGuhggvSmzhtiTHJsA",
         "eval_credentials": "",
         "tags": [
-            "Swirl"
+            "Swirl",
+            "Dev"
         ]
     }
 ]
diff --git a/SearchProviders/hacker_news.json b/SearchProviders/hacker_news.json
@@ -26,7 +26,7 @@
         "tags": [
             "HackerNews",
             "Stories",
-            "Development"
+            "Dev"
         ]
     },
     {
@@ -56,7 +56,7 @@
         "tags": [
             "HackerNews",
             "Comments",
-            "Development"
+            "Dev"
         ]
     }
 

diff --git a/SearchProviders/microsoft.json b/SearchProviders/microsoft.json
@@ -13,8 +13,8 @@
         "query_mappings": "NOT=true,NOT_CHAR=-",
         "result_grouping_field": "conversationId",
         "result_processors": [
-            "MappingResultProcessor",
             "DedupeByFieldResultProcessor",
+            "MappingResultProcessor",
             "CosineRelevancyResultProcessor"
         ],
         "response_mappings": "",
@@ -75,7 +75,7 @@
             "CosineRelevancyResultProcessor"
         ],
         "response_mappings": "",
-        "result_mappings": "title=resource.name,body='{resource.name} - {summary}',date_published=resource.createdDateTime,url=resource.webUrl,author=resource.createdBy.user.displayName,resource.lastModifiedBy.user.displayName,resource.lastModifiedDateTime,FILE_SYSTEM,NO_PAYLOAD",
+        "result_mappings": "title=resource.name,body='{resource.name} - {summary}',date_published=resource.createdDateTime,url=resource.webUrl,author=resource.createdBy.user.displayName,resource.lastModifiedBy.user.displayName,resource.lastModifiedDateTime,NO_PAYLOAD",
         "results_per_query": 10,
         "credentials": "",
         "eval_credentials": "",
-Original file line number
+Diff line change
@@ Expand Up / @@ -130,7 +130,6 @@ dmypy.json @@
     /static
-    swirl/migrations/
     # emacs
     *~
@@ Expand Down @@