Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into simple-prompt-studio-crud
Browse files Browse the repository at this point in the history
tahierhussain authored Jul 27, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
2 parents 11cd25d + e1fff33 commit e4ce52a
Showing 24 changed files with 700 additions and 700 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -648,3 +648,6 @@ unstract/unstract-sdk/

# BE pluggable-apps
backend/pluggable_apps/*

# Backend requirements.txt
backend/requirements.txt
10 changes: 5 additions & 5 deletions backend/adapter_processor/adapter_processor.py
Original file line number Diff line number Diff line change
@@ -13,11 +13,11 @@
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from platform_settings.platform_auth_service import PlatformAuthenticationService
from unstract.adapters.adapterkit import Adapterkit
from unstract.adapters.base import Adapter
from unstract.adapters.enums import AdapterTypes
from unstract.adapters.exceptions import AdapterError
from unstract.adapters.x2text.constants import X2TextConstants
from unstract.sdk.adapters.adapterkit import Adapterkit
from unstract.sdk.adapters.base import Adapter
from unstract.sdk.adapters.enums import AdapterTypes
from unstract.sdk.adapters.exceptions import AdapterError
from unstract.sdk.adapters.x2text.constants import X2TextConstants

from .models import AdapterInstance, UserDefaultAdapter

6 changes: 3 additions & 3 deletions backend/adapter_processor/models.py
Original file line number Diff line number Diff line change
@@ -8,9 +8,9 @@
from django.conf import settings
from django.db import models
from django.db.models import QuerySet
from unstract.adapters.adapterkit import Adapterkit
from unstract.adapters.enums import AdapterTypes
from unstract.adapters.exceptions import AdapterError
from unstract.sdk.adapters.adapterkit import Adapterkit
from unstract.sdk.adapters.enums import AdapterTypes
from unstract.sdk.adapters.exceptions import AdapterError
from utils.models.base_model import BaseModel

ADAPTER_NAME_SIZE = 128
4 changes: 2 additions & 2 deletions backend/adapter_processor/serializers.py
Original file line number Diff line number Diff line change
@@ -9,8 +9,8 @@
from django.conf import settings
from rest_framework import serializers
from rest_framework.serializers import ModelSerializer
from unstract.adapters.constants import Common as common
from unstract.adapters.enums import AdapterTypes
from unstract.sdk.adapters.constants import Common as common
from unstract.sdk.adapters.enums import AdapterTypes

from backend.constants import FieldLengthConstants as FLC
from backend.serializers import AuditSerializer
398 changes: 188 additions & 210 deletions backend/pdm.lock

Large diffs are not rendered by default.

3 changes: 1 addition & 2 deletions backend/pyproject.toml
Original file line number Diff line number Diff line change
@@ -31,8 +31,7 @@ dependencies = [
"python-socketio==5.9.0", # For log_events
"social-auth-app-django==5.3.0", # For OAuth
"social-auth-core==4.4.2", # For OAuth
"unstract-sdk~=0.38.0",
"unstract-adapters~=0.22.0",
"unstract-sdk~=0.39.0",
# ! IMPORTANT!
# Indirect local dependencies usually need to be added in their own projects
# as: https://pdm-project.org/latest/usage/dependency/#local-dependencies.
2 changes: 1 addition & 1 deletion backend/tool_instance/tool_instance_helper.py
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@
from tool_instance.exceptions import ToolSettingValidationError
from tool_instance.models import ToolInstance
from tool_instance.tool_processor import ToolProcessor
from unstract.adapters.enums import AdapterTypes
from unstract.sdk.adapters.enums import AdapterTypes
from unstract.sdk.tool.validator import DefaultsGeneratingValidator
from unstract.tool_registry.constants import AdapterPropertyKey
from unstract.tool_registry.dto import Spec, Tool
2 changes: 1 addition & 1 deletion backend/tool_instance/tool_processor.py
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@
PromptStudioRegistryHelper,
)
from tool_instance.exceptions import ToolDoesNotExist
from unstract.adapters.enums import AdapterTypes
from unstract.sdk.adapters.enums import AdapterTypes
from unstract.tool_registry.dto import Spec, Tool
from unstract.tool_registry.tool_registry import ToolRegistry
from unstract.tool_registry.tool_utils import ToolUtils
19 changes: 17 additions & 2 deletions backend/workflow_manager/endpoint/constants.py
Original file line number Diff line number Diff line change
@@ -73,8 +73,23 @@ class FileType:

class FilePattern:
PDF_DOCUMENTS = ["*.pdf", "*.PDF"]
TEXT_DOCUMENTS = ["*.txt"]
IMAGES = ["*.jpg", "*.jpeg", "*.png", "*.gif", "*.bmp", "*.tif", "*.tiff"]
TEXT_DOCUMENTS = ["*.txt", "*.TXT"]
IMAGES = [
"*.jpg",
"*.jpeg",
"*.png",
"*.gif",
"*.bmp",
"*.tif",
"*.tiff",
"*.JPG",
"*.JPEG",
"*.PNG",
"*.GIF",
"*.BMP",
"*.TIF",
"*.TIFF",
]


class SourceConstant:
13 changes: 13 additions & 0 deletions docker/connector-db/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Docker Run

Useful while running database connectors like mysql, mariadb, mssql

**NOTE**: Copy `sample.env` into `.env` and update the necessary variables.

```bash
# Up all connector db
docker compose -f docker-compose-db-connector.yaml up -d

# Up a specific service alone
docker compose -f docker-compose-db-connector.yaml up -d mysql
```
37 changes: 37 additions & 0 deletions docker/connector-db/docker-compose-db-connector.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
version: '3.7'

services:
mariadb:
image: 'mariadb:11.2.4'
container_name: unstract-mariadb
ports:
- "3306:3306"
volumes:
- mariadb_data:/var/lib/mysql
env_file:
- .env

mysql:
image: 'mysql:9.0.1'
container_name: unstract-mysql
env_file:
- .env
ports:
- "3307:3306"
volumes:
- mysql_data:/var/lib/mysql

mssql:
image: mcr.microsoft.com/mssql/server:2022-preview-ubuntu-22.04
container_name: unstract-mssql
env_file:
- .env
ports:
- "1433:1433"
volumes:
- mssql_data:/var/opt/mssql

volumes:
mariadb_data:
mysql_data:
mssql_data:
6 changes: 6 additions & 0 deletions docker/connector-db/sample.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
MYSQL_ROOT_PASSWORD=unstract_root_pass
MYSQL_DATABASE=unstract_db
MYSQL_USER=unstract_dev
MYSQL_PASSWORD=unstract_pass
ACCEPT_EULA=Y
MSSQL_SA_PASSWORD=unstractPassword@123
4 changes: 1 addition & 3 deletions docker/docker-compose-dev-essentials.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
version: '3.7'

services:
db:
image: 'postgres:15.6'
image: 'pgvector/pgvector:pg15'
container_name: unstract-db
restart: unless-stopped
# set shared memory limit when using docker-compose
2 changes: 0 additions & 2 deletions docker/docker-compose.build.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
version: '3.7'

services:
frontend:
image: unstract/frontend:${VERSION}
1 change: 0 additions & 1 deletion docker/docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
version: '3.7'
include:
- docker-compose-dev-essentials.yaml

394 changes: 186 additions & 208 deletions pdm.lock

Large diffs are not rendered by default.

453 changes: 213 additions & 240 deletions prompt-service/pdm.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion prompt-service/pyproject.toml
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ dependencies = [
"flask~=3.0",
"llama-index==0.10.38",
"python-dotenv==1.0.0",
"unstract-sdk~=0.38.0",
"unstract-sdk~=0.39.0",
"redis>=5.0.3",
"unstract-core @ file:///${PROJECT_ROOT}/../unstract/core",
"unstract-flags @ file:///${PROJECT_ROOT}/../unstract/flags",
3 changes: 1 addition & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -40,8 +40,7 @@ hook-check-django-migrations = [
"psycopg2-binary==2.9.9",
"python-dotenv==1.0.0",
"python-magic==0.4.27",
"unstract-sdk~=0.38.0",
"unstract-adapters~=0.22.0",
"unstract-sdk~=0.39.0",
"-e unstract-connectors @ file:///${PROJECT_ROOT}/unstract/connectors",
"-e unstract-core @ file:///${PROJECT_ROOT}/unstract/core",
"-e unstract-flags @ file:///${PROJECT_ROOT}/unstract/flags",
30 changes: 17 additions & 13 deletions run-platform.sh
Original file line number Diff line number Diff line change
@@ -174,6 +174,7 @@ setup_env() {
if [ -e "$sample_env_path" ] && [ ! -e "$env_path" ]; then
first_setup=true
cp "$sample_env_path" "$env_path"

# Add encryption secret for backend and platform-service.
if [[ "$service" == "backend" || "$service" == "platform-service" ]]; then
echo -e "$blue_text""Adding encryption secret to $service""$default_text"
@@ -189,17 +190,17 @@ setup_env() {
if [[ "$OSTYPE" == "darwin"* ]]; then
sed -i '' "s/DEFAULT_AUTH_USERNAME.*/DEFAULT_AUTH_USERNAME=\"$DEFAULT_AUTH_KEY\"/" $env_path
sed -i '' "s/DEFAULT_AUTH_PASSWORD.*/DEFAULT_AUTH_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path
# sed -i '' "s/SYSTEM_ADMIN_USERNAME.*/SYSTEM_ADMIN_USERNAME=\"$DEFAULT_AUTH_KEY\"/" $env_path
# sed -i '' "s/SYSTEM_ADMIN_PASSWORD.*/SYSTEM_ADMIN_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path
# sed -i '' "s/SYSTEM_ADMIN_USERNAME.*/SYSTEM_ADMIN_USERNAME=\"$DEFAULT_AUTH_KEY\"/" $env_path
# sed -i '' "s/SYSTEM_ADMIN_PASSWORD.*/SYSTEM_ADMIN_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path
else
sed -i "s/DEFAULT_AUTH_USERNAME.*/DEFAULT_AUTH_USERNAME=\"$DEFAULT_AUTH_KEY\"/" $env_path
sed -i "s/DEFAULT_AUTH_PASSWORD.*/DEFAULT_AUTH_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path
# sed -i "s/SYSTEM_ADMIN_USERNAME.*/SYSTEM_ADMIN_USERNAME=\"$DEFAULT_AUTH_KEY\"/" $env_path
# sed -i "s/SYSTEM_ADMIN_PASSWORD.*/SYSTEM_ADMIN_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path
# sed -i "s/SYSTEM_ADMIN_USERNAME.*/SYSTEM_ADMIN_USERNAME=\"$DEFAULT_AUTH_KEY\"/" $env_path
# sed -i "s/SYSTEM_ADMIN_PASSWORD.*/SYSTEM_ADMIN_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path
fi
fi
echo -e "Created env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"."
elif [ "$opt_update" = true ]; then
elif [ "$opt_only_env" = true ] || [ "$opt_update" = true ]; then
python3 $script_dir/docker/scripts/merge_env.py $sample_env_path $env_path
if [ $? -ne 0 ]; then
exit 1
@@ -211,21 +212,14 @@ setup_env() {
if [ ! -e "$script_dir/docker/essentials.env" ]; then
cp "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env"
echo -e "Created env for ""$blue_text""essential services""$default_text"" at ""$blue_text""$script_dir/docker/essentials.env""$default_text""."
elif [ "$opt_update" = true ]; then
elif [ "$opt_only_env" = true ] || [ "$opt_update" = true ]; then
python3 $script_dir/docker/scripts/merge_env.py "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env"
if [ $? -ne 0 ]; then
exit 1
fi
echo -e "Merged env for ""$blue_text""essential services""$default_text"" at ""$blue_text""$script_dir/docker/essentials.env""$default_text""."
fi

# Not part of an upgrade.
if [ ! -e "$script_dir/docker/proxy_overrides.yaml" ]; then
echo -e "NOTE: Reverse proxy config can be overridden via ""$blue_text""$script_dir/docker/proxy_overrides.yaml""$default_text""."
else
echo -e "Found ""$blue_text""$script_dir/docker/proxy_overrides.yaml""$default_text"". ""$yellow_text""Reverse proxy config will be overridden.""$default_text"
fi

if [ "$opt_only_env" = true ]; then
echo -e "$green_text""Done.""$default_text" && exit 0
fi
@@ -279,6 +273,16 @@ run_services() {
echo -e "Configure services by updating corresponding ""$yellow_text""<service>/.env""$default_text"" files."
echo -e "Make sure to ""$yellow_text""restart""$default_text"" the services with:"
echo -e " ""$blue_text""$docker_compose_cmd -f docker/docker-compose.yaml up -d""$default_text"
if [ "$first_setup" = true ]; then
echo -e "\n###################### BACKUP ENCRYPTION KEY ######################"
echo -e "Copy the value of ""$yellow_text""ENCRYPTION_KEY""$default_text"" in any of the following env files"
echo -e "to a secure location:\n"
echo -e "- ""$red_text""backend/.env""$default_text"
echo -e "- ""$red_text""platform-service/.env""$default_text"
echo -e "\nAapter credentials are encrypted by the platform using this key."
echo -e "Its loss or change will make all existing adapters inaccessible!"
echo -e "###################################################################"
fi

popd 1>/dev/null
}
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@
"port": {
"type": "string",
"title": "Port",
"default": "",
"default": "1433",
"description": "default port: 1433"
},
"database": {
2 changes: 1 addition & 1 deletion unstract/tool-registry/pyproject.toml
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@ dependencies = [
"docker~=6.1.3",
"jsonschema~=4.18.2",
"PyYAML~=6.0.1",
"unstract-adapters~=0.22.0",
"unstract-sdk~=0.39.0",
# ! IMPORTANT!
# Local dependencies usually need to be added as:
# https://pdm-project.org/latest/usage/dependency/#local-dependencies
2 changes: 1 addition & 1 deletion unstract/tool-registry/src/unstract/tool_registry/dto.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from dataclasses import asdict, dataclass, field
from typing import Any, Optional

from unstract.adapters.enums import AdapterTypes
from unstract.sdk.adapters.enums import AdapterTypes
from unstract.tool_registry.constants import AdapterPropertyKey


Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@
from typing import Any, Optional

import yaml
from unstract.adapters.enums import AdapterTypes
from unstract.sdk.adapters.enums import AdapterTypes
from unstract.tool_registry.constants import AdapterPropertyKey, Tools
from unstract.tool_registry.dto import AdapterProperties, Spec, Tool, ToolMeta
from unstract.tool_registry.exceptions import InvalidToolURLException

0 comments on commit e4ce52a

Please sign in to comment.