Merge pull request #14 from decisionfacts/db-connector-semantic-ai

Db structure db - semantic ai

Author: SyedHabibM

.env

@@ -4,36 +4,50 @@ EXTRACTED_DIR_PATH=
 
 # Embeddings
 EMBEDDING_TYPE=
-EMBED__MODEL_NAME=
+EMBED_MODEL_NAME=
 
 # Connector
 CONNECTOR_TYPE=
-SHAREPOINT__CLIENT_ID=
-SHAREPOINT__CLIENT_SECRET=
-SHAREPOINT__TENANT_ID=
-SHAREPOINT__HOST_NAME=
-SHAREPOINT__SCOPE=
-SHAREPOINT__SITE_ID=
-SHAREPOINT__DRIVE_ID=
-SHAREPOINT__FOLDER_URL=
+SHAREPOINT_CLIENT_ID=
+SHAREPOINT_CLIENT_SECRET=
+SHAREPOINT_TENANT_ID=
+SHAREPOINT_HOST_NAME=
+SHAREPOINT_SCOPE=
+SHAREPOINT_SITE_ID=
+SHAREPOINT_DRIVE_ID=
+SHAREPOINT_FOLDER_URL=
 
 # Indexer
 INDEXER_TYPE=
 
 # Elasticsearch
-ELASTICSEARCH__URL=
-ELASTICSEARCH__USER=
-ELASTICSEARCH__PASSWORD=
-ELASTICSEARCH__INDEX_NAME=
-ELASTICSEARCH__SSL_VERIFY=False
+ELASTICSEARCH_URL=
+ELASTICSEARCH_USER=
+ELASTICSEARCH_PASSWORD=
+ELASTICSEARCH_INDEX_NAME=
+ELASTICSEARCH_SSL_VERIFY=False
 
 # Qdrant
-QDRANT__URL=
-QDRANT__INDEX_NAME=
-QDRANT__API_KEY=
+QDRANT_URL=
+QDRANT_INDEX_NAME=
+QDRANT_API_KEY=
 
+# IBM
+IBM_URL=
+IBM_API_KEY=
+IBM_PROJECT_ID=
 
 # LLM
-LLM__MODEL=
-LLM__MODEL_NAME_OR_PATH=
-OPENAI_API_KEY=
+LLM_MODEL=
+LLM_MODEL_NAME_OR_PATH=
+OPENAI_API_KEY=
+
+# SQL
+SQLITE_SQL_PATH=''
+
+# MYSQL
+MYSQL_HOST=
+MYSQL_USER=
+MYSQL_PASSWORD=
+MYSQL_DATABASE=
+MYSQL_PORT=""

README.md

@@ -28,32 +28,43 @@ EXTRACTED_DIR_PATH= # default directory name 'extracted_dir'
 
 # Connector (SharePoint, S3, GCP Bucket, GDrive, Confluence etc.,)
 CONNECTOR_TYPE="connector_name" # sharepoint
-SHAREPOINT__CLIENT_ID="client_id"
-SHAREPOINT__CLIENT_SECRET="client_secret"
-SHAREPOINT__TENANT_ID="tenant_id"
-SHAREPOINT__HOST_NAME='<tenant_name>.sharepoint.com'
-SHAREPOINT__SCOPE='https://graph.microsoft.com/.default'
-SHAREPOINT__SITE_ID="site_id"
-SHAREPOINT__DRIVE_ID="drive_id"
-SHAREPOINT__FOLDER_URL="folder_url" # /My_folder/child_folder/
+SHAREPOINT_CLIENT_ID="client_id"
+SHAREPOINT_CLIENT_SECRET="client_secret"
+SHAREPOINT_TENANT_ID="tenant_id"
+SHAREPOINT_HOST_NAME='<tenant_name>.sharepoint.com'
+SHAREPOINT_SCOPE='https://graph.microsoft.com/.default'
+SHAREPOINT_SITE_ID="site_id"
+SHAREPOINT_DRIVE_ID="drive_id"
+SHAREPOINT_FOLDER_URL="folder_url" # /My_folder/child_folder/
 
 # Indexer
 INDEXER_TYPE="vector_db_name" # elasticsearch, qdrant
-ELASTICSEARCH__URL="elasticsearch_url" # give valid url
-ELASTICSEARCH__USER="elasticsearch_user" # give valid user
-ELASTICSEARCH__PASSWORD="elasticsearch_password" # give valid password
-ELASTICSEARCH__INDEX_NAME="index_name"
-ELASTICSEARCH__SSL_VERIFY="ssl_verify" # True or False
+ELASTICSEARCH_URL="elasticsearch_url" # give valid url
+ELASTICSEARCH_USER="elasticsearch_user" # give valid user
+ELASTICSEARCH_PASSWORD="elasticsearch_password" # give valid password
+ELASTICSEARCH_INDEX_NAME="index_name"
+ELASTICSEARCH_SSL_VERIFY="ssl_verify" # True or False
 
 # Qdrant
-QDRANT__URL="<qdrant_url>"
-QDRANT__INDEX_NAME="<index_name>"
-QDRANT__API_KEY="<apikey>"
+QDRANT_URL="<qdrant_url>"
+QDRANT_INDEX_NAME="<index_name>"
+QDRANT_API_KEY="<apikey>"
 
 # LLM
-LLM__MODEL="<llm_model>" # llama, openai
-LLM__MODEL_NAME_OR_PATH="" # model name
+LLM_MODEL="<llm_model>" # llama, openai
+LLM_MODEL_NAME_OR_PATH="" # model name
 OPENAI_API_KEY="<openai_api_key>" # if using openai
+
+# SQL
+SQLITE_SQL_PATH="<database_path>" # sqlit db path
+
+# MYSQL
+MYSQL_HOST="<host_name>" # localhost or Ip Address
+MYSQL_USER="<user_name>"
+MYSQL_PASSWORD="<password>"
+MYSQL_DATABASE="<database_name>"
+MYSQL_PORT="<port>" # default port is 3306
+
 ```
 Method 1:
     To load the .env file. Env file should have the credentials
@@ -72,6 +83,7 @@ from semantic_ai.config import Settings
 settings = Settings()
 ```
 
+# Un-Structure 
 ### 1. Import the module
 ```python
 import asyncio
@@ -106,12 +118,53 @@ SCOPE = 'https://graph.microsoft.com/.default'  # scope
 HOST_NAME = "<tenant_name>.sharepoint.com"  # for example 'contoso.sharepoint.com'
 
 # Sharepoint object creation
-connection = Sharepoint(client_id=CLIENT_ID,
-                        client_secret=CLIENT_SECRET,
-                        tenant_id=TENANT_ID,
-                        host_name=HOST_NAME,
-                        scope=SCOPE)
+connection = Sharepoint(
+    client_id=CLIENT_ID,
+    client_secret=CLIENT_SECRET,
+    tenant_id=TENANT_ID,
+    host_name=HOST_NAME,
+    scope=SCOPE
+)
+```
+
+# Structure
+
+### 1. Import the module
+```python
+import asyncio
+import semantic_ai
+```
+
+### 2. The database connection  
+
+#### Sqlite:
+```
+from semantic_ai.connectors import Sqlite
+
+file_path= <database_file_path>
+
+sql = Sqlite(sql_path=file_path)
+```
+
+#### Mysql:
 ```
+from semantic_ai.connectors import Mysql
+
+sql = Mysql(
+    host=<host_name>,
+    user=<user_name>,
+    password=<password>,
+    database=<database>,
+    port=<port_number>  # 3306 is default port
+)
+```
+
+### 3. To generate the answer from db using retrieval LLM model.
+```
+query = ""
+search_obj = await semantic_ai.db_search(query=query)
+```
+
 ## Run in the server
 ```shell
 $ semantic_ai serve -f .env

docs/source/_static/images/logo/NLP.png

@@ -7,4 +7,6 @@ Connectors
    :maxdepth: 2
    :hidden:
 
-   sharepoint
+   sharepoint
+   sqlite
+   mysql

docs/source/_static/images/logo/Sqlite.jpeg

@@ -0,0 +1,57 @@
+MYSQL
+=====
+
+.. figure:: https://github.com/decisionfacts/semantic-ai/blob/master/docs/source/_static/images/logo/mysql.png?raw=true
+   :alt: Logo
+   :align: left
+   :width: 100px
+   :target: https://www.office.com/
+
+This documents covers how to interact with Mysql database using our library.
+
+**Pre-requisites:**
+
+    Please make sure the credentials of Mysql DB.
+
+**Setup:**
+
+**To create a mysql connection using credentials**
+
+    Please setup the value to the following environment object
+
+.. code-block:: python
+
+    MYSQL_HOST='<host>'
+    MYSQL_USER='<user>'
+    MYSQL_PASSWORD='<password>'
+    MYSQL_DATABASE='<database>'
+    MYSQL_PORT='<port>'
+
+.. code-block:: python
+
+    from semantic_ai.connectors import Mysql
+
+    conn = Mysql(
+        host='<host>',
+        user='<user_name>',
+        password='<password>',
+        database='<database>',
+        port="<6033>" # it's default port
+    )
+    cursor = await conn.connect_db()
+
+**To generate the sql query from NLP text**
+
+.. code-block:: python
+
+    from semantic_ai.nlp.prompt import Prompt
+
+    prompt = Prompt()
+    prompt_res = await prompt.nlp_to_sql(data_base=cursor, normal_text="query")
+
+**To get answer from NLP**
+
+.. code-block:: python
+
+    data = json.loads(prompt_res)
+    result = sql.execution(data)

docs/source/_static/images/logo/mysql.png

@@ -0,0 +1,48 @@
+SQLITE
+======
+
+.. figure:: https://github.com/decisionfacts/semantic-ai/blob/master/docs/source/_static/images/logo/Sqlite.jpeg?raw=true
+   :alt: Logo
+   :align: left
+   :width: 100px
+
+SQLite is an in-process library that implements a self-contained, serverless, zero-configuration, transactional SQL database engine.
+SQLite is an embedded SQL database engine. Unlike most other SQL databases, SQLite does not have a separate server process. SQLite reads and writes directly to ordinary disk files.
+
+This documents covers how to interact with SQLITE database using our library.
+
+**Pre-requisites:**
+
+    Please make sure the accessible Sqlite DB path.
+
+**Setup:**
+
+**To create a sqlite connection using path**
+
+.. code-block:: python
+
+    SQLITE_SQL_PATH = '<DB_PATH>'
+
+.. code-block:: python
+
+    from semantic_ai.connectors import Sqlite
+
+    file_path = f'<DB_PATH>'
+    sqlite = Sqlite(sql_path=file_path)
+    conn = await sqlite.connect_db()
+
+**To create natural language processing for normal text**
+
+.. code-block:: python
+
+    from semantic_ai.nlp.prompt import Prompt
+
+    prompt = Prompt()
+    prompt_res = await prompt.nlp_to_sql(data_base=conn, normal_text="query")
+
+**To get answer from NLP**
+
+.. code-block:: python
+
+    data = json.loads(prompt_res)
+    result = sql.execution(data)

docs/source/connectors.rst

@@ -26,4 +26,6 @@ Library Documentation
    vector_indexer
    llm
    llm/index
+   nlp
+   nlp/index
    semantic_search

docs/source/connectors/index.rst

@@ -1,6 +1,8 @@
 LLM Models
 ==========
 
+A Large Language Model (LLM) typically refers to advanced natural language processing models that are characterized by their size and complexity. These models are designed to understand and generate human-like text at a sophisticated level
+
 
 .. toctree::
    :maxdepth: 2

docs/source/connectors/mysql.rst

@@ -0,0 +1,10 @@
+NLP to Text
+===========
+
+    Natural language processing (NLP) is a machine learning technology that gives computers the ability to interpret, manipulate, and comprehend human language.
+
+.. toctree::
+   :maxdepth: 2
+   :hidden:
+
+   prompt

docs/source/connectors/sqlite.rst

@@ -0,0 +1,38 @@
+Natural Language Processing
+===========================
+
+.. figure:: https://github.com/decisionfacts/semantic-ai/blob/master/docs/source/_static/images/logo/NLP.png?raw=true
+   :alt: Logo
+   :align: left
+   :width: 100px
+   :target: https://www.office.com/
+
+Natural language processing (NLP) is a machine learning technology that gives computers the ability to interpret,
+manipulate, and comprehend human language.
+
+**Prompt:**
+
+    An artificial intelligence (AI) prompt is a mode of interaction between a human and a large language model that lets
+the model generate the intended output. This interaction can be in the form of a question, text, code snippets or examples.
+
+**Normal text to SQL:**
+
+.. code-block:: python
+
+    async def nlp_to_sql(
+        self,
+        data_base: SQLDatabase,
+        normal_text: str,
+        prompt: str = None
+   ):
+
+
+The `nlp_to_sql()` function is used for generate the sql query from the normal text. It's return the sql response
+object. The function have three params first one is for get Database object next one is for get user query text and
+the last one for get prompt
+
+.. code-block:: python
+
+    get_llm_chain()
+
+i the prompt value is `None` then the default prompt will be apply.