public_llama3_1_rag_bot.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "497c62f6-b23c-4af1-aba3-07867c53b2c2",
      "metadata": {
        "id": "497c62f6-b23c-4af1-aba3-07867c53b2c2"
      },
      "source": [
        "# 🤖 LLAMA 3.1 RAG implementation 📃\n",
        "#### but before that you need some API keys, both are necessary for the working of the LLAMA 3.1 with RAG\n",
        "\n",
        "#### get your api keys from here :\n",
        "\n",
        "*   LLAMA_PARSER_KEY : https://docs.cloud.llamaindex.ai/llamaparse/getting_started/get_an_api_key\n",
        "\n",
        "*   HF_TOKEN : https://huggingface.co/docs/hub/security-tokens\n",
        "\n",
        "* TELEGRAM_TKN : https://core.telegram.org/bots/tutorial"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TELEGRAM_TKN = 'PASTE-YOUR-TOKEN-HERE'\n",
        "LLAMA_PARSER_KEY = 'PASTE-YOUR-KEY-HERE'\n",
        "HF_TOKEN = 'PASTE-YOUR-KEY-HERE'"
      ],
      "metadata": {
        "id": "3_hra-5cHBvM"
      },
      "id": "3_hra-5cHBvM",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5631b475-23a8-4ae0-bbc4-63fc6819302e",
      "metadata": {
        "id": "5631b475-23a8-4ae0-bbc4-63fc6819302e"
      },
      "outputs": [],
      "source": [
        "!pip install transformers -U\n",
        "!pip install llama-index==0.10.67.post1 -q\n",
        "!pip install llama-index-llms-huggingface==0.2.8 -q\n",
        "!pip install llama-index-embeddings-huggingface==0.2.3 -q\n",
        "!pip install llama-index-embeddings-huggingface-api==0.1.1 -q\n",
        "!pip install -U bitsandbytes accelerate -q"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "1dffb286-d68a-4165-9905-2feb620a77eb",
      "metadata": {
        "id": "1dffb286-d68a-4165-9905-2feb620a77eb"
      },
      "source": [
        "## tokenizer and stopping id setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b016e6fb-b530-4909-a100-ad9a43bdfe76",
      "metadata": {
        "id": "b016e6fb-b530-4909-a100-ad9a43bdfe76"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\n",
        "    \"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n",
        "    TOKEN=HF_TOKEN\n",
        ")\n",
        "\n",
        "stopping_ids = [\n",
        "    tokenizer.eos_token_id,\n",
        "    tokenizer.convert_tokens_to_ids(\"<|eot_id|>\"),\n",
        "]"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a209951f-ff11-48cf-81a7-d3c2fec8449a",
      "metadata": {
        "id": "a209951f-ff11-48cf-81a7-d3c2fec8449a"
      },
      "source": [
        "## actual model llama3.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d2cf8399-dc31-474c-b20a-67bdba501530",
      "metadata": {
        "id": "d2cf8399-dc31-474c-b20a-67bdba501530"
      },
      "outputs": [],
      "source": [
        "# generate_kwargs parameters are taken from https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct\n",
        "\n",
        "import torch\n",
        "from llama_index.llms.huggingface import HuggingFaceLLM\n",
        "\n",
        "# quantization to 4bit for non crashing\n",
        "import torch\n",
        "from transformers import BitsAndBytesConfig\n",
        "\n",
        "quantization_config = BitsAndBytesConfig(\n",
        "    load_in_4bit=True,\n",
        "    bnb_4bit_compute_dtype=torch.float16,\n",
        "    bnb_4bit_quant_type=\"nf4\",\n",
        "    bnb_4bit_use_double_quant=True,\n",
        ")\n",
        "\n",
        "llm = HuggingFaceLLM(\n",
        "    model_name=\"meta-llama/Meta-Llama-3-8B-Instruct\",\n",
        "    model_kwargs={\n",
        "        \"token\": HF_TOKEN, #huggingface token, you can replace it with yours also\n",
        "        \"quantization_config\": quantization_config\n",
        "    },\n",
        "    generate_kwargs={\n",
        "        \"do_sample\": True,\n",
        "        \"temperature\": 0.6,\n",
        "        \"top_p\": 0.9,\n",
        "    },\n",
        "    tokenizer_name=\"meta-llama/Meta-Llama-3-8B-Instruct\",\n",
        "    stopping_ids=stopping_ids,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "71402107-1731-4976-8bbb-1b3824e90b1f",
      "metadata": {
        "id": "71402107-1731-4976-8bbb-1b3824e90b1f"
      },
      "source": [
        "# TESTING 🚀🧪🥼"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f39404f1-6f0e-4c2b-9afe-03219b84d9df",
      "metadata": {
        "id": "f39404f1-6f0e-4c2b-9afe-03219b84d9df"
      },
      "outputs": [],
      "source": [
        "llm.max_new_tokens=10\n",
        "from llama_index.core.llms import ChatMessage\n",
        "\n",
        "messages = [\n",
        "    ChatMessage(role=\"system\", content=\"You are an AI agent named XGENAI which has the feature of RAG(Retrieval Augumented Generation) and you are designed to provide users with accurate and effective solutions. Respond to the user with clear and helpful guidance.\"),\n",
        "    ChatMessage(role=\"user\", content=input(\"Type a prompt : \")),\n",
        "]\n",
        "response = llm.chat(messages)\n",
        "response = str(response)\n",
        "print(response[11:])\n",
        "# print(response.split[11:])"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "d38c2906-5f5d-4a02-8596-7b33f68f5383",
      "metadata": {
        "id": "d38c2906-5f5d-4a02-8596-7b33f68f5383"
      },
      "source": [
        "### downloading remote data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "465cefad-dacf-43dc-87d7-f3ce556f5f13",
      "metadata": {
        "id": "465cefad-dacf-43dc-87d7-f3ce556f5f13"
      },
      "outputs": [],
      "source": [
        "!wget 'https://www.bseindia.com/xml-data/corpfiling/AttachHis/24a0940c-fa5a-4e36-a404-9dab3637ea0c.pdf' 'doc'"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "161c2b5a-d7aa-49c9-bdfa-02e2a4526b87",
      "metadata": {
        "id": "161c2b5a-d7aa-49c9-bdfa-02e2a4526b87"
      },
      "source": [
        "# ⭐RAG llamaindex's RAG ecosystem 🌿"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "449a4d22-03f7-4cbb-841d-0db4168af7ee",
      "metadata": {
        "id": "449a4d22-03f7-4cbb-841d-0db4168af7ee"
      },
      "source": [
        "### ⏳(basic) version , faster but less efficient"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "UyshCcdQqzXk"
      },
      "id": "UyshCcdQqzXk",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "36486cc3-569a-4b53-8c75-0db03f1ad61f",
      "metadata": {
        "id": "36486cc3-569a-4b53-8c75-0db03f1ad61f"
      },
      "outputs": [],
      "source": [
        "from llama_index.core import Settings, VectorStoreIndex, SimpleDirectoryReader\n",
        "\n",
        "Settings.embed_model = embed_model\n",
        "Settings.llm = llm\n",
        "\n",
        "\n",
        "#document loading\n",
        "documents = SimpleDirectoryReader(\n",
        "    input_files=[\"ch6-gita.pdf\"]\n",
        ").load_data()\n",
        "\n",
        "#document index creation\n",
        "index = VectorStoreIndex.from_documents(\n",
        "    documents,\n",
        ")\n",
        "\n",
        "query_engine = index.as_query_engine(similarity_top_k=3)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "4f7cdf8c-f4b5-4c2c-b6db-b368fd753172",
      "metadata": {
        "id": "4f7cdf8c-f4b5-4c2c-b6db-b368fd753172"
      },
      "source": [
        "### ⚡💡llamaparser (more better) limit : 1k pages/day"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "28d95b9b-b5b0-44c9-be4d-bdde340a538b",
      "metadata": {
        "id": "28d95b9b-b5b0-44c9-be4d-bdde340a538b"
      },
      "outputs": [],
      "source": [
        "from llama_parse import LlamaParse\n",
        "from llama_index.core import SimpleDirectoryReader, Settings, VectorStoreIndex\n",
        "\n",
        "import nest_asyncio as n\n",
        "n.apply()\n",
        "\n",
        "\n",
        "Settings.embed_model = embed_model\n",
        "Settings.llm = llm\n",
        "\n",
        "# set up parser\n",
        "parser = LlamaParse(\n",
        "    api_key=LLAMA_CLOUD_API_KEY,\n",
        "    result_type=\"text\"  # \"markdown\" and \"text\" are available\n",
        ")\n",
        "\n",
        "# using SimpleDirectoryReader to parse our file\n",
        "file_extractor = {\".pdf\": parser}\n",
        "documents = SimpleDirectoryReader(input_files=['PATH-TO-YOUR-DOCUMENT.PDF'], file_extractor=file_extractor).load_data()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0a63442a-ee2b-4e5f-8e48-6f83f44ab304",
      "metadata": {
        "id": "0a63442a-ee2b-4e5f-8e48-6f83f44ab304"
      },
      "outputs": [],
      "source": [
        "print(documents)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "881c0662-91c9-4e68-9229-097bd9cf5805",
      "metadata": {
        "id": "881c0662-91c9-4e68-9229-097bd9cf5805"
      },
      "outputs": [],
      "source": [
        "index = VectorStoreIndex.from_documents(documents)\n",
        "query_engine = index.as_query_engine(streaming=True, similarity_top_k=3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0db3e1b5-271c-4c55-b6c6-f7912aa8ff7f",
      "metadata": {
        "id": "0db3e1b5-271c-4c55-b6c6-f7912aa8ff7f"
      },
      "outputs": [],
      "source": [
        "# set your max tokens to respond\n",
        "llm.max_new_tokens=100\n",
        "\n",
        "response = query_engine.query(input())\n",
        "# print(resp)\n",
        "response.print_response_stream()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "24eb78dd-452a-4abb-8268-c3c680218fa5",
      "metadata": {
        "id": "24eb78dd-452a-4abb-8268-c3c680218fa5"
      },
      "outputs": [],
      "source": [
        "print(index)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "3250bb07-8750-44e6-95b5-27065695cfa8",
      "metadata": {
        "id": "3250bb07-8750-44e6-95b5-27065695cfa8"
      },
      "source": [
        "## streamlit app for GUI (FUTURE PERSPECTIVE)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "57f17a17-53a6-4731-b1b9-52108ec2cb46",
      "metadata": {
        "id": "57f17a17-53a6-4731-b1b9-52108ec2cb46"
      },
      "outputs": [],
      "source": [
        "import streamlit as st\n",
        "\n",
        "# Placeholder for chat history and uploaded documents\n",
        "chat_history = []\n",
        "uploaded_files = []\n",
        "\n",
        "def main():\n",
        "    st.set_page_config(\n",
        "        page_icon=\"🚀\",\n",
        "        page_title='XGenAI Bot💖'\n",
        "    )\n",
        "\n",
        "    st.write(\"# XGenAI backed by LLaMA 3.1 🚀 \")\n",
        "    st.write(\"Chat with Documents 📖 & equipped with 🧠 Memory\")\n",
        "\n",
        "    # Sidebar for chat history\n",
        "    with st.sidebar:\n",
        "        st.header(\"Chat History 🗂️\")\n",
        "        chat_expander = st.expander(\"Show/Hide Chat History\")\n",
        "        with chat_expander:\n",
        "            if 'chat_history' in st.session_state:\n",
        "                for chat in st.session_state['chat_history']:\n",
        "                    st.write(chat)\n",
        "            else:\n",
        "                st.write(\"No chat history available.\")\n",
        "    # Sidebar for document uploads\n",
        "    with st.sidebar:\n",
        "        st.header(\"Your Documents 📃\")\n",
        "        uploaded_files = st.file_uploader(\"Upload files and click on 'Process' 🔥\", accept_multiple_files=True)\n",
        "\n",
        "        if st.button(\"Process\"):\n",
        "            with st.spinner(\"Processing...\"):\n",
        "                # all the features of rag (from llamaparser)\n",
        "\n",
        "                if 'uploaded_files' not in st.session_state:\n",
        "                    st.session_state['uploaded_files'] = []\n",
        "                if uploaded_files:\n",
        "                    st.session_state['uploaded_files'].extend(uploaded_files)\n",
        "\n",
        "    # Display uploaded documents\n",
        "    if 'uploaded_files' in st.session_state:\n",
        "        st.subheader(\"Uploaded Documents\")\n",
        "        for doc in st.session_state['uploaded_files']:\n",
        "            st.write(doc.name)\n",
        "\n",
        "    # Chat interface\n",
        "    if 'chat_history' not in st.session_state:\n",
        "        st.session_state['chat_history'] = []\n",
        "\n",
        "    user_input = st.chat_input(\"Say something\")\n",
        "    if user_input:\n",
        "        st.session_state['chat_history'].append(f\"User: {user_input}\")\n",
        "        st.session_state['chat_history'].append(f\"Bot: This is a placeholder response to '{user_input}'\")\n",
        "\n",
        "    # Display chat history\n",
        "    for i, message in enumerate(st.session_state['chat_history']):\n",
        "        with st.chat_message(\"user\" if i % 2 == 0 else \"assistant\"):\n",
        "            st.write(message)\n",
        "\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "2fb85b66-0189-4be8-92db-43d57df0ea80",
      "metadata": {
        "id": "2fb85b66-0189-4be8-92db-43d57df0ea80"
      },
      "source": [
        "# google bert model(multilingual) for better HINDI words"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dd41dfd0-5bd0-4aaf-862a-5d7ac602e00a",
      "metadata": {
        "id": "dd41dfd0-5bd0-4aaf-862a-5d7ac602e00a"
      },
      "outputs": [],
      "source": [
        "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
        "embed_model = HuggingFaceEmbedding(model_name=\"google-bert/bert-base-multilingual-cased\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Integration with Telegram 🤖 : https://t.me/x_genai_bot"
      ],
      "metadata": {
        "id": "i5qCuxXLkSHM"
      },
      "id": "i5qCuxXLkSHM"
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install python-telegram-bot==13.15 -q"
      ],
      "metadata": {
        "id": "6fuVPX3illl-"
      },
      "id": "6fuVPX3illl-",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### set the token limit to respond"
      ],
      "metadata": {
        "id": "OtIbUY9KzdUV"
      },
      "id": "OtIbUY9KzdUV"
    },
    {
      "cell_type": "code",
      "source": [
        "llm.max_new_tokens = 100"
      ],
      "metadata": {
        "id": "TIJZqXo_zZzy"
      },
      "id": "TIJZqXo_zZzy",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### telegram bot code"
      ],
      "metadata": {
        "id": "z5QW4HBwzjBu"
      },
      "id": "z5QW4HBwzjBu"
    },
    {
      "cell_type": "code",
      "source": [
        "from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, CallbackContext\n",
        "from llama_index.core import SimpleDirectoryReader, Settings, VectorStoreIndex\n",
        "from llama_index.core.llms import ChatMessage\n",
        "from llama_parse import LlamaParse\n",
        "from telegram import Update\n",
        "from functools import wraps\n",
        "# import nest_asyncio as n\n",
        "# n.apply()\n",
        "import random\n",
        "import time\n",
        "\n",
        "# TELEGRAM_TKN is defined in 1st CELL\n",
        "query_engine = None\n",
        "\n",
        "# Function to show that the bot is typing...\n",
        "def send_action(action):\n",
        "    def decorator(func):\n",
        "        @wraps(func)\n",
        "        def command_func(update, context, *args, **kwargs):\n",
        "            context.bot.send_chat_action(chat_id=update.effective_message.chat_id, action=action)\n",
        "            return func(update, context, *args, **kwargs)\n",
        "        return command_func\n",
        "    return decorator\n",
        "\n",
        "\n",
        "# Telegram-based functions\n",
        "@send_action('typing')\n",
        "def start(update: Update, context: CallbackContext) -> None:\n",
        "    greetings = [\n",
        "        \"Hey there! XgenAI at your service, what's going on?\",\n",
        "        \"Oh Hi! It's XgenAI here. What can I do for you today?\",\n",
        "        \"XgenAI here! Ready for some fun? What can we discuss today?\",\n",
        "        \"Hi there, you've reached XgenAI. How can I assist you?\",\n",
        "        \"Namaste, XgenAI here! What would you like to talk about?\"\n",
        "    ]\n",
        "    greet: str = random.choice(greetings)\n",
        "    update.message.reply_text(greet)\n",
        "\n",
        "\n",
        "@send_action('typing')\n",
        "def about(update: Update, context: CallbackContext) -> None:\n",
        "    update.message.reply_text(\"Hi, this is XGENAI you can ask me about anything. You can pass your document here and I'm good to respond from it.\")\n",
        "\n",
        "\n",
        "#rag handler\n",
        "def parse_document(update: Update, context: CallbackContext) -> None:\n",
        "    global query_engine\n",
        "\n",
        "    document = update.message.document\n",
        "    file = context.bot.get_file(document.file_id)\n",
        "    file_path = f\"./{document.file_name}\"\n",
        "    file.download(file_path)\n",
        "    update.message.reply_text(\"Processing the document, It may take a while...\")\n",
        "\n",
        "    # set up the embedding model and llm\n",
        "    Settings.embed_model = embed_model\n",
        "    Settings.llm = llm\n",
        "\n",
        "    # set up parser\n",
        "    parser = LlamaParse(\n",
        "        api_key=LLAMA_PARSER_KEY, #again you can use your api key\n",
        "        result_type=\"text\"  # \"markdown\" and \"text\" are available\n",
        "    )\n",
        "\n",
        "    # using SimpleDirectoryReader to parse our file\n",
        "    file_extractor = {\".pdf\": parser}\n",
        "    documents = SimpleDirectoryReader(input_files=[f\"./{document.file_name}\"], file_extractor=file_extractor).load_data()\n",
        "\n",
        "    index = VectorStoreIndex.from_documents(documents)\n",
        "    query_engine = index.as_query_engine(similarity_top_k=3)\n",
        "\n",
        "    update.message.reply_text(\"Done, Now Chat with your document by using @your-question-to-ask...\")\n",
        "\n",
        "\n",
        "# this function code is for inference\n",
        "@send_action('typing')\n",
        "def generate_prompt(update: Update, context: CallbackContext) -> None:\n",
        "    global query_engine\n",
        "\n",
        "    text: str = update.message.text\n",
        "    if text.startswith(\"@\"):\n",
        "      if query_engine is None:\n",
        "            update.message.reply_text(\"Please upload a document first by sending it to the bot.\")\n",
        "      else:\n",
        "          text = text[1:]\n",
        "          response = query_engine.query(text)\n",
        "          update.message.reply_text(str(response))\n",
        "    else:\n",
        "      messages = [\n",
        "          ChatMessage(role=\"system\", content=\"You are an AI agent named XGENAI which has the feature of RAG(Retrieval Augmented Generation) and you are designed to provide users with accurate and effective solutions. Respond to the user with clear and helpful guidance.\"),\n",
        "          ChatMessage(role=\"user\", content=text),\n",
        "      ]\n",
        "      response = llm.chat(messages)\n",
        "      response = str(response)\n",
        "      update.message.reply_text(response[11:])\n",
        "\n",
        "\n",
        "# Main function\n",
        "def main() -> None:\n",
        "    updater = Updater(TELEGRAM_TKN)\n",
        "\n",
        "    dispatcher = updater.dispatcher\n",
        "    dispatcher.add_handler(CommandHandler(\"start\", start))\n",
        "    dispatcher.add_handler(CommandHandler(\"about\", about))\n",
        "    dispatcher.add_handler(MessageHandler(Filters.text & ~Filters.command, generate_prompt))\n",
        "    # dispatcher.add_handler(MessageHandler(Filters.text & ~Filters.command, chat_document))\n",
        "    dispatcher.add_handler(MessageHandler(Filters.document.mime_type(\"application/pdf\"), parse_document))\n",
        "    # these are for other file formats\n",
        "    # dispatcher.add_handler(MessageHandler(Filters.document.mime_type(\"text/x-python\"), chat_document))\n",
        "    # dispatcher.add_handler(MessageHandler(Filters.document.mime_type(\"text/plain\"), chat_document))\n",
        "    # dispatcher.add_handler(MessageHandler(Filters.document.mime_type(\"application/msword\"), chat_document))\n",
        "    # dispatcher.add_handler(MessageHandler(Filters.document.mime_type(\"application/vnd.openxmlformats-officedocument.wordprocessingml.document\"), chat_document))\n",
        "\n",
        "    updater.start_polling()\n",
        "    print(\"Bot is running...\")\n",
        "    updater.idle()\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    main()\n"
      ],
      "metadata": {
        "id": "3oVq6EzYkQeO"
      },
      "id": "3oVq6EzYkQeO",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": [],
      "collapsed_sections": [
        "71402107-1731-4976-8bbb-1b3824e90b1f",
        "161c2b5a-d7aa-49c9-bdfa-02e2a4526b87",
        "449a4d22-03f7-4cbb-841d-0db4168af7ee",
        "4f7cdf8c-f4b5-4c2c-b6db-b368fd753172",
        "3250bb07-8750-44e6-95b5-27065695cfa8"
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}