Copy_of_Local_vs_OpenAI_Embeddings.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bdiu29/BasicWODGenerator/blob/master/Copy_of_Local_vs_OpenAI_Embeddings.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "25BD_nZqRU8a"
      },
      "source": [
        "## OpenAI vs. Local Embeddings\n",
        "Performance Comparison\n",
        "- OpenAI's Embedding Model\n",
        "- InstructorEmbedding (https://huggingface.co/hkunlp/instructor-xl)"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "2ubnQs3zLjJu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "I-a7hN1OKY8O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WjR3P6LORXQP"
      },
      "outputs": [],
      "source": [
        "!pip -q install langchain openai tiktoken chromadb pypdf sentence_transformers InstructorEmbedding faiss-cpu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zhDZSwumRZhM"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"YOUR-API-KEY\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Up3o9DB3Rdpm"
      },
      "outputs": [],
      "source": [
        "from langchain.vectorstores import Chroma\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.llms import OpenAI\n",
        "from langchain.chains import RetrievalQA\n",
        "from langchain.document_loaders import PyPDFLoader\n",
        "from langchain.document_loaders import DirectoryLoader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8nDH7kaURj5I",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8e532fcf-cc37-4c90-8ee1-dbea41d2b7ed"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/InstructorEmbedding/instructor.py:7: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
            "  from tqdm.autonotebook import trange\n"
          ]
        }
      ],
      "source": [
        "# InstructorEmbedding\n",
        "from InstructorEmbedding import INSTRUCTOR\n",
        "from langchain.embeddings import HuggingFaceInstructEmbeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vruCPSuxRkrL"
      },
      "outputs": [],
      "source": [
        "# OpenAI Embedding\n",
        "from langchain.embeddings import OpenAIEmbeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d26el4XSRp2N"
      },
      "source": [
        "### Load Multiple files from Directory"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xEW8tZezRqZx",
        "outputId": "2af87b19-7ff1-4a63-ed75-7649575debe9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ],
      "source": [
        "# connect your Google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive', force_remount=True)\n",
        "root_dir = \"/content/gdrive/My Drive\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l6Z2FfiWSIw3"
      },
      "outputs": [],
      "source": [
        "# loader = TextLoader('single_text_file.txt')\n",
        "loader = DirectoryLoader(f'{root_dir}/Documents/', glob=\"./*.pdf\", loader_cls=PyPDFLoader)\n",
        "documents = loader.load()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zA1SXgawTm0x"
      },
      "outputs": [],
      "source": [
        "# documents"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3C9YZXJST2uh"
      },
      "source": [
        "### Divide and Conquer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bis-3BD4Txoo"
      },
      "outputs": [],
      "source": [
        "text_splitter = RecursiveCharacterTextSplitter(\n",
        "                                               chunk_size=1000,\n",
        "                                               chunk_overlap=200)\n",
        "\n",
        "texts = text_splitter.split_documents(documents)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A3625-PyT_0O",
        "outputId": "82593d0b-7163-4e3f-848b-ea35425dea8e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Document(page_content='GPT4All: Training an Assistant-style Chatbot with Large Scale Data\\nDistillation from GPT-3.5-Turbo\\nYuvanesh Anand\\nyuvanesh@nomic.aiZach Nussbaum\\nzanussbaum@gmail.com\\nBrandon Duderstadt\\nbrandon@nomic.aiBenjamin Schmidt\\nben@nomic.aiAndriy Mulyar\\nandriy@nomic.ai\\nAbstract\\nThis preliminary technical report describes the\\ndevelopment of GPT4All, a chatbot trained\\nover a massive curated corpus of assistant in-\\nteractions including word problems, story de-\\nscriptions, multi-turn dialogue, and code. We\\nopenly release the collected data, data cura-\\ntion procedure, training code, and final model\\nweights to promote open research and repro-\\nducibility. Additionally, we release quantized\\n4-bit versions of the model allowing virtually\\nanyone to run the model on CPU.\\n1 Data Collection and Curation\\nWe collected roughly one million prompt-\\nresponse pairs using the GPT-3.5-Turbo OpenAI\\nAPI between March 20, 2023 and March 26th,\\n2023. To do this, we first gathered a diverse sam-', metadata={'source': '/content/gdrive/My Drive/Documents/2023_GPT4All_Technical_Report.pdf', 'page': 0})"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "texts[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CfEjgcSmUA6d",
        "outputId": "98f79b81-b32a-47b3-beb6-e12e49003bbb"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "22"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "len(texts)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bwfVo4oSUHy1"
      },
      "source": [
        "### Get Embeddings for OUR Documents"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qdp-IdzFUNXh"
      },
      "outputs": [],
      "source": [
        "# !pip install faiss-cpu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Kv0XDq1zUFEd"
      },
      "outputs": [],
      "source": [
        "import pickle\n",
        "import faiss\n",
        "from langchain.vectorstores import FAISS"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bqTbURk-UKj1"
      },
      "outputs": [],
      "source": [
        "def store_embeddings(docs, embeddings, sotre_name, path):\n",
        "\n",
        "    vectorStore = FAISS.from_documents(docs, embeddings)\n",
        "\n",
        "    with open(f\"{path}/faiss_{sotre_name}.pkl\", \"wb\") as f:\n",
        "        pickle.dump(vectorStore, f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UA3gl00wUWfk"
      },
      "outputs": [],
      "source": [
        "def load_embeddings(sotre_name, path):\n",
        "    with open(f\"{path}/faiss_{sotre_name}.pkl\", \"rb\") as f:\n",
        "        VectorStore = pickle.load(f)\n",
        "    return VectorStore"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "siKL5Hb7HaQz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "esaW2OyiHabx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "51zi5anzUbZW"
      },
      "source": [
        "### HF Instructor Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CRSnS73iUX91",
        "outputId": "c60dbd83-5908-4aac-f392-12617579fbe8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "load INSTRUCTOR_Transformer\n",
            "max_seq_length  512\n"
          ]
        }
      ],
      "source": [
        "from langchain.embeddings import HuggingFaceInstructEmbeddings\n",
        "\n",
        "instructor_embeddings = HuggingFaceInstructEmbeddings(model_name=\"hkunlp/instructor-xl\",\n",
        "                                                      model_kwargs={\"device\": \"cuda\"})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_QCyoIjUaGoM"
      },
      "outputs": [],
      "source": [
        "Embedding_store_path = f\"{root_dir}/Embedding_store\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QK9b7v97UgaF"
      },
      "outputs": [],
      "source": [
        "# store_embeddings(texts,\n",
        "#                  instructor_embeddings,\n",
        "#                  sotre_name='instructEmbeddings',\n",
        "#                  path=Embedding_store_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oPykF4P5VHjU"
      },
      "outputs": [],
      "source": [
        "# db_instructEmbedd = load_embeddings(sotre_name='instructEmbeddings',\n",
        "#                                     path=Embedding_store_path)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "db_instructEmbedd = FAISS.from_documents(texts, instructor_embeddings)"
      ],
      "metadata": {
        "id": "ZP0cn7ywKLaf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ia4Fhe3tbZQ0"
      },
      "outputs": [],
      "source": [
        "retriever = db_instructEmbedd.as_retriever(search_kwargs={\"k\": 3})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6_lC4EB-bj5N",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "3cc773bd-e484-4456-8b81-0f399fb923ff"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'similarity'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "retriever.search_type"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hDU0cE8UbpBF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0c73637f-af16-4380-84b9-071cb0744220"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'k': 3}"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ],
      "source": [
        "retriever.search_kwargs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Fr1bteubbtoe"
      },
      "outputs": [],
      "source": [
        "docs = retriever.get_relevant_documents(\"Who are the authors of GPT4All report?\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "docs[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IDlW6ZtOLjrF",
        "outputId": "3e581937-efaa-4361-bca1-b312122b0be9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Document(page_content='We accompany this paper with the 800k point\\nGPT4All-J dataset that is a superset of the origi-\\nnal 400k points GPT4All dataset. We dedicated\\nsubstantial attention to data preparation and cura-\\ntion.Building on the GPT4All dataset, we curated\\nthe GPT4All-J dataset by augmenting the origi-\\nnal 400k GPT4All examples with new samples\\nencompassing additional multi-turn QA samples\\nand creative writing such as poetry, rap, and short\\nstories. We designed prompt templates to create\\ndifferent scenarios for creative writing. The cre-\\native prompt template was inspired by Mad Libs\\nstyle variations of ‘Write a [creative story type]\\nabout [NOUN] in the style of [PERSON]‘. In ear-\\nlier versions of GPT4All, we found that rather than\\nwriting actual creative content, the model would\\ndiscuss how it would go about writing the content.\\nTraining on this new dataset allows GPT4All-J to\\nwrite poems, songs, and plays with increased com-\\npetence.\\nWe used Atlas to inform our data cleaning and', metadata={'source': '/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf', 'page': 0})"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "-wfOpSsjLj5K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r7hIC2WucAVf"
      },
      "outputs": [],
      "source": [
        "# create the chain to answer questions\n",
        "qa_chain_instrucEmbed = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0.2, ),\n",
        "                                  chain_type=\"stuff\",\n",
        "                                  retriever=retriever,\n",
        "                                  return_source_documents=True)"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "71Vx3LA0L6QP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "S64QLK6ZL6xv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n1JGbzGNc-t4"
      },
      "source": [
        "### OpenAI's Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2J49di9ncByE"
      },
      "outputs": [],
      "source": [
        "from langchain.embeddings import OpenAIEmbeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GCZssU67ClpY"
      },
      "outputs": [],
      "source": [
        "embeddings = OpenAIEmbeddings()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cMgpDyjNcI7E"
      },
      "outputs": [],
      "source": [
        "# store_embeddings(texts,\n",
        "#                  embeddings,\n",
        "#                  sotre_name='openAIEmbeddings',\n",
        "#                  path=Embedding_store_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FRYrIf6ZdWqF"
      },
      "outputs": [],
      "source": [
        "# db_openAIEmbedd = load_embeddings(sotre_name='openAIEmbeddings',\n",
        "#                                     path=Embedding_store_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hlL7QEhOdlxr"
      },
      "outputs": [],
      "source": [
        "db_openAIEmbedd = FAISS.from_documents(texts, embeddings)\n",
        "retriever_openai = db_openAIEmbedd.as_retriever(search_kwargs={\"k\": 3})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jWv6T5ZsduuS"
      },
      "outputs": [],
      "source": [
        "# create the chain to answer questions\n",
        "qa_chain_openai = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0.2, ),\n",
        "                                  chain_type=\"stuff\",\n",
        "                                  retriever=retriever_openai,\n",
        "                                  return_source_documents=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dKRWeYvId5po"
      },
      "source": [
        "### Testing both MODELS"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XAOlpma2d20q"
      },
      "outputs": [],
      "source": [
        "## Cite sources\n",
        "\n",
        "import textwrap\n",
        "\n",
        "def wrap_text_preserve_newlines(text, width=110):\n",
        "    # Split the input text into lines based on newline characters\n",
        "    lines = text.split('\\n')\n",
        "\n",
        "    # Wrap each line individually\n",
        "    wrapped_lines = [textwrap.fill(line, width=width) for line in lines]\n",
        "\n",
        "    # Join the wrapped lines back together using newline characters\n",
        "    wrapped_text = '\\n'.join(wrapped_lines)\n",
        "\n",
        "    return wrapped_text\n",
        "\n",
        "def process_llm_response(llm_response):\n",
        "    print(wrap_text_preserve_newlines(llm_response['result']))\n",
        "    print('\\nSources:')\n",
        "    for source in llm_response[\"source_documents\"]:\n",
        "        print(source.metadata['source'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "D21M6zCUeAd7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "22e69b02-0aed-421e-96cf-e00250180b8e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------Instructor Embeddings------------------\n",
            "\n",
            " The authors of the GPT4all technical report are Yuvanesh Anand, Zach Nussbaum, Brandon Duderstadt, Benjamin\n",
            "M. Schmidt, Adam Treat, and Andriy Mulyar.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n"
          ]
        }
      ],
      "source": [
        "query = 'who are the authors of GPT4all technical report?'\n",
        "\n",
        "print('-------------------Instructor Embeddings------------------\\n')\n",
        "llm_response = qa_chain_instrucEmbed(query)\n",
        "process_llm_response(llm_response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QIknq_4MAh1v",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ab3e3ebc-6a6c-4502-9074-884a2d7b0b0f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------OpenAI Embeddings------------------\n",
            " Yuvanesh Anand, Zach Nussbaum, Brandon Duderstadt, Benjamin M. Schmidt, Adam Treat, and Andriy Mulyar.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "query = 'who are the authors of GPT4all technical report?'\n",
        "\n",
        "print('-------------------OpenAI Embeddings------------------')\n",
        "llm_response = qa_chain_openai(query)\n",
        "process_llm_response(llm_response)\n",
        "print('\\n\\n\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HOjsoKvCef3J",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b6471946-a655-44fc-b2e6-6a4a74194452"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------Instructor Embeddings------------------\n",
            "\n",
            " The GPT4All-J model was trained over a massive curated corpus of assistant interactions including word\n",
            "problems, multi-turn dialogue, code, poems, songs, and stories.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n"
          ]
        }
      ],
      "source": [
        "query = 'How was the GPT4All-J model trained?'\n",
        "\n",
        "print('-------------------Instructor Embeddings------------------\\n')\n",
        "llm_response = qa_chain_instrucEmbed(query)\n",
        "process_llm_response(llm_response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-g1LTybWAp90",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f6511972-e298-4986-8cc7-752054c93e71"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------OpenAI Embeddings------------------\n",
            " The GPT4All-J model was trained with LoRA (Hu et al., 2021) on the 437,605 post-processed examples for four\n",
            "epochs.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "query = 'How was the GPT4All-J model trained?'\n",
        "\n",
        "print('-------------------OpenAI Embeddings------------------')\n",
        "llm_response = qa_chain_openai(query)\n",
        "process_llm_response(llm_response)\n",
        "print('\\n\\n\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8HGgdB0ee23d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3bd4dec3-aa81-4e72-a87f-075b4c62dce4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------Instructor Embeddings------------------\n",
            "\n",
            " The cost of training the GPT4all model was about $800 in OpenAI API credits and $100 for a Lambda Labs DGX\n",
            "A100 8x 80GB.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All_Technical_Report.pdf\n"
          ]
        }
      ],
      "source": [
        "query = '\"What was the cost of training the GPT4all model?\"'\n",
        "\n",
        "print('-------------------Instructor Embeddings------------------\\n')\n",
        "llm_response = qa_chain_instrucEmbed(query)\n",
        "process_llm_response(llm_response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uI9sBQrwA58N",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "73749a6b-83fe-4623-fb37-9e47c7dc9ac2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------OpenAI Embeddings------------------\n",
            " $200\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All_Technical_Report.pdf\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "query = '\"What was the cost of training the GPT4all model?\"'\n",
        "\n",
        "print('-------------------OpenAI Embeddings------------------')\n",
        "llm_response = qa_chain_openai(query)\n",
        "process_llm_response(llm_response)\n",
        "print('\\n\\n\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SlBOGAQafEG_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "91ce58ba-b4e7-432d-c011-2290081cd692"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------Instructor Embeddings------------------\n",
            "\n",
            " GPT4All-J is using an Apache 2 license.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All_Technical_Report.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n"
          ]
        }
      ],
      "source": [
        "query = \"what license is GPT4All-J using?\"\n",
        "\n",
        "# print('-------------------OpenAI Embeddings------------------')\n",
        "# llm_response = qa_chain_openai(query)\n",
        "# process_llm_response(llm_response)\n",
        "# print('\\n\\n\\n')\n",
        "print('-------------------Instructor Embeddings------------------\\n')\n",
        "llm_response = qa_chain_instrucEmbed(query)\n",
        "process_llm_response(llm_response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WPuH7LypA-hg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a8bb9839-5f4f-404d-861f-21e03ae7bf93"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------OpenAI Embeddings------------------\n",
            " GPT4All-J is using an Apache 2 license.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "query = \"what license is GPT4All-J using?\"\n",
        "\n",
        "print('-------------------OpenAI Embeddings------------------')\n",
        "llm_response = qa_chain_openai(query)\n",
        "process_llm_response(llm_response)\n",
        "print('\\n\\n\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yESZ6IQojO5D",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ff9b9a1f-c0c2-4766-8af2-35581a04db73"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------Instructor Embeddings------------------\n",
            "\n",
            " Roughly one million prompt-response pairs.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All_Technical_Report.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n"
          ]
        }
      ],
      "source": [
        "query = \"what was the size of training dataset used for training GPT4All?\"\n",
        "\n",
        "# print('-------------------OpenAI Embeddings------------------')\n",
        "# llm_response = qa_chain_openai(query)\n",
        "# process_llm_response(llm_response)\n",
        "# print('\\n\\n\\n')\n",
        "print('-------------------Instructor Embeddings------------------\\n')\n",
        "llm_response = qa_chain_instrucEmbed(query)\n",
        "process_llm_response(llm_response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jBW4mxGFBDVA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0d1b884d-e9ca-4c84-df8c-38b324236957"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------OpenAI Embeddings------------------\n",
            " The final training dataset used for training GPT4All was 437,605 post-processed examples.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All_Technical_Report.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "query = \"what was the size of training dataset used for training GPT4All?\"\n",
        "\n",
        "print('-------------------OpenAI Embeddings------------------')\n",
        "llm_response = qa_chain_openai(query)\n",
        "process_llm_response(llm_response)\n",
        "print('\\n\\n\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5pqc0bC2jjwT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3081494e-9015-43ba-ba3f-17e58e6dc6b5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------Instructor Embeddings------------------\n",
            "\n",
            " The training dataset used for training GPT4All-J was 437,605 post-processed examples.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n"
          ]
        }
      ],
      "source": [
        "query = \"what was the size of training dataset used for training GPT4All-J?\"\n",
        "\n",
        "# print('-------------------OpenAI Embeddings------------------')\n",
        "# llm_response = qa_chain_openai(query)\n",
        "# process_llm_response(llm_response)\n",
        "# print('\\n\\n\\n')\n",
        "print('-------------------Instructor Embeddings------------------\\n')\n",
        "llm_response = qa_chain_instrucEmbed(query)\n",
        "process_llm_response(llm_response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "etIbdKNhBItk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b28865ca-0f8d-4d37-f7c8-4dbd3c246759"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------OpenAI Embeddings------------------\n",
            " The final training dataset used for training GPT4All-J was 437,605 post-processed examples.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All_Technical_Report.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "query = \"what was the size of training dataset used for training GPT4All-J?\"\n",
        "\n",
        "print('-------------------OpenAI Embeddings------------------')\n",
        "llm_response = qa_chain_openai(query)\n",
        "process_llm_response(llm_response)\n",
        "print('\\n\\n\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VVKOjCXJjXYg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "00ab68f4-ebb3-41c9-a475-7167341f48b7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------Instructor Embeddings------------------\n",
            "\n",
            " GPT4All is using an Apache 2 license.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All_Technical_Report.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n"
          ]
        }
      ],
      "source": [
        "query = \"what license is GPT4All using?\"\n",
        "\n",
        "# print('-------------------OpenAI Embeddings------------------')\n",
        "# llm_response = qa_chain_openai(query)\n",
        "# process_llm_response(llm_response)\n",
        "# print('\\n\\n\\n')\n",
        "print('-------------------Instructor Embeddings------------------\\n')\n",
        "llm_response = qa_chain_instrucEmbed(query)\n",
        "process_llm_response(llm_response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jV3P7as9BNWJ"
      },
      "outputs": [],
      "source": [
        "query = \"what license is GPT4All using?\"\n",
        "\n",
        "print('-------------------OpenAI Embeddings------------------')\n",
        "llm_response = qa_chain_openai(query)\n",
        "process_llm_response(llm_response)\n",
        "print('\\n\\n\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b_7Mc9WfgSAW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d382f503-b921-456b-a624-0d99d0ad62b2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------Instructor Embeddings------------------\n",
            "\n",
            " According to the evaluation data from the Self-Instruct paper (Wang et al., 2022), the best openly available\n",
            "alpaca-lora model provided by user chainyo on huggingface has the lowest ground truth perplexity.\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All_Technical_Report.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n"
          ]
        }
      ],
      "source": [
        "query = \"Which MPT-7B model is the best?\"\n",
        "\n",
        "# print('-------------------OpenAI Embeddings------------------')\n",
        "# llm_response = qa_chain_openai(query)\n",
        "# process_llm_response(llm_response)\n",
        "# print('\\n\\n\\n')\n",
        "print('-------------------Instructor Embeddings------------------\\n')\n",
        "llm_response = qa_chain_instrucEmbed(query)\n",
        "process_llm_response(llm_response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BYrf2Gd0g9ja",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cd567249-485e-403b-f074-f62770fb1ab3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------OpenAI Embeddings------------------\n",
            " Alpaca Lora 7B is the best MPT-7B model according to the evaluation data from the Self-Instruct paper (Wang\n",
            "et al., 2022).\n",
            "\n",
            "Sources:\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "/content/gdrive/My Drive/Documents/2023_GPT4All-J_Technical_Report_2.pdf\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "query = \"Which MPT-7B model is the best?\"\n",
        "\n",
        "print('-------------------OpenAI Embeddings------------------')\n",
        "llm_response = qa_chain_openai(query)\n",
        "process_llm_response(llm_response)\n",
        "print('\\n\\n\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "33tnoTXzBRk5"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}