RAG_with_ReRank.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Will be using:\n",
        " 1. Langchain framework for RAG Architecture.\n",
        " 2. Faiss for Vector DB\n",
        " 3. PyPDF for pdf text extraction.\n",
        " 4. Sentence transformer for vector embedding.\n",
        " 5. Free Chatgpt API from RAPID API.\n",
        " 6. FlashRank(OpenSource) to improve the retrieval Performance"
      ],
      "metadata": {
        "id": "aXPzcsSPNXTf"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9nhe8nxTGkY0",
        "outputId": "6959a7a4-0962-4894-999a-f51e22fb84d6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting langchain\n",
            "  Downloading langchain-0.1.16-py3-none-any.whl (817 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m817.7/817.7 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0.1)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.29)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.9.5)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n",
            "Collecting dataclasses-json<0.7,>=0.5.7 (from langchain)\n",
            "  Downloading dataclasses_json-0.6.4-py3-none-any.whl (28 kB)\n",
            "Collecting jsonpatch<2.0,>=1.33 (from langchain)\n",
            "  Downloading jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n",
            "Collecting langchain-community<0.1,>=0.0.32 (from langchain)\n",
            "  Downloading langchain_community-0.0.34-py3-none-any.whl (1.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langchain-core<0.2.0,>=0.1.42 (from langchain)\n",
            "  Downloading langchain_core-0.1.45-py3-none-any.whl (291 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m291.3/291.3 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langchain-text-splitters<0.1,>=0.0.1 (from langchain)\n",
            "  Downloading langchain_text_splitters-0.0.1-py3-none-any.whl (21 kB)\n",
            "Collecting langsmith<0.2.0,>=0.1.17 (from langchain)\n",
            "  Downloading langsmith-0.1.49-py3-none-any.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.2/115.2 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.25.2)\n",
            "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.7.0)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.31.0)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.7,>=0.5.7->langchain)\n",
            "  Downloading marshmallow-3.21.1-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m766.3 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.7,>=0.5.7->langchain)\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain)\n",
            "  Downloading jsonpointer-2.4-py2.py3-none-any.whl (7.8 kB)\n",
            "Collecting packaging<24.0,>=23.2 (from langchain-core<0.2.0,>=0.1.42->langchain)\n",
            "  Downloading packaging-23.2-py3-none-any.whl (53 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.0/53.0 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting orjson<4.0.0,>=3.9.14 (from langsmith<0.2.0,>=0.1.17->langchain)\n",
            "  Downloading orjson-3.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (141 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.1/141.1 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (2.18.1)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (4.11.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2024.2.2)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.3)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Installing collected packages: packaging, orjson, mypy-extensions, jsonpointer, typing-inspect, marshmallow, jsonpatch, langsmith, dataclasses-json, langchain-core, langchain-text-splitters, langchain-community, langchain\n",
            "  Attempting uninstall: packaging\n",
            "    Found existing installation: packaging 24.0\n",
            "    Uninstalling packaging-24.0:\n",
            "      Successfully uninstalled packaging-24.0\n",
            "Successfully installed dataclasses-json-0.6.4 jsonpatch-1.33 jsonpointer-2.4 langchain-0.1.16 langchain-community-0.0.34 langchain-core-0.1.45 langchain-text-splitters-0.0.1 langsmith-0.1.49 marshmallow-3.21.1 mypy-extensions-1.0.0 orjson-3.10.1 packaging-23.2 typing-inspect-0.9.0\n",
            "Collecting faiss-gpu\n",
            "  Downloading faiss_gpu-1.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (85.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.5/85.5 MB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: faiss-gpu\n",
            "Successfully installed faiss-gpu-1.7.2\n",
            "Collecting pypdf\n",
            "  Downloading pypdf-4.2.0-py3-none-any.whl (290 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.4/290.4 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing_extensions>=4.0 in /usr/local/lib/python3.10/dist-packages (from pypdf) (4.11.0)\n",
            "Installing collected packages: pypdf\n",
            "Successfully installed pypdf-4.2.0\n",
            "Collecting sentence-transformers\n",
            "  Downloading sentence_transformers-2.7.0-py3-none-any.whl (171 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m171.5/171.5 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: transformers<5.0.0,>=4.34.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.38.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.66.2)\n",
            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (2.2.1+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.25.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.11.4)\n",
            "Requirement already satisfied: huggingface-hub>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (0.20.3)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (9.4.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (3.13.4)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (2.31.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (4.11.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (23.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (3.1.3)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.19.3 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (2.2.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.11.0->sentence-transformers)\n",
            "  Using cached nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers) (0.4.3)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers) (1.4.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers) (3.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.11.0->sentence-transformers) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence-transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence-transformers) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence-transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence-transformers) (2024.2.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.11.0->sentence-transformers) (1.3.0)\n",
            "Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, sentence-transformers\n",
            "Successfully installed nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.1.105 sentence-transformers-2.7.0\n"
          ]
        }
      ],
      "source": [
        "!pip install langchain\n",
        "!pip install faiss-gpu\n",
        "!pip install pypdf\n",
        "!pip install sentence-transformers\n",
        "!pip install flashrank"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.schema.output_parser import StrOutputParser\n",
        "from langchain_community.document_loaders import PyPDFLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.prompts import PromptTemplate\n",
        "from langchain.vectorstores.utils import filter_complex_metadata\n",
        "from langchain.document_loaders import TextLoader\n",
        "from langchain.vectorstores import DocArrayInMemorySearch\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "from langchain.vectorstores import FAISS\n",
        "from langchain.chains import ConversationalRetrievalChain\n",
        "from langchain.indexes import VectorstoreIndexCreator\n",
        "from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings"
      ],
      "metadata": {
        "id": "UoUgdBq6Gpyg"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()\n",
        "# Uploading the file named Doc1.pdf\n",
        "txt_file_path = 'Doc1.pdf'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "id": "wiHj5i1IGsDq",
        "outputId": "ddabd90f-db46-45ff-8962-9cc8d99dcb9a"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-071de63d-f4d5-495b-94aa-ee9bdcca607e\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-071de63d-f4d5-495b-94aa-ee9bdcca607e\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving Doc1.pdf to Doc1.pdf\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Extracting the text and splitting into chunks to be vectorized.\n",
        "docs = PyPDFLoader(file_path=txt_file_path).load()\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=250, chunk_overlap=80)\n",
        "chunks = text_splitter.split_documents(docs)\n",
        "chunks = filter_complex_metadata(chunks)\n",
        "#Using the embedding model named all-MiniLM-L6-v2, more can be read in HuggingFace\n",
        "embedding_function = SentenceTransformerEmbeddings(model_name=\"all-MiniLM-L6-v2\")\n",
        "#Storing the chunks in vector DB\n",
        "db = FAISS.from_documents(documents=chunks, embedding=embedding_function)"
      ],
      "metadata": {
        "id": "z29IRiRNGwDq"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "retriever = db.as_retriever(\n",
        "    search_type=\"similarity_score_threshold\",\n",
        "    search_kwargs={\n",
        "        \"score_threshold\": 0.4,\n",
        "        \"k\":3},\n",
        ")"
      ],
      "metadata": {
        "id": "XbUaHQfiGyZe"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## This question can be found in Page 12 in 3rd Paragraph."
      ],
      "metadata": {
        "id": "J7WEa5cwNPaC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "what_to_ask=\"What are the definitions of visualization?\""
      ],
      "metadata": {
        "id": "dXKnun2FG4xx"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "retrieves=retriever.invoke(what_to_ask)"
      ],
      "metadata": {
        "id": "z1KxxHnlG9Li"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Only one chunk got retrieved that crosses the pre-defined threshold. Which does not contain the answer we need.**"
      ],
      "metadata": {
        "id": "wBDZHtFhOHgp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "retrieves"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xa5h_hq-G_QD",
        "outputId": "956af2f9-0c9c-4cd2-9f00-470e171301fa"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Document(page_content=\"phenomena. Scientists need a representation form that can show all the \\ncorrelations. Static maps are not the best presentation form for displaying \\nthese relationship s. Often, maps are overloaded with information layers \\nto show the many correlations. ln an animated map sequence, map \\nelements can be presented in different orders and combinations to make \\nthe spatial relationships more apparent. The map user can be directed \\nthrough the presented subject, and the correlations can be brought to the \\nuser's attention. \\nIn the late 1980s, the sciences discovered scientific visualization. It is \\nused for data analysis to see patterns that either answer questions or that \\npose new and unexpected questions. Scientific visualization requires \\ncomputer animation, especially interactive animation, that can show the Cartographic \\nAnimation: \\nPotential and \\nResearch \\nIssues \\nDoris Karl \\nTHE EEO FOR ANIMA TIO \\nIN CARTOGRAPHY \\nDoris Karl is a stude11t at the \\nFreie Universitiit Berlin,\", metadata={'source': 'Doc1.pdf', 'page': 3})]"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Using ReRank to improve the Performance"
      ],
      "metadata": {
        "id": "JG512PyZORlb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Decreasing the threshold and increasing the number of chunks to obtain more chunks.\n",
        "retriever = db.as_retriever(\n",
        "    search_type=\"similarity_score_threshold\",\n",
        "    search_kwargs={\n",
        "        \"score_threshold\": 0.2,\n",
        "        \"k\":10},\n",
        ")"
      ],
      "metadata": {
        "id": "nDepOiV4Kf0C"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "retrieves=retriever.invoke(what_to_ask)\n",
        "retrieves"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0EGztuM6KuCk",
        "outputId": "b513b8c7-31cc-4bf6-9b1c-b1814e3588ef"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Document(page_content=\"phenomena. Scientists need a representation form that can show all the \\ncorrelations. Static maps are not the best presentation form for displaying \\nthese relationship s. Often, maps are overloaded with information layers \\nto show the many correlations. ln an animated map sequence, map \\nelements can be presented in different orders and combinations to make \\nthe spatial relationships more apparent. The map user can be directed \\nthrough the presented subject, and the correlations can be brought to the \\nuser's attention. \\nIn the late 1980s, the sciences discovered scientific visualization. It is \\nused for data analysis to see patterns that either answer questions or that \\npose new and unexpected questions. Scientific visualization requires \\ncomputer animation, especially interactive animation, that can show the Cartographic \\nAnimation: \\nPotential and \\nResearch \\nIssues \\nDoris Karl \\nTHE EEO FOR ANIMA TIO \\nIN CARTOGRAPHY \\nDoris Karl is a stude11t at the \\nFreie Universitiit Berlin,\", metadata={'source': 'Doc1.pdf', 'page': 3}),\n",
              " Document(page_content=\"10 cartographic perspectives Number 13, Fall 1992 \\nAlan M. MacEachren \\nAln11 M. MncEnchre11 is n \\nProfe:;sor i11 the Department of \\nGcogrnphy, 302 Wn/ka B11ildi11g, The \\nPc1111sylm11in Stntc U11hicrsity, \\nU111per-.1ftf Pnrk, PA 16802. \\nc-mnil: \\\\JYB'a PSUVM.PSU.EOU \\nWhen n GJS is used to compile, \\nnnalyz.e, nnd display infor111n\\xad\\ntio11, the chance for 111zaccept\\xad\\nable or variable data quality is \\nhigh due to the merging of \\n11111/tiple data layers. Visualizing Uncertain Information \\nWhen a GJS is used to drive map-based visualization, exploration of \\npotential relationships takes precedence over presentation of facts. In \\nthese early stages of scientific analysis or policy formulation, providing \\na way for analysts to assess uncertainty in the data they are exploring is \\ncritical to the perspectives they form and the approaches they decide to \\npursue. As a basis from which to develop methods for visualizing \\nuncertain information , this paper addresses the difference between data\", metadata={'source': 'Doc1.pdf', 'page': 10}),\n",
              " Document(page_content='Number 13, Fall 1992 cartographic perspectives 13 \\nquality needs specifically. David \\nDiBiasc (1990) recently developed \\na graphic model of the range of \\nuses to which graphics might be \\nput in scientific research (fig. 2). I \\nbelieve that his basic model is \\nrelevant, not only to science, but to \\napplied spatial decision making \\nwith a GIS. \\nAs we begin to consider the \\nvisualization of w1certainty , we \\nneed to be cognizant of this range \\nof visualization goals and environ\\xad\\nments and the varying information \\nrequirement s it implies. The kind \\nof uncertainty and the tools used \\nto visualize it are likely to vary \\nacross this range, from the use of \\nGIS by an EPA scientist exploring \\nthe spatial distribution of a pollut\\xad\\nant to the use of GIS driven map \\ndisplays by policy makers trying to \\ndecide which industrie s to add to \\nthe list of those regulated for toxic \\nwaste emission. \\nGraphics variables \\nBecause few GIS users are trained in cartographic symbolization and', metadata={'source': 'Doc1.pdf', 'page': 13}),\n",
              " Document(page_content='metaphors and their use. Further work should expand this subject. \\nCartographic animation requires a variety of research on design, \\ncreation, and use of animated map sequences. Because of the potential of \\nanimation for cartography, more attention to research is required in this \\nfield. \\nThe potential of computer animation has already been realized by differ\\xad\\nent disciplines. It can also be used for cartography as a powerful visual\\xad\\nization tool that transcends the potential of the printed map. With anima\\xad\\ntion, dynamic and interactive map sequences can be created that show \\nchanges over time or can be used to depict or explore spatial and statisti\\xad\\ncal relationships. CO CLUSIO>J', metadata={'source': 'Doc1.pdf', 'page': 7}),\n",
              " Document(page_content=\"8 cartographic perspectives Number 13, Fall 1992 \\nREFERE. CES e\\\\'\\\\' approaches and methods in spatial sciences such as the strong \\nemphasis on examining processes and correlations and the use of scientific \\nvisualization require a more dynamic and user-oriented form of informa\\xad\\ntion display that is not possible ,,·ith the printed map. Also, the map users \\nof the future, the children of today, who are more accustomed to interac\\xad\\ntive computers, will probably ask for more dynamic and interactive forms \\nof information display. \\nIn spite of the need for computer animation in cartography, it has been \\nneglected until recently. There arc two main reasons for this. The first is \\nthe fixation on the traditional printed map that is strongly present within \\ncartography. As a result of this thinking, other and perhaps better forms \\nof information display such as computer animation ha\\\\'e been overlooked. \\nThe second reason is the absence of a comprehensive approach to carto\\xad\", metadata={'source': 'Doc1.pdf', 'page': 8}),\n",
              " Document(page_content=\"the flexibility of data manipulation that makes GIS so powerful can lead to \\nconsiderable uncertainty in map displays produced at various stages of \\nG!S analysi .... This paper addresses a variety of conceptual issues underly\\xad\\ning de\\\\'elopment of \\\\'i<;ualization tools that allow analyo.,t<; to take this \\nuncertainty into account in their research and policy formulation acti,·i\\xad\\nties. \\nThere io., a o.,trong tradition in cartography of attention to data quality. \\nOnly rudimentary steps, however, have been made thus far to deal \\\\·\\\\'ith \\nthe complex issues of \\\\'isualizing data quality for multidimensional data \\ndisplays used in image analysis and GIS application<;. The importance of \\nthis topic is evidenced by the decision of the National Center for Geo\\xad\\ngraphic Information and Analysis(. CGIA) to make visualiza tion of data \\nquality the first visualization initiative undertak en by the center. \\nKate Beard and Barbara Buttenfield (1991 ), presenting the CGIA\", metadata={'source': 'Doc1.pdf', 'page': 10}),\n",
              " Document(page_content='tools allo\"v our visual and cognitive processes to almost automatically \\nfocus on the patterns depicted rather than on mentally generating those \\npatterns. \\nFollowing from the above conception of visualization, a research \\nagenda to address visualizing uncertain information should include \\nattention to the cognitive issues of what it means to understand attribute, \\nspatial, and temporal uncertainty and the implications of this understand\\xad\\ning for decision making and for symbolizing and categorizing uncertainty. \\nAt the most basic level, uncertainty can be divided into two components \\nthat might require different visualization strategies: \\\\\\'isualizing accuracy \\nand visualizing precision. In addition, attention should be directed toward \\nthe methodological, technical, and ergonomic issue:; of generating dis\\xad\\nplays and creating interfaces that work. It is, of course, also essential to \\ndevelop methods for assessing and measuring uncertainty before we can', metadata={'source': 'Doc1.pdf', 'page': 12}),\n",
              " Document(page_content=\"the potential to compose complex animation sequences. Ideas for this \\nhave been suggested by Monmonier (1989). A second and very important \\npoint is the interactivity of an animation. If animation is to be more than \\njust a film, it must have the possibility of interaction. Animation systems \\nshould have the minimum ability to stop and restart the sequence and to \\nchange the speed. The animation for graphical exploration of a data set \\nmust have a \\\\'ar1ety of controlling functions. We have to think about the \\nwhole range of interactions that cartographic animation requires. \\nAnimation techniques \\nWe also need more knowledge of the different animation techniques and \\nfor what types of animation they work best. Our experience in this area is \\nlimited because only a few cartographic animations ha,·e been realized. \\nGersmehl (1990) has examined this issue and describes nine animation \\nmetaphors and their use. Further work should expand this subject.\", metadata={'source': 'Doc1.pdf', 'page': 7}),\n",
              " Document(page_content='develop methods for assessing and measuring uncertainty before we can \\nrepresent it. This latter topic, howe\\\\\\'er , will not be addressed here. \\nVaried goals and needs -categories of interaction with data \\nIf we continue to attack cartographic questions with our communication \\nmodel \\\\\\'isors on, \\\\.Ve will fail to take advantage of the power that GIS and \\nvisualization tools provide. The search for the \"optimal\" data quality \\nvisualization tool might prove as fruitless as the search for the optimal \\ngraduated circle map. lt is critical to recognize that GIS and visualization \\ntools attached to them are used for a range of problem types that may \\nhave quite different visualization needs in general, and visualization', metadata={'source': 'Doc1.pdf', 'page': 12}),\n",
              " Document(page_content='(fig. 5). \\nFog-The transparency of the \"atmosphere\" thnt an analyst views a \\nmap through can be controlled on some computer display devices. lt is \\npossible to create what, in effect, looks like a fog passing between the \\nanalyst and the map -the thicker the fog, the more uncertain that part of \\nthe map (fig. 6).3 \\nResolution-Often maps are produced in which attribute data, geo\\xad\\ngraphic position, and temporal position are depicted with very different \\nresolutions. One method of communicating uncertainty would be to \\nadjust the resolution of geographic detail so that it corresponds to that of \\nattributes or time (e.g., adjust resolution with which coastlines are de\\xad\\npicted on a world map to correspond to the resolution of thematic infor\\xad\\nmation depicted) (fig. 7). \\nLinking visualization tools to models of uncertainty \\nDifferent uncertainty visualization issues will arise when dealing with \\ndifferent kinds of data (e.g., qualitative data on land use/land cover', metadata={'source': 'Doc1.pdf', 'page': 14})]"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# importing open-source Flash-Rank which uses natural language to filter out irrelevant chunks\n",
        "from flashrank import Ranker, RerankRequest\n",
        "ranker = Ranker()"
      ],
      "metadata": {
        "id": "dTVQiMdCKxTe"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Converting into required list to be passed into Flashrank.\n",
        "passages = []\n",
        "for element in retrieves:\n",
        "  passage = {\n",
        "    \"text\": element.page_content,\n",
        "    \"meta\": element.metadata\n",
        "  }\n",
        "  passages.append(passage)"
      ],
      "metadata": {
        "id": "yFH3IKofK77B"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## After using Flash Rank we can see the context which contains answer to our question is placed as the first chunk. Hence we can confidently pass it into Chat-GPT"
      ],
      "metadata": {
        "id": "2VllzAnfPS_9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Using Flash Rank to Re - Rank the chunks\n",
        "rerankrequest = RerankRequest(query=what_to_ask, passages=passages)\n",
        "results = ranker.rerank(rerankrequest)\n",
        "results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XDH2a9rALFaA",
        "outputId": "d8d80701-f585-460f-fe20-cae56d8b425b"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'text': 'tools allo\"v our visual and cognitive processes to almost automatically \\nfocus on the patterns depicted rather than on mentally generating those \\npatterns. \\nFollowing from the above conception of visualization, a research \\nagenda to address visualizing uncertain information should include \\nattention to the cognitive issues of what it means to understand attribute, \\nspatial, and temporal uncertainty and the implications of this understand\\xad\\ning for decision making and for symbolizing and categorizing uncertainty. \\nAt the most basic level, uncertainty can be divided into two components \\nthat might require different visualization strategies: \\\\\\'isualizing accuracy \\nand visualizing precision. In addition, attention should be directed toward \\nthe methodological, technical, and ergonomic issue:; of generating dis\\xad\\nplays and creating interfaces that work. It is, of course, also essential to \\ndevelop methods for assessing and measuring uncertainty before we can',\n",
              "  'meta': {'source': 'Doc1.pdf', 'page': 12},\n",
              "  'score': 0.7468776},\n",
              " {'text': \"8 cartographic perspectives Number 13, Fall 1992 \\nREFERE. CES e\\\\'\\\\' approaches and methods in spatial sciences such as the strong \\nemphasis on examining processes and correlations and the use of scientific \\nvisualization require a more dynamic and user-oriented form of informa\\xad\\ntion display that is not possible ,,·ith the printed map. Also, the map users \\nof the future, the children of today, who are more accustomed to interac\\xad\\ntive computers, will probably ask for more dynamic and interactive forms \\nof information display. \\nIn spite of the need for computer animation in cartography, it has been \\nneglected until recently. There arc two main reasons for this. The first is \\nthe fixation on the traditional printed map that is strongly present within \\ncartography. As a result of this thinking, other and perhaps better forms \\nof information display such as computer animation ha\\\\'e been overlooked. \\nThe second reason is the absence of a comprehensive approach to carto\\xad\",\n",
              "  'meta': {'source': 'Doc1.pdf', 'page': 8},\n",
              "  'score': 0.08601928},\n",
              " {'text': 'Number 13, Fall 1992 cartographic perspectives 13 \\nquality needs specifically. David \\nDiBiasc (1990) recently developed \\na graphic model of the range of \\nuses to which graphics might be \\nput in scientific research (fig. 2). I \\nbelieve that his basic model is \\nrelevant, not only to science, but to \\napplied spatial decision making \\nwith a GIS. \\nAs we begin to consider the \\nvisualization of w1certainty , we \\nneed to be cognizant of this range \\nof visualization goals and environ\\xad\\nments and the varying information \\nrequirement s it implies. The kind \\nof uncertainty and the tools used \\nto visualize it are likely to vary \\nacross this range, from the use of \\nGIS by an EPA scientist exploring \\nthe spatial distribution of a pollut\\xad\\nant to the use of GIS driven map \\ndisplays by policy makers trying to \\ndecide which industrie s to add to \\nthe list of those regulated for toxic \\nwaste emission. \\nGraphics variables \\nBecause few GIS users are trained in cartographic symbolization and',\n",
              "  'meta': {'source': 'Doc1.pdf', 'page': 13},\n",
              "  'score': 0.05499399},\n",
              " {'text': \"phenomena. Scientists need a representation form that can show all the \\ncorrelations. Static maps are not the best presentation form for displaying \\nthese relationship s. Often, maps are overloaded with information layers \\nto show the many correlations. ln an animated map sequence, map \\nelements can be presented in different orders and combinations to make \\nthe spatial relationships more apparent. The map user can be directed \\nthrough the presented subject, and the correlations can be brought to the \\nuser's attention. \\nIn the late 1980s, the sciences discovered scientific visualization. It is \\nused for data analysis to see patterns that either answer questions or that \\npose new and unexpected questions. Scientific visualization requires \\ncomputer animation, especially interactive animation, that can show the Cartographic \\nAnimation: \\nPotential and \\nResearch \\nIssues \\nDoris Karl \\nTHE EEO FOR ANIMA TIO \\nIN CARTOGRAPHY \\nDoris Karl is a stude11t at the \\nFreie Universitiit Berlin,\",\n",
              "  'meta': {'source': 'Doc1.pdf', 'page': 3},\n",
              "  'score': 0.05321406},\n",
              " {'text': 'develop methods for assessing and measuring uncertainty before we can \\nrepresent it. This latter topic, howe\\\\\\'er , will not be addressed here. \\nVaried goals and needs -categories of interaction with data \\nIf we continue to attack cartographic questions with our communication \\nmodel \\\\\\'isors on, \\\\.Ve will fail to take advantage of the power that GIS and \\nvisualization tools provide. The search for the \"optimal\" data quality \\nvisualization tool might prove as fruitless as the search for the optimal \\ngraduated circle map. lt is critical to recognize that GIS and visualization \\ntools attached to them are used for a range of problem types that may \\nhave quite different visualization needs in general, and visualization',\n",
              "  'meta': {'source': 'Doc1.pdf', 'page': 12},\n",
              "  'score': 0.042233557},\n",
              " {'text': '(fig. 5). \\nFog-The transparency of the \"atmosphere\" thnt an analyst views a \\nmap through can be controlled on some computer display devices. lt is \\npossible to create what, in effect, looks like a fog passing between the \\nanalyst and the map -the thicker the fog, the more uncertain that part of \\nthe map (fig. 6).3 \\nResolution-Often maps are produced in which attribute data, geo\\xad\\ngraphic position, and temporal position are depicted with very different \\nresolutions. One method of communicating uncertainty would be to \\nadjust the resolution of geographic detail so that it corresponds to that of \\nattributes or time (e.g., adjust resolution with which coastlines are de\\xad\\npicted on a world map to correspond to the resolution of thematic infor\\xad\\nmation depicted) (fig. 7). \\nLinking visualization tools to models of uncertainty \\nDifferent uncertainty visualization issues will arise when dealing with \\ndifferent kinds of data (e.g., qualitative data on land use/land cover',\n",
              "  'meta': {'source': 'Doc1.pdf', 'page': 14},\n",
              "  'score': 0.0036402822},\n",
              " {'text': \"the flexibility of data manipulation that makes GIS so powerful can lead to \\nconsiderable uncertainty in map displays produced at various stages of \\nG!S analysi .... This paper addresses a variety of conceptual issues underly\\xad\\ning de\\\\'elopment of \\\\'i<;ualization tools that allow analyo.,t<; to take this \\nuncertainty into account in their research and policy formulation acti,·i\\xad\\nties. \\nThere io., a o.,trong tradition in cartography of attention to data quality. \\nOnly rudimentary steps, however, have been made thus far to deal \\\\·\\\\'ith \\nthe complex issues of \\\\'isualizing data quality for multidimensional data \\ndisplays used in image analysis and GIS application<;. The importance of \\nthis topic is evidenced by the decision of the National Center for Geo\\xad\\ngraphic Information and Analysis(. CGIA) to make visualiza tion of data \\nquality the first visualization initiative undertak en by the center. \\nKate Beard and Barbara Buttenfield (1991 ), presenting the CGIA\",\n",
              "  'meta': {'source': 'Doc1.pdf', 'page': 10},\n",
              "  'score': 0.0026656615},\n",
              " {'text': \"10 cartographic perspectives Number 13, Fall 1992 \\nAlan M. MacEachren \\nAln11 M. MncEnchre11 is n \\nProfe:;sor i11 the Department of \\nGcogrnphy, 302 Wn/ka B11ildi11g, The \\nPc1111sylm11in Stntc U11hicrsity, \\nU111per-.1ftf Pnrk, PA 16802. \\nc-mnil: \\\\JYB'a PSUVM.PSU.EOU \\nWhen n GJS is used to compile, \\nnnalyz.e, nnd display infor111n\\xad\\ntio11, the chance for 111zaccept\\xad\\nable or variable data quality is \\nhigh due to the merging of \\n11111/tiple data layers. Visualizing Uncertain Information \\nWhen a GJS is used to drive map-based visualization, exploration of \\npotential relationships takes precedence over presentation of facts. In \\nthese early stages of scientific analysis or policy formulation, providing \\na way for analysts to assess uncertainty in the data they are exploring is \\ncritical to the perspectives they form and the approaches they decide to \\npursue. As a basis from which to develop methods for visualizing \\nuncertain information , this paper addresses the difference between data\",\n",
              "  'meta': {'source': 'Doc1.pdf', 'page': 10},\n",
              "  'score': 0.00052375725},\n",
              " {'text': 'metaphors and their use. Further work should expand this subject. \\nCartographic animation requires a variety of research on design, \\ncreation, and use of animated map sequences. Because of the potential of \\nanimation for cartography, more attention to research is required in this \\nfield. \\nThe potential of computer animation has already been realized by differ\\xad\\nent disciplines. It can also be used for cartography as a powerful visual\\xad\\nization tool that transcends the potential of the printed map. With anima\\xad\\ntion, dynamic and interactive map sequences can be created that show \\nchanges over time or can be used to depict or explore spatial and statisti\\xad\\ncal relationships. CO CLUSIO>J',\n",
              "  'meta': {'source': 'Doc1.pdf', 'page': 7},\n",
              "  'score': 0.0004069523},\n",
              " {'text': \"the potential to compose complex animation sequences. Ideas for this \\nhave been suggested by Monmonier (1989). A second and very important \\npoint is the interactivity of an animation. If animation is to be more than \\njust a film, it must have the possibility of interaction. Animation systems \\nshould have the minimum ability to stop and restart the sequence and to \\nchange the speed. The animation for graphical exploration of a data set \\nmust have a \\\\'ar1ety of controlling functions. We have to think about the \\nwhole range of interactions that cartographic animation requires. \\nAnimation techniques \\nWe also need more knowledge of the different animation techniques and \\nfor what types of animation they work best. Our experience in this area is \\nlimited because only a few cartographic animations ha,·e been realized. \\nGersmehl (1990) has examined this issue and describes nine animation \\nmetaphors and their use. Further work should expand this subject.\",\n",
              "  'meta': {'source': 'Doc1.pdf', 'page': 7},\n",
              "  'score': 4.8687696e-05}]"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Extracting only the first 3 chunks from the Re- Ranked List\n",
        "\n",
        "texts = [item['text'] for item in results[:3]]\n",
        "\n",
        "#Converting into strings\n",
        "\n",
        "concatenated_text = '\\n'.join(texts)\n",
        "\n",
        "print(concatenated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KpR-GUR6LJGZ",
        "outputId": "0d73576e-94fd-418c-d861-f5c354f89a5b"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tools allo\"v our visual and cognitive processes to almost automatically \n",
            "focus on the patterns depicted rather than on mentally generating those \n",
            "patterns. \n",
            "Following from the above conception of visualization, a research \n",
            "agenda to address visualizing uncertain information should include \n",
            "attention to the cognitive issues of what it means to understand attribute, \n",
            "spatial, and temporal uncertainty and the implications of this understand­\n",
            "ing for decision making and for symbolizing and categorizing uncertainty. \n",
            "At the most basic level, uncertainty can be divided into two components \n",
            "that might require different visualization strategies: \\'isualizing accuracy \n",
            "and visualizing precision. In addition, attention should be directed toward \n",
            "the methodological, technical, and ergonomic issue:; of generating dis­\n",
            "plays and creating interfaces that work. It is, of course, also essential to \n",
            "develop methods for assessing and measuring uncertainty before we can\n",
            "8 cartographic perspectives Number 13, Fall 1992 \n",
            "REFERE. CES e\\'\\' approaches and methods in spatial sciences such as the strong \n",
            "emphasis on examining processes and correlations and the use of scientific \n",
            "visualization require a more dynamic and user-oriented form of informa­\n",
            "tion display that is not possible ,,·ith the printed map. Also, the map users \n",
            "of the future, the children of today, who are more accustomed to interac­\n",
            "tive computers, will probably ask for more dynamic and interactive forms \n",
            "of information display. \n",
            "In spite of the need for computer animation in cartography, it has been \n",
            "neglected until recently. There arc two main reasons for this. The first is \n",
            "the fixation on the traditional printed map that is strongly present within \n",
            "cartography. As a result of this thinking, other and perhaps better forms \n",
            "of information display such as computer animation ha\\'e been overlooked. \n",
            "The second reason is the absence of a comprehensive approach to carto­\n",
            "Number 13, Fall 1992 cartographic perspectives 13 \n",
            "quality needs specifically. David \n",
            "DiBiasc (1990) recently developed \n",
            "a graphic model of the range of \n",
            "uses to which graphics might be \n",
            "put in scientific research (fig. 2). I \n",
            "believe that his basic model is \n",
            "relevant, not only to science, but to \n",
            "applied spatial decision making \n",
            "with a GIS. \n",
            "As we begin to consider the \n",
            "visualization of w1certainty , we \n",
            "need to be cognizant of this range \n",
            "of visualization goals and environ­\n",
            "ments and the varying information \n",
            "requirement s it implies. The kind \n",
            "of uncertainty and the tools used \n",
            "to visualize it are likely to vary \n",
            "across this range, from the use of \n",
            "GIS by an EPA scientist exploring \n",
            "the spatial distribution of a pollut­\n",
            "ant to the use of GIS driven map \n",
            "displays by policy makers trying to \n",
            "decide which industrie s to add to \n",
            "the list of those regulated for toxic \n",
            "waste emission. \n",
            "Graphics variables \n",
            "Because few GIS users are trained in cartographic symbolization and\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Instuction to be passed to the chatgpt.\n",
        "instruction=\"Answer the user quesestion as accuractely as possible, based on the following context :\"+concatenated_text"
      ],
      "metadata": {
        "id": "ukoQOVmQMX6d"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Using RAPID API's free Chatgpt API to get answer based on user query.\n",
        "\n",
        "# You can get your key from here : https://rapidapi.com/haxednet/api/chatgpt-api8\n",
        "import requests\n",
        "\n",
        "url = \"https://chatgpt-api8.p.rapidapi.com/\"\n",
        "\n",
        "payload = [\n",
        "\t{\n",
        "\t\t\"content\": instruction,\n",
        "\t\t\"role\": \"system\"\n",
        "\t},\n",
        "\t{\n",
        "\t\t\"content\": what_to_ask,\n",
        "\t\t\"role\": \"user\"\n",
        "\t}\n",
        "]\n",
        "headers = {\n",
        "\t\"content-type\": \"...\",\n",
        "\t\"X-RapidAPI-Key\": \"...\",\n",
        "\t\"X-RapidAPI-Host\": \"...\"\n",
        "}\n",
        "\n",
        "\n",
        "response = requests.post(url, json=payload, headers=headers)\n",
        "\n",
        "print(response.json())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9Tzp8BMaMn7M",
        "outputId": "e19ecb21-59d8-4427-a58c-1b1f987fef2a"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'text': 'Visualization refers to the process of creating visual representations of data or information. In the context of uncertain information, visualization involves using tools to automatically focus on patterns depicted rather than mentally generating those patterns. It helps in understanding attributes, spatial, and temporal uncertainty, which can be divided into visualizing accuracy and visualizing precision. Visualization also involves creating dynamic and user-oriented forms of information display, especially important in the realm of computer animation in cartography.', 'finish_reason': 'stop', 'model': 'gpt-3.5-turbo-030', 'server': 'backup-K'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## The response we get is very accurate to the answer from the pdf. (Page 12 - 3rd Paragraph)"
      ],
      "metadata": {
        "id": "Qgv7hLCwPtws"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(response.json()['text'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9m_tWqAmMqSI",
        "outputId": "705c5498-b024-46ef-ba89-54d4f2f574ba"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Visualization refers to the process of creating visual representations of data or information. In the context of uncertain information, visualization involves using tools to automatically focus on patterns depicted rather than mentally generating those patterns. It helps in understanding attributes, spatial, and temporal uncertainty, which can be divided into visualizing accuracy and visualizing precision. Visualization also involves creating dynamic and user-oriented forms of information display, especially important in the realm of computer animation in cartography.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "prKMpKbjMskd"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}