evaluate_langchain_rag_pipeline_beyondllm.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i9ge7Vk6G8ZV"
      },
      "source": [
        "# BeyondLLM integration with LangChain\n",
        "\n",
        "This notebook shows the integration of BeyondLLM with LangChain. By combining the strengths of these two tools, we demonstrate how to create and evaluate a simple document retrieval and question-answering system powered by Retrieval-Augmented Generation (RAG)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "De8sHJmPEbBH"
      },
      "source": [
        "# Install necessary packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "L18fYAOPJzZX",
        "outputId": "5777f2cb-627d-4253-891f-7814f546cce9"
      },
      "outputs": [],
      "source": [
        "!pip install langchain sentence-transformers chromadb llama-cpp-python langchain_community pypdf langchain-groq\n",
        "!pip install beyondllm\n",
        "!pip install faiss-gpu"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C7j1cysyElA2"
      },
      "source": [
        "# Import necessary libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "id": "kGXeRCdILsha"
      },
      "outputs": [],
      "source": [
        "from langchain_community.document_loaders import PyPDFDirectoryLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.vectorstores import FAISS\n",
        "from langchain_groq import ChatGroq\n",
        "from langchain.schema.runnable import RunnablePassthrough\n",
        "from langchain.schema.output_parser import StrOutputParser\n",
        "from langchain_core.prompts import ChatPromptTemplate\n",
        "from beyondllm.utils import CONTEXT_RELEVENCE, GROUNDEDNESS, ANSWER_RELEVENCE\n",
        "import re\n",
        "import numpy as np\n",
        "import pysbd"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NYjv7E6ZEz8A"
      },
      "source": [
        "# Set your API keys here"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 126,
      "metadata": {
        "id": "_u2X2AvFgkun"
      },
      "outputs": [],
      "source": [
        "GROQ_API_KEY = \"<your groq api key>\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SKx3mdzNE_9S"
      },
      "source": [
        "# Load PDF documents from a directory"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 99,
      "metadata": {
        "id": "bOrnY8dgMdxt"
      },
      "outputs": [],
      "source": [
        "loader = PyPDFDirectoryLoader(\"/content/sample_data/Data\")\n",
        "docs = loader.load()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0yoEe5e9FC4q"
      },
      "source": [
        "# Split documents into manageable chunks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 100,
      "metadata": {
        "id": "Bouw0M5ANVqL"
      },
      "outputs": [],
      "source": [
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=756, chunk_overlap=50)\n",
        "chunks = text_splitter.split_documents(docs)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NxAFovHFFGtR"
      },
      "source": [
        "# Create embeddings for the document chunks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 101,
      "metadata": {
        "collapsed": true,
        "id": "QcUdFP8cPz1L"
      },
      "outputs": [],
      "source": [
        "embeddings = HuggingFaceEmbeddings(model_name=\"BAAI/bge-base-en-v1.5\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 102,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U4EC5DNopmVQ",
        "outputId": "28233885-90b6-456f-f932-077717131d08"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "client=SentenceTransformer(\n",
            "  (0): Transformer({'max_seq_length': 512, 'do_lower_case': True}) with Transformer model: BertModel \n",
            "  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': True, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})\n",
            "  (2): Normalize()\n",
            ") model_name='BAAI/bge-base-en-v1.5' cache_folder=None model_kwargs={} encode_kwargs={} multi_process=False show_progress=False\n"
          ]
        }
      ],
      "source": [
        "print(embeddings)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oezIkv-pFJw9"
      },
      "source": [
        "# Create a vector store from the document chunks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 103,
      "metadata": {
        "id": "c2ZmC8zq1PW5"
      },
      "outputs": [],
      "source": [
        "vectorstore = FAISS.from_documents(chunks, embeddings)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 104,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FmJth21AQx8V",
        "outputId": "a3660645-8a61-4703-b520-395848069994"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<langchain_community.vectorstores.faiss.FAISS at 0x7926724f1f30>"
            ]
          },
          "execution_count": 104,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "vectorstore"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 105,
      "metadata": {
        "id": "MxUNoeFeR7-u"
      },
      "outputs": [],
      "source": [
        "query = \"what causes heart diseases\"\n",
        "search = vectorstore.similarity_search(query)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hOJQAq89FPpO"
      },
      "source": [
        "# Set up the retriever for similarity search\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 106,
      "metadata": {
        "id": "rK67OkSQRbnk"
      },
      "outputs": [],
      "source": [
        "retriever = vectorstore.as_retriever(\n",
        "    search_kwargs={'k': 3}\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 107,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "HC4Xly8XSf26",
        "outputId": "62a2f3b7-1881-451e-bd30-f7a1b613100a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[Document(metadata={'source': '/content/sample_data/Data/healthyheart.pdf', 'page': 7}, page_content='What Is Heart Disease? \\nCoronary heart disease—often simply called heart disease—occurs\\nwhen the arteries that supply blood to the heart muscle becomehardened and narrowed due to a buildup of plaque on the arteries’inner walls. Plaque is the accumulation of fat, cholesterol, and othersubstances. As plaque continues to build up in the arteries, bloodflow to the heart is reduced.\\nHeart disease can lead to a heart attack. A heart attack happens\\nwhen an artery becomes totally blocked with plaque, preventingvital oxygen and nutrients from getting to the heart. A heart attackcan cause permanent damage to the heart muscle.\\nHeart disease is one of several cardiovascular diseases, which are'),\n",
              " Document(metadata={'source': '/content/sample_data/Data/healthyheart.pdf', 'page': 40}, page_content='36\\nA number of other factors affect heart disease, including certain\\nhealth conditions, medicines, and other substances. Here is whatyou need to know: \\nStress'),\n",
              " Document(metadata={'source': '/content/sample_data/Data/healthyheart.pdf', 'page': 7}, page_content='Many people die of complications\\nfrom heart disease, or become \\npermanently disabled. That’s\\nwhy it is so vital to take action to \\nprevent this disease.3What You Need To Know About Heart DiseaseWhat You Need To Know\\nAbout Heart Disease')]"
            ]
          },
          "execution_count": 107,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "retriever.invoke(query)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1LukLTD9FUL0"
      },
      "source": [
        "# Initialize the language model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 117,
      "metadata": {
        "id": "2mihcpTwkYkI"
      },
      "outputs": [],
      "source": [
        "from langchain_groq import ChatGroq\n",
        "\n",
        "llm = ChatGroq(\n",
        "    model=\"llama3-8b-8192\",\n",
        "    groq_api_key=GROQ_API_KEY,\n",
        "    temperature=0.1  # Set the temperature to 0.1\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Y2ukOe5TS6u"
      },
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eOm3anhXTwg-"
      },
      "source": [
        "# Define the prompt template for the RAG chain"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 119,
      "metadata": {
        "id": "aBDfjug4U61T"
      },
      "outputs": [],
      "source": [
        "template = \"\"\"\n",
        "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n",
        "You are an AI assistant that follows instruction extremely well. Please be truthful and give direct answers.<|eot_id|><|start_header_id|>user<|end_header_id|>\n",
        "{query}<|eot_id|>\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 120,
      "metadata": {
        "id": "oGSLkCc7U8YI"
      },
      "outputs": [],
      "source": [
        "prompt = ChatPromptTemplate.from_template(template)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-1Nszr8rFwV6"
      },
      "source": [
        "# Define the RAG chain\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 121,
      "metadata": {
        "id": "hLfDSrrzU1kE"
      },
      "outputs": [],
      "source": [
        "rag_chain = (\n",
        "    {\"context\": retriever,  \"query\": RunnablePassthrough()}\n",
        "    | prompt\n",
        "    | llm\n",
        "    | StrOutputParser()\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ABcAASBMF1_X"
      },
      "source": [
        "# Function to extract numbers from the response\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 127,
      "metadata": {
        "id": "jZfGywHHtX4t"
      },
      "outputs": [],
      "source": [
        "def extract_number(response):\n",
        "    match = re.search(r'\\b(10|[0-9]+)\\b', response)\n",
        "    if match:\n",
        "        return int(match.group(0))\n",
        "    return np.nan\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XJndu2jqGD53"
      },
      "source": [
        "# Function to tokenize sentences\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 128,
      "metadata": {
        "id": "KZPZmvw9GCE2"
      },
      "outputs": [],
      "source": [
        "def sent_tokenize(text: str):\n",
        "    seg = pysbd.Segmenter(language=\"en\", clean=False)\n",
        "    return seg.segment(text)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4f6RnnjZIR6e"
      },
      "source": [
        "# Evaluate Langchain RAG using BeyondLLM evals"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Z7t9Jz_GMBH"
      },
      "source": [
        "## Get Context Relevancy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 129,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5MBo4kpUGJsB",
        "outputId": "3f7fd1aa-1725-488f-a71e-a11c8fda686f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Context Relevancy Score: 7.7\n"
          ]
        }
      ],
      "source": [
        "def get_context_relevancy(llm, query, context):\n",
        "    total_score = 0\n",
        "    score_count = 0\n",
        "\n",
        "    for content in context:\n",
        "        score_response = llm.invoke(CONTEXT_RELEVENCE.format(question=query, context=content))\n",
        "\n",
        "        # Access the content attribute directly\n",
        "        score_str = score_response.content\n",
        "\n",
        "        # Accumulate the score\n",
        "        score = float(extract_number(score_str))\n",
        "        total_score += score\n",
        "        score_count += 1\n",
        "\n",
        "    average_score = total_score / score_count if score_count > 0 else 0\n",
        "    return f\"Context Relevancy Score: {round(average_score, 1)}\"\n",
        "\n",
        "# Example query\n",
        "query = \"what causes heart diseases?\"\n",
        "\n",
        "# Retrieve relevant documents based on the user query using the updated method\n",
        "retrieved_docs = retriever.invoke(query)\n",
        "\n",
        "# Prepare the context from the retrieved documents\n",
        "context = [doc.page_content for doc in retrieved_docs]\n",
        "\n",
        "# Get context relevancy score\n",
        "print(get_context_relevancy(llm, query, context))\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "trOlL-V9GblQ"
      },
      "source": [
        "## Get answer relevancy score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 130,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gV0dZC5_5yYi",
        "outputId": "a22ba6db-cbc3-4e27-8032-7d3904c46ada"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Answer Relevancy Score: content='9' response_metadata={'token_usage': {'completion_tokens': 2, 'prompt_tokens': 795, 'total_tokens': 797, 'completion_time': 0.000814979, 'prompt_time': 0.158120416, 'queue_time': None, 'total_time': 0.158935395}, 'model_name': 'llama3-8b-8192', 'system_fingerprint': 'fp_a97cfe35ae', 'finish_reason': 'stop', 'logprobs': None} id='run-03100f78-f53c-4d56-aba1-72afb2621385-0' usage_metadata={'input_tokens': 795, 'output_tokens': 2, 'total_tokens': 797}\n"
          ]
        }
      ],
      "source": [
        "response=rag_chain.invoke(query)\n",
        "answer_relevancy_score = llm.invoke(ANSWER_RELEVENCE.format(question=query, context=response))\n",
        "print(f\"Answer Relevancy Score: {answer_relevancy_score}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HGDK0BjnGduP"
      },
      "source": [
        "## Calculate groundedness score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 125,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iv03YsSV55nc",
        "outputId": "c5678aeb-4cea-472e-cd9e-d9072567712c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Groundedness Score: 7.9\n"
          ]
        }
      ],
      "source": [
        "total_score = 0\n",
        "score_count = 0\n",
        "\n",
        "# Tokenize the response into sentences\n",
        "statements = sent_tokenize(response)\n",
        "\n",
        "for statement in statements:\n",
        "    score_response = llm.invoke(GROUNDEDNESS.format(statement=statement, context=\" \".join(context)))\n",
        "\n",
        "    # Access the content attribute directly\n",
        "    score_str = score_response.content\n",
        "\n",
        "    # Accumulate the score\n",
        "    score = float(extract_number(score_str))\n",
        "    total_score += score\n",
        "    score_count += 1\n",
        "\n",
        "average_groundedness = total_score / score_count if score_count > 0 else 0\n",
        "print(f\"Groundedness Score: {round(average_groundedness, 1)}\")\n",
        "\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}