TLM-SimpleQA-Benchmark/evaluate_response.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2a62d828-e8bc-471c-aa4e-e47a3441a01e",
   "metadata": {},
   "source": [
    "# Evaluate LLM Responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "651cc58f-df2f-42b3-aaf5-46ae2fde6317",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"<openai-api-key\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a13ba28c-3f89-489a-bcd3-d49c4aeccd04",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random \n",
    "import re\n",
    "import pandas as pd\n",
    "\n",
    "import openai_utils.common as common\n",
    "from openai_utils.types import Eval, EvalResult, SamplerBase, SingleEvalResult\n",
    "from openai_utils.chat_completion_sampler import ChatCompletionSampler\n",
    "from openai_utils.simpleqa_constants import GRADER_TEMPLATE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fcd3609-3281-497a-94e6-cd3b9b075535",
   "metadata": {},
   "source": [
    "## Read LLM responeses and use an LLM to evaluate them\n",
    "\n",
    "The `SimpleQAEval` class below was obtained from https://github.com/openai/simple-evals/blob/main/simpleqa_eval.py, and then slightly modified so that it directly reads the prompt-response pairs from a file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "69f55c0a-8b43-4f5e-8072-825ae735962e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleQAEval(Eval):\n",
    "    def __init__(self, grader_model: SamplerBase, dataset: pd.DataFrame, num_examples: int | None = None, n_repeats: int = 1):\n",
    "        examples = [row.to_dict() for _, row in dataset.iterrows()]\n",
    "        if num_examples:\n",
    "            assert n_repeats == 1, \"n_repeats only supported when max_examples = None\"\n",
    "            rng = random.Random(0)\n",
    "            examples = rng.sample(examples, num_examples)\n",
    "        self.examples = examples * n_repeats\n",
    "        self.grader_model = grader_model\n",
    "\n",
    "    def grade_sample(self, question: str, target: str, predicted_answer: str) -> str:\n",
    "        grader_prompt = GRADER_TEMPLATE.format(\n",
    "            question=question,\n",
    "            target=target,\n",
    "            predicted_answer=predicted_answer,\n",
    "        )\n",
    "        \n",
    "        prompt_messages = [\n",
    "            self.grader_model._pack_message(content=grader_prompt, role=\"user\")\n",
    "        ]\n",
    "        grading_response = self.grader_model(prompt_messages)\n",
    "        \n",
    "        match = re.search(r\"(A|B|C)\", grading_response)\n",
    "        return match.group(0) if match else \"C\"  # Default to \"NOT_ATTEMPTED\" if no match\n",
    "\n",
    "    def __call__(self) -> EvalResult:\n",
    "            def fn(row: dict):\n",
    "                prompt_messages = [\n",
    "                    self.grader_model._pack_message(content=row.get(\"problem\", \"\"), role=\"user\")\n",
    "                ]\n",
    "                response_text = row.get(\"response\", \"\")\n",
    "                grade_letter = self.grade_sample(row.get(\"problem\", \"\"), row.get(\"answer\", \"\"), response_text)\n",
    "                \n",
    "                # Metrics based on grading response\n",
    "                is_correct = grade_letter == \"A\"\n",
    "                is_incorrect = grade_letter == \"B\"\n",
    "                is_not_attempted = grade_letter == \"C\"\n",
    "                \n",
    "                score = is_correct\n",
    "\n",
    "                # Create HTML for each sample result\n",
    "                html = common.jinja_env.from_string(common.HTML_JINJA).render(\n",
    "                    prompt_messages=prompt_messages,\n",
    "                    next_message=dict(content=response_text, role=\"assistant\"),\n",
    "                    score=score,\n",
    "                    correct_answer=row[\"answer\"],\n",
    "                    extracted_answer=response_text,\n",
    "                )\n",
    "                convo = prompt_messages + [dict(content=response_text, role=\"assistant\")]\n",
    "                return SingleEvalResult(html=html, score=score, convo=convo, metrics={\n",
    "                    \"is_correct\": is_correct,\n",
    "                    \"is_incorrect\": is_incorrect,\n",
    "                    \"is_not_attempted\": is_not_attempted\n",
    "                })\n",
    "\n",
    "            # Run evaluation and collect results\n",
    "            results = common.map_with_progress(fn, self.examples)\n",
    "\n",
    "            # Aggregate metrics\n",
    "            aggregate_metrics = {\n",
    "                \"is_correct\": sum(result.metrics[\"is_correct\"] for result in results) / len(results),\n",
    "                \"is_incorrect\": sum(result.metrics[\"is_incorrect\"] for result in results) / len(results),\n",
    "                \"is_not_attempted\": sum(result.metrics[\"is_not_attempted\"] for result in results) / len(results),\n",
    "            }\n",
    "            aggregate_metrics[\"is_given_attempted\"] = aggregate_metrics[\"is_correct\"] + aggregate_metrics[\"is_incorrect\"]\n",
    "            # Calculate accuracy_given_attempted\n",
    "            aggregate_metrics[\"accuracy_given_attempted\"] = (\n",
    "                aggregate_metrics[\"is_correct\"]\n",
    "                / aggregate_metrics[\"is_given_attempted\"]\n",
    "                if aggregate_metrics[\"is_given_attempted\"] > 0\n",
    "                else 0\n",
    "            )\n",
    "            print(\"AGGREGATE METRICS\") \n",
    "            print(aggregate_metrics) \n",
    "            print(\"##################\")\n",
    "\n",
    "            output_d = {\n",
    "                \"accuracy_given_attempted\": aggregate_metrics[\"accuracy_given_attempted\"],\n",
    "                \"f1\": (\n",
    "                    2 * aggregate_metrics[\"accuracy_given_attempted\"] * aggregate_metrics[\"is_correct\"]\n",
    "                    / (aggregate_metrics[\"accuracy_given_attempted\"] + aggregate_metrics[\"is_correct\"])\n",
    "                    if (aggregate_metrics[\"accuracy_given_attempted\"] + aggregate_metrics[\"is_correct\"]) > 0\n",
    "                    else 0\n",
    "                )\n",
    "            }\n",
    "            \n",
    "            print(f\"Accuracy Given Attempted: {output_d['accuracy_given_attempted']:.3f}\")\n",
    "            print(f\"F1 Score: {output_d['f1']:.3f}\")\n",
    "            \n",
    "            return common.aggregate_results(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2954184a-2b21-43fa-819a-72ff5b27fb35",
   "metadata": {},
   "source": [
    "In the next cell we show the evaluation for the GPT-4o baseline responses. Here we only run the evaluation on the first 10 examples as a sample, to run it on all examples, set `num_examples` to None)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b015b3c6-8b0d-4f76-9fb9-75175d95d983",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:00<00:00, 12.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AGGREGATE METRICS\n",
      "{'is_correct': 0.3, 'is_incorrect': 0.7, 'is_not_attempted': 0.0, 'is_given_attempted': 1.0, 'accuracy_given_attempted': 0.3}\n",
      "##################\n",
      "Accuracy Given Attempted: 0.300\n",
      "F1 Score: 0.300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "gpt_4o_baseline = pd.read_csv(\"results/gpt-4o-baseline-responses.csv\")\n",
    "grading_sampler = ChatCompletionSampler(model=\"gpt-4o\")\n",
    "\n",
    "simple_qa_eval = SimpleQAEval(grading_sampler, dataset=gpt_4o_baseline, num_examples=10)\n",
    "res = simple_qa_eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f222b44-91f3-4ae4-810f-60cf525ba74e",
   "metadata": {},
   "source": [
    "Execute the function below to evaluate all datasets generated in the [get_tlm_response.ipynb](get_tlm_response.ipynb) script."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c3f64dbb-8f85-4d04-9401-ccf532428335",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_all_datasets():\n",
    "    dataset_list = [\n",
    "        \"results/gpt-4o-baseline-responses.csv\",\n",
    "        \"results/gpt-4o-baseline-25-responses.csv\",\n",
    "        \"results/gpt-4o-baseline-80-responses.csv\",\n",
    "        \"results/gpt-4o-best-responses.csv\",\n",
    "        \"results/gpt-4o-best-25-responses.csv\",\n",
    "        \"results/gpt-4o-best-80-responses.csv\",\n",
    "    ]\n",
    "\n",
    "    grading_sampler = ChatCompletionSampler(model=\"gpt-4o\")\n",
    "\n",
    "    for dataset in dataset_list:\n",
    "        print(dataset)\n",
    "        df = pd.read_csv(dataset)\n",
    "        simple_qa_eval = SimpleQAEval(grading_sampler, dataset=df)\n",
    "        res = simple_qa_eval()\n",
    "        print()\n",
    "\n",
    "# evaluate_all_datasets()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}